From: Anthony Geay Date: Mon, 13 Feb 2023 13:34:40 +0000 (+0100) Subject: [EDF26936] : Management of inputs with more than 2 GB X-Git-Url: http://git.salome-platform.org/gitweb/?a=commitdiff_plain;h=bd50032a744db61240dccf11cb359c9a41de8f6b;p=modules%2Fkernel.git [EDF26936] : Management of inputs with more than 2 GB --- diff --git a/idl/SALOME_PyNode.idl b/idl/SALOME_PyNode.idl index 58e9d3898..3a9747995 100644 --- a/idl/SALOME_PyNode.idl +++ b/idl/SALOME_PyNode.idl @@ -92,7 +92,7 @@ module Engines /*! \brief first part of whole execute method. This split is to reduce the memory peak. */ - void executeFirst(in pickledArgs inargs) raises (SALOME::SALOME_Exception); + void executeFirst(in SALOME::SenderByte inargs) raises (SALOME::SALOME_Exception); /*! \brief second and last part of execute method. This split is to reduce the memory peak. */ diff --git a/src/Communication/CMakeLists.txt b/src/Communication/CMakeLists.txt index 3bb29256b..038c4756d 100755 --- a/src/Communication/CMakeLists.txt +++ b/src/Communication/CMakeLists.txt @@ -26,6 +26,7 @@ INCLUDE_DIRECTORIES( ${CMAKE_CURRENT_SOURCE_DIR}/../Basics ${CMAKE_CURRENT_SOURCE_DIR}/../SALOMELocalTrace ${CMAKE_CURRENT_SOURCE_DIR}/../Utils + ${CMAKE_CURRENT_SOURCE_DIR}/../GenericObj ${PROJECT_BINARY_DIR}/idl ) @@ -33,6 +34,7 @@ SET(COMMON_LIBS OpUtil SALOMELocalTrace SalomeIDLKernel + SalomeGenericObj ${PYTHON_LIBRARIES} ${MPI_CXX_LIBRARIES} ) @@ -45,6 +47,7 @@ SET(SalomeCommunication_SOURCES SALOMEMultiComm.cxx ReceiverFactory.cxx MatrixClient.cxx + SenderByteImpl.cxx ) ADD_DEFINITIONS(${OMNIORB_DEFINITIONS} ${MPI_DEFINITIONS}) @@ -64,6 +67,7 @@ SET(COMMON_HEADERS_HXX SALOME_Communication.hxx SALOME_Matrix_i.hxx SenderFactory.hxx + SenderByteImpl.hxx ) INSTALL(FILES ${COMMON_HEADERS_HXX} DESTINATION ${SALOME_INSTALL_HEADERS}) diff --git a/src/Communication/SenderByteImpl.cxx b/src/Communication/SenderByteImpl.cxx new file mode 100644 index 000000000..d79a6d276 --- /dev/null +++ b/src/Communication/SenderByteImpl.cxx @@ -0,0 +1,35 @@ +// Copyright (C) 2023 CEA/DEN, EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +#include "SenderByteImpl.hxx" + +CORBA::ULongLong SenderByteImpl::getSize() +{ + return _size; +} + +SALOME::vectorOfByte *SenderByteImpl::sendPart(CORBA::ULongLong n1, CORBA::ULongLong n2) +{ + SALOME::vectorOfByte *ret = new SALOME::vectorOfByte; + CORBA::ULongLong retSize(n2-n1); + ret->length(retSize); + for(CORBA::ULongLong i = 0 ; i < retSize ; ++i) + (*ret)[i] = _data[n1+i]; + return ret; +} diff --git a/src/Communication/SenderByteImpl.hxx b/src/Communication/SenderByteImpl.hxx new file mode 100644 index 000000000..18b4a41fa --- /dev/null +++ b/src/Communication/SenderByteImpl.hxx @@ -0,0 +1,42 @@ +// Copyright (C) 2023 CEA/DEN, EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +#pragma once + +#include "SALOME_Communication.hxx" + +#include +#include CORBA_SERVER_HEADER(SALOME_Comm) +#include "SALOME_GenericObj_i.hh" + +#include + +class COMMUNICATION_EXPORT SenderByteImpl : public virtual POA_SALOME::SenderByte, + public virtual PortableServer::ServantBase, + public virtual SALOME::GenericObj_i +{ +public: + //! SenderByteImpl instance does not have ownership of data + SenderByteImpl(char *data, std::size_t size):_data(data),_size(size) { } + CORBA::ULongLong getSize() override; + SALOME::vectorOfByte *sendPart(CORBA::ULongLong n1, CORBA::ULongLong n2) override; +private: + char *_data = nullptr; + std::size_t _size = 0; +}; diff --git a/src/Container/SALOME_PyNode.py b/src/Container/SALOME_PyNode.py index abe082376..4279063d0 100644 --- a/src/Container/SALOME_PyNode.py +++ b/src/Container/SALOME_PyNode.py @@ -113,6 +113,31 @@ class SenderByte_i(SALOME__POA.SenderByte,Generic): def sendPart(self,n1,n2): return self.bytesToSend[n1:n2] + +class SeqByteReceiver: + CHUNK_SIZE = 2000000000 + def __init__(self,sender): + self._obj = sender + def __del__(self): + self._obj.UnRegister() + pass + def data(self): + size = self._obj.getSize() + if size <= SeqByteReceiver.CHUNK_SIZE: + return self.fetchOneShot( size ) + else: + return self.fetchByChunks( size ) + def fetchOneShot(self,size): + return self._obj.sendPart(0,size) + def fetchByChunks(self,size): + data_for_split_case = bytes(0) + EFF_CHUNK_SIZE = SeqByteReceiver.CHUNK_SIZE // 8 + iStart = 0 ; iEnd = EFF_CHUNK_SIZE + while iStart!=iEnd and iEnd <= size: + part = self._obj.sendPart(iStart,iEnd) + data_for_split_case = bytes(0).join( [data_for_split_case,part] ) + iStart = iEnd; iEnd = min(iStart + EFF_CHUNK_SIZE,size) + return data_for_split_case class PyScriptNode_i (Engines__POA.PyScriptNode,Generic): """The implementation of the PyScriptNode CORBA IDL that executes a script""" @@ -177,7 +202,11 @@ class PyScriptNode_i (Engines__POA.PyScriptNode,Generic): """ Same than first part of self.execute to reduce memory peak.""" import time try: - _,kws=pickle.loads(argsin) + data = None + if True: # to force call of SeqByteReceiver's destructor + argsInPy = SeqByteReceiver( argsin ) + data = argsInPy.data() + _,kws=pickle.loads(data) self.context.update(kws) except Exception: exc_typ,exc_val,exc_fr=sys.exc_info()