]> SALOME platform Git repositories - modules/kernel.git/commitdiff
Salome HOME
[EDF26936] : Management of inputs with more than 2 GB agy/edf26936
authorAnthony Geay <anthony.geay@edf.fr>
Mon, 13 Feb 2023 13:34:40 +0000 (14:34 +0100)
committerAnthony Geay <anthony.geay@edf.fr>
Mon, 13 Feb 2023 13:34:40 +0000 (14:34 +0100)
idl/SALOME_PyNode.idl
src/Communication/CMakeLists.txt
src/Communication/SenderByteImpl.cxx [new file with mode: 0644]
src/Communication/SenderByteImpl.hxx [new file with mode: 0644]
src/Container/SALOME_PyNode.py

index 58e9d3898fa22f909d55db0660a6e42172246729..3a974799548229ef4c0a7a2a857c3bfdbf9efc01 100644 (file)
@@ -92,7 +92,7 @@ module Engines
 
     /*! \brief first part of whole execute method. This split is to reduce the memory peak.
     */
-    void executeFirst(in pickledArgs inargs) raises (SALOME::SALOME_Exception);
+    void executeFirst(in SALOME::SenderByte inargs) raises (SALOME::SALOME_Exception);
 
     /*! \brief second and last part of execute method. This split is to reduce the memory peak.
     */
index 3bb29256bb05231998292689076bf1e39e2dd434..038c4756d9a5875dbae973d0ad44b46603c10c18 100755 (executable)
@@ -26,6 +26,7 @@ INCLUDE_DIRECTORIES(
   ${CMAKE_CURRENT_SOURCE_DIR}/../Basics
   ${CMAKE_CURRENT_SOURCE_DIR}/../SALOMELocalTrace
   ${CMAKE_CURRENT_SOURCE_DIR}/../Utils
+  ${CMAKE_CURRENT_SOURCE_DIR}/../GenericObj
   ${PROJECT_BINARY_DIR}/idl
 )
 
@@ -33,6 +34,7 @@ SET(COMMON_LIBS
   OpUtil
   SALOMELocalTrace
   SalomeIDLKernel
+  SalomeGenericObj
   ${PYTHON_LIBRARIES}
   ${MPI_CXX_LIBRARIES}
 )
@@ -45,6 +47,7 @@ SET(SalomeCommunication_SOURCES
   SALOMEMultiComm.cxx
   ReceiverFactory.cxx
   MatrixClient.cxx
+  SenderByteImpl.cxx
 )
 
 ADD_DEFINITIONS(${OMNIORB_DEFINITIONS} ${MPI_DEFINITIONS})
@@ -64,6 +67,7 @@ SET(COMMON_HEADERS_HXX
   SALOME_Communication.hxx
   SALOME_Matrix_i.hxx
   SenderFactory.hxx
+  SenderByteImpl.hxx
 )
 
 INSTALL(FILES ${COMMON_HEADERS_HXX} DESTINATION ${SALOME_INSTALL_HEADERS})
diff --git a/src/Communication/SenderByteImpl.cxx b/src/Communication/SenderByteImpl.cxx
new file mode 100644 (file)
index 0000000..d79a6d2
--- /dev/null
@@ -0,0 +1,35 @@
+// Copyright (C) 2023  CEA/DEN, EDF R&D
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+//
+// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
+//
+
+#include "SenderByteImpl.hxx"
+
+CORBA::ULongLong SenderByteImpl::getSize()
+{
+  return _size;
+}
+
+SALOME::vectorOfByte *SenderByteImpl::sendPart(CORBA::ULongLong n1, CORBA::ULongLong n2)
+{
+  SALOME::vectorOfByte *ret = new SALOME::vectorOfByte;
+  CORBA::ULongLong retSize(n2-n1);
+  ret->length(retSize);
+  for(CORBA::ULongLong i = 0 ; i < retSize ; ++i)
+    (*ret)[i] = _data[n1+i];
+  return ret;
+}
diff --git a/src/Communication/SenderByteImpl.hxx b/src/Communication/SenderByteImpl.hxx
new file mode 100644 (file)
index 0000000..18b4a41
--- /dev/null
@@ -0,0 +1,42 @@
+// Copyright (C) 2023  CEA/DEN, EDF R&D
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+//
+// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
+//
+
+#pragma once
+
+#include "SALOME_Communication.hxx"
+
+#include <SALOMEconfig.h>
+#include CORBA_SERVER_HEADER(SALOME_Comm)
+#include "SALOME_GenericObj_i.hh"
+
+#include <cstddef>
+
+class COMMUNICATION_EXPORT SenderByteImpl : public virtual POA_SALOME::SenderByte,
+                                            public virtual PortableServer::ServantBase,
+                                            public virtual SALOME::GenericObj_i
+{
+public:
+  //! SenderByteImpl instance does not have ownership of data
+  SenderByteImpl(char *data, std::size_t size):_data(data),_size(size) { }
+  CORBA::ULongLong getSize() override;
+  SALOME::vectorOfByte *sendPart(CORBA::ULongLong n1, CORBA::ULongLong n2) override;
+private:
+  char *_data = nullptr;
+  std::size_t _size = 0;
+};
index abe0823769403fd1854a36df4a8aaa39e39fbf44..4279063d080f12b85465da4af398b256ea1c4f19 100644 (file)
@@ -113,6 +113,31 @@ class SenderByte_i(SALOME__POA.SenderByte,Generic):
 
   def sendPart(self,n1,n2):
     return self.bytesToSend[n1:n2]
+    
+class SeqByteReceiver:
+  CHUNK_SIZE = 2000000000
+  def __init__(self,sender):
+    self._obj = sender
+  def __del__(self):
+    self._obj.UnRegister()
+    pass
+  def data(self):
+    size = self._obj.getSize()
+    if size <= SeqByteReceiver.CHUNK_SIZE:
+      return self.fetchOneShot( size )
+    else:
+      return self.fetchByChunks( size )
+  def fetchOneShot(self,size):
+    return self._obj.sendPart(0,size)
+  def fetchByChunks(self,size):
+      data_for_split_case = bytes(0)
+      EFF_CHUNK_SIZE = SeqByteReceiver.CHUNK_SIZE // 8
+      iStart = 0 ; iEnd = EFF_CHUNK_SIZE
+      while iStart!=iEnd and iEnd <= size:
+        part = self._obj.sendPart(iStart,iEnd)
+        data_for_split_case = bytes(0).join( [data_for_split_case,part] )
+        iStart = iEnd; iEnd = min(iStart + EFF_CHUNK_SIZE,size)
+      return data_for_split_case
 
 class PyScriptNode_i (Engines__POA.PyScriptNode,Generic):
   """The implementation of the PyScriptNode CORBA IDL that executes a script"""
@@ -177,7 +202,11 @@ class PyScriptNode_i (Engines__POA.PyScriptNode,Generic):
     """ Same than first part of self.execute to reduce memory peak."""
     import time
     try:
-      _,kws=pickle.loads(argsin)
+      data = None
+      if True: # to force call of SeqByteReceiver's destructor
+        argsInPy = SeqByteReceiver( argsin )
+        data = argsInPy.data()
+      _,kws=pickle.loads(data)
       self.context.update(kws)
     except Exception:
       exc_typ,exc_val,exc_fr=sys.exc_info()