SALOME_ComponentPy.py
SALOME_PyNode.py
SALOME_Container.py
+ SALOME_ContainerHelper.py
SALOME_ContainerPy.py
)
from SALOME_NamingServicePy import *
from SALOME_ComponentPy import *
import SALOME_PyNode
-from collections import defaultdict
from SALOME_utilities import *
from Utils_Identity import getShortHostName
from launchConfigureParser import verbose
from KernelBasis import VerbosityActivated
-
-class ScriptExecInfo:
- @classmethod
- def GetRepresentationOfTimeDelta(cls,endTime, startTime):
- if endTime is None and startTime is None:
- return "not measured"
- td = endTime - startTime
- import time
- ts_of_td = time.gmtime(td.total_seconds())
- return "{}.{:06d}".format(time.strftime("%H:%M:%S",ts_of_td),td.microseconds)
-
- def __init__(self):
- self._start_exec_time = None
- self._end_exec_time = None
- self._start_input_time = None
- self._end_input_time = None
- self._start_output_time = None
- self._end_output_time = None
-
- @property
- def startInputTime(self):
- return self._start_input_time
-
- @startInputTime.setter
- def startInputTime(self,value):
- self._start_input_time = value
-
- @property
- def endInputTime(self):
- return self._end_input_time
-
- @endInputTime.setter
- def endInputTime(self,value):
- self._end_input_time = value
-
- @property
- def startExecTime(self):
- return self._start_exec_time
-
- @startExecTime.setter
- def startExecTime(self,value):
- self._start_exec_time = value
-
- @property
- def endExecTime(self):
- return self._end_exec_time
-
- @endExecTime.setter
- def endExecTime(self,value):
- self._end_exec_time = value
-
- @property
- def startOutputTime(self):
- return self._start_output_time
-
- @startOutputTime.setter
- def startOutputTime(self,value):
- self._start_output_time = value
-
- @property
- def endOutputTime(self):
- return self._end_output_time
-
- @endOutputTime.setter
- def endOutputTime(self,value):
- self._end_output_time = value
-
- @property
- def execTimeStr(self):
- return ScriptExecInfo.GetRepresentationOfTimeDelta(self.endExecTime,self.startExecTime)
-
- @property
- def inputTimeStr(self):
- return ScriptExecInfo.GetRepresentationOfTimeDelta(self.endInputTime,self.startInputTime)
-
- def __str__(self):
- return """start exec time = {self.startExecTime}
-end exec time = {self.endExecTime}
-exec_time = {self.execTimeStr}
-input unpickling and ev load from disk time = {self.inputTimeStr}
-output serialization and ev write to disk time = {self.inputTimeStr}""".format(**locals())
-
-class ScriptInfo:
- def __init__(self, nodeName):
- self._node_name = nodeName
- self._code = ""
- self._exec = defaultdict(ScriptExecInfo)
-
- @property
- def execs(self):
- return self._exec
-
- @property
- def nodeName(self):
- return self._node_name
-
- @property
- def code(self):
- return self._code
-
- @code.setter
- def code(self,value):
- self._code = value
-
- def __str__(self):
- return """code = {self.code}\nexecs = {self.execs}""".format(**locals())
-
- def __repr__(self):
- return """ScriptInfo \"{self.nodeName}\"""".format(**locals())
+from SALOME_ContainerHelper import ScriptInfo
#=============================================================================
return 1,"".join(l)
def create_pyscriptnode(self,nodeName,code):
- import pickle
try:
self._dbg_info.append( ScriptInfo(nodeName) )
node=SALOME_PyNode.PyScriptNode_i(nodeName,code,self._poa,self)
except Exception:
exc_typ,exc_val,exc_fr=sys.exc_info()
l=traceback.format_exception(exc_typ,exc_val,exc_fr)
+ print("".join(l)) ; sys.stdout.flush() # print error also in logs of remote container
return 1,"".join(l)
def positionVerbosityOfLogger(self):
--- /dev/null
+# -*- coding: iso-8859-1 -*-
+# Copyright (C) 2023 CEA, EDF
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
+#
+
+from collections import defaultdict
+
+class ScriptExecInfo:
+ @classmethod
+ def GetRepresentationOfTimeDelta(cls,endTime, startTime):
+ if endTime is None and startTime is None:
+ return "not measured"
+ td = endTime - startTime
+ import time
+ ts_of_td = time.gmtime(td.total_seconds())
+ return "{}.{:06d}".format(time.strftime("%H:%M:%S",ts_of_td),td.microseconds)
+
+ @classmethod
+ def MemRepr(cls,memInByte):
+ m = memInByte
+ UNITS=["B","kB","MB","GB"]
+ remain = 0
+ oss = ""
+ for i in range( len(UNITS) ):
+ if m<1024:
+ oss = "{:03d}".format( int( (remain/1024)*1000 ) )
+ oss = "{}.{} {}".format(m,oss,UNITS[i])
+ return oss
+ else:
+ if i!=3:
+ remain = m%1024
+ m//=1024
+ return "{} {}".format(m,UNITS[3])
+
+ def __init__(self):
+ self._start_exec_time = None
+ self._end_exec_time = None
+ self._start_input_time = None
+ self._end_input_time = None
+ self._start_output_time = None
+ self._end_output_time = None
+ self._input_mem = 0
+ self._input_hdd_mem = None
+ self._output_mem = 0
+ self._output_hdd_mem = None
+
+ @property
+ def inputMem(self):
+ return self._input_mem
+
+ @inputMem.setter
+ def inputMem(self,value):
+ self._input_mem = value
+
+ @property
+ def inputMemStr(self):
+ return ScriptExecInfo.MemRepr( self.inputMem )
+
+ @property
+ def outputMem(self):
+ return self._output_mem
+
+ @outputMem.setter
+ def outputMem(self,value):
+ self._output_mem = value
+
+ @property
+ def outputMemStr(self):
+ return ScriptExecInfo.MemRepr( self.outputMem )
+
+ @property
+ def inputHDDMem(self):
+ return self._input_hdd_mem
+
+ @inputHDDMem.setter
+ def inputHDDMem(self,value):
+ self._input_hdd_mem = value
+
+ @property
+ def inputHDDMemStr(self):
+ if self._input_hdd_mem is None:
+ return "not computed"
+ return " ".join( [ ScriptExecInfo.MemRepr( elt.getSizeOfFileRead() ) for elt in self._input_hdd_mem] )
+
+ @property
+ def outputHDDMem(self):
+ return self._output_hdd_mem
+
+ @outputHDDMem.setter
+ def outputHDDMem(self,value):
+ self._output_hdd_mem = value
+
+ @property
+ def outputHDDMemStr(self):
+ if self._output_hdd_mem is None:
+ return "not computed"
+ return " ".join( [ ScriptExecInfo.MemRepr( elt.getSizeOfFileRead() ) for elt in self._output_hdd_mem] )
+
+ @property
+ def startInputTime(self):
+ return self._start_input_time
+
+ @startInputTime.setter
+ def startInputTime(self,value):
+ self._start_input_time = value
+
+ @property
+ def endInputTime(self):
+ return self._end_input_time
+
+ @endInputTime.setter
+ def endInputTime(self,value):
+ self._end_input_time = value
+
+ @property
+ def startExecTime(self):
+ return self._start_exec_time
+
+ @startExecTime.setter
+ def startExecTime(self,value):
+ self._start_exec_time = value
+
+ @property
+ def endExecTime(self):
+ return self._end_exec_time
+
+ @endExecTime.setter
+ def endExecTime(self,value):
+ self._end_exec_time = value
+
+ @property
+ def startOutputTime(self):
+ return self._start_output_time
+
+ @startOutputTime.setter
+ def startOutputTime(self,value):
+ self._start_output_time = value
+
+ @property
+ def endOutputTime(self):
+ return self._end_output_time
+
+ @endOutputTime.setter
+ def endOutputTime(self,value):
+ self._end_output_time = value
+
+ @property
+ def execTimeStr(self):
+ return ScriptExecInfo.GetRepresentationOfTimeDelta(self.endExecTime,self.startExecTime)
+
+ @property
+ def inputTimeStr(self):
+ return ScriptExecInfo.GetRepresentationOfTimeDelta(self.endInputTime,self.startInputTime)
+
+ def __str__(self):
+ return """start exec time = {self.startExecTime}
+end exec time = {self.endExecTime}
+exec_time = {self.execTimeStr}
+input unpickling and ev load from disk time = {self.inputTimeStr}
+output serialization and ev write to disk time = {self.inputTimeStr}
+input memory size before exec (MemoryPeak 2x) = {self.inputMemStr}
+input memory size from HDD = {self.inputHDDMemStr}
+output memory size after exec (MemoryPeak 2x) = {self.outputMemStr}
+output memory size from HDD = {self.outputHDDMemStr}""".format(**locals())
+
+class ScriptInfo:
+ def __init__(self, nodeName):
+ self._node_name = nodeName
+ self._code = ""
+ self._exec = defaultdict(ScriptExecInfo)
+
+ @property
+ def execs(self):
+ return self._exec
+
+ @property
+ def nodeName(self):
+ return self._node_name
+
+ @property
+ def code(self):
+ return self._code
+
+ @code.setter
+ def code(self,value):
+ self._code = value
+
+ def __str__(self):
+ return """code = {self.code}\nexecs = {self.execs}""".format(**locals())
+
+ def __repr__(self):
+ return """ScriptInfo \"{self.nodeName}\"""".format(**locals())
+
+from abc import ABC, abstractmethod
+
+class InOutputObjVisitorAbstract(ABC):
+ def __init__(self):
+ self._cur_obj = None
+ self._data = []
+
+ def enter(self):
+ self._cur_obj = ObjMemModel()
+ return self._cur_obj
+
+ def leave(self):
+ self._data.append( self._cur_obj )
+ self._cur_obj = None
+
+ def getSizeOfFileRead(self):
+ return sum( [elt.getSizeOfFileRead() for elt in self._data] )
+
+ def visitor(self):
+ return self
+
+ def setHDDMem(self, v):
+ pass
+
+ def setFileName(self, fileName):
+ pass
+
+ @abstractmethod
+ def getRepr(self):
+ pass
+
+class InOutputObjVisitorIter:
+ def __init__(self, visitor):
+ self._visitor = visitor
+ self._current = 0
+
+ def __next__(self):
+ if self._current >= len(self._visitor._data):
+ raise StopIteration
+ else:
+ ret = self._visitor._data[ self._current ]
+ self._current += 1
+ return ret
+
+class InOutputObjVisitor(InOutputObjVisitorAbstract):
+ def __init__(self):
+ super().__init__()
+
+ def getRepr(self):
+ return self.getSizeOfFileRead()
+
+ def __iter__(self):
+ return InOutputObjVisitorIter(self)
+
+class ObjMemModel(InOutputObjVisitorAbstract):
+ def __init__(self):
+ super().__init__()
+ self._hdd_mem = 0
+ self._file_name = None
+
+ def setHDDMem(self, v):
+ self._hdd_mem = v
+ del self._data
+
+ def setFileName(self, fileName):
+ self._file_name = fileName
+ pass
+
+ def getSizeOfFileRead(self):
+ if hasattr(self,"_data"):
+ return super().getSizeOfFileRead()
+ else:
+ return self._hdd_mem
+
+ def getRepr(self):
+ return self.getSizeOfFileRead()
+
+class FakeObjVisitor:
+ def setHDDMem(self, v):
+ pass
+
+ def visitor(self):
+ return None
+
+class InOutputObjVisitorCM:
+ def __init__(self, visitor):
+ self._visitor = visitor
+ def __enter__(self):
+ if self._visitor:
+ r = self._visitor.enter()
+ return r
+ else:
+ return FakeObjVisitor()
+ def __exit__(self,exctype, exc, tb):
+ if self._visitor:
+ self._visitor.leave()
+ pass
}
cont->override_environment_python( envCorba );
if( !_code_to_exe_on_startup.empty() )
+ {
+ INFOS("[GiveContainer] container " << containerNameInNS << " python code executed " << _code_to_exe_on_startup);
cont->execute_python_code( _code_to_exe_on_startup.c_str() );
+ }
+ INFOS("[GiveContainer] container " << containerNameInNS << " verbosity positionning Activation = " << SALOME::VerbosityActivated() << " Verbosity Level = " << SALOME::VerbosityLevelStr());
cont->setVerbosity( SALOME::VerbosityActivated(), SALOME::VerbosityLevelStr().c_str() );
return cont._retn();
}
import Engines__POA
import SALOME__POA
import SALOME
+import logging
MY_CONTAINER_ENTRY_IN_GLBS = "my_container"
def GetSizeOfTCnt():
return len( bytes(TypeCounter(0) ) )
-def GetObjectFromFile(fname):
+def GetSizeOfBufferedReader(f):
+ """
+ This method returns in bytes size of a file openned.
+
+ Args:
+ ----
+ f (io.IOBase): buffered reader returned by open
+
+ Returns
+ -------
+ int: number of bytes
+ """
+ import io
+ pos = f.tell()
+ f.seek(0,io.SEEK_END)
+ pos2 = f.tell()
+ f.seek(pos,io.SEEK_SET)
+ return pos2-pos
+
+def GetObjectFromFile(fname, visitor = None):
with open(fname,"rb") as f:
cntb = f.read( GetSizeOfTCnt() )
cnt = TypeCounter.from_buffer_copy( cntb ).value
+ if visitor:
+ visitor.setHDDMem( GetSizeOfBufferedReader(f) )
+ visitor.setFileName( fname )
obj = pickle.load(f)
return obj,cnt
def __dumpIntoFile(self, objSerialized):
DumpInFile( objSerialized, self._filename )
- def get(self):
- obj, _ = GetObjectFromFile( self._filename )
+ def get(self, visitor = None):
+ obj, _ = GetObjectFromFile( self._filename, visitor )
return obj
def __float__(self):
self._pos = pos
self._length = length
- def get(self):
- fullObj = BigObjectOnDiskBase.get(self)
+ def get(self, visitor = None):
+ fullObj = BigObjectOnDiskBase.get(self, visitor)
return fullObj[ self._pos ]
def __getitem__(self, i):
def __init__(self, length, fileName, objSerialized):
BigObjectOnDiskSequence.__init__(self, length, fileName, objSerialized)
-def SpoolPickleObject( obj ):
- import pickle
- pickleObjInit = pickle.dumps( obj , pickle.HIGHEST_PROTOCOL )
- if not ActivateProxyMecanismOrNot( len(pickleObjInit) ):
- return pickleObjInit
+def ProxyfyPickeled( obj, pickleObjInit = None, visitor = None ):
+ """
+ This method return a proxy instance of pickled form of object given in input.
+
+ Args:
+ ----
+ obj (pickelable type) : object to be proxified
+ pickleObjInit (bytes) : Optionnal. Original pickeled form of object to be proxyfied if already computed. If not this method generate it
+
+ Returns
+ -------
+ BigObjectOnDiskBase: proxy instance
+ """
+ pickleObj = pickleObjInit
+ if pickleObj is None:
+ pickleObj = pickle.dumps( obj , pickle.HIGHEST_PROTOCOL )
+ fileName = GetBigObjectFileName()
+ if visitor:
+ visitor.setHDDMem( len(pickleObj) )
+ visitor.setFileName(fileName)
+ if isinstance( obj, list):
+ proxyObj = BigObjectOnDiskList( len(obj), fileName, pickleObj )
+ elif isinstance( obj, tuple):
+ proxyObj = BigObjectOnDiskTuple( len(obj), fileName , pickleObj )
else:
- if isinstance( obj, list):
- proxyObj = BigObjectOnDiskList( len(obj), GetBigObjectFileName() , pickleObjInit )
- elif isinstance( obj, tuple):
- proxyObj = BigObjectOnDiskTuple( len(obj), GetBigObjectFileName() , pickleObjInit )
+ proxyObj = BigObjectOnDisk( fileName , pickleObj )
+ return proxyObj
+
+def SpoolPickleObject( obj, visitor = None ):
+ import pickle
+ with InOutputObjVisitorCM(visitor) as v:
+ pickleObjInit = pickle.dumps( obj , pickle.HIGHEST_PROTOCOL )
+ if not ActivateProxyMecanismOrNot( len(pickleObjInit) ):
+ return pickleObjInit
else:
- proxyObj = BigObjectOnDisk( GetBigObjectFileName() , pickleObjInit )
- pickleProxy = pickle.dumps( proxyObj , pickle.HIGHEST_PROTOCOL )
- return pickleProxy
+ proxyObj = ProxyfyPickeled( obj, pickleObjInit, v.visitor() )
+ pickleProxy = pickle.dumps( proxyObj , pickle.HIGHEST_PROTOCOL )
+ return pickleProxy
+
+from SALOME_ContainerHelper import InOutputObjVisitorCM, InOutputObjVisitor
-def UnProxyObjectSimple( obj ):
+def UnProxyObjectSimple( obj, visitor = None ):
"""
Method to be called in Remote mode. Alterate the obj _status attribute.
Because the slave process does not participate in the reference counting
+
+ Args:
+ ----
+ visitor (InOutputObjVisitor): A visitor to keep track of amount of memory on chip and those on HDD
+
"""
- if isinstance(obj,BigObjectOnDiskBase):
- obj.doNotTouchFile()
- return obj.get()
- elif isinstance( obj, list):
- retObj = []
- for elt in obj:
- retObj.append( UnProxyObjectSimple(elt) )
- return retObj
- else:
- return obj
+ with InOutputObjVisitorCM(visitor) as v:
+ logging.debug( "UnProxyObjectSimple {}".format(type(obj)) )
+ if isinstance(obj,BigObjectOnDiskBase):
+ obj.doNotTouchFile()
+ return obj.get( v )
+ elif isinstance( obj, list):
+ retObj = []
+ for elt in obj:
+ retObj.append( UnProxyObjectSimple(elt,v.visitor()) )
+ return retObj
+ else:
+ return obj
def UnProxyObjectSimpleLocal( obj ):
"""
def executeFirst(self,argsin):
""" Same than first part of self.execute to reduce memory peak."""
- import time
try:
data = None
self.my_container_py.addTimeInfoOnLevel2(self.getIDInContainer(),self._current_exec,"startInputTime")
if True: # to force call of SeqByteReceiver's destructor
argsInPy = SeqByteReceiver( argsin )
data = argsInPy.data()
+ self.my_container_py.addInfoOnLevel2(self.getIDInContainer(),self._current_exec,"inputMem",len(data))
_,kws=pickle.loads(data)
+ vis = InOutputObjVisitor()
for elt in kws:
# fetch real data if necessary
- kws[elt] = UnProxyObjectSimple( kws[elt] )
+ kws[elt] = UnProxyObjectSimple( kws[elt],vis)
+ self.my_container_py.addInfoOnLevel2(self.getIDInContainer(),self._current_exec,"inputHDDMem",vis)
self.context.update(kws)
self.my_container_py.addTimeInfoOnLevel2(self.getIDInContainer(),self._current_exec,"endInputTime")
except Exception:
raise KeyError("There is no variable %s in context" % arg)
argsout.append(self.context[arg])
ret = [ ]
+ outputMem = 0
+ vis = InOutputObjVisitor()
for arg in argsout:
# the proxy mecanism is catched here
- argPickle = SpoolPickleObject( arg )
+ argPickle = SpoolPickleObject( arg, vis )
retArg = SenderByte_i( self.poa,argPickle )
id_o = self.poa.activate_object(retArg)
retObj = self.poa.id_to_reference(id_o)
ret.append( retObj._narrow( SALOME.SenderByte ) )
+ outputMem += len(argPickle)
+ self.my_container_py.addInfoOnLevel2(self.getIDInContainer(),self._current_exec,"outputMem",outputMem)
+ self.my_container_py.addInfoOnLevel2(self.getIDInContainer(),self._current_exec,"outputHDDMem",vis)
self.my_container_py.addTimeInfoOnLevel2(self.getIDInContainer(),self._current_exec,"endOutputTime")
return ret
except Exception:
