remain = m%1024
m//=1024
return "{} {}".format(m,UNITS[3])
+
+ @classmethod
+ def SpeedRepr(cls,memInBytePerS):
+ return "{}/s".format( cls.MemRepr(memInBytePerS) )
def __init__(self):
self._measure_time_resolution_ms = None
self._output_hdd_mem = None
self._start_pos_log = None
self._stop_pos_log = None
+ self._freestyle_log = []
+
+ @property
+ def freestyle(self):
+ return self._freestyle_log
+
+ @freestyle.setter
+ def freestyle(self, value):
+ self._freestyle_log.append( value )
@property
def measureTimeResolution(self):
@property
def CPUMemDuringExecStr(self):
- cpu = self._cpu_mem_during_exec[::2]
- mem_rss = self._cpu_mem_during_exec[1::2]
- return [(a,ScriptExecInfo.MemRepr(b)) for a,b in self._cpu_mem_during_exec]
+ return [(a,ScriptExecInfo.MemRepr(b)) for a,b in self._cpu_mem_during_exec.data]
@property
def inputMem(self):
def outputMemStr(self):
return ScriptExecInfo.MemRepr( self.outputMem )
+ def inputReadHDDSize(self):
+ return self.inputHDDMem.getSizeOfFileRead()
+
+ def inputReadHDDSizeRepr(self):
+ return ScriptExecInfo.MemRepr( self.inputReadHDDSize() )
+
+ def inputReadHDDSpeed(self):
+ return self.inputReadHDDSize() / ( self.endInputTime - self.startInputTime ).total_seconds()
+
+ def inputReadHDDSpeedRepr(self):
+ return ScriptExecInfo.SpeedRepr( self.inputReadHDDSpeed() )
+
+ def outputWriteHDDSize(self):
+ return self.outputHDDMem.getSizeOfFileRead()
+
+ def outputWriteHDDSizeRepr(self):
+ return ScriptExecInfo.MemRepr( self.outputWriteHDDSize() )
+
+ def outputWriteHDDSpeed(self):
+ return self.outputWriteHDDSize() / ( self.endOutputTime - self.startOutputTime ).total_seconds()
+
+ def outputWriteHDDSpeedRepr(self):
+ return ScriptExecInfo.SpeedRepr( self.outputWriteHDDSpeed() )
+
@property
def inputHDDMem(self):
return self._input_hdd_mem
def endExecTime(self,value):
self._end_exec_time = value
+ @property
+ def execTime(self):
+ if ( self.endExecTime is not None ) and (self.startExecTime is not None):
+ return self.endExecTime - self.startExecTime
+ return None
+
+ @property
+ def fullExecTime(self):
+ return self.endOutputTime - self.startInputTime
+
@property
def startOutputTime(self):
return self._start_output_time
return ScriptExecInfo.GetRepresentationOfTimeDelta(self.endOutputTime,self.startOutputTime)
def __str__(self):
+ CPUMemDuringExecForStr = self.CPUMemDuringExecStr
+ if len( CPUMemDuringExecForStr ) > 30:
+ CPUMemDuringExecForStr = "{} ...".format( str(CPUMemDuringExecForStr[:30]) )
+ else:
+ CPUMemDuringExecForStr = str( CPUMemDuringExecForStr )
return """start exec time = {self.startExecTime}
end exec time = {self.endExecTime}
exec_time = {self.execTimeStr}
Measure time resolution = {self.measureTimeResolution} ms
-CPU and mem monitoring = {self.CPUMemDuringExecStr}
+CPU and mem monitoring = {CPUMemDuringExecForStr}
input unpickling and ev load from disk time = {self.inputTimeStr}
output serialization and ev write to disk time = {self.outputTimeStr}
input memory size before exec (MemoryPeak 2x) = {self.inputMemStr}
with open(self.father.father.logfile,"rb") as f:
cont = f.read()
return cont[self._eff.tracePosStart:self._eff.tracePosStop].decode()
+ @property
+ def nodeName(self):
+ return self.father.get().nodeName
+ @property
+ def code(self):
+ return self.father.code
+ @property
+ def ns_entry(self):
+ return self.father.father.ns_entry
+ @property
+ def computingNode(self):
+ return self.father.computingNode
+
+ @property
+ def freestyle(self):
+ return self.get().freestyle
+
+ @property
+ def measureTimeResolution(self):
+ return self.get().measureTimeResolution
+
+ @property
+ def CPUMemDuringExec(self):
+ return self.get().CPUMemDuringExec
+
+ @property
+ def inputMem(self):
+ return self.get().inputMem
+
+ @property
+ def outputMem(self):
+ return self.get().outputMem
+
+ @property
+ def inputHDDMem(self):
+ return self.get().inputHDDMem
+
+ @property
+ def outputHDDMem(self):
+ return self.get().outputHDDMem
+
+ def inputReadHDDSize(self):
+ return self.get().inputReadHDDSize()
+
+ def inputReadHDDSizeRepr(self):
+ return self.get().inputReadHDDSizeRepr()
+
+ def inputReadHDDSpeed(self):
+ return self.get().inputReadHDDSpeed()
+
+ def inputReadHDDSpeedRepr(self):
+ return self.get().inputReadHDDSpeedRepr()
+
+ def outputWriteHDDSize(self):
+ return self.get().outputWriteHDDSize()
+
+ def outputWriteHDDSizeRepr(self):
+ return self.get().outputWriteHDDSizeRepr()
+
+ def outputWriteHDDSpeed(self):
+ return self.get().outputWriteHDDSpeed()
+
+ def outputWriteHDDSpeedRepr(self):
+ return self.get().outputWriteHDDSpeedRepr()
+
+ @property
+ def startInputTime(self):
+ return self.get().startInputTime
+
+ @property
+ def endInputTime(self):
+ return self.get().endInputTime
+
+ @property
+ def startExecTime(self):
+ return self.get().startExecTime
+
+ @property
+ def endExecTime(self):
+ return self.get().endExecTime
+
+ @property
+ def execTime(self):
+ return self.get().execTime
+
+ @property
+ def fullExecTime(self):
+ return self.get().fullExecTime
+
+ @property
+ def startOutputTime(self):
+ return self.get().startOutputTime
+
+ @property
+ def endOutputTime(self):
+ return self.get().endOutputTime
class ScriptInfoAbstract:
def __init__(self, scriptPtr):
return self._father
def get(self):
return self._eff
+ @property
+ def nodeName(self):
+ return self.get().nodeName
+ @property
+ def code(self):
+ return self.get().code
+ @property
+ def ns_entry(self):
+ return self.father.ns_entry
+ @property
+ def computingNode(self):
+ return self.father.computingNode
def __getitem__(self,i):
return ScriptExecInfoDeco( self._eff[i], self )
def __len__(self):
return self._eff.__repr__()
class ContainerLogInfoAbstract:
-
+
+ @property
def log(self):
with open(self.logfile,"rb") as f:
cont = f.read()
def logfile(self):
return self._log_file
+ @property
+ def computingNode(self):
+ return ContainerLogInfoAbstract.ComputingNodeFromNSEntry( self.ns_entry )
+
def __len__(self):
return len( self._scripts )
return ScriptInfoDeco( self._scripts[i], self)
def __str__(self):
- return """NS entry = {self.ns_entry} LogFile = {self.logfile}""".format(**locals())
+ return """NS entry = {self.ns_entry} LogFile = {self.logfile}""".format(**locals())
+
+ @classmethod
+ def ComputingNodeFromNSEntry(cls, nsEntry):
+ return nsEntry.split("/")[2]
class ContainerLogInfoClt(ContainerLogInfoAbstract):
def __init__(self,contLogPtr):
class FakeObjVisitor:
def setHDDMem(self, v):
pass
+
+ def setFileName(self, fileName):
+ pass
def visitor(self):
return None
if int(offset) != len(structData):
raise RuntimeError("Something went wrong during unserialization phase.")
return logManagerInst
+
+def ListAllExecContainIn( listOfContainerLogInfo ):
+ """
+ For all ContainerLogInfo contained in listOfContainerLogInfo extract all ScriptExecInfo contained recursively
+ in it. This method filters all "side" executions like those positionning environment for exemple.
+
+ See also : ListAllExecFinishedContainIn
+
+ Args:
+ -----
+
+ listOfContainerLogInfo (list<ContainerLogInfo>) : instance typically returned by salome.LogManagerLoadFromFile
+
+ Returns
+ -------
+
+ list<ScriptExecInfoDeco> : all ScriptExecInfoDeco instance contained recursively in all input ContainerLogInfo instances
+
+ """
+ allexecs = sum( [sum( [[myexec for myexec in ps] for ps in cont],[] ) for cont in listOfContainerLogInfo], [] )
+ return [elt for elt in allexecs if elt.get() is not None]
+
+def ListAllExecFinishedContainIn( listOfContainerLogInfo ):
+ """
+ For all ContainerLogInfo contained in listOfContainerLogInfo extract all ScriptExecInfo contained recursively
+ in it. This method filters all "side" executions like those positionning environment for exemple and those not finished.
+
+ See also : ListAllExecContainIn
+
+ Args:
+ -----
+
+ listOfContainerLogInfo (list<ContainerLogInfo>) : instance typically returned by salome.LogManagerLoadFromFile
+
+ Returns
+ -------
+
+ list<ScriptExecInfoDeco> : all ScriptExecInfoDeco instance contained recursively in all input ContainerLogInfo instances
+
+ """
+ beforeFiltering = ListAllExecContainIn( listOfContainerLogInfo )
+ return [elt for elt in beforeFiltering if elt.get().execTime is not None]
+
+def IsExecTimeHigherThan( execInstDeco, limitDuration ):
+ """
+ Example of Usage :
+
+ [elt for elt in allexecs if IsExecTimeHigherThan(elt,datetime.timedelta(hours=1))]
+
+ Args:
+ -----
+
+ execInstDeco (ScriptExecInfoDeco)
+ limitDuration (datetime.timedelta) Ex (datetime.timedelta(hours=1))
+
+ """
+ if execInstDeco.get() is not None:
+ return execInstDeco.get().execTime > limitDuration
+ else:
+ return False
+
+def GetMaxTimeExec( listOfFinishedExecs ):
+ """
+ Returns the instance among listOfFinishedExecs that spends the max time to be executed
+
+ Args:
+ -----
+
+ listOfFinishedExecs ( list<ScriptExecInfoDeco> ) : instance of ScriptExecInfoDeco that have finished to be executed
+ Typically returned by ListAllExecFinishedContainIn
+
+ Returns
+ -------
+
+ ScriptExecInfoDeco :
+ """
+ return listOfFinishedExecs[ max([(i,elt.execTime) for i,elt in enumerate( listOfFinishedExecs) ],key = lambda x : x[1])[0] ]
+
+def GetMinTimeExec( listOfFinishedExecs ):
+ """
+ Returns the instance among listOfFinishedExecs that spends the min time to be executed
+
+ Args:
+ -----
+
+ listOfFinishedExecs ( list<ScriptExecInfoDeco> ) : instance of ScriptExecInfoDeco that have finished to be executed
+ Typically returned by ListAllExecFinishedContainIn
+
+ Returns
+ -------
+
+ ScriptExecInfoDeco :
+ """
+ return listOfFinishedExecs[ min([(i,elt.execTime) for i,elt in enumerate( listOfFinishedExecs) ],key = lambda x : x[1])[0] ]
+
+class DistriutionClass:
+ def __init__(self, begin, end, listOfExecs):
+ self._begin = begin
+ self._end = end
+ self._list_of_execs = sorted( listOfExecs, key = lambda x : x.execTime)
+ def __getitem__(self, *args):
+ return self._list_of_execs.__getitem__( *args )
+ @property
+ def begin(self):
+ return self._begin
+ @property
+ def end(self):
+ return self._end
+ @property
+ def listOfExecs(self):
+ return self._list_of_execs
+ @property
+ def size(self):
+ return len( self.listOfExecs )
+ @property
+ def reprGlobal(self):
+ return f"[{self.begin}->{self.end}] : {self.size} execs"
+
+class DistributionOfExecs:
+ """
+ Class in charge to distribute execution log instance into equi-classes
+ """
+ def __init__(self, listOfFinishedExecs, nbOfClasses):
+ self._list_of_finished_execs = listOfFinishedExecs
+ self._nb_of_classes = nbOfClasses
+ self._classes = DistributionOfExecs.ComputeDistributionOfExecTime(self._list_of_finished_execs,self._nb_of_classes)
+
+ def __getitem__(self, *args):
+ return self._classes.__getitem__( *args )
+ def reprOfExecs(self):
+ import numpy as np
+ cs = np.cumsum( [elt.size for elt in self._classes] )
+ ret = []
+ for i,(elt,cum_nb) in enumerate( zip(self._classes,cs) ):
+ ratio = (cum_nb / cs[-1])*100.0
+ ret.append( f"- For class {i} - {elt.reprGlobal} ( {ratio:.1f}% )" )
+ return "\n".join( ret )
+ @classmethod
+ def ComputeDistributionOfExecTime( cls, listOfFinishedExecs, nbOfClasses ):
+ maxt = GetMaxTimeExec( listOfFinishedExecs )
+ mint = GetMinTimeExec( listOfFinishedExecs )
+ deltaTime = ( maxt.execTime - mint.execTime ) / nbOfClasses
+ res = []
+ for i in range( nbOfClasses ):
+ begin = mint.execTime + i * deltaTime
+ end = mint.execTime + (i+1) * deltaTime
+ res.append( DistriutionClass(begin,end,[elt for elt in listOfFinishedExecs if elt.execTime >=begin and elt.execTime <= end]) )
+ return res
