1 # -*- coding: iso-8859-1 -*-
2 # Copyright (C) 2023-2024 CEA, EDF
4 # This library is free software; you can redistribute it and/or
5 # modify it under the terms of the GNU Lesser General Public
6 # License as published by the Free Software Foundation; either
7 # version 2.1 of the License, or (at your option) any later version.
9 # This library is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 # Lesser General Public License for more details.
14 # You should have received a copy of the GNU Lesser General Public
15 # License along with this library; if not, write to the Free Software
16 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 # See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
21 from collections import defaultdict
27 def GetRepresentationOfTimeDelta(cls,endTime, startTime):
28 if endTime is None and startTime is None:
30 td = endTime - startTime
32 ts_of_td = time.gmtime(td.total_seconds())
33 return "{}.{:06d}".format(time.strftime("%H:%M:%S",ts_of_td),td.microseconds)
36 def MemRepr(cls,memInByte):
38 UNITS=["B","kB","MB","GB"]
41 for i in range( len(UNITS) ):
43 oss = "{:03d}".format( int( (remain/1024)*1000 ) )
44 oss = "{}.{} {}".format(m,oss,UNITS[i])
50 return "{} {}".format(m,UNITS[3])
53 def SpeedRepr(cls,memInBytePerS):
54 return "{}/s".format( cls.MemRepr(memInBytePerS) )
57 self._measure_time_resolution_ms = None
58 self._cpu_mem_during_exec = None
59 self._start_exec_time = None
60 self._end_exec_time = None
61 self._start_input_time = None
62 self._end_input_time = None
63 self._start_output_time = None
64 self._end_output_time = None
66 self._input_hdd_mem = None
68 self._output_hdd_mem = None
69 self._start_pos_log = None
70 self._stop_pos_log = None
71 self._freestyle_log = []
75 return self._freestyle_log
78 def freestyle(self, value):
79 self._freestyle_log.append( value )
82 def measureTimeResolution(self):
83 return self._measure_time_resolution_ms
85 @measureTimeResolution.setter
86 def measureTimeResolution(self, value):
87 self._measure_time_resolution_ms = value
90 def tracePosStart(self):
91 return self._start_pos_log
94 def tracePosStart(self,value):
95 self._start_pos_log = value
98 def tracePosStop(self):
99 return self._stop_pos_log
102 def tracePosStop(self,value):
103 self._stop_pos_log = value
106 def CPUMemDuringExec(self):
107 return self._cpu_mem_during_exec
109 @CPUMemDuringExec.setter
110 def CPUMemDuringExec(self,value):
111 self._cpu_mem_during_exec = value
114 def CPUMemDuringExecStr(self):
115 return [(a,ScriptExecInfo.MemRepr(b)) for a,b in self._cpu_mem_during_exec.data]
119 return self._input_mem
122 def inputMem(self,value):
123 self._input_mem = value
126 def inputMemStr(self):
127 return ScriptExecInfo.MemRepr( self.inputMem )
131 return self._output_mem
134 def outputMem(self,value):
135 self._output_mem = value
138 def outputMemStr(self):
139 return ScriptExecInfo.MemRepr( self.outputMem )
141 def inputReadHDDSize(self):
142 return self.inputHDDMem.getSizeOfFileRead()
144 def inputReadHDDSizeRepr(self):
145 return ScriptExecInfo.MemRepr( self.inputReadHDDSize() )
147 def inputReadHDDSpeed(self):
148 return self.inputReadHDDSize() / ( self.endInputTime - self.startInputTime ).total_seconds()
150 def inputReadHDDSpeedRepr(self):
151 return ScriptExecInfo.SpeedRepr( self.inputReadHDDSpeed() )
153 def outputWriteHDDSize(self):
154 return self.outputHDDMem.getSizeOfFileRead()
156 def outputWriteHDDSizeRepr(self):
157 return ScriptExecInfo.MemRepr( self.outputWriteHDDSize() )
159 def outputWriteHDDSpeed(self):
160 return self.outputWriteHDDSize() / ( self.endOutputTime - self.startOutputTime ).total_seconds()
162 def outputWriteHDDSpeedRepr(self):
163 return ScriptExecInfo.SpeedRepr( self.outputWriteHDDSpeed() )
166 def inputHDDMem(self):
167 return self._input_hdd_mem
170 def inputHDDMem(self,value):
171 self._input_hdd_mem = value
174 def inputHDDMemStr(self):
175 if self._input_hdd_mem is None:
176 return "not computed"
177 return " ".join( [ ScriptExecInfo.MemRepr( elt.getSizeOfFileRead() ) for elt in self._input_hdd_mem] )
180 def outputHDDMem(self):
181 return self._output_hdd_mem
184 def outputHDDMem(self,value):
185 self._output_hdd_mem = value
188 def outputHDDMemStr(self):
189 if self._output_hdd_mem is None:
190 return "not computed"
191 return " ".join( [ ScriptExecInfo.MemRepr( elt.getSizeOfFileRead() ) for elt in self._output_hdd_mem] )
194 def startInputTime(self):
195 return self._start_input_time
197 @startInputTime.setter
198 def startInputTime(self,value):
199 self._start_input_time = value
202 def endInputTime(self):
203 return self._end_input_time
206 def endInputTime(self,value):
207 self._end_input_time = value
210 def startExecTime(self):
211 return self._start_exec_time
213 @startExecTime.setter
214 def startExecTime(self,value):
215 self._start_exec_time = value
218 def endExecTime(self):
219 return self._end_exec_time
222 def endExecTime(self,value):
223 self._end_exec_time = value
227 if ( self.endExecTime is not None ) and (self.startExecTime is not None):
228 return self.endExecTime - self.startExecTime
232 def fullExecTime(self):
233 return self.endOutputTime - self.startInputTime
236 def startOutputTime(self):
237 return self._start_output_time
239 @startOutputTime.setter
240 def startOutputTime(self,value):
241 self._start_output_time = value
244 def endOutputTime(self):
245 return self._end_output_time
247 @endOutputTime.setter
248 def endOutputTime(self,value):
249 self._end_output_time = value
252 def execTimeStr(self):
253 return ScriptExecInfo.GetRepresentationOfTimeDelta(self.endExecTime,self.startExecTime)
256 def inputTimeStr(self):
257 return ScriptExecInfo.GetRepresentationOfTimeDelta(self.endInputTime,self.startInputTime)
260 def outputTimeStr(self):
261 return ScriptExecInfo.GetRepresentationOfTimeDelta(self.endOutputTime,self.startOutputTime)
264 CPUMemDuringExecForStr = self.CPUMemDuringExecStr
265 if len( CPUMemDuringExecForStr ) > 30:
266 CPUMemDuringExecForStr = "{} ...".format( str(CPUMemDuringExecForStr[:30]) )
268 CPUMemDuringExecForStr = str( CPUMemDuringExecForStr )
269 return """start exec time = {self.startExecTime}
270 end exec time = {self.endExecTime}
271 exec_time = {self.execTimeStr}
272 Measure time resolution = {self.measureTimeResolution} ms
273 CPU and mem monitoring = {CPUMemDuringExecForStr}
274 input unpickling and ev load from disk time = {self.inputTimeStr}
275 output serialization and ev write to disk time = {self.outputTimeStr}
276 input memory size before exec (MemoryPeak 2x) = {self.inputMemStr}
277 input memory size from HDD = {self.inputHDDMemStr}
278 output memory size after exec (MemoryPeak 2x) = {self.outputMemStr}
279 output memory size from HDD = {self.outputHDDMemStr}
280 Start position in log = {self.tracePosStart}
281 End position in log = {self.tracePosStop}""".format(**locals())
283 class ScriptExecInfoDeco:
284 def __init__(self, eff, father):
286 self._father = father
292 def __getitem__(self,i):
295 return self._eff.__str__()
297 return self._eff.__repr__()
299 with open(self.father.father.logfile,"rb") as f:
301 return cont[self._eff.tracePosStart:self._eff.tracePosStop].decode()
304 return self.father.get().nodeName
307 return self.father.code
310 return self.father.father.ns_entry
312 def computingNode(self):
313 return self.father.computingNode
317 return self.get().freestyle
320 def measureTimeResolution(self):
321 return self.get().measureTimeResolution
324 def CPUMemDuringExec(self):
325 return self.get().CPUMemDuringExec
329 return self.get().inputMem
333 return self.get().outputMem
336 def inputHDDMem(self):
337 return self.get().inputHDDMem
340 def outputHDDMem(self):
341 return self.get().outputHDDMem
343 def inputReadHDDSize(self):
344 return self.get().inputReadHDDSize()
346 def inputReadHDDSizeRepr(self):
347 return self.get().inputReadHDDSizeRepr()
349 def inputReadHDDSpeed(self):
350 return self.get().inputReadHDDSpeed()
352 def inputReadHDDSpeedRepr(self):
353 return self.get().inputReadHDDSpeedRepr()
355 def outputWriteHDDSize(self):
356 return self.get().outputWriteHDDSize()
358 def outputWriteHDDSizeRepr(self):
359 return self.get().outputWriteHDDSizeRepr()
361 def outputWriteHDDSpeed(self):
362 return self.get().outputWriteHDDSpeed()
364 def outputWriteHDDSpeedRepr(self):
365 return self.get().outputWriteHDDSpeedRepr()
368 def startInputTime(self):
369 return self.get().startInputTime
372 def endInputTime(self):
373 return self.get().endInputTime
376 def startExecTime(self):
377 return self.get().startExecTime
380 def endExecTime(self):
381 return self.get().endExecTime
385 return self.get().execTime
388 def fullExecTime(self):
389 return self.get().fullExecTime
392 def startOutputTime(self):
393 return self.get().startOutputTime
396 def endOutputTime(self):
397 return self.get().endOutputTime
399 class ScriptInfoAbstract:
400 def __init__(self, scriptPtr):
401 self._node_name = scriptPtr.getName()
402 self._code = scriptPtr.getCode()
403 self._exec = [pickle.loads(elt.getObj()) for elt in scriptPtr.listOfExecs()]
411 return self._node_name
418 def code(self,value):
422 return len( self._exec )
424 def __getitem__(self,i):
428 return """name = {self.nodeName}\ncode = {self.code}\nexecs = {self.execs}""".format(**locals())
431 return """ScriptInfo \"{self.nodeName}\"""".format(**locals())
433 class ScriptInfoClt(ScriptInfoAbstract):
434 def __init__(self, scriptPtr):
435 self._node_name = scriptPtr.getName()
436 self._code = scriptPtr.getCode()
437 self._exec = [pickle.loads(elt.getObj()) for elt in scriptPtr.listOfExecs()]
439 class ScriptInfo(ScriptInfoAbstract):
440 def __init__(self, nodeName, code, execs):
441 self._node_name = nodeName
445 class ScriptInfoDeco:
446 def __init__(self, eff, father):
448 self._father = father
456 return self.get().nodeName
459 return self.get().code
462 return self.father.ns_entry
464 def computingNode(self):
465 return self.father.computingNode
466 def __getitem__(self,i):
467 return ScriptExecInfoDeco( self._eff[i], self )
469 return self._eff.__len__()
471 return self._eff.__str__()
473 return self._eff.__repr__()
475 class ContainerLogInfoAbstract:
479 with open(self.logfile,"rb") as f:
485 return self._ns_entry
489 return self._log_file
492 def computingNode(self):
493 return ContainerLogInfoAbstract.ComputingNodeFromNSEntry( self.ns_entry )
496 return len( self._scripts )
498 def __getitem__(self,i):
499 return ScriptInfoDeco( self._scripts[i], self)
502 return """NS entry = {self.ns_entry} LogFile = {self.logfile}""".format(**locals())
505 def ComputingNodeFromNSEntry(cls, nsEntry):
506 return nsEntry.split("/")[2]
508 class ContainerLogInfoClt(ContainerLogInfoAbstract):
509 def __init__(self,contLogPtr):
510 self._log_file = contLogPtr.getLogFile()
511 self._ns_entry = contLogPtr.getContainerEntryInNS()
512 self._scripts = [ScriptInfoClt(elt) for elt in contLogPtr.listOfScripts()]
514 class ContainerLogInfo(ContainerLogInfoAbstract):
515 def __init__(self, nsEntry, logFile, scripts):
516 self._log_file = logFile
517 self._ns_entry = nsEntry
518 self._scripts = scripts
520 from abc import ABC, abstractmethod
522 class InOutputObjVisitorAbstract(ABC):
528 self._cur_obj = ObjMemModel()
532 self._data.append( self._cur_obj )
535 def getSizeOfFileRead(self):
536 return sum( [elt.getSizeOfFileRead() for elt in self._data] )
541 def setHDDMem(self, v):
544 def setFileName(self, fileName):
551 class InOutputObjVisitorIter:
552 def __init__(self, visitor):
553 self._visitor = visitor
557 if self._current >= len(self._visitor._data):
560 ret = self._visitor._data[ self._current ]
564 class InOutputObjVisitor(InOutputObjVisitorAbstract):
569 return self.getSizeOfFileRead()
572 return InOutputObjVisitorIter(self)
574 class ObjMemModel(InOutputObjVisitorAbstract):
578 self._file_name = None
580 def setHDDMem(self, v):
584 def setFileName(self, fileName):
585 self._file_name = fileName
588 def getSizeOfFileRead(self):
589 if hasattr(self,"_data"):
590 return super().getSizeOfFileRead()
595 return self.getSizeOfFileRead()
597 class FakeObjVisitor:
598 def setHDDMem(self, v):
601 def setFileName(self, fileName):
607 class InOutputObjVisitorCM:
608 def __init__(self, visitor):
609 self._visitor = visitor
612 r = self._visitor.enter()
615 return FakeObjVisitor()
616 def __exit__(self,exctype, exc, tb):
618 self._visitor.leave()
622 def __init__(self,i):
626 def __iadd__(self,delta):
630 def unserializeInt(structData, offset):
631 from ctypes import c_int
633 sz = c_int.from_buffer_copy( structData[int(offset):int(offset)+sz_of_cint] ).value
637 def unserializeString(structData,offset):
638 sz = unserializeInt(structData,offset)
639 ret = structData[int(offset):int(offset)+sz].decode()
643 def unserializeContainerScriptExecPerfLog(structData, offset):
645 sz = unserializeInt(structData,offset)
648 inst = pickle.loads( structData[int(offset):int(offset)+sz] )
652 def unserializeContainerScriptPerfLog(structData, offset):
653 name = unserializeString(structData,offset)
654 code = unserializeString(structData,offset)
655 numberOfSessions = unserializeInt(structData,offset)
657 for _ in range(numberOfSessions):
658 session = unserializeContainerScriptExecPerfLog(structData,offset)
659 sessions.append( session )
660 return ScriptInfo(name,code,sessions)
662 def unserializeContainerPerfLog(structData, offset):
663 nsEntry = unserializeString(structData,offset)
664 logFile = unserializeString(structData,offset)
666 nbScripts = unserializeInt(structData,offset)
667 for _ in range(nbScripts):
668 script = unserializeContainerScriptPerfLog(structData,offset)
669 scripts.append( script )
670 return ContainerLogInfo(nsEntry,logFile,scripts)
672 def unserializeLogManager(structData):
673 offset = OffsetType(0)
674 numberOfScripts = unserializeInt(structData,offset)
676 for _ in range(numberOfScripts):
677 containerPerfLogInst = unserializeContainerPerfLog(structData,offset)
678 logManagerInst.append( containerPerfLogInst )
679 if int(offset) != len(structData):
680 raise RuntimeError("Something went wrong during unserialization phase.")
681 return logManagerInst
683 def ListAllExecContainIn( listOfContainerLogInfo ):
685 For all ContainerLogInfo contained in listOfContainerLogInfo extract all ScriptExecInfo contained recursively
686 in it. This method filters all "side" executions like those positionning environment for exemple.
688 See also : ListAllExecFinishedContainIn
693 listOfContainerLogInfo (list<ContainerLogInfo>) : instance typically returned by salome.LogManagerLoadFromFile
698 list<ScriptExecInfoDeco> : all ScriptExecInfoDeco instance contained recursively in all input ContainerLogInfo instances
701 allexecs = sum( [sum( [[myexec for myexec in ps] for ps in cont],[] ) for cont in listOfContainerLogInfo], [] )
702 return [elt for elt in allexecs if elt.get() is not None]
704 def ListAllExecFinishedContainIn( listOfContainerLogInfo ):
706 For all ContainerLogInfo contained in listOfContainerLogInfo extract all ScriptExecInfo contained recursively
707 in it. This method filters all "side" executions like those positionning environment for exemple and those not finished.
709 See also : ListAllExecContainIn
714 listOfContainerLogInfo (list<ContainerLogInfo>) : instance typically returned by salome.LogManagerLoadFromFile
719 list<ScriptExecInfoDeco> : all ScriptExecInfoDeco instance contained recursively in all input ContainerLogInfo instances
722 beforeFiltering = ListAllExecContainIn( listOfContainerLogInfo )
723 return [elt for elt in beforeFiltering if elt.get().execTime is not None]
725 def IsExecTimeHigherThan( execInstDeco, limitDuration ):
729 [elt for elt in allexecs if IsExecTimeHigherThan(elt,datetime.timedelta(hours=1))]
734 execInstDeco (ScriptExecInfoDeco)
735 limitDuration (datetime.timedelta) Ex (datetime.timedelta(hours=1))
738 if execInstDeco.get() is not None:
739 return execInstDeco.get().execTime > limitDuration
743 def GetMaxTimeExec( listOfFinishedExecs ):
745 Returns the instance among listOfFinishedExecs that spends the max time to be executed
750 listOfFinishedExecs ( list<ScriptExecInfoDeco> ) : instance of ScriptExecInfoDeco that have finished to be executed
751 Typically returned by ListAllExecFinishedContainIn
758 return listOfFinishedExecs[ max([(i,elt.execTime) for i,elt in enumerate( listOfFinishedExecs) ],key = lambda x : x[1])[0] ]
760 def GetMinTimeExec( listOfFinishedExecs ):
762 Returns the instance among listOfFinishedExecs that spends the min time to be executed
767 listOfFinishedExecs ( list<ScriptExecInfoDeco> ) : instance of ScriptExecInfoDeco that have finished to be executed
768 Typically returned by ListAllExecFinishedContainIn
775 return listOfFinishedExecs[ min([(i,elt.execTime) for i,elt in enumerate( listOfFinishedExecs) ],key = lambda x : x[1])[0] ]
777 class DistriutionClass:
778 def __init__(self, begin, end, listOfExecs):
781 self._list_of_execs = sorted( listOfExecs, key = lambda x : x.execTime)
782 def __getitem__(self, *args):
783 return self._list_of_execs.__getitem__( *args )
791 def listOfExecs(self):
792 return self._list_of_execs
795 return len( self.listOfExecs )
797 def reprGlobal(self):
798 return f"[{self.begin}->{self.end}] : {self.size} execs"
800 class DistributionOfExecs:
802 Class in charge to distribute execution log instance into equi-classes
804 def __init__(self, listOfFinishedExecs, nbOfClasses):
805 self._list_of_finished_execs = listOfFinishedExecs
806 self._nb_of_classes = nbOfClasses
807 self._classes = DistributionOfExecs.ComputeDistributionOfExecTime(self._list_of_finished_execs,self._nb_of_classes)
809 def __getitem__(self, *args):
810 return self._classes.__getitem__( *args )
811 def reprOfExecs(self):
813 cs = np.cumsum( [elt.size for elt in self._classes] )
815 for i,(elt,cum_nb) in enumerate( zip(self._classes,cs) ):
816 ratio = (cum_nb / cs[-1])*100.0
817 ret.append( f"- For class {i} - {elt.reprGlobal} ( {ratio:.1f}% )" )
818 return "\n".join( ret )
820 def ComputeDistributionOfExecTime( cls, listOfFinishedExecs, nbOfClasses ):
821 maxt = GetMaxTimeExec( listOfFinishedExecs )
822 mint = GetMinTimeExec( listOfFinishedExecs )
823 deltaTime = ( maxt.execTime - mint.execTime ) / nbOfClasses
825 for i in range( nbOfClasses ):
826 begin = mint.execTime + i * deltaTime
827 end = mint.execTime + (i+1) * deltaTime
828 res.append( DistriutionClass(begin,end,[elt for elt in listOfFinishedExecs if elt.execTime >=begin and elt.execTime <= end]) )