From 6db7f01cf129c108f282c2a0456c647dce1b781c Mon Sep 17 00:00:00 2001 From: Anthony Geay Date: Fri, 26 Jan 2024 17:30:41 +0100 Subject: [PATCH] [EDF29150] : Implementation ready for the test batch #2 --- src/Container/SALOME_PyNode.py | 44 +++++++++--- src/Launcher/Test/testPerfLogManager1.py | 87 ++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 10 deletions(-) diff --git a/src/Container/SALOME_PyNode.py b/src/Container/SALOME_PyNode.py index d36e36908..cb0f24511 100644 --- a/src/Container/SALOME_PyNode.py +++ b/src/Container/SALOME_PyNode.py @@ -396,12 +396,16 @@ def UnProxyObjectSimpleLocal( obj ): else: return obj -class FileDeleter: +class FileHolder: def __init__(self, fileName): self._filename = fileName @property def filename(self): return self._filename + +class FileDeleter(FileHolder): + def __init__(self, fileName): + super().__init__( fileName ) def __del__(self): import os if os.path.exists( self._filename ): @@ -429,11 +433,16 @@ class MonitoringInfo: def outFileName(self): return self._out_file_name -def FileSystemMonitoring(intervalInMs, dirNameToInspect): +def FileSystemMonitoring(intervalInMs, dirNameToInspect, outFileName = None): """ This method loops indefinitely every intervalInMs milliseconds to scan number of inodes and size of content recursively included into the in input directory. + Args: + ---- + + outFileName (str) : name of file inside the results will be written. If None a new file is generated + See also CPUMemoryMonitoring """ global orb @@ -447,7 +456,9 @@ def FileSystemMonitoring(intervalInMs, dirNameToInspect): outFileNameSave = f.name with tempfile.NamedTemporaryFile(prefix="fs_monitor_",suffix=".py") as f: tempPyFile = f.name - tempOutFile = "{}.txt".format( os.path.splitext( tempPyFile )[0] ) + tempOutFile = outFileName + if tempOutFile is None: + tempOutFile = "{}.txt".format( os.path.splitext( tempPyFile )[0] ) with open(tempPyFile,"w") as f: f.write(""" import subprocess as sp @@ -467,26 +478,35 @@ with open("{tempOutFile}","a") as f: """.format( **locals())) logging.debug( "File for FS monitoring dump file : {}".format(tempPyFile) ) pyFileName = FileDeleter( tempPyFile ) - outFileName = FileDeleter( tempOutFile ) + if outFileName is None: + outFileName = FileDeleter( tempOutFile ) + else: + outFileName = FileHolder(outFileName) pid = KernelBasis.LaunchMonitoring( tempPyFile ) return MonitoringInfo(pyFileName,outFileName,pid) -def CPUMemoryMonitoring( intervalInMs ): +def CPUMemoryMonitoring( intervalInMs, outFileName = None ): """ Launch a subprocess monitoring self process. This monitoring subprocess is a python process lauching every intervalInMs ms evaluation of CPU usage and RSS memory of the calling process. Communication between subprocess and self is done by file. - + + Args: + ---- + outFileName (str) : name of file inside the results will be written. If None a new file is generated + See also FileSystemMonitoring """ import KernelBasis - def BuildPythonFileForCPUPercent( intervalInMs ): + def BuildPythonFileForCPUPercent( intervalInMs, outFileName): import os import tempfile with tempfile.NamedTemporaryFile(prefix="cpu_mem_monitor_",suffix=".py") as f: tempPyFile = f.name - tempOutFile = "{}.txt".format( os.path.splitext( tempPyFile )[0] ) + tempOutFile = outFileName + if tempOutFile is None: + tempOutFile = "{}.txt".format( os.path.splitext( tempPyFile )[0] ) pid = os.getpid() with open(tempPyFile,"w") as f: f.write("""import psutil @@ -500,8 +520,12 @@ with open("{}","a") as f: f.flush() time.sleep( {} / 1000.0 ) """.format(pid, tempOutFile, intervalInMs)) - return FileDeleter(tempPyFile), FileDeleter(tempOutFile) - pyFileName, outFileName = BuildPythonFileForCPUPercent( intervalInMs ) + if outFileName is None: + autoOutFile = FileDeleter(tempOutFile) + else: + autoOutFile = FileHolder(tempOutFile) + return FileDeleter(tempPyFile),autoOutFile + pyFileName, outFileName = BuildPythonFileForCPUPercent( intervalInMs, outFileName ) return MonitoringInfo(pyFileName, outFileName, None) class GenericPythonMonitoringLauncherCtxMgr: diff --git a/src/Launcher/Test/testPerfLogManager1.py b/src/Launcher/Test/testPerfLogManager1.py index c2f27fdec..d5e216365 100644 --- a/src/Launcher/Test/testPerfLogManager1.py +++ b/src/Launcher/Test/testPerfLogManager1.py @@ -191,10 +191,97 @@ time.sleep(1) import gc ; gc.collect() self.assertFalse( os.path.exists(pyFileContainingCodeOfMonitoring) ) cont.Shutdown() + fnToTest0 = monitoringParamsMaster.outFileName.filename cpumeminfo = SALOME_PyNode.ReadCPUMemInfo( monitoringParamsMaster ) + self.assertTrue( isinstance(monitoringParamsMaster.outFileName,SALOME_PyNode.FileDeleter) ) + del monitoringParamsMaster + import gc + gc.collect() + self.assertTrue( os.path.exists( fnToTest0 ) ) dirInfo = SALOME_PyNode.ReadInodeSizeInfo( monitoringParamsForFileMaster ) + self.assertTrue( isinstance(monitoringParamsForFileMaster.outFileName,SALOME_PyNode.FileDeleter) ) + self.assertFalse( os.path.exists( fnToTest0 ) ) logging.debug( cpumeminfo ) logging.debug( dirInfo ) + + def testPerfLogManager1(self): + """ + [EDF29150] : Similar than testPerfLogManager1 but with out fileName precised for CPU/Mem monitoring and FS monitoring + """ + hostname = "localhost" + cp = pylauncher.GetRequestForGiveContainer(hostname,"container_test_three") + salome.logm.clear() + #PROXY_THRES = "-1" + PROXY_THRES = "1" + with tempfile.TemporaryDirectory() as tmpdirnameMonitoring: + fsMonitoringFile = os.path.join( str( tmpdirnameMonitoring ), "zeFS.txt" ) + cpuMemMonitoringFile = os.path.join( str( tmpdirnameMonitoring ), "zeCPUMem.txt" ) + with SALOME_PyNode.GenericPythonMonitoringLauncherCtxMgr( SALOME_PyNode.FileSystemMonitoring(1000,salome.__file__,fsMonitoringFile) ) as monitoringParamsForFileMaster: + with SALOME_PyNode.GenericPythonMonitoringLauncherCtxMgr( SALOME_PyNode.CPUMemoryMonitoring(1000,cpuMemMonitoringFile) ) as monitoringParamsMaster: + monitoringFile = os.path.join( str( tmpdirnameMonitoring ), "zeHtop.pckl" ) + monitoringFileTwo = os.path.join( str( tmpdirnameMonitoring ), "zeHtopTwo.pckl" ) + iorLogm = os.path.join( str( tmpdirnameMonitoring ), "logm.ior" ) + with open(iorLogm,"w") as f: + f.write( salome.orb.object_to_string(salome.logm) ) + logging.debug("Monitoring file : {}".format(monitoringFile)) + with tempfile.TemporaryDirectory() as tmpdirname: + with salome.LogManagerLaunchMonitoringFileCtxMgr(250,monitoringFile) as monitoringParams: + pyFileContainingCodeOfMonitoring = monitoringParams.pyFileName.filename + logging.debug("Python file containing code of monitoring : {}".format(pyFileContainingCodeOfMonitoring)) + val_for_big_obj = str( tmpdirname ) + os.environ["SALOME_FILE_BIG_OBJ_DIR"] = val_for_big_obj + # Override environement for all containers launched + salome.cm.SetOverrideEnvForContainersSimple(env = [("SALOME_FILE_BIG_OBJ_DIR",val_for_big_obj),("SALOME_BIG_OBJ_ON_DISK_THRES",PROXY_THRES)]) + salome.cm.SetDeltaTimeBetweenCPUMemMeasureInMilliSecond( 250 ) + cont = salome.cm.GiveContainer(cp) + logging.debug("{} {}".format(40*"*",cont.getPID())) + script_st = """ +import logging +import sys +import KernelBasis +from datetime import datetime +cst = KernelBasis.GetTimeAdjustmentCst() +logging.debug("constant = {}".format(cst)) +nbcore = 3 +my_log_4_this_session.addFreestyleAndFlush( ("a",datetime.now()) ) # exemple of custom tracking +print("coucou {} {}".format(len(zeinput0),len(zeinput1))) +logging.debug("debug or not debug") +ob = [ [ bytes(3000000) ] ] +pihm, ts = KernelBasis.HeatMarcel(1 * nbcore * cst,nbcore) +print("Time ellapse spent : {} s".format(ts)) +ob2 = [ [ bytes(100000) ] ] +pihm, ts = KernelBasis.HeatMarcel(1 * nbcore * cst,nbcore) +print("Time ellapse spent : {} s".format(ts)) +sys.stderr.write("fake error message\\n") +""" + poa = salome.orb.resolve_initial_references("RootPOA") + zeinput0 = [ bytes(100000000) ] + if SALOME_PyNode.GetBigObjectOnDiskThreshold() != -1: + zeinput0 = SALOME_PyNode.ProxyfyPickeled( zeinput0 ) + zeinput0.unlinkOnDestructor() + obj = SALOME_PyNode.SenderByte_i(poa,pickle.dumps( (["zeinput0"],{"zeinput0": [zeinput0], "zeinput1": [ [zeinput0], [zeinput0] ] }) )) + id_o = poa.activate_object(obj) + refPtr = poa.id_to_reference(id_o) + pyscript2 = cont.createPyScriptNode("testScript2",script_st) + pyscript2.executeFirst(refPtr) + ret2 = pyscript2.executeSecond(["ob","ob2"])# generate a DeprecationWarning: PY_SSIZE_T_CLEAN will be required for '#' formats on debian11 ? + ret3, fileNamesProxyOut = unProxyfy( ret2 ) + logging.getLogger().debug("test logging 1") + logging.debug("test logging 2") + logging.debug( salome.orb.object_to_string( salome.logm ) ) + a = salome.logm.NaiveFetch() + logging.debug(a) + cont.Shutdown() + cpumeminfo = SALOME_PyNode.ReadCPUMemInfo( monitoringParamsMaster ) + self.assertTrue( isinstance(monitoringParamsMaster.outFileName,SALOME_PyNode.FileHolder) ) + dirInfo = SALOME_PyNode.ReadInodeSizeInfo( monitoringParamsForFileMaster ) + self.assertTrue( isinstance(monitoringParamsForFileMaster.outFileName,SALOME_PyNode.FileHolder) ) + self.assertTrue( os.path.exists( fsMonitoringFile ) ) + self.assertTrue( os.path.exists( cpuMemMonitoringFile ) ) + logging.debug( cpumeminfo ) + logging.debug( dirInfo ) + self.assertFalse( os.path.exists( fsMonitoringFile ) ) + self.assertFalse( os.path.exists( cpuMemMonitoringFile ) ) def testEasyNamingService(self): """ -- 2.39.2