src/Launcher/Test/testPerfLogManager1.py

   1 #  -*- coding: iso-8859-1 -*-
   2 # Copyright (C) 2024  CEA/DEN, EDF R&D
   3 #
   4 # This library is free software; you can redistribute it and/or
   5 # modify it under the terms of the GNU Lesser General Public
   6 # License as published by the Free Software Foundation; either
   7 # version 2.1 of the License, or (at your option) any later version.
   8 #
   9 # This library is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12 # Lesser General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Lesser General Public
  15 # License along with this library; if not, write to the Free Software
  16 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  17 #
  18 # See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
  19 #
  20
  21 import unittest
  22 import os
  23 import salome
  24 import Engines
  25 import pylauncher
  26 import SALOME_PyNode
  27 import KernelBasis
  28
  29 import glob
  30 import pickle
  31 import tempfile
  32 import logging
  33 from datetime import datetime
  34
  35 def flush():
  36     import sys
  37     sys.stdout.flush()
  38     sys.stderr.flush()
  39
  40 def unProxyfy( objs ):
  41     """
  42     objs is a list of SenderByte
  43     """
  44     ret = []
  45     fileNamesProxyOut = []
  46     for ret2 in objs:
  47         ret3 = pickle.loads( SALOME_PyNode.SeqByteReceiver(ret2).data() )
  48         ret4 = ret3
  49         fileNameProxyOut = None
  50         if SALOME_PyNode.GetBigObjectOnDiskThreshold() != -1:
  51             if isinstance( ret3, SALOME_PyNode.BigObjectOnDiskBase ):
  52                 ret4 = ret3.get()
  53             else:
  54                 raise RuntimeError("Oooops")
  55             ret3.unlinkOnDestructor()
  56             fileNameProxyOut = ret3.getFileName()
  57         logging.debug("Prxy file : {}".format( fileNameProxyOut ))
  58         ret.append( ret3 )
  59         fileNamesProxyOut.append( fileNameProxyOut )
  60     return ret, fileNamesProxyOut
  61
  62 class testPerfLogManager1(unittest.TestCase):
  63     def testPerfLogManager0(self):
  64         """
  65         [EDF29150] : This test checks measure performance methods
  66         """
  67         hostname = "localhost"
  68         cp = pylauncher.GetRequestForGiveContainer(hostname,"container_test")
  69         salome.logm.clear()
  70         #PROXY_THRES = "-1"
  71         PROXY_THRES = 1
  72         with SALOME_PyNode.GenericPythonMonitoringLauncherCtxMgr( SALOME_PyNode.FileSystemMonitoring(1000,os.path.dirname( salome.__file__ )) ) as monitoringParamsForFileMaster:
  73             with SALOME_PyNode.GenericPythonMonitoringLauncherCtxMgr( SALOME_PyNode.CPUMemoryMonitoring(1000) ) as monitoringParamsMaster:
  74                 with tempfile.TemporaryDirectory() as tmpdirnameMonitoring:
  75                     monitoringFile = os.path.join( str( tmpdirnameMonitoring ), "zeHtop.pckl" )
  76                     monitoringFileTwo = os.path.join( str( tmpdirnameMonitoring ), "zeHtopTwo.pckl" )
  77                     iorLogm = os.path.join( str( tmpdirnameMonitoring ), "logm.ior" )
  78                     with open(iorLogm,"w") as f:
  79                         f.write( salome.orb.object_to_string(salome.logm) )
  80                     logging.debug("Monitoring file : {}".format(monitoringFile))
  81                     with tempfile.TemporaryDirectory() as tmpdirname:
  82                         with salome.LogManagerLaunchMonitoringFileCtxMgr(250,monitoringFile) as monitoringParams:
  83                             pyFileContainingCodeOfMonitoring = monitoringParams.pyFileName.filename
  84                             logging.debug("Python file containing code of monitoring : {}".format(pyFileContainingCodeOfMonitoring))
  85                             val_for_big_obj = str( tmpdirname )
  86                             KernelBasis.SetBigObjOnDiskDirectory( val_for_big_obj )
  87                             # Override environement for all containers launched
  88                             salome.cm.SetBigObjOnDiskDirectory(val_for_big_obj)
  89                             salome.cm.SetBigObjOnDiskThreshold(PROXY_THRES)
  90                             salome.cm.SetOverrideEnvForContainersSimple(env = [])
  91                             salome.cm.SetDeltaTimeBetweenCPUMemMeasureInMilliSecond( 250 )
  92                             cont = salome.cm.GiveContainer(cp)
  93                             logging.debug("{} {}".format(40*"*",cont.getPID()))
  94                             script_st = """
  95 import logging
  96 import sys
  97 import KernelBasis
  98 from datetime import datetime
  99 cst = KernelBasis.GetTimeAdjustmentCst()
 100 logging.debug("constant = {}".format(cst))
 101 nbcore = 3
 102 my_log_4_this_session.addFreestyleAndFlush( ("a",datetime.now()) ) # exemple of custom tracking
 103 print("coucou {} {}".format(len(zeinput0),len(zeinput1)))
 104 logging.debug("debug or not debug")
 105 ob = [ [ bytes(3000000) ] ]
 106 pihm, ts = KernelBasis.HeatMarcel(1 * nbcore * cst,nbcore)
 107 print("Time ellapse spent : {} s".format(ts))
 108 ob2 = [ [ bytes(100000) ] ]
 109 pihm, ts = KernelBasis.HeatMarcel(1 * nbcore * cst,nbcore)
 110 print("Time ellapse spent : {} s".format(ts))
 111 sys.stderr.write("fake error message\\n")
 112 """
 113                             poa = salome.orb.resolve_initial_references("RootPOA")
 114                             zeinput0 = [ bytes(100000000) ]
 115                             if SALOME_PyNode.GetBigObjectOnDiskThreshold() != -1:
 116                                 zeinput0 = SALOME_PyNode.ProxyfyPickeled( zeinput0 )
 117                                 zeinput0.unlinkOnDestructor()
 118                             obj = SALOME_PyNode.SenderByte_i(poa,pickle.dumps( (["zeinput0"],{"zeinput0": [zeinput0], "zeinput1": [ [zeinput0], [zeinput0] ] }) ))
 119                             id_o = poa.activate_object(obj)
 120                             refPtr = poa.id_to_reference(id_o)
 121                             pyscript2 = cont.createPyScriptNode("testScript2",script_st)
 122                             pyscript2.executeFirst(refPtr)
 123                             ret2 = pyscript2.executeSecond(["ob","ob2"])# generate a DeprecationWarning: PY_SSIZE_T_CLEAN will be required for '#' formats on debian11 ?
 124                             ret3, fileNamesProxyOut = unProxyfy( ret2 )
 125                             logging.getLogger().debug("test logging 1")
 126                             logging.debug("test logging 2")
 127                             logging.debug( salome.orb.object_to_string( salome.logm ) )
 128                             a = salome.logm.NaiveFetch()
 129                             logging.debug(a)
 130                             fs = a[0][1][0].get().freestyle
 131                             self.assertEqual(len(fs),1)
 132                             self.assertEqual(fs[0][0],"a")
 133                             self.assertTrue( isinstance( fs[0][1], datetime ) )
 134                             logging.debug(a[0][1][0])
 135                             logging.debug( a[0][1][0].get()._input_hdd_mem._data[0]._data[0]._hdd_mem ) # important
 136                             logging.debug( a[0][1][0].get()._input_hdd_mem._data[1]._data[0]._data[0]._hdd_mem ) # important
 137                             fileNameProxyIn = a[0][1][0].get()._input_hdd_mem._data[0]._data[0]._file_name
 138                             logging.debug( fileNameProxyIn )
 139                             del zeinput0
 140                             del ret3
 141                             import gc ; gc.collect()
 142                             if fileNameProxyIn is not None:
 143                                 if os.path.exists(fileNameProxyIn):
 144                                     raise RuntimeError("Oooops 2")
 145                             for fileNameProxyOut in fileNamesProxyOut:
 146                                 if fileNameProxyOut is not None:
 147                                     if os.path.exists(fileNameProxyOut):
 148                                         raise RuntimeError("Oooops 3")
 149                             # execution #2 inside last
 150                             script_st2 = """
 151 import logging
 152 b = 7+a
 153 logging.debug("Execution 2")
 154 import time
 155 time.sleep(1)
 156 """
 157                             obj2 = SALOME_PyNode.SenderByte_i(poa,pickle.dumps((["a"],{"a":3})))
 158                             id2_o = poa.activate_object(obj2)
 159                             refPtr2 = poa.id_to_reference(id2_o)
 160                             pyscript2.assignNewCompiledCode(script_st2)
 161                             pyscript2.executeFirst(refPtr2)
 162                             ret2_0 = pyscript2.executeSecond(["b"])
 163                             ret2_1, fileNamesProxyOut2 = unProxyfy( ret2_0 )
 164                             logging.debug( fileNamesProxyOut2 )
 165                             a = salome.logm.NaiveFetch()
 166                             del ret2_1
 167                             import gc ; gc.collect()
 168                             for fileNameProxyOut in fileNamesProxyOut2:
 169                                 if fileNameProxyOut is not None:
 170                                     if os.path.exists(fileNameProxyOut):
 171                                         raise RuntimeError("Oooops 3")
 172                             #
 173                             fname = os.path.join(str( tmpdirname ),"perf.log")
 174                             salome.logm.DumpInFile( fname )
 175                             logManagerInst0 = salome.LogManagerLoadFromFile( fname )
 176                             logging.debug( logManagerInst0[0][1][0].get()._input_hdd_mem._data[1]._data[0]._data[0]._hdd_mem ) # important
 177                             logManagerInst = salome.logm.Fetch(True)
 178                             logManagerInst2 = salome.logm.Fetch(True)
 179                             logging.debug( salome.LogManagerLoadFromIORFile( iorLogm )[0][1][0].get() )
 180                             salome.logm.putStructInFileAtomic(False,monitoringFileTwo)
 181                             logging.debug( salome.LogManagerLoadFromFile(monitoringFileTwo)[0][1][0].get() )
 182                             logging.debug( logManagerInst[0][1][0].get()._input_hdd_mem._data[1]._data[0]._data[0]._hdd_mem ) # important
 183                             self.assertTrue( logManagerInst2[0][1][0].get() is None )
 184                             self.assertTrue( logManagerInst[0][1][1].get()._output_hdd_mem._data[0]._file_name == fileNamesProxyOut2[0] )
 185                             logging.debug( logManagerInst[0][1][1].log() )
 186                             # 2 files because a backup file is stored in case of unexpected kill during
 187                             self.assertEqual( len( glob.glob("{}*".format(monitoringFile) ) ) , 2 )
 188                             # leaving MonitoringFile Manager -> backup file is killed
 189                             pass
 190                     #self.assertEqual(monitoringFileSafe, monitoringFile)
 191                     self.assertEqual( len( glob.glob("{}*".format(monitoringFile) ) ) , 1 )
 192                     logging.debug( salome.LogManagerLoadFromFile(monitoringFile)[0][1][0].get() )
 193                     del monitoringParams
 194                     import gc ; gc.collect()
 195                     self.assertFalse( os.path.exists(pyFileContainingCodeOfMonitoring) )
 196                     cont.Shutdown()
 197                 fnToTest0 = monitoringParamsMaster.outFileName.filename
 198                 cpumeminfo = SALOME_PyNode.ReadCPUMemInfo( monitoringParamsMaster )
 199                 self.assertTrue( isinstance(monitoringParamsMaster.outFileName,SALOME_PyNode.FileDeleter) )
 200                 del monitoringParamsMaster
 201                 import gc
 202                 gc.collect()
 203                 self.assertTrue( os.path.exists( fnToTest0 ) )
 204             dirInfo = SALOME_PyNode.ReadInodeSizeInfo( monitoringParamsForFileMaster )
 205             self.assertTrue( isinstance(monitoringParamsForFileMaster.outFileName,SALOME_PyNode.FileDeleter) )
 206             self.assertFalse( os.path.exists( fnToTest0 ) )
 207         logging.debug( cpumeminfo )
 208         logging.debug( dirInfo )
 209
 210     def testPerfLogManager1(self):
 211         """
 212         [EDF29150] : Similar than testPerfLogManager1 but with out fileName precised for CPU/Mem monitoring and FS monitoring
 213         """
 214         hostname = "localhost"
 215         cp = pylauncher.GetRequestForGiveContainer(hostname,"container_test_three")
 216         salome.logm.clear()
 217         #PROXY_THRES = -1
 218         PROXY_THRES = 1
 219         with tempfile.TemporaryDirectory() as tmpdirnameMonitoring:
 220             fsMonitoringFile = os.path.join( str( tmpdirnameMonitoring ), "zeFS.txt" )
 221             cpuMemMonitoringFile = os.path.join( str( tmpdirnameMonitoring ), "zeCPUMem.txt" )
 222             with SALOME_PyNode.GenericPythonMonitoringLauncherCtxMgr( SALOME_PyNode.FileSystemMonitoring(1000,os.path.dirname( salome.__file__ ),fsMonitoringFile) ) as monitoringParamsForFileMaster:
 223                 with SALOME_PyNode.GenericPythonMonitoringLauncherCtxMgr( SALOME_PyNode.CPUMemoryMonitoring(1000,cpuMemMonitoringFile) ) as monitoringParamsMaster:
 224                     monitoringFile = os.path.join( str( tmpdirnameMonitoring ), "zeHtop.pckl" )
 225                     monitoringFileTwo = os.path.join( str( tmpdirnameMonitoring ), "zeHtopTwo.pckl" )
 226                     iorLogm = os.path.join( str( tmpdirnameMonitoring ), "logm.ior" )
 227                     with open(iorLogm,"w") as f:
 228                         f.write( salome.orb.object_to_string(salome.logm) )
 229                     logging.debug("Monitoring file : {}".format(monitoringFile))
 230                     with tempfile.TemporaryDirectory() as tmpdirname:
 231                         with salome.LogManagerLaunchMonitoringFileCtxMgr(250,monitoringFile) as monitoringParams:
 232                             pyFileContainingCodeOfMonitoring = monitoringParams.pyFileName.filename
 233                             logging.debug("Python file containing code of monitoring : {}".format(pyFileContainingCodeOfMonitoring))
 234                             val_for_big_obj = str( tmpdirname )
 235                             KernelBasis.SetBigObjOnDiskDirectory( val_for_big_obj )
 236                             salome.cm.SetBigObjOnDiskDirectory(val_for_big_obj)
 237                             salome.cm.SetBigObjOnDiskThreshold(PROXY_THRES)
 238                             # Override environement for all containers launched
 239                             salome.cm.SetOverrideEnvForContainersSimple(env = [])
 240                             salome.cm.SetDeltaTimeBetweenCPUMemMeasureInMilliSecond( 250 )
 241                             cont = salome.cm.GiveContainer(cp)
 242                             logging.debug("{} {}".format(40*"*",cont.getPID()))
 243                             script_st = """
 244 import logging
 245 import sys
 246 import KernelBasis
 247 from datetime import datetime
 248 cst = KernelBasis.GetTimeAdjustmentCst()
 249 logging.debug("constant = {}".format(cst))
 250 nbcore = 3
 251 my_log_4_this_session.addFreestyleAndFlush( ("a",datetime.now()) ) # exemple of custom tracking
 252 print("coucou {} {}".format(len(zeinput0),len(zeinput1)))
 253 logging.debug("debug or not debug")
 254 ob = [ [ bytes(3000000) ] ]
 255 pihm, ts = KernelBasis.HeatMarcel(1 * nbcore * cst,nbcore)
 256 print("Time ellapse spent : {} s".format(ts))
 257 ob2 = [ [ bytes(100000) ] ]
 258 pihm, ts = KernelBasis.HeatMarcel(1 * nbcore * cst,nbcore)
 259 print("Time ellapse spent : {} s".format(ts))
 260 sys.stderr.write("fake error message\\n")
 261 """
 262                             poa = salome.orb.resolve_initial_references("RootPOA")
 263                             zeinput0 = [ bytes(100000000) ]
 264                             if SALOME_PyNode.GetBigObjectOnDiskThreshold() != -1:
 265                                 zeinput0 = SALOME_PyNode.ProxyfyPickeled( zeinput0 )
 266                                 zeinput0.unlinkOnDestructor()
 267                             obj = SALOME_PyNode.SenderByte_i(poa,pickle.dumps( (["zeinput0"],{"zeinput0": [zeinput0], "zeinput1": [ [zeinput0], [zeinput0] ] }) ))
 268                             id_o = poa.activate_object(obj)
 269                             refPtr = poa.id_to_reference(id_o)
 270                             pyscript2 = cont.createPyScriptNode("testScript2",script_st)
 271                             pyscript2.executeFirst(refPtr)
 272                             ret2 = pyscript2.executeSecond(["ob","ob2"])# generate a DeprecationWarning: PY_SSIZE_T_CLEAN will be required for '#' formats on debian11 ?
 273                             ret3, fileNamesProxyOut = unProxyfy( ret2 )
 274                             logging.getLogger().debug("test logging 1")
 275                             logging.debug("test logging 2")
 276                             logging.debug( salome.orb.object_to_string( salome.logm ) )
 277                             a = salome.logm.NaiveFetch()
 278                             logging.debug(a)
 279                             cont.Shutdown()
 280                 cpumeminfo = SALOME_PyNode.ReadCPUMemInfo( monitoringParamsMaster )
 281                 self.assertTrue( isinstance(monitoringParamsMaster.outFileName,SALOME_PyNode.FileHolder) )
 282             dirInfo = SALOME_PyNode.ReadInodeSizeInfo( monitoringParamsForFileMaster )
 283             self.assertTrue( isinstance(monitoringParamsForFileMaster.outFileName,SALOME_PyNode.FileHolder) )
 284             self.assertTrue( os.path.exists( fsMonitoringFile ) )
 285             self.assertTrue( os.path.exists( cpuMemMonitoringFile ) )
 286         logging.debug( cpumeminfo )
 287         logging.debug( dirInfo )
 288         self.assertFalse( os.path.exists( fsMonitoringFile ) )
 289         self.assertFalse( os.path.exists( cpuMemMonitoringFile ) )
 290
 291     def testEasyNamingService(self):
 292         """
 293         [EDF29150] : This test checks measure performance methods
 294         """
 295         hostname = "localhost"
 296         cp = pylauncher.GetRequestForGiveContainer(hostname,"container_test_two")
 297         salome.logm.clear()
 298         PROXY_THRES = 1
 299         with tempfile.TemporaryDirectory() as tmpdirname:
 300             ior_ns_file = os.path.join(tmpdirname,"ns.ior")
 301             val_for_big_obj = str( tmpdirname )
 302             salome.cm.SetBigObjOnDiskDirectory(val_for_big_obj)
 303             salome.cm.SetBigObjOnDiskThreshold(PROXY_THRES)
 304             salome.cm.SetOverrideEnvForContainersSimple(env = [])
 305             salome.cm.SetDeltaTimeBetweenCPUMemMeasureInMilliSecond( 250 )
 306             salome.naming_service.DumpIORInFile( ior_ns_file )
 307             cont = salome.cm.GiveContainer(cp)
 308             script_st = """
 309 from SALOME_Embedded_NamingService_ClientPy import SALOME_Embedded_NamingService_ClientPy
 310 ior_ns_file = "{ior_ns_file}"
 311 ns = SALOME_Embedded_NamingService_ClientPy.BuildFromIORFile( ior_ns_file )
 312 ret = ns.repr()
 313 """.format(**locals())
 314             cont = salome.cm.GiveContainer(cp)
 315             pyscript2 = cont.createPyScriptNode("testScript3",script_st)
 316             retCoarse = pickle.loads( pyscript2.execute( ["ret"], pickle.dumps( ([],{} ) ) ) )
 317             ret = retCoarse[0]
 318             self.assertTrue( isinstance(ret,list) and isinstance(ret[0],str) )
 319             cont.Shutdown()
 320         pass
 321
 322 if __name__ == '__main__':
 323     from salome_utils import positionVerbosityOfLoggerRegardingState,setVerboseLevel,setVerbose
 324     salome.standalone()
 325     salome.salome_init()
 326     setVerbose(True)
 327     setVerboseLevel(logging.DEBUG)
 328     positionVerbosityOfLoggerRegardingState()
 329     unittest.main()