From d1070d09cfbba6d9b72d8f2ca8d07d23f296686c Mon Sep 17 00:00:00 2001 From: Anthony Geay Date: Mon, 6 May 2024 12:38:29 +0200 Subject: [PATCH] [EDF30062] : Steering proxy threshold/directory without environement considerations to avoid unconsistent behaviour between different internal protocols. --- idl/SALOME_Component.idl | 4 +++ idl/SALOME_ContainerManager.idl | 4 +++ src/Basics/KernelBasis.cxx | 31 ++++++++++++++++++++ src/Basics/KernelBasis.hxx | 5 ++++ src/Basics/KernelBasis.i | 30 +++++++++++++++++++ src/Container/Container_i.cxx | 11 +++++++ src/Container/SALOME_ContainerManager.cxx | 12 ++++++++ src/Container/SALOME_ContainerManager.hxx | 4 +++ src/Container/SALOME_Container_i.hxx | 4 +++ src/Container/SALOME_PyNode.py | 18 +++--------- src/Container/Test/testProxy.py | 13 ++++---- src/Launcher/Test/testCrashProofContainer.py | 11 ++++--- src/Launcher/Test/testPerfLogManager1.py | 25 ++++++++++------ 13 files changed, 140 insertions(+), 32 deletions(-) diff --git a/idl/SALOME_Component.idl b/idl/SALOME_Component.idl index f31438844..04ac5b340 100644 --- a/idl/SALOME_Component.idl +++ b/idl/SALOME_Component.idl @@ -86,6 +86,10 @@ module Engines FieldsDict get_os_environment(); + void set_big_obj_on_disk_threshold(in long thresholdInByte); + + void set_big_obj_on_disk_directory(in string directory); + void addLogFileNameGroup(in vectorOfString groupOfLogFileNames); vectorOfVectorOfString getAllLogFileNameGroups(); diff --git a/idl/SALOME_ContainerManager.idl b/idl/SALOME_ContainerManager.idl index d06c3c06b..199ee7560 100644 --- a/idl/SALOME_ContainerManager.idl +++ b/idl/SALOME_ContainerManager.idl @@ -108,6 +108,10 @@ interface ContainerManager KeyValDict GetOverrideEnvForContainers(); + void SetBigObjOnDiskThreshold(in long thresholdInByte); + + void SetBigObjOnDiskDirectory(in string directory); + void SetCodeOnContainerStartUp(in string code); } ; diff --git a/src/Basics/KernelBasis.cxx b/src/Basics/KernelBasis.cxx index db29c9d87..649f51556 100644 --- a/src/Basics/KernelBasis.cxx +++ b/src/Basics/KernelBasis.cxx @@ -123,6 +123,37 @@ namespace SALOME } } +constexpr int SALOME_BIG_OBJ_ON_DISK_THRES_DFT = 50000000; + +static int SALOME_BIG_OBJ_ON_DISK_THRES = SALOME_BIG_OBJ_ON_DISK_THRES_DFT; + +int SALOME::GetBigObjOnDiskThreshold() +{ + return SALOME_BIG_OBJ_ON_DISK_THRES; +} + +void SALOME::SetBigObjOnDiskThreshold(int newThresholdInByte) +{ + SALOME_BIG_OBJ_ON_DISK_THRES = newThresholdInByte; +} + +static std::string SALOME_FILE_BIG_OBJ_DIR; + +std::string SALOME::GetBigObjOnDiskDirectory() +{ + return SALOME_FILE_BIG_OBJ_DIR; +} + +void SALOME::SetBigObjOnDiskDirectory(const std::string& directory) +{ + SALOME_FILE_BIG_OBJ_DIR = directory; +} + +bool SALOME::BigObjOnDiskDirectoryDefined() +{ + return ! SALOME_FILE_BIG_OBJ_DIR.empty(); +} + static SALOME::PyExecutionMode DefaultPyExecMode = SALOME::PyExecutionMode::NotSet; void SALOME::SetPyExecutionMode(PyExecutionMode mode) diff --git a/src/Basics/KernelBasis.hxx b/src/Basics/KernelBasis.hxx index 4a42e4257..cbecfe72e 100644 --- a/src/Basics/KernelBasis.hxx +++ b/src/Basics/KernelBasis.hxx @@ -43,4 +43,9 @@ namespace SALOME std::vector BASICS_EXPORT GetAllPyExecutionModes(); std::string BASICS_EXPORT GetPyExecutionModeStr(); PyExecutionMode BASICS_EXPORT GetPyExecutionMode(); + int BASICS_EXPORT GetBigObjOnDiskThreshold(); + void BASICS_EXPORT SetBigObjOnDiskThreshold(int newThresholdInByte); + std::string BASICS_EXPORT GetBigObjOnDiskDirectory(); + void BASICS_EXPORT SetBigObjOnDiskDirectory(const std::string& directory); + bool BASICS_EXPORT BigObjOnDiskDirectoryDefined(); } diff --git a/src/Basics/KernelBasis.i b/src/Basics/KernelBasis.i index 8979e3eab..092c7537f 100644 --- a/src/Basics/KernelBasis.i +++ b/src/Basics/KernelBasis.i @@ -51,6 +51,11 @@ using namespace SALOME; } %rename (HeatMarcel) HeatMarcelSwig; +%rename (GetBigObjOnDiskThreshold) GetBigObjOnDiskThresholdSwig; +%rename (SetBigObjOnDiskThreshold) SetBigObjOnDiskThresholdSwig; +%rename (GetBigObjOnDiskDirectory) GetBigObjOnDiskDirectorySwig; +%rename (SetBigObjOnDiskDirectory) SetBigObjOnDiskDirectorySwig; +%rename (BigObjOnDiskDirectoryDefined) BigObjOnDiskDirectoryDefinedSwig; bool getSSLMode(); void setSSLMode(bool sslMode); @@ -112,6 +117,31 @@ std::vector ReadFloatsInFileSwig(const std::string& fileName) return ret; } +int GetBigObjOnDiskThresholdSwig() +{ + return SALOME::GetBigObjOnDiskThreshold(); +} + +void SetBigObjOnDiskThresholdSwig(int newThreshold) +{ + return SALOME::SetBigObjOnDiskThreshold(newThreshold); +} + +std::string GetBigObjOnDiskDirectorySwig() +{ + return SALOME::GetBigObjOnDiskDirectory(); +} + +void SetBigObjOnDiskDirectorySwig(const std::string& directory) +{ + return SALOME::SetBigObjOnDiskDirectory(directory); +} + +bool BigObjOnDiskDirectoryDefinedSwig() +{ + return SALOME::BigObjOnDiskDirectoryDefined(); +} + void SetVerbosityLevelSwig(const std::string& level) { SetVerbosityLevelStr(level); diff --git a/src/Container/Container_i.cxx b/src/Container/Container_i.cxx index aee98c06c..a12edd4ff 100644 --- a/src/Container/Container_i.cxx +++ b/src/Container/Container_i.cxx @@ -63,6 +63,7 @@ int SIGUSR1 = 1000; #include "SALOME_Embedded_NamingService_Client.hxx" #include "SALOME_Embedded_NamingService.hxx" #include "Basics_Utils.hxx" +#include "KernelBasis.hxx" #include "PythonCppUtils.hxx" #include "Utils_CorbaException.hxx" @@ -1169,6 +1170,16 @@ Engines::FieldsDict *Abstract_Engines_Container_i::get_os_environment() return ret.release(); } +void Abstract_Engines_Container_i::set_big_obj_on_disk_threshold(CORBA::Long thresholdInByte) +{ + SALOME::SetBigObjOnDiskThreshold(thresholdInByte); +} + +void Abstract_Engines_Container_i::set_big_obj_on_disk_directory(const char *directory) +{ + SALOME::SetBigObjOnDiskDirectory(directory); +} + Engines::vectorOfString_var FromVecStringCppToCORBA( const std::vector& group) { Engines::vectorOfString_var ret( new Engines::vectorOfString ); diff --git a/src/Container/SALOME_ContainerManager.cxx b/src/Container/SALOME_ContainerManager.cxx index 232f47eb1..5b3c04986 100644 --- a/src/Container/SALOME_ContainerManager.cxx +++ b/src/Container/SALOME_ContainerManager.cxx @@ -236,6 +236,16 @@ void SALOME_ContainerManager::SetDeltaTimeBetweenCPUMemMeasureInMilliSecond(CORB this->_delta_time_measure_in_ms = timeInMS; } +void SALOME_ContainerManager::SetBigObjOnDiskThreshold(CORBA::Long thresholdInByte) +{ + SALOME::SetBigObjOnDiskThreshold(thresholdInByte); +} + +void SALOME_ContainerManager::SetBigObjOnDiskDirectory(const char *directory) +{ + SALOME::SetBigObjOnDiskDirectory(directory); +} + //============================================================================= //! Loop on all the containers listed in naming service, ask shutdown on each /*! CORBA Method: @@ -524,6 +534,8 @@ Engines::Container_ptr SALOME_ContainerManager::GiveContainer(const Engines::Con std::ostringstream envInfo; std::for_each( _override_env.begin(), _override_env.end(), [&envInfo](const std::pair& p) { envInfo << p.first << " = " << p.second << " "; } ); INFOS("[GiveContainer] container " << containerNameInNS << " override " << envInfo.str()); + cont->set_big_obj_on_disk_directory( SALOME::GetBigObjOnDiskDirectory().c_str() ); + cont->set_big_obj_on_disk_threshold( SALOME::GetBigObjOnDiskThreshold() ); Engines::FieldsDict envCorba; { auto sz = _override_env.size(); diff --git a/src/Container/SALOME_ContainerManager.hxx b/src/Container/SALOME_ContainerManager.hxx index 1ca71fef2..2584f70ba 100644 --- a/src/Container/SALOME_ContainerManager.hxx +++ b/src/Container/SALOME_ContainerManager.hxx @@ -78,6 +78,10 @@ public: void SetDeltaTimeBetweenCPUMemMeasureInMilliSecond(CORBA::Long timeInMS) override; + void SetBigObjOnDiskThreshold(CORBA::Long thresholdInByte) override; + + void SetBigObjOnDiskDirectory(const char *directory) override; + static const char *_ContainerManagerNameInNS; private: diff --git a/src/Container/SALOME_Container_i.hxx b/src/Container/SALOME_Container_i.hxx index 81643dc17..d929939f0 100644 --- a/src/Container/SALOME_Container_i.hxx +++ b/src/Container/SALOME_Container_i.hxx @@ -82,6 +82,10 @@ public: Engines::FieldsDict *get_os_environment() override; + void set_big_obj_on_disk_threshold(CORBA::Long thresholdInByte) override; + + void set_big_obj_on_disk_directory(const char *directory) override; + void addLogFileNameGroup(const Engines::vectorOfString& groupOfLogFileNames) override; Engines::vectorOfVectorOfString *getAllLogFileNameGroups() override; diff --git a/src/Container/SALOME_PyNode.py b/src/Container/SALOME_PyNode.py index e7f3cb625..312c7bf5d 100644 --- a/src/Container/SALOME_PyNode.py +++ b/src/Container/SALOME_PyNode.py @@ -30,6 +30,7 @@ import Engines__POA import SALOME__POA import SALOME import logging +import KernelBasis import abc import os import sys @@ -123,13 +124,6 @@ class SenderByte_i(SALOME__POA.SenderByte,Generic): def sendPart(self,n1,n2): return self.bytesToSend[n1:n2] -SALOME_FILE_BIG_OBJ_DIR = "SALOME_FILE_BIG_OBJ_DIR" - -SALOME_BIG_OBJ_ON_DISK_THRES_VAR = "SALOME_BIG_OBJ_ON_DISK_THRES" - -# default is 50 MB -SALOME_BIG_OBJ_ON_DISK_THRES_DFT = 50000000 - DicoForProxyFile = { } def GetSizeOfBufferedReader(f): @@ -184,11 +178,7 @@ def DecrRefInFile(fname): pass def GetBigObjectOnDiskThreshold(): - import os - if SALOME_BIG_OBJ_ON_DISK_THRES_VAR in os.environ: - return int( os.environ[SALOME_BIG_OBJ_ON_DISK_THRES_VAR] ) - else: - return SALOME_BIG_OBJ_ON_DISK_THRES_DFT + return KernelBasis.GetBigObjOnDiskThreshold() def ActivateProxyMecanismOrNot( sizeInByte ): thres = GetBigObjectOnDiskThreshold() @@ -199,9 +189,9 @@ def ActivateProxyMecanismOrNot( sizeInByte ): def GetBigObjectDirectory(): import os - if SALOME_FILE_BIG_OBJ_DIR not in os.environ: + if not KernelBasis.BigObjOnDiskDirectoryDefined(): raise RuntimeError("An object of size higher than limit detected and no directory specified to dump it in file !") - return os.path.expanduser( os.path.expandvars( os.environ[SALOME_FILE_BIG_OBJ_DIR] ) ) + return os.path.expanduser( os.path.expandvars( KernelBasis.GetBigObjOnDiskDirectory() ) ) def GetBigObjectFileName(): """ diff --git a/src/Container/Test/testProxy.py b/src/Container/Test/testProxy.py index 0e7c1e64d..07ff9b6de 100644 --- a/src/Container/Test/testProxy.py +++ b/src/Container/Test/testProxy.py @@ -39,15 +39,18 @@ class TestProxy(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpdirname: val_for_jj = "3333" val_for_big_obj = str( tmpdirname ) - val_for_thres = "100" # force proxy file + val_for_thres = 100 # force proxy file # Override environement for all containers launched - salome.cm.SetOverrideEnvForContainersSimple(env = [("jj",val_for_jj),("SALOME_FILE_BIG_OBJ_DIR",val_for_big_obj),("SALOME_BIG_OBJ_ON_DISK_THRES",val_for_thres)]) + salome.cm.SetBigObjOnDiskDirectory(val_for_big_obj) + salome.cm.SetBigObjOnDiskThreshold(val_for_thres) + salome.cm.SetOverrideEnvForContainersSimple(env = [("jj",val_for_jj)]) cont = salome.cm.GiveContainer(cp) ## Time to test it script_st = """import os -a = os.environ["SALOME_FILE_BIG_OBJ_DIR"] +import KernelBasis +a = KernelBasis.GetBigObjOnDiskDirectory() b = os.environ["jj"] -c = os.environ["SALOME_BIG_OBJ_ON_DISK_THRES"] +c = KernelBasis.GetBigObjOnDiskThreshold() j = a,b,c""" pyscript = cont.createPyScriptNode("testScript",script_st) a,b,c = pickle.loads(pyscript.execute(["j"],pickle.dumps(([],{}))))[0] @@ -55,7 +58,7 @@ j = a,b,c""" self.assertTrue( b == val_for_jj ) self.assertTrue( c == val_for_thres ) # check environment using POSIX API in the container process - for k,v in [("SALOME_FILE_BIG_OBJ_DIR",val_for_big_obj),("SALOME_BIG_OBJ_ON_DISK_THRES",val_for_thres),("jj",val_for_jj)]: + for k,v in [("jj",val_for_jj)]: assert( {elt.key:elt.value.value() for elt in cont.get_os_environment()}[k] == v ) # import SALOME_PyNode diff --git a/src/Launcher/Test/testCrashProofContainer.py b/src/Launcher/Test/testCrashProofContainer.py index 7bad47407..e8b7a9730 100644 --- a/src/Launcher/Test/testCrashProofContainer.py +++ b/src/Launcher/Test/testCrashProofContainer.py @@ -63,7 +63,8 @@ class testPerfLogManager1(unittest.TestCase): KernelBasis.SetPyExecutionMode("OutOfProcessNoReplay") # the aim of test is here hostname = "localhost" cp = pylauncher.GetRequestForGiveContainer(hostname,"container_crash_test") - salome.cm.SetOverrideEnvForContainersSimple(env = [("SALOME_BIG_OBJ_ON_DISK_THRES","1000")]) + salome.cm.SetBigObjOnDiskThreshold(1000) + salome.cm.SetOverrideEnvForContainersSimple(env = []) cont = salome.cm.GiveContainer(cp) poa = salome.orb.resolve_initial_references("RootPOA") obj = SALOME_PyNode.SenderByte_i(poa,pickle.dumps( (["i"],{"i": 3} ) )) ; id_o = poa.activate_object(obj) ; refPtr = poa.id_to_reference(id_o) @@ -90,7 +91,8 @@ class testPerfLogManager1(unittest.TestCase): KernelBasis.SetPyExecutionMode("OutOfProcessWithReplay") # the aim of test is here hostname = "localhost" cp = pylauncher.GetRequestForGiveContainer(hostname,"container_crash_test") - salome.cm.SetOverrideEnvForContainersSimple(env = [("SALOME_BIG_OBJ_ON_DISK_THRES","1000")]) + salome.cm.SetBigObjOnDiskThreshold(1000) + salome.cm.SetOverrideEnvForContainersSimple(env = []) cont = salome.cm.GiveContainer(cp) poa = salome.orb.resolve_initial_references("RootPOA") obj = SALOME_PyNode.SenderByte_i(poa,pickle.dumps( (["i"],{"i": 3} ) )) ; id_o = poa.activate_object(obj) ; refPtr = poa.id_to_reference(id_o) @@ -131,9 +133,10 @@ class testPerfLogManager1(unittest.TestCase): assert(isinstance(KernelBasis.GetAllPyExecutionModes(),tuple)) KernelBasis.SetPyExecutionMode("OutOfProcessNoReplay") # the aim of test is here hostname = "localhost" - PROXY_THRES = "-1" + PROXY_THRES = -1 # - salome.cm.SetOverrideEnvForContainersSimple(env = [("SALOME_BIG_OBJ_ON_DISK_THRES",PROXY_THRES)]) + salome.cm.SetBigObjOnDiskThreshold(PROXY_THRES) + salome.cm.SetOverrideEnvForContainersSimple(env = []) salome.cm.SetDeltaTimeBetweenCPUMemMeasureInMilliSecond( 250 ) cp = pylauncher.GetRequestForGiveContainer(hostname,"container_cpu_mem_out_process_test") cont = salome.cm.GiveContainer(cp) diff --git a/src/Launcher/Test/testPerfLogManager1.py b/src/Launcher/Test/testPerfLogManager1.py index 7df51a851..668a1e4f7 100644 --- a/src/Launcher/Test/testPerfLogManager1.py +++ b/src/Launcher/Test/testPerfLogManager1.py @@ -24,6 +24,7 @@ import salome import Engines import pylauncher import SALOME_PyNode +import KernelBasis import glob import pickle @@ -67,7 +68,7 @@ class testPerfLogManager1(unittest.TestCase): cp = pylauncher.GetRequestForGiveContainer(hostname,"container_test") salome.logm.clear() #PROXY_THRES = "-1" - PROXY_THRES = "1" + PROXY_THRES = 1 with SALOME_PyNode.GenericPythonMonitoringLauncherCtxMgr( SALOME_PyNode.FileSystemMonitoring(1000,os.path.dirname( salome.__file__ )) ) as monitoringParamsForFileMaster: with SALOME_PyNode.GenericPythonMonitoringLauncherCtxMgr( SALOME_PyNode.CPUMemoryMonitoring(1000) ) as monitoringParamsMaster: with tempfile.TemporaryDirectory() as tmpdirnameMonitoring: @@ -82,9 +83,11 @@ class testPerfLogManager1(unittest.TestCase): pyFileContainingCodeOfMonitoring = monitoringParams.pyFileName.filename logging.debug("Python file containing code of monitoring : {}".format(pyFileContainingCodeOfMonitoring)) val_for_big_obj = str( tmpdirname ) - os.environ["SALOME_FILE_BIG_OBJ_DIR"] = val_for_big_obj + KernelBasis.SetBigObjOnDiskDirectory( val_for_big_obj ) # Override environement for all containers launched - salome.cm.SetOverrideEnvForContainersSimple(env = [("SALOME_FILE_BIG_OBJ_DIR",val_for_big_obj),("SALOME_BIG_OBJ_ON_DISK_THRES",PROXY_THRES)]) + salome.cm.SetBigObjOnDiskDirectory(val_for_big_obj) + salome.cm.SetBigObjOnDiskThreshold(PROXY_THRES) + salome.cm.SetOverrideEnvForContainersSimple(env = []) salome.cm.SetDeltaTimeBetweenCPUMemMeasureInMilliSecond( 250 ) cont = salome.cm.GiveContainer(cp) logging.debug("{} {}".format(40*"*",cont.getPID())) @@ -211,8 +214,8 @@ time.sleep(1) hostname = "localhost" cp = pylauncher.GetRequestForGiveContainer(hostname,"container_test_three") salome.logm.clear() - #PROXY_THRES = "-1" - PROXY_THRES = "1" + #PROXY_THRES = -1 + PROXY_THRES = 1 with tempfile.TemporaryDirectory() as tmpdirnameMonitoring: fsMonitoringFile = os.path.join( str( tmpdirnameMonitoring ), "zeFS.txt" ) cpuMemMonitoringFile = os.path.join( str( tmpdirnameMonitoring ), "zeCPUMem.txt" ) @@ -229,9 +232,11 @@ time.sleep(1) pyFileContainingCodeOfMonitoring = monitoringParams.pyFileName.filename logging.debug("Python file containing code of monitoring : {}".format(pyFileContainingCodeOfMonitoring)) val_for_big_obj = str( tmpdirname ) - os.environ["SALOME_FILE_BIG_OBJ_DIR"] = val_for_big_obj + KernelBasis.SetBigObjOnDiskDirectory( val_for_big_obj ) + salome.cm.SetBigObjOnDiskDirectory(val_for_big_obj) + salome.cm.SetBigObjOnDiskThreshold(PROXY_THRES) # Override environement for all containers launched - salome.cm.SetOverrideEnvForContainersSimple(env = [("SALOME_FILE_BIG_OBJ_DIR",val_for_big_obj),("SALOME_BIG_OBJ_ON_DISK_THRES",PROXY_THRES)]) + salome.cm.SetOverrideEnvForContainersSimple(env = []) salome.cm.SetDeltaTimeBetweenCPUMemMeasureInMilliSecond( 250 ) cont = salome.cm.GiveContainer(cp) logging.debug("{} {}".format(40*"*",cont.getPID())) @@ -290,11 +295,13 @@ sys.stderr.write("fake error message\\n") hostname = "localhost" cp = pylauncher.GetRequestForGiveContainer(hostname,"container_test_two") salome.logm.clear() - PROXY_THRES = "1" + PROXY_THRES = 1 with tempfile.TemporaryDirectory() as tmpdirname: ior_ns_file = os.path.join(tmpdirname,"ns.ior") val_for_big_obj = str( tmpdirname ) - salome.cm.SetOverrideEnvForContainersSimple(env = [("SALOME_FILE_BIG_OBJ_DIR",val_for_big_obj),("SALOME_BIG_OBJ_ON_DISK_THRES",PROXY_THRES)]) + salome.cm.SetBigObjOnDiskDirectory(val_for_big_obj) + salome.cm.SetBigObjOnDiskThreshold(PROXY_THRES) + salome.cm.SetOverrideEnvForContainersSimple(env = []) salome.cm.SetDeltaTimeBetweenCPUMemMeasureInMilliSecond( 250 ) salome.naming_service.DumpIORInFile( ior_ns_file ) cont = salome.cm.GiveContainer(cp) -- 2.39.2