{
ContainerPerfLog declareContainer(in string contInNS, in string logfile);
ListOfContainerPerfLog listOfContainerLogs();
- SALOME::vectorOfByte getAllStruct();
+ SALOME::vectorOfByte getAllStruct( in boolean clearMemory );
};
};
return self._eff
def __getitem__(self,i):
return ScriptExecInfoDeco( self._eff[i], self )
+ def __len__(self):
+ return self._eff.__len__()
def __str__(self):
return self._eff.__str__()
def __repr__(self):
self._scripts = [ScriptInfoClt(elt) for elt in contLogPtr.listOfScripts()]
class ContainerLogInfo(ContainerLogInfoAbstract):
- def __init__(self, logFile, nsEntry, scripts):
+ def __init__(self, nsEntry, logFile, scripts):
self._log_file = logFile
self._ns_entry = nsEntry
self._scripts = scripts
if self._visitor:
self._visitor.leave()
pass
+
+class OffsetType:
+ def __init__(self,i):
+ self._i = i
+ def __int__(self):
+ return self._i
+ def __iadd__(self,delta):
+ self._i += delta
+ return self
+
+def unserializeInt(structData, offset):
+ from ctypes import c_int
+ sz_of_cint = 4
+ sz = c_int.from_buffer_copy( structData[int(offset):int(offset)+sz_of_cint] ).value
+ offset += sz_of_cint
+ return sz
+
+def unserializeString(structData,offset):
+ sz = unserializeInt(structData,offset)
+ ret = structData[int(offset):int(offset)+sz].decode()
+ offset += sz
+ return ret
+
+def unserializeContainerScriptExecPerfLog(structData, offset):
+ import pickle
+ sz = unserializeInt(structData,offset)
+ inst = None
+ if sz > 0:
+ inst = pickle.loads( structData[int(offset):int(offset)+sz] )
+ offset += sz
+ return inst
+
+def unserializeContainerScriptPerfLog(structData, offset):
+ name = unserializeString(structData,offset)
+ code = unserializeString(structData,offset)
+ numberOfSessions = unserializeInt(structData,offset)
+ sessions = []
+ for _ in range(numberOfSessions):
+ session = unserializeContainerScriptExecPerfLog(structData,offset)
+ sessions.append( session )
+ return ScriptInfo(name,code,sessions)
+
+def unserializeContainerPerfLog(structData, offset):
+ nsEntry = unserializeString(structData,offset)
+ logFile = unserializeString(structData,offset)
+ scripts = []
+ nbScripts = unserializeInt(structData,offset)
+ for _ in range(nbScripts):
+ script = unserializeContainerScriptPerfLog(structData,offset)
+ scripts.append( script )
+ return ContainerLogInfo(nsEntry,logFile,scripts)
+
+def unserializeLogManager(structData):
+ offset = OffsetType(0)
+ numberOfScripts = unserializeInt(structData,offset)
+ logManagerInst = []
+ for _ in range(numberOfScripts):
+ containerPerfLogInst = unserializeContainerPerfLog(structData,offset)
+ logManagerInst.append( containerPerfLogInst )
+ if int(offset) != len(structData):
+ raise RuntimeError("Something went wrong during unserialization phase.")
+ return logManagerInst
cont->monitoringtimeresms( this->_delta_time_measure_in_ms );
INFOS("[GiveContainer] container " << containerNameInNS << " first CORBA invocation OK");
std::ostringstream envInfo;
- std::for_each( _override_env.begin(), _override_env.end(), [&envInfo](const std::pair<std::string,std::string>& p) { envInfo << p.first << " = " << p.second << std::endl; } );
+ std::for_each( _override_env.begin(), _override_env.end(), [&envInfo](const std::pair<std::string,std::string>& p) { envInfo << p.first << " = " << p.second << " "; } );
INFOS("[GiveContainer] container " << containerNameInNS << " override " << envInfo.str());
Engines::FieldsDict envCorba;
{
return value
def __str__(self):
- str = []
- str.append("nb_proc: %s" % self.nb_proc)
- str.append("workdir: %s" % self.workdir)
- str.append("isTmpDir: %s" % self.isTmpDir)
- str.append("name_server: %s" % self.name_server)
- str.append("container: %s" % self.container)
- str.append("container_name: %s" % self.container_name)
- str.append("libbatch_nodefile: %s" % self.libbatch_nodefile)
- str.append("machine_file: %s" % self.machine_file)
- str.append("ompi_uri_file: %s" % self.ompi_uri_file)
- str.append("--")
- return "\n".join(str)
\ No newline at end of file
+ str = """nb_proc: {self.nb_proc} workdir: {self.workdir} isTmpDir: {self.isTmpDir} name_server: {self.name_server} container: {self.container} container_name: {self.container_name} libbatch_nodefile: {self.libbatch_nodefile} machine_file: {self.machine_file} ompi_uri_file: {self.ompi_uri_file}""".format(**locals())
+ return str
esm = KernelLauncher.GetExternalServer()
# idem for logm
logm = KernelLauncher.myLogManager()
+ type(logm).NaiveFetch = LogManagerNaiveFetch
type(logm).Fetch = LogManagerFetch
+ type(logm).DumpInFile = LogManagerDumpInFile
+ type(logm).LoadFromFile = LogManagerLoadFromFile
#
import KernelLogger
naming_service.Register(KernelLogger.myLogger(),"/Logger")
envEff = [ Engines.KeyValPairString(key=k,val=v) for k,v in env ]
return self.SetOverrideEnvForContainers( envEff )
-def LogManagerFetch(self):
+def LogManagerNaiveFetch(self):
+ """
+ Fetch data from server with multiple CORBA invokations.
+ """
import SALOME_ContainerHelper
return [SALOME_ContainerHelper.ContainerLogInfoClt(elt) for elt in self.listOfContainerLogs()]
+def LogManagerFetch(self,clearMemory = False):
+ """
+ Fetch data from server in one shot mode.
+ """
+ from SALOME_ContainerHelper import unserializeLogManager
+ return unserializeLogManager( self.getAllStruct(clearMemory) )
+
+def LogManagerDumpInFile(self,fileName,clearMemory = False):
+ with open(fileName,"wb") as f:
+ f.write( self.getAllStruct( clearMemory ) )
+
+def LogManagerLoadFromFile(self,fileName):
+ from SALOME_ContainerHelper import unserializeLogManager
+ with open(fileName,"rb") as f:
+ data = f.read()
+ return unserializeLogManager( data )
+
#to expose all objects to pydoc
__all__=dir()
SALOME_ContainerScriptExecPerfLog *servC = dynamic_cast<SALOME_ContainerScriptExecPerfLog *>(serv);
visitor.visitContainerScriptExecPerfLog( *servC );
}
- visitor.enterContainerScriptPerfLog( *this );
+ visitor.leaveContainerScriptPerfLog( *this );
}
Engines::ContainerScriptExecPerfLog_ptr SALOME_ContainerScriptPerfLog::addExecutionSession()
visitor.leaveLogManager( *this );
}
-SALOME::vectorOfByte *SALOME_LogManager::getAllStruct()
+/*!
+ \param [in] unloadMemory - specify if big part of struct data (SALOME_ContainerScriptExecPerfLog) is cleared after retrieving data
+ */
+SALOME::vectorOfByte *SALOME_LogManager::getAllStruct(bool clearMemory)
{
- std::vector<char> data = this->dumpCppInternalFrmt();
+ std::vector<char> data = this->dumpCppInternalFrmt(clearMemory);
return FromVectCharToCorba(data);
}
class InternalFormatVisitorDump : public SALOME_VisitorContainerLog
{
public:
- InternalFormatVisitorDump(std::vector<char> *data):_data(data) { }
+ InternalFormatVisitorDump(bool clearMemory, std::vector<char> *data):_clear_memory(clearMemory),_data(data) { }
void enterLogManager(SALOME_LogManager& inst) override;
void leaveLogManager(SALOME_LogManager& inst) override { }
void enterContainerPerfLog(SALOME_ContainerPerfLog& inst) override;
void leaveContainerScriptPerfLog(SALOME_ContainerScriptPerfLog& inst) override { }
void visitContainerScriptExecPerfLog(SALOME_ContainerScriptExecPerfLog& inst) override;
private:
+ bool _clear_memory = false;
std::vector<char> *_data = nullptr;
};
void InternalFormatVisitorDump::visitContainerScriptExecPerfLog(SALOME_ContainerScriptExecPerfLog& inst)
{
PushStringInVC<std::vector<char>>(inst.data(),*_data);
+ if( _clear_memory )
+ inst.clear();
}
void InternalFormatVisitorDump::enterContainerScriptPerfLog(SALOME_ContainerScriptPerfLog& inst)
PushIntInVC((std::uint32_t)inst.getNumberOfContainers(),*_data);
}
-///////////////////////
-
-std::vector<char> FetchVCFromVC(std::vector<char>& data, std::size_t& offset)
-{
- std::uint32_t *sz = reinterpret_cast<std::uint32_t *>( data.data()+offset);
- std::vector<char> ret(data.data()+offset+sizeof(std::uint32_t),data.data()+offset+sizeof(std::uint32_t)+*sz);
- offset += sizeof(std::uint32_t)+*sz;
- return ret;
-}
-
-std::string FetchStringFromVC(std::vector<char>& data, std::size_t& offset)
-{
- std::uint32_t *sz = reinterpret_cast<std::uint32_t *>( data.data() );
- std::string ret(data.data()+sizeof(std::uint32_t),*sz);
- offset += sizeof(std::uint32_t)+*sz;
- return ret;
-}
-
-class InternalFormatVisitorLoad : public SALOME_VisitorContainerLog
-{
-public:
- InternalFormatVisitorLoad(std::vector<char> *data):_data(data) { }
- void enterContainerPerfLog(SALOME_ContainerPerfLog& inst) override;
- void leaveContainerPerfLog(SALOME_ContainerPerfLog& inst) override { }
- void enterContainerScriptPerfLog(SALOME_ContainerScriptPerfLog& inst) override;
- void leaveContainerScriptPerfLog(SALOME_ContainerScriptPerfLog& inst) override { }
- void visitContainerScriptExecPerfLog(SALOME_ContainerScriptExecPerfLog& inst) override;
-private:
- std::vector<char> *_data = nullptr;
- std::size_t _offset = 0;
-};
-
-void InternalFormatVisitorLoad::visitContainerScriptExecPerfLog(SALOME_ContainerScriptExecPerfLog& inst)
-{
- inst.setData( FetchVCFromVC( *_data, _offset ) );
-}
-
-void InternalFormatVisitorLoad::enterContainerScriptPerfLog(SALOME_ContainerScriptPerfLog& inst)
-{
- inst.setName( FetchStringFromVC(*_data, _offset) );
- inst.setCode( FetchStringFromVC(*_data, _offset) );
-}
-
-void InternalFormatVisitorLoad::enterContainerPerfLog(SALOME_ContainerPerfLog& inst)
-{
- inst.setNameInNS( FetchStringFromVC(*_data, _offset) );
- inst.setLogFile( FetchStringFromVC(*_data, _offset) );
-}
-
-///////////////////////
-
-std::vector<char> SALOME_LogManager::dumpCppInternalFrmt()
+std::vector<char> SALOME_LogManager::dumpCppInternalFrmt(bool clearMemory)
{
std::vector<char> ret;
- InternalFormatVisitorDump visitor(&ret);
+ InternalFormatVisitorDump visitor(clearMemory,&ret);
this->accept( visitor );
return ret;
}
public:
void start();
AutoPyRef end();
+ void clear() { _data.clear(); }
private:
AutoPyRef _pyExecutionLog;
SALOME_ContainerScriptPerfLog *_father = nullptr;
virtual ~SALOME_LogManager() = default;
Engines::ContainerPerfLog_ptr declareContainer(const char *contInNS, const char *logfile) override;
Engines::ListOfContainerPerfLog *listOfContainerLogs() override;
- SALOME::vectorOfByte *getAllStruct() override;
+ SALOME::vectorOfByte *getAllStruct(bool clearMemory) override;
std::size_t getNumberOfContainers() const { return _containers.size(); }
public:
void accept(SALOME_VisitorContainerLog &visitor);
private:
- std::vector<char> dumpCppInternalFrmt();
+ std::vector<char> dumpCppInternalFrmt(bool clearMemory);
private:
PyObject *_pyLogManager = nullptr;
std::unique_ptr<SALOME_NamingService_Abstract> _NS;
# --- rules ---
IF(NOT WIN32)
-# ** Now in CTestTestfileInstall.cmake **
-# ** In this file only remain unit tests (no SALOME session is needed) **
-# ADD_TEST(NAME SalomeLauncher
-# COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/../../UnitTests/prepare_test.py
-# ${CMAKE_CURRENT_SOURCE_DIR}/test_launcher.py
-# -d KERNEL_ROOT_DIR=${CMAKE_INSTALL_PREFIX}
-# )
+
INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/test_launcher.py ${CMAKE_CURRENT_SOURCE_DIR}/TestSSLAttached.py
- DESTINATION ${KERNEL_TEST_DIR}/Launcher)
+ ${CMAKE_CURRENT_SOURCE_DIR}/testPerfLogManager1.py DESTINATION ${KERNEL_TEST_DIR}/Launcher)
INSTALL(FILES CTestTestfileInstall.cmake
DESTINATION ${KERNEL_TEST_DIR}/Launcher
IF(NOT WIN32)
SET(TEST_NAME ${COMPONENT_NAME}_Launcher)
ADD_TEST(${TEST_NAME} ${PYTHON_TEST_DRIVER} 2000 test_launcher.py)
- SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}"
- # TIMEOUT 500
- )
+ SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}")
SET(TEST_NAME ${COMPONENT_NAME}_AttachedLauncher)
ADD_TEST(${TEST_NAME} ${PYTHON_TEST_DRIVER} 2000 TestSSLAttached.py)
- SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}"
- # TIMEOUT 500
- )
+ SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}")
SET(TEST_NAME ${COMPONENT_NAME}_StressLauncher)
ADD_TEST(${TEST_NAME} ${PYTHON_TEST_DRIVER} 2000 ./test_stress.sh)
- SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}"
- # TIMEOUT 500
- )
+ SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}")
+
+ SET(TEST_NAME ${COMPONENT_NAME}_PerfLogManager1)
+ ADD_TEST(${TEST_NAME} ${PYTHON_TEST_DRIVER} 2000 testPerfLogManager1.py)
+ SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}")
# /!\ DO NOT SET TIMEOUT PROPERTY IF USING ${SALOME_TEST_DRIVER}
# BUT PASS TIMEOUT VALUE TO THE DRIVER
--- /dev/null
+# -*- coding: iso-8859-1 -*-
+# Copyright (C) 2024 CEA/DEN, EDF R&D
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
+#
+
+import unittest
+import os
+import salome
+import Engines
+import pylauncher
+import SALOME_PyNode
+
+import pickle
+import tempfile
+import logging
+
+def flush():
+ import sys
+ sys.stdout.flush()
+ sys.stderr.flush()
+
+def unProxyfy( objs ):
+ """
+ objs is a list of SenderByte
+ """
+ ret = []
+ fileNamesProxyOut = []
+ for ret2 in objs:
+ ret3 = pickle.loads( SALOME_PyNode.SeqByteReceiver(ret2).data() )
+ ret4 = ret3
+ fileNameProxyOut = None
+ if SALOME_PyNode.GetBigObjectOnDiskThreshold() != -1:
+ if isinstance( ret3, SALOME_PyNode.BigObjectOnDiskBase ):
+ ret4 = ret3.get()
+ else:
+ raise RuntimeError("Oooops")
+ ret3.unlinkOnDestructor()
+ fileNameProxyOut = ret3.getFileName()
+ logging.debug("Prxy file : {}".format( fileNameProxyOut ))
+ ret.append( ret3 )
+ fileNamesProxyOut.append( fileNameProxyOut )
+ return ret, fileNamesProxyOut
+
+class testPerfLogManager1(unittest.TestCase):
+ def testPerfLogManager0(self):
+ """
+ [EDF29150] : This test checks measure performance methods
+ """
+ hostname = "localhost"
+ cp = pylauncher.GetRequestForGiveContainer(hostname,"container_test")
+ #PROXY_THRES = "-1"
+ PROXY_THRES = "1"
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ val_for_big_obj = str( tmpdirname )
+ # Override environement for all containers launched
+ salome.cm.SetOverrideEnvForContainersSimple(env = [("SALOME_FILE_BIG_OBJ_DIR",val_for_big_obj),("SALOME_BIG_OBJ_ON_DISK_THRES",PROXY_THRES)])
+ salome.cm.SetDeltaTimeBetweenCPUMemMeasureInMilliSecond( 250 )
+ cont = salome.cm.GiveContainer(cp)
+ logging.debug("{} {}".format(40*"*",cont.getPID()))
+ script_st = """
+import logging
+import sys
+import KernelBasis
+cst = KernelBasis.GetTimeAdjustmentCst()
+logging.debug("constant = {}".format(cst))
+nbcore = 3
+print("coucou {} {}".format(len(zeinput0),len(zeinput1)))
+logging.debug("debug or not debug")
+ob = [ [ bytes(3000000) ] ]
+pihm, ts = KernelBasis.HeatMarcel(1 * nbcore * cst,nbcore)
+print("Time ellapse spent : {} s".format(ts))
+ob2 = [ [ bytes(100000) ] ]
+pihm, ts = KernelBasis.HeatMarcel(1 * nbcore * cst,nbcore)
+print("Time ellapse spent : {} s".format(ts))
+sys.stderr.write("fake error message\\n")
+"""
+ poa = salome.orb.resolve_initial_references("RootPOA")
+ zeinput0 = [ bytes(100000000) ]
+ if SALOME_PyNode.GetBigObjectOnDiskThreshold() != -1:
+ zeinput0 = SALOME_PyNode.ProxyfyPickeled( zeinput0 )
+ zeinput0.unlinkOnDestructor()
+ obj = SALOME_PyNode.SenderByte_i(poa,pickle.dumps( (["zeinput0"],{"zeinput0": [zeinput0], "zeinput1": [ [zeinput0], [zeinput0] ] }) ))
+ id_o = poa.activate_object(obj)
+ refPtr = poa.id_to_reference(id_o)
+ pyscript2 = cont.createPyScriptNode("testScript2",script_st)
+ pyscript2.executeFirst(refPtr)
+ ret2 = pyscript2.executeSecond(["ob","ob2"])# generate a DeprecationWarning: PY_SSIZE_T_CLEAN will be required for '#' formats on debian11 ?
+ ret3, fileNamesProxyOut = unProxyfy( ret2 )
+ logging.getLogger().debug("test logging 1")
+ logging.debug("test logging 2")
+ logging.debug( salome.orb.object_to_string( salome.logm ) )
+ a = salome.logm.NaiveFetch()
+ logging.debug(a)
+ logging.debug(a[0][1][0])
+ logging.debug( a[0][1][0].get()._input_hdd_mem._data[0]._data[0]._hdd_mem ) # important
+ logging.debug( a[0][1][0].get()._input_hdd_mem._data[1]._data[0]._data[0]._hdd_mem ) # important
+ fileNameProxyIn = a[0][1][0].get()._input_hdd_mem._data[0]._data[0]._file_name
+ logging.debug( fileNameProxyIn )
+ del zeinput0
+ del ret3
+ import gc ; gc.collect()
+ if fileNameProxyIn is not None:
+ if os.path.exists(fileNameProxyIn):
+ raise RuntimeError("Oooops 2")
+ for fileNameProxyOut in fileNamesProxyOut:
+ if fileNameProxyOut is not None:
+ if os.path.exists(fileNameProxyOut):
+ raise RuntimeError("Oooops 3")
+
+ # execution #2 inside last
+ script_st2 = """
+import logging
+b = 7+a
+logging.debug("Execution 2")
+import time
+time.sleep(1)
+"""
+ obj2 = SALOME_PyNode.SenderByte_i(poa,pickle.dumps((["a"],{"a":3})))
+ id2_o = poa.activate_object(obj2)
+ refPtr2 = poa.id_to_reference(id2_o)
+ pyscript2.assignNewCompiledCode(script_st2)
+ pyscript2.executeFirst(refPtr2)
+ ret2_0 = pyscript2.executeSecond(["b"])
+ ret2_1, fileNamesProxyOut2 = unProxyfy( ret2_0 )
+ logging.debug( fileNamesProxyOut2 )
+ a = salome.logm.NaiveFetch()
+ del ret2_1
+ import gc ; gc.collect()
+ for fileNameProxyOut in fileNamesProxyOut2:
+ if fileNameProxyOut is not None:
+ if os.path.exists(fileNameProxyOut):
+ raise RuntimeError("Oooops 3")
+ #
+ fname = os.path.join(str( tmpdirname ),"perf.log")
+ salome.logm.DumpInFile( fname )
+ logManagerInst0 = salome.logm.LoadFromFile( fname )
+ logging.debug( logManagerInst0[0][1][0].get()._input_hdd_mem._data[1]._data[0]._data[0]._hdd_mem ) # important
+ logManagerInst = salome.logm.Fetch(True)
+ logManagerInst2 = salome.logm.Fetch(True)
+ logging.debug( logManagerInst[0][1][0].get()._input_hdd_mem._data[1]._data[0]._data[0]._hdd_mem ) # important
+ self.assertTrue( logManagerInst2[0][1][0].get() is None )
+ self.assertTrue( logManagerInst[0][1][1].get()._output_hdd_mem._data[0]._file_name == fileNamesProxyOut2[0] )
+ logging.debug( logManagerInst[0][1][1].log() )
+ cont.Shutdown()
+
+if __name__ == '__main__':
+ from salome_utils import positionVerbosityOfLoggerRegardingState,setVerboseLevel,setVerbose
+ salome.standalone()
+ salome.salome_init()
+ setVerboseLevel(logging.DEBUG)
+ positionVerbosityOfLoggerRegardingState()
+ unittest.main()
