From 461f7bd0f18eda1bf99df8598c12b967f302c479 Mon Sep 17 00:00:00 2001 From: Anthony Geay Date: Fri, 22 Dec 2023 13:35:34 +0100 Subject: [PATCH] [EDF29150] : log performance of python scripts run inside SALOME container + verbosity level --- bin/runSalomeCommon.py | 53 +- bin/salome_utils.py | 112 +++- idl/CMakeLists.txt | 1 + idl/SALOME_Component.idl | 11 + idl/SALOME_ContainerManager.idl | 4 + idl/SALOME_LogManager.idl | 64 +++ src/Basics/CMakeLists.txt | 1 + src/Basics/KernelBasis.i | 47 +- src/Basics/Monitoring.cxx | 137 +++++ src/Basics/Monitoring.hxx | 34 ++ src/Basics/PythonCppUtils.hxx | 1 + src/Basics/libSALOMELog.cxx | 124 ++++- src/Basics/libSALOMELog.hxx | 13 + src/Container/CMakeLists.txt | 1 + src/Container/Container_i.cxx | 143 +++-- src/Container/Container_init_python.cxx | 5 +- src/Container/SALOME_Container.py | 58 +- src/Container/SALOME_ContainerHelper.py | 511 ++++++++++++++++++ src/Container/SALOME_ContainerManager.cxx | 32 +- src/Container/SALOME_ContainerManager.hxx | 5 + src/Container/SALOME_Container_i.hxx | 14 +- src/Container/SALOME_FileTransfer_i.cxx | 7 - src/Container/SALOME_PyNode.py | 240 ++++++-- .../ScriptsTemplate/script_parameters.py | 14 +- src/KERNEL_PY/__init__.py | 40 +- src/KERNEL_PY/salome_study.py | 8 - src/Launcher/CMakeLists.txt | 4 +- src/Launcher/KernelLauncher.cxx | 27 + src/Launcher/KernelLauncher.hxx | 1 + src/Launcher/KernelLauncher.i | 7 + src/Launcher/SALOME_LogManager.cxx | 382 +++++++++++++ src/Launcher/SALOME_LogManager.hxx | 154 ++++++ src/Launcher/SALOME_LogManager.py | 79 +++ src/Launcher/Test/CMakeLists.txt | 10 +- src/Launcher/Test/CTestTestfileInstall.cmake | 16 +- src/Launcher/Test/testPerfLogManager1.py | 167 ++++++ .../SALOME_ModuleCatalog_Handler.cxx | 18 +- .../SALOME_ModuleCatalog_impl.cxx | 11 +- src/ResourcesManager/ResourcesManager.cxx | 36 +- .../SALOME_ResourcesCatalog_Handler.cxx | 18 +- .../SALOME_ResourcesCatalog_Parser.cxx | 61 ++- .../SALOME_ResourcesCatalog_Parser.hxx | 1 + .../SALOME_ResourcesManager.cxx | 3 - src/SALOMEDS/SALOMEDS_Study_i.cxx | 1 - src/SALOMELocalTrace/LocalTraceCollector.cxx | 7 +- src/SALOMELocalTrace/utilities.h | 8 +- .../SALOMETraceCollector.cxx | 52 +- 47 files changed, 2388 insertions(+), 355 deletions(-) create mode 100644 idl/SALOME_LogManager.idl create mode 100644 src/Basics/Monitoring.cxx create mode 100644 src/Basics/Monitoring.hxx create mode 100644 src/Container/SALOME_ContainerHelper.py create mode 100644 src/Launcher/SALOME_LogManager.cxx create mode 100644 src/Launcher/SALOME_LogManager.hxx create mode 100644 src/Launcher/SALOME_LogManager.py create mode 100644 src/Launcher/Test/testPerfLogManager1.py diff --git a/bin/runSalomeCommon.py b/bin/runSalomeCommon.py index 841f62a65..6466c9a3e 100755 --- a/bin/runSalomeCommon.py +++ b/bin/runSalomeCommon.py @@ -34,62 +34,13 @@ import subprocess from salomeContextUtils import ScriptAndArgsObjectEncoder import platform import logging +from salome_utils import positionVerbosityOfLogger -## Setting formatter in setVerbose() was commented because adding of handler -## breaks using of root logger in other modules and cause many double lines in logs. -#FORMAT = '%(levelname)s : %(asctime)s : [%(filename)s:%(funcName)s:%(lineno)s] : %(message)s' -#logging.basicConfig(format=FORMAT) logger = logging.getLogger() -class ColoredFormatter(logging.Formatter): - BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30,38) - COLORS = { 'WARNING': YELLOW, 'INFO': WHITE, 'DEBUG': BLUE, 'CRITICAL': YELLOW, 'ERROR': RED } - def __init__(self, *args, **kwargs): - logging.Formatter.__init__(self, *args, **kwargs) - def format(self, record): - RESET_SEQ = "\033[0m" - COLOR_SEQ = "\033[1;%dm" - import inspect - frame = inspect.currentframe() - for i in range(8): - frame = frame.f_back - record.levelname = COLOR_SEQ % ColoredFormatter.COLORS[record.levelname] + record.levelname + RESET_SEQ - record.msg = "{} ( callsite is {} of file \"{}\" at line {} )".format(record.msg, frame.f_code.co_name,inspect.getsourcefile(frame),inspect.getlineno(frame) ) - return logging.Formatter.format(self, record) - -class BackTraceFormatter(logging.Formatter): - def __init__(self, *args, **kwargs): - logging.Formatter.__init__(self, *args, **kwargs) - def format(self, record): - import inspect - frame = inspect.currentframe() - # go upward of the stack to catch the effective callsite. Not very steady.... - # should be replaced by an analysis of frame.f_code - for i in range(8): - frame = frame.f_back - record.msg = "{} ( callsite is {} of file \"{}\" at line {} )".format(record.msg, frame.f_code.co_name,inspect.getsourcefile(frame),inspect.getlineno(frame) ) - return logging.Formatter.format(self, record) - def setVerbose(verbose): - from packaging import version - current_version = version.parse("{}.{}".format(sys.version_info.major,sys.version_info.minor)) - version_ref = version.parse("3.5.0") - global logger - formatter = None - if current_version >= version_ref: - formatter = BackTraceFormatter('%(levelname)s : %(asctime)s : %(message)s ',style='%') - else: - formatter = logging.Formatter('%(levelname)s : %(asctime)s : %(message)s ',style='%') - formatter.default_time_format = '%H:%M:%S' - formatter.default_msec_format = "%s.%03d" - stream_handler = logging.StreamHandler() - stream_handler.setFormatter(formatter) - logger.addHandler(stream_handler) - verbose_map = { "0": logging.WARNING, "1": logging.INFO, "2": logging.DEBUG} - if verbose in verbose_map: - logger.setLevel(verbose_map[verbose]) - + positionVerbosityOfLogger( verbose_map[verbose] ) # ----------------------------------------------------------------------------- # # Class definitions to launch CORBA Servers diff --git a/bin/salome_utils.py b/bin/salome_utils.py index 20af63b4b..7963b4e9f 100644 --- a/bin/salome_utils.py +++ b/bin/salome_utils.py @@ -34,10 +34,17 @@ import shutil import socket import sys import tempfile +import logging from contextlib import suppress import psutil +## Setting formatter in setVerbose() was commented because adding of handler +## breaks using of root logger in other modules and cause many double lines in logs. +#FORMAT = '%(levelname)s : %(asctime)s : [%(filename)s:%(funcName)s:%(lineno)s] : %(message)s' +#logging.basicConfig(format=FORMAT) +logger = logging.getLogger() + def _try_bool(arg): """ Convert given `arg` to a boolean value. @@ -384,35 +391,116 @@ def uniteFiles(src_file, dest_file): # -- +class ColoredFormatter(logging.Formatter): + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30,38) + COLORS = { 'WARNING': YELLOW, 'INFO': WHITE, 'DEBUG': BLUE, 'CRITICAL': YELLOW, 'ERROR': RED } + def __init__(self, *args, **kwargs): + logging.Formatter.__init__(self, *args, **kwargs) + def format(self, record): + RESET_SEQ = "\033[0m" + COLOR_SEQ = "\033[1;%dm" + import inspect + frame = inspect.currentframe() + for i in range(8): + frame = frame.f_back + record.levelname = COLOR_SEQ % ColoredFormatter.COLORS[record.levelname] + record.levelname + RESET_SEQ + record.msg = "{} ( callsite is {} of file \"{}\" at line {} )".format(record.msg, frame.f_code.co_name,inspect.getsourcefile(frame),inspect.getlineno(frame) ) + return logging.Formatter.format(self, record) + +class BackTraceFormatter(logging.Formatter): + def __init__(self, *args, **kwargs): + logging.Formatter.__init__(self, *args, **kwargs) + def format(self, record): + import inspect + frame = inspect.currentframe() + # go upward with ( a limit of 10 steps ) of the stack to catch the effective callsite. Not very steady.... + # should be replaced by an analysis of frame.f_code + for i in range(10): + frame = frame.f_back + if inspect.getsourcefile(frame) != logging.__file__: + break + record.msg = "{} ( callsite is {} of file \"{}\" at line {} )".format(record.msg, frame.f_code.co_name,inspect.getsourcefile(frame),inspect.getlineno(frame) ) + return logging.Formatter.format(self, record) + +def positionVerbosityOfLogger( verboseLevel ): + from packaging import version + current_version = version.parse("{}.{}".format(sys.version_info.major,sys.version_info.minor)) + version_ref = version.parse("3.5.0") + global logger + formatter = None + if current_version >= version_ref: + formatter = BackTraceFormatter('%(levelname)s : %(asctime)s : %(message)s ',style='%') + else: + formatter = logging.Formatter('%(levelname)s : %(asctime)s : %(message)s ',style='%') + formatter.default_time_format = '%H:%M:%S' + formatter.default_msec_format = "%s.%03d" + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + logger.addHandler(stream_handler) + logger.setLevel(verboseLevel) + +def positionVerbosityOfLoggerRegardingState(): + positionVerbosityOfLogger( verboseLevel() ) + def verbose(): """ - Get current verbosity level. + Get current verbosity activation. Default verbosity level is specified via the environment variable SALOME_VERBOSE, e.g. in bash: $ export SALOME_VERBOSE=1 - The function `setVerbose()` can be used to explicitly set verbosity level. + The function `setVerbose()` can be used to explicitly set verbosity activation. :return current verbosity level """ - if not hasattr(verbose, 'verbosity_level'): - verbose.verbosity_level = 0 # default value - with suppress(TypeError, ValueError): - # from SALOME_VERBOSE environment variable - verbose.verbosity_level = int(os.getenv('SALOME_VERBOSE', '0')) - return verbose.verbosity_level + import KernelBasis + return KernelBasis.VerbosityActivated() + # -- -def setVerbose(level): +def setVerbose(status): """ - Change verbosity level. + Change verbosity activation status. The function `verbose()` can be used to get current verbosity level. + :param status : verbosity status + :type status: bool + """ + import KernelBasis + return KernelBasis.SetVerbosityActivated( status ) + +# -- + +KernelLogLevelToLogging = {"INFO":logging.INFO, "DEBUG":logging.DEBUG, "WARNING":logging.WARNING, "ERROR":logging.ERROR} + +LoggingToKernelLogLevel = {v: k for k, v in KernelLogLevelToLogging.items()} + +def verboseLevel(): + """ + Get current verbosity level. + + Default verbosity level is specified via the environment variable SALOME_VERBOSE, + e.g. in bash: + + $ export SALOME_VERBOSE_LEVEL=7 + + The function `setVerboseLevel()` can be used to explicitly set verbosity level. + + :return current verbosity level + """ + import KernelBasis + return KernelLogLevelToLogging[ KernelBasis.VerbosityLevel() ] + +def setVerboseLevel(level): + """ + Change verbosity level. + The function `verboseLevel()` can be used to get current verbosity level. :param level : verbosity level """ - with suppress(TypeError, ValueError): - verbose.verbosity_level = int(level) + import KernelBasis + KernelBasis.SetVerbosityLevel(LoggingToKernelLogLevel[ level ]) + # -- def killPid(pid, sig=9): diff --git a/idl/CMakeLists.txt b/idl/CMakeLists.txt index 43cd37e9a..ea026e5fd 100644 --- a/idl/CMakeLists.txt +++ b/idl/CMakeLists.txt @@ -55,6 +55,7 @@ SET(SalomeIDLKernel_IDLSOURCES SALOME_TestModuleCatalog.idl SALOME_CommonTypes.idl SALOME_ExternalServerLauncher.idl + SALOME_LogManager.idl SALOME_Embedded_NamingService.idl ${CMAKE_CURRENT_BINARY_DIR}/Calcium_Ports.idl ) diff --git a/idl/SALOME_Component.idl b/idl/SALOME_Component.idl index 2e3b662f4..8c543d82b 100644 --- a/idl/SALOME_Component.idl +++ b/idl/SALOME_Component.idl @@ -31,6 +31,7 @@ #include "SALOME_Exception.idl" #include "SALOME_PyNode.idl" #include "SALOME_Embedded_NamingService.idl" +#include "SALOME_Comm.idl" /*! \file SALOME_Component.idl \brief interfaces for EngineComponent and Container */ @@ -178,6 +179,16 @@ module Engines //! name of the %container log file (this has been set by the launcher) attribute string logfilename ; + + //! name of the %container log file + attribute string locallogfilename ; + + //! interval of time between two measures of CPU/time process container + attribute long monitoringtimeresms; + + void verbosity(out boolean activated, out string level); + + void setVerbosity(in boolean activated, in string level); //! Shutdown the Container process. void Shutdown(); diff --git a/idl/SALOME_ContainerManager.idl b/idl/SALOME_ContainerManager.idl index 474d51642..3a58f620b 100644 --- a/idl/SALOME_ContainerManager.idl +++ b/idl/SALOME_ContainerManager.idl @@ -96,6 +96,10 @@ interface ContainerManager long GetDeltaTimeBetweenNSLookupAtLaunchTimeInMilliSecond(); void SetDeltaTimeBetweenNSLookupAtLaunchTimeInMilliSecond(in long timeInMS); + + long GetDeltaTimeBetweenCPUMemMeasureInMilliSecond(); + + void SetDeltaTimeBetweenCPUMemMeasureInMilliSecond(in long timeInMS); void SetOverrideEnvForContainers(in KeyValDict env); diff --git a/idl/SALOME_LogManager.idl b/idl/SALOME_LogManager.idl new file mode 100644 index 000000000..3e56d300f --- /dev/null +++ b/idl/SALOME_LogManager.idl @@ -0,0 +1,64 @@ +// Copyright (C) 2024 CEA, EDF +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +#ifndef __SALOME_LOGMANAGER_IDL__ +#define __SALOME_LOGMANAGER_IDL__ + +#include "SALOME_Exception.idl" +#include "SALOME_Comm.idl" + +module Engines +{ + interface ContainerScriptExecPerfLog + { + void assign(in SALOME::vectorOfByte value); + SALOME::vectorOfByte getObj(); + }; + + typedef sequence ListOfContainerScriptExecPerfLog; + + interface ContainerScriptPerfLog + { + string getCode(); + string getName(); + ContainerScriptExecPerfLog addExecutionSession(); + ListOfContainerScriptExecPerfLog listOfExecs(); + }; + + typedef sequence ListOfContainerScriptPerfLog; + + interface ContainerPerfLog + { + string getLogFile(); + string getContainerEntryInNS(); + ContainerScriptPerfLog addScript(in string name, in string code); + ListOfContainerScriptPerfLog listOfScripts(); + }; + + typedef sequence ListOfContainerPerfLog; + + interface LogManager + { + ContainerPerfLog declareContainer(in string contInNS, in string logfile); + ListOfContainerPerfLog listOfContainerLogs(); + SALOME::vectorOfByte getAllStruct( in boolean clearMemory ); + }; +}; + +#endif diff --git a/src/Basics/CMakeLists.txt b/src/Basics/CMakeLists.txt index 721defd8b..b3d781511 100644 --- a/src/Basics/CMakeLists.txt +++ b/src/Basics/CMakeLists.txt @@ -35,6 +35,7 @@ SET(SALOMEBasics_SOURCES Basics_DirUtils.cxx KernelBasis.cxx HeatMarcel.cxx + Monitoring.cxx ) ADD_LIBRARY(SALOMELog ${SALOMELog_SOURCES}) diff --git a/src/Basics/KernelBasis.i b/src/Basics/KernelBasis.i index 45908234b..3d070b68c 100644 --- a/src/Basics/KernelBasis.i +++ b/src/Basics/KernelBasis.i @@ -23,10 +23,31 @@ #include "KernelBasis.hxx" #include "HeatMarcel.hxx" #include "libSALOMELog.hxx" +#include "Monitoring.hxx" using namespace SALOME; %} -%include "std_string.i" +%include std_string.i +%include std_set.i +%include std_except.i +%include std_vector.i + +%template(dvec) std::vector; + +%exception { + try + { + $action + } + catch(std::exception& e) + { + SWIG_exception(SWIG_SystemError, e.what() ); + } + catch(...) + { + SWIG_exception(SWIG_UnknownError, "Unknown exception"); + } +} %rename (HeatMarcel) HeatMarcelSwig; @@ -41,14 +62,28 @@ void setIOROfEmbeddedNS(const std::string& ior); double GetTimeAdjustmentCst(); +void LaunchMonitoring(const std::string& pyScriptToEvaluate, const std::string& outFileName); + +std::vector StopMonitoring(); + bool VerbosityActivated(); void SetVerbosityActivated(bool flag); +bool IsDebugLevel(); + +bool IsInfoLevel(); + +bool IsWarningLevel(); + +bool IsErrorLevel(); + void WriteInStdout(const std::string& msg); void WriteInStderr(const std::string& msg); +%rename (SetVerbosityLevel) SetVerbosityLevelSwig; +%rename (VerbosityLevel) VerbosityLevelSwig; %inline { @@ -61,4 +96,14 @@ PyObject *HeatMarcelSwig(double timeAjustment, unsigned int nbThreads = 0) PyTuple_SetItem(ret,1,SWIG_From_double(timeInS)); return ret; } + +void SetVerbosityLevelSwig(const std::string& level) +{ + SetVerbosityLevelStr(level); +} + +std::string VerbosityLevelSwig() +{ + return VerbosityLevelStr(); +} } diff --git a/src/Basics/Monitoring.cxx b/src/Basics/Monitoring.cxx new file mode 100644 index 000000000..0fa97dda7 --- /dev/null +++ b/src/Basics/Monitoring.cxx @@ -0,0 +1,137 @@ +// Copyright (C) 2023 CEA/DEN, EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +#include "Monitoring.hxx" + +#include "baseutilities.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef WIN32 +#include +#include +#include +#include +#endif + +static std::string _out_filename; +#ifndef WIN32 +static pid_t pid_of_subprocess = 0; +#endif + +#ifndef WIN32 +static void LaunchMonitoringLinux(const std::string& pyScriptToEvaluate, const std::string& outFileName) +{ + constexpr char PYTHON_EXEC[] = "python3"; + pid_t pid = fork(); + if (pid == -1) + { + throw std::runtime_error("LaunchMonitoring : Error at creation of sub process !"); + } + else if( pid == 0) + { + execlp(PYTHON_EXEC,PYTHON_EXEC,pyScriptToEvaluate.c_str(),nullptr); + std::ostringstream oss; oss << "LaunchMonitoring : Error during exe : " << sys_errlist[errno]; + throw std::runtime_error( oss.str() ); + } + else + { + pid_of_subprocess = pid; + } +} +#endif + +void SALOME::LaunchMonitoring(const std::string& pyScriptToEvaluate, const std::string& outFileName) +{ + _out_filename = outFileName; +#ifndef WIN32 + LaunchMonitoringLinux(pyScriptToEvaluate,outFileName); +#else + throw std::runtime_error("LaunchMonitoring not implemented for Windows !"); +#endif +} + +std::vector SALOME::ReadFloatsInFile(const std::string& fileName) +{ + std::ifstream inputFile( fileName ); + + if(!inputFile.is_open()) + { + std::ostringstream oss; oss << "Impossible to open file \"" << _out_filename<< "\" !"; + throw std::runtime_error( oss.str() ); + } + std::vector ret; + std::string line; + try + { + while (std::getline(inputFile, line)) + { + std::istringstream iss(line); + double floatValue; + if( !(iss >> floatValue) ) + throw std::invalid_argument("Conversion into FP failed !"); + if( !iss.eof() ) + throw std::invalid_argument("Conversion into FP failed !"); + ret.push_back(floatValue); + } + inputFile.close(); + } + catch (const std::exception& e) + { + } + return ret; +} + +#ifndef WIN32 +static std::vector StopMonitoringLinux() +{ + kill( pid_of_subprocess, SIGTERM ); + std::vector ret; + try + { + ret = SALOME::ReadFloatsInFile( _out_filename ); + } + catch(std::exception& e) { } + pid_of_subprocess = 0; + _out_filename.clear(); + return ret; +} +#endif + +std::vector SALOME::StopMonitoring() +{ +#ifndef WIN32 + return StopMonitoringLinux(); +#else + throw std::runtime_error("StopMonitoring not implemented for Windows !"); +#endif +} diff --git a/src/Basics/Monitoring.hxx b/src/Basics/Monitoring.hxx new file mode 100644 index 000000000..379218844 --- /dev/null +++ b/src/Basics/Monitoring.hxx @@ -0,0 +1,34 @@ +// Copyright (C) 2023 CEA/DEN, EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +#pragma once + +#include "SALOME_Basics.hxx" + +#include +#include + +namespace SALOME +{ + void BASICS_EXPORT LaunchMonitoring(const std::string& pyScriptToEvaluate, const std::string& outFileName); + + std::vector BASICS_EXPORT ReadFloatsInFile(const std::string& fileName); + + std::vector BASICS_EXPORT StopMonitoring(); +} diff --git a/src/Basics/PythonCppUtils.hxx b/src/Basics/PythonCppUtils.hxx index 8ef4904a2..93a3cfb59 100644 --- a/src/Basics/PythonCppUtils.hxx +++ b/src/Basics/PythonCppUtils.hxx @@ -36,6 +36,7 @@ public: AutoPyRef(PyObject *pyobj=nullptr):_pyobj(pyobj) { } ~AutoPyRef() { release(); } AutoPyRef(const AutoPyRef& other):_pyobj(other._pyobj) { if(_pyobj) Py_XINCREF(_pyobj); } + AutoPyRef(AutoPyRef&& other) = default; AutoPyRef& operator=(const AutoPyRef& other) { if(_pyobj==other._pyobj) return *this; release(); _pyobj=other._pyobj; Py_XINCREF(_pyobj); return *this; } operator PyObject *() { return _pyobj; } void set(PyObject *pyobj) { if(pyobj==_pyobj) return ; release(); _pyobj=pyobj; } diff --git a/src/Basics/libSALOMELog.cxx b/src/Basics/libSALOMELog.cxx index b1d1332e6..6c0cd55a8 100644 --- a/src/Basics/libSALOMELog.cxx +++ b/src/Basics/libSALOMELog.cxx @@ -26,6 +26,7 @@ #include #include +#include enum class VerbosityMode { undefined, nolog, withlog }; @@ -33,6 +34,64 @@ static VerbosityMode isActivated = VerbosityMode::undefined; namespace SALOME { + static constexpr char ERROR_LEVEL_VALUE = 0; + static constexpr char ERROR_LEVEL_VALUE_STR[] = "ERROR"; + static constexpr char WARNING_LEVEL_VALUE = 1; + static constexpr char WARNING_LEVEL_VALUE_STR[] = "WARNING"; + static constexpr char INFO_LEVEL_VALUE = 2; + static constexpr char INFO_LEVEL_VALUE_STR[] = "INFO"; + static constexpr char DEBUG_LEVEL_VALUE = 7; + static constexpr char DEBUG_LEVEL_VALUE_STR[] = "DEBUG"; + static constexpr char UNDEFINED_LEVEL_VALUE=99; + + enum class VerbosityLevelType { error_level=ERROR_LEVEL_VALUE, warning_level=WARNING_LEVEL_VALUE, info_level=INFO_LEVEL_VALUE, debug_level=DEBUG_LEVEL_VALUE, undefined_level=UNDEFINED_LEVEL_VALUE }; + static VerbosityLevelType verbosityLevel = VerbosityLevelType::undefined_level; + + static VerbosityLevelType FromIntToVerbosityLevel(char value) + { + switch(value) + { + case ERROR_LEVEL_VALUE: + return VerbosityLevelType::error_level; + case WARNING_LEVEL_VALUE: + return VerbosityLevelType::warning_level; + case INFO_LEVEL_VALUE: + return VerbosityLevelType::info_level; + case DEBUG_LEVEL_VALUE: + return VerbosityLevelType::debug_level; + } + throw std::range_error("FromIntToVerbosityLevel : Invalid value for verbosity level !"); + } + + static VerbosityLevelType FromStrToVerbosityLevel(const std::string& val) + { + if(val == ERROR_LEVEL_VALUE_STR) + return VerbosityLevelType::error_level; + if(val == WARNING_LEVEL_VALUE_STR) + return VerbosityLevelType::warning_level; + if(val == INFO_LEVEL_VALUE_STR) + return VerbosityLevelType::info_level; + if(val == DEBUG_LEVEL_VALUE_STR) + return VerbosityLevelType::debug_level; + throw std::range_error("FromStrToVerbosityLevel : Invalid str value for verbosity level !"); + } + + static std::string FromVerbosityLevelToStr(VerbosityLevelType level) + { + switch(level) + { + case VerbosityLevelType::error_level: + return std::string(ERROR_LEVEL_VALUE_STR); + case VerbosityLevelType::warning_level: + return std::string(WARNING_LEVEL_VALUE_STR); + case VerbosityLevelType::info_level: + return std::string(INFO_LEVEL_VALUE_STR); + case VerbosityLevelType::debug_level: + return std::string(DEBUG_LEVEL_VALUE_STR); + default: + throw std::range_error("FromVerbosityLevelToStr : not managed verbosity level !"); + } + } // ============================================================================ /*! @@ -49,19 +108,12 @@ namespace SALOME { auto isEnvVarSet = []() -> VerbosityMode { - const char* envVar = std::getenv("SALOME_VERBOSE"); + const char *envVar = std::getenv("SALOME_VERBOSE"); if (envVar && (envVar[0] != '\0')) { - try - { - const long long numValue = std::stoll(envVar); - return numValue > 0?VerbosityMode::withlog:VerbosityMode::nolog; - } - catch(const std::exception& e) - { - std::cerr << e.what() << '\n'; - } + const int numValue = std::stoi(envVar); + return numValue > 0?VerbosityMode::withlog:VerbosityMode::nolog; } return VerbosityMode::nolog; @@ -76,4 +128,56 @@ namespace SALOME { isActivated = flag ? VerbosityMode::withlog:VerbosityMode::nolog; } + + VerbosityLevelType VerbosityLevel() + { + auto isEnvVarSet = []() -> VerbosityLevelType + { + const char *envVar = std::getenv("SALOME_VERBOSE_LEVEL"); + if (envVar && (envVar[0] != '\0')) + { + const int numValue = std::stoi(envVar); + return FromIntToVerbosityLevel( static_cast(numValue) ); + } + return VerbosityLevelType::info_level; + }; + if(verbosityLevel == VerbosityLevelType::undefined_level) + verbosityLevel = isEnvVarSet(); + return verbosityLevel; + } + + void BASICS_EXPORT SetVerbosityLevel(VerbosityLevelType level) + { + verbosityLevel = level; + } + + void SetVerbosityLevelStr(const std::string& level) + { + verbosityLevel = FromStrToVerbosityLevel(level); + } + + std::string VerbosityLevelStr() + { + return FromVerbosityLevelToStr( VerbosityLevel() ); + } + + bool IsDebugLevel() + { + return VerbosityLevel() >= VerbosityLevelType::debug_level; + } + + bool IsInfoLevel() + { + return VerbosityLevel() >= VerbosityLevelType::info_level; + } + + bool IsWarningLevel() + { + return VerbosityLevel() >= VerbosityLevelType::warning_level; + } + + bool IsErrorLevel() + { + return VerbosityLevel() >= VerbosityLevelType::error_level; + } } diff --git a/src/Basics/libSALOMELog.hxx b/src/Basics/libSALOMELog.hxx index 7d4b70611..f4d91193e 100644 --- a/src/Basics/libSALOMELog.hxx +++ b/src/Basics/libSALOMELog.hxx @@ -26,8 +26,21 @@ #include "SALOME_Basics.hxx" +#include + namespace SALOME { + enum class VerbosityLevelType; + bool BASICS_EXPORT VerbosityActivated(); void BASICS_EXPORT SetVerbosityActivated(bool); + + void BASICS_EXPORT SetVerbosityLevel(VerbosityLevelType level); + void BASICS_EXPORT SetVerbosityLevelStr(const std::string& level); + bool BASICS_EXPORT IsDebugLevel(); + bool BASICS_EXPORT IsInfoLevel(); + bool BASICS_EXPORT IsWarningLevel(); + bool BASICS_EXPORT IsErrorLevel(); + VerbosityLevelType BASICS_EXPORT VerbosityLevel(); + std::string BASICS_EXPORT VerbosityLevelStr(); } diff --git a/src/Container/CMakeLists.txt b/src/Container/CMakeLists.txt index cc291a4a0..dfc050ba8 100644 --- a/src/Container/CMakeLists.txt +++ b/src/Container/CMakeLists.txt @@ -45,6 +45,7 @@ SET(SCRIPTS SALOME_ComponentPy.py SALOME_PyNode.py SALOME_Container.py + SALOME_ContainerHelper.py SALOME_ContainerPy.py ) diff --git a/src/Container/Container_i.cxx b/src/Container/Container_i.cxx index dd8ea1843..87c92afe2 100644 --- a/src/Container/Container_i.cxx +++ b/src/Container/Container_i.cxx @@ -110,6 +110,8 @@ extern "C" {void SigIntHandler( int ) ; } #define SLASH '/' #endif +const int Abstract_Engines_Container_i::DFT_TIME_INTERVAL_BTW_MEASURE = 500; + std::map Abstract_Engines_Container_i::_cntInstances_map; std::map Abstract_Engines_Container_i::_library_map; std::map Abstract_Engines_Container_i::_toRemove_map; @@ -162,27 +164,11 @@ Abstract_Engines_Container_i::Abstract_Engines_Container_i (CORBA::ORB_ptr orb, std::string hostname = Kernel_Utils::GetHostname(); #ifndef WIN32 - MESSAGE(hostname << " " << getpid() << - " Engines_Container_i starting argc " << - _argc << " Thread " << pthread_self() ) ; + INFO_MESSAGE("Starting Container servant instance on Hostname :" << hostname << " with PID : " << getpid() ) ; #else MESSAGE(hostname << " " << _getpid() << " Engines_Container_i starting argc " << _argc<< " Thread " << pthread_self().p ) ; #endif - - int i = 0 ; - while ( _argv[ i ] ) - { - MESSAGE(" argv" << i << " " << _argv[ i ]) ; - i++ ; - } - - if ( argc < 2 ) - { - INFOS("SALOME_Container usage : SALOME_Container ServerName"); - ASSERT(0) ; - } - SCRUTE(argv[1]); _isSupervContainer = false; _orb = CORBA::ORB::_duplicate(orb) ; @@ -202,20 +188,15 @@ Abstract_Engines_Container_i::Abstract_Engines_Container_i (CORBA::ORB_ptr orb, _remove_ref(); _containerName = SALOME_NamingService_Abstract::BuildContainerNameForNS(containerName, hostname.c_str()); - SCRUTE(_containerName); - _NS->Register(pCont, _containerName.c_str()); - MESSAGE("Engines_Container_i::Engines_Container_i : Container name " << _containerName); // Python: // import SALOME_Container // pycont = SALOME_Container.SALOME_Container_i(containerIORStr) CORBA::String_var sior = _orb->object_to_string(pCont); - std::string myCommand="pyCont = SALOME_Container.SALOME_Container_i('"; - myCommand += _containerName + "','"; - myCommand += sior; - myCommand += "')\n"; - SCRUTE(myCommand); + std::ostringstream myCommand; + myCommand << "pyCont = SALOME_Container.SALOME_Container_i('" << _containerName << "','" << sior << "'," << DFT_TIME_INTERVAL_BTW_MEASURE << ")\n"; + INFO_MESSAGE("Python command executed : " << myCommand.str()); //[RNV]: Comment the PyEval_AcquireLock() and PyEval_ReleaseLock() because this //approach leads to the deadlock of the main thread of the application on Windows platform @@ -241,14 +222,17 @@ Abstract_Engines_Container_i::Abstract_Engines_Container_i (CORBA::ORB_ptr orb, PyRun_SimpleString("sys.path = sys.path[1:]\n"); #endif PyRun_SimpleString("import SALOME_Container\n"); - PyRun_SimpleString((char*)myCommand.c_str()); + PyRun_SimpleString((char*)myCommand.str().c_str()); PyObject *mainmod = PyImport_AddModule("__main__"); PyObject *globals = PyModule_GetDict(mainmod); _pyCont = PyDict_GetItemString(globals, "pyCont"); //PyThreadState_Swap(NULL); //PyEval_ReleaseLock(); } - + {// register to NS after python initialization to be sure that client invoke after py constructor execution + _NS->Register(pCont, _containerName.c_str()); + DEBUG_MESSAGE("Container registred in NS as : " << _containerName); + } fileTransfer_i* aFileTransfer = new fileTransfer_i(); CORBA::Object_var obref=aFileTransfer->_this(); _fileTransfer = Engines::fileTransfer::_narrow(obref); @@ -305,7 +289,7 @@ char* Abstract_Engines_Container_i::workingdir() */ //============================================================================= -char* Abstract_Engines_Container_i::logfilename() +char *Abstract_Engines_Container_i::logfilename() { return CORBA::string_dup(_logfilename.c_str()) ; } @@ -316,6 +300,76 @@ void Abstract_Engines_Container_i::logfilename(const char* name) _logfilename=name; } +char *Abstract_Engines_Container_i::locallogfilename() +{ + return CORBA::string_dup( _localfilename.c_str() ); +} + +void Abstract_Engines_Container_i::locallogfilename(const char *name) +{ + AutoGIL gstate; + _localfilename = name; + AutoPyRef result = PyObject_CallMethod(_pyCont,(char*)"setLogFileName","s",name,nullptr); + if (PyErr_Occurred()) + { + std::string error("can not set logfilename"); + PyErr_Print(); + THROW_SALOME_CORBA_EXCEPTION(error.c_str(),SALOME::INTERNAL_ERROR); + } +} + +CORBA::Long Abstract_Engines_Container_i::monitoringtimeresms() +{ + AutoGIL gstate; + AutoPyRef result = PyObject_CallMethod(_pyCont,(char*)"monitoringtimeresms",nullptr); + if (PyErr_Occurred()) + { + std::string error("can not retrieve time interval between 2 measures"); + PyErr_Print(); + THROW_SALOME_CORBA_EXCEPTION(error.c_str(),SALOME::INTERNAL_ERROR); + } + CORBA::Long ret = PyLong_AsLong( result ); + return ret; +} + +void Abstract_Engines_Container_i::monitoringtimeresms(CORBA::Long intervalInMs) +{ + AutoGIL gstate; + AutoPyRef result = PyObject_CallMethod(_pyCont,(char*)"SetMonitoringtimeresms","i",intervalInMs,nullptr); + if (PyErr_Occurred()) + { + std::string error("can not set time interval between 2 measures"); + PyErr_Print(); + THROW_SALOME_CORBA_EXCEPTION(error.c_str(),SALOME::INTERNAL_ERROR); + } +} + +void Abstract_Engines_Container_i::verbosity(bool& activated, CORBA::String_out level) +{ + activated = SALOME::VerbosityActivated(); + level = CORBA::string_dup( SALOME::VerbosityLevelStr().c_str() ); +} + +void Abstract_Engines_Container_i::setVerbosity(bool activated, const char *level) +{ + SALOME::SetVerbosityActivated( activated ); + SALOME::SetVerbosityLevelStr( level ); + { + AutoGIL gstate; + AutoPyRef res = PyObject_CallMethod(_pyCont, + (char*)"positionVerbosityOfLogger",NULL); + if(res==NULL) + { + //internal error + PyErr_Print(); + SALOME::ExceptionStruct es; + es.type = SALOME::INTERNAL_ERROR; + es.text = "can not create a python node"; + throw SALOME::SALOME_Exception(es); + } + } +} + //============================================================================= //! Get container host name /*! @@ -364,11 +418,9 @@ CORBA::Long Abstract_Engines_Container_i::getNumberOfCPUCores() { AutoGIL gstate; PyObject *module = PyImport_ImportModuleNoBlock((char*)"salome_psutil"); - PyObject *result = PyObject_CallMethod(module, + AutoPyRef result = PyObject_CallMethod(module, (char*)"getNumberOfCPUCores", NULL); int n = PyLong_AsLong(result); - Py_DECREF(result); - return (CORBA::Long)n; } @@ -526,7 +578,7 @@ Engines::vectorOfDouble* Abstract_Engines_Container_i::loadOfCPUCores() { AutoGIL gstate; PyObject *module = PyImport_ImportModuleNoBlock((char*)"salome_psutil"); - PyObject *result = PyObject_CallMethod(module, + AutoPyRef result = PyObject_CallMethod(module, (char*)"loadOfCPUCores", "s", _load_script.c_str()); if (PyErr_Occurred()) @@ -542,7 +594,6 @@ Engines::vectorOfDouble* Abstract_Engines_Container_i::loadOfCPUCores() int n = this->getNumberOfCPUCores(); if (!PyList_Check(result) || PyList_Size(result) != n) { // bad number of cores - Py_DECREF(result); SALOME::ExceptionStruct es; es.type = SALOME::INTERNAL_ERROR; es.text = "wrong number of cores"; @@ -557,7 +608,6 @@ Engines::vectorOfDouble* Abstract_Engines_Container_i::loadOfCPUCores() if (foo < 0.0 || foo > 1.0) { // value not in [0, 1] range - Py_DECREF(result); SALOME::ExceptionStruct es; es.type = SALOME::INTERNAL_ERROR; es.text = "load not in [0, 1] range"; @@ -566,8 +616,6 @@ Engines::vectorOfDouble* Abstract_Engines_Container_i::loadOfCPUCores() loads[i] = foo; } - Py_DECREF(result); - return loads._retn(); } @@ -607,10 +655,9 @@ CORBA::Long Abstract_Engines_Container_i::getTotalPhysicalMemory() { AutoGIL gstate; PyObject *module = PyImport_ImportModuleNoBlock((char*)"salome_psutil"); - PyObject *result = PyObject_CallMethod(module, + AutoPyRef result = PyObject_CallMethod(module, (char*)"getTotalPhysicalMemory", NULL); int n = PyLong_AsLong(result); - Py_DECREF(result); return (CORBA::Long)n; } @@ -626,10 +673,9 @@ CORBA::Long Abstract_Engines_Container_i::getTotalPhysicalMemoryInUse() { AutoGIL gstate; PyObject *module = PyImport_ImportModuleNoBlock((char*)"salome_psutil"); - PyObject *result = PyObject_CallMethod(module, + AutoPyRef result = PyObject_CallMethod(module, (char*)"getTotalPhysicalMemoryInUse", NULL); int n = PyLong_AsLong(result); - Py_DECREF(result); return (CORBA::Long)n; } @@ -645,10 +691,9 @@ CORBA::Long Abstract_Engines_Container_i::getTotalPhysicalMemoryInUseByMe() { AutoGIL gstate; PyObject *module = PyImport_ImportModuleNoBlock((char*)"salome_psutil"); - PyObject *result = PyObject_CallMethod(module, + AutoPyRef result = PyObject_CallMethod(module, (char*)"getTotalPhysicalMemoryInUseByMe", NULL); int n = PyLong_AsLong(result); - Py_DECREF(result); return (CORBA::Long)n; } @@ -914,12 +959,11 @@ Abstract_Engines_Container_i::load_component_PythonImplementation(const char* co { AutoGIL gstate; - PyObject *result = PyObject_CallMethod(_pyCont, + AutoPyRef result = PyObject_CallMethod(_pyCont, (char*)"import_component", (char*)"s",componentName); reason=PyUnicode_AsUTF8(result); - Py_XDECREF(result); SCRUTE(reason); } @@ -1331,7 +1375,7 @@ Abstract_Engines_Container_i::createPythonInstance(std::string CompName, std::string iors; { AutoGIL gstate; - PyObject *result = PyObject_CallMethod(_pyCont, + AutoPyRef result = PyObject_CallMethod(_pyCont, (char*)"create_component_instance", (char*)"ss", CompName.c_str(), @@ -1341,7 +1385,6 @@ Abstract_Engines_Container_i::createPythonInstance(std::string CompName, PyArg_ParseTuple(result,"ss", &ior, &error); iors = ior; reason=error; - Py_DECREF(result); } if( iors!="" ) @@ -1372,7 +1415,7 @@ Abstract_Engines_Container_i::create_python_service_instance(const char * CompNa char * _ior = nullptr; { AutoGIL gstate; - PyObject *result = PyObject_CallMethod(_pyCont, + AutoPyRef result = PyObject_CallMethod(_pyCont, (char*)"create_component_instance", (char*)"ss", CompName, @@ -1382,7 +1425,6 @@ Abstract_Engines_Container_i::create_python_service_instance(const char * CompNa PyArg_ParseTuple(result,"ss", &ior, &error); reason = CORBA::string_dup(error); _ior = CORBA::string_dup(ior); - Py_DECREF(result); } return _ior; } @@ -2128,12 +2170,12 @@ Engines::PyScriptNode_ptr Abstract_Engines_Container_i::createPyScriptNode(const std::string astr; { AutoGIL gstate; - PyObject *res = PyObject_CallMethod(_pyCont, + AutoPyRef res = PyObject_CallMethod(_pyCont, (char*)"create_pyscriptnode", (char*)"ss", nodeName, code); - if(res==NULL) + if( res.isNull() ) { //internal error PyErr_Print(); @@ -2145,7 +2187,6 @@ Engines::PyScriptNode_ptr Abstract_Engines_Container_i::createPyScriptNode(const ierr=PyLong_AsLong(PyTuple_GetItem(res,0)); PyObject* result=PyTuple_GetItem(res,1); astr = PyUnicode_AsUTF8(result); - Py_DECREF(res); } if(ierr==0) diff --git a/src/Container/Container_init_python.cxx b/src/Container/Container_init_python.cxx index 93f54ac04..b3ba17777 100644 --- a/src/Container/Container_init_python.cxx +++ b/src/Container/Container_init_python.cxx @@ -165,9 +165,6 @@ void KERNEL_PYTHON::init_python(int argc, char **argv) MESSAGE("Python already initialized"); return; } - MESSAGE("================================================================="); - MESSAGE("Python Initialization..."); - MESSAGE("================================================================="); // set stdout to line buffering (aka C++ std::cout) setvbuf(stdout, (char *)NULL, _IOLBF, BUFSIZ); char* env_python=getenv("SALOME_PYTHON"); @@ -179,7 +176,7 @@ void KERNEL_PYTHON::init_python(int argc, char **argv) Py_Initialize(); // Initialize the interpreter if (Py_IsInitialized()) { - MESSAGE("Python initialized eh eh eh"); + MESSAGE("Python initialized"); } wchar_t **changed_argv = new wchar_t*[argc]; // Setting arguments for (int i = 0; i < argc; i++) diff --git a/src/Container/SALOME_Container.py b/src/Container/SALOME_Container.py index 8abee5f50..0745548d6 100644 --- a/src/Container/SALOME_Container.py +++ b/src/Container/SALOME_Container.py @@ -41,10 +41,12 @@ import Engines, Engines__POA from SALOME_NamingServicePy import * from SALOME_ComponentPy import * import SALOME_PyNode +import logging from SALOME_utilities import * from Utils_Identity import getShortHostName -from launchConfigureParser import verbose +from salome_utils import verbose +from KernelBasis import VerbosityActivated,getSSLMode #============================================================================= @@ -58,8 +60,9 @@ class SALOME_Container_i: #------------------------------------------------------------------------- - def __init__(self ,containerName, containerIORStr): - MESSAGE( "SALOME_Container_i::__init__" ) + def __init__(self ,containerName, containerIORStr, dftTimeIntervalInMs): + # Warning this part of code is called at the very first step of container launching + # so logging is not instanciate. So use verbose method to discrimine if a message should be printed or not try: argv = sys.argv except AttributeError : @@ -70,18 +73,28 @@ class SALOME_Container_i: self._orb = CORBA.ORB_init(argv, CORBA.ORB_ID) self._poa = self._orb.resolve_initial_references("RootPOA") self._containerName = containerName - if verbose(): print("SALOME_Container.SALOME_Container_i : _containerName ",self._containerName) + self._logFileName = None + self._timeIntervalInMs = dftTimeIntervalInMs + self._logm = None + self._log = None self._container = self._orb.string_to_object(containerIORStr) + @property + def logm(self): + import salome + if self._logm is None: + salome.salome_init() + self._logm = salome.logm + return self._logm + #------------------------------------------------------------------------- def import_component(self, componentName): - MESSAGE( "SALOME_Container_i::import_component" ) ret="" try: - if verbose(): print("try import ",componentName) + logging.debug("try import ",componentName) importlib.import_module(componentName) - if verbose(): print("import ",componentName," successful") + logging.debug("import ",componentName," successful") except ImportError: #can't import python module componentName #try to find it in python path @@ -95,20 +108,18 @@ class SALOME_Container_i: except ImportError as ee: ret="ImplementationNotFound" except Exception: - if verbose():print("error when calling find_module") + print("error when calling find_module") ret="ImplementationNotFound" except Exception: ret="Component "+componentName+": Python implementation found but it can't be loaded\n" ret=ret+traceback.format_exc(10) - if verbose(): - traceback.print_exc() - print("import ",componentName," not possible") + traceback.print_exc() + print("import ",componentName," not possible") return ret #------------------------------------------------------------------------- def create_component_instance(self, componentName, instanceName): - MESSAGE( "SALOME_Container_i::create_component_instance" ) comp_iors="" ret="" try: @@ -121,13 +132,11 @@ class SALOME_Container_i: instanceName, componentName) - MESSAGE( "SALOME_Container_i::create_component_instance : OK") comp_o = comp_i._this() comp_iors = self._orb.object_to_string(comp_o) except Exception: ret=traceback.format_exc(10) traceback.print_exc() - MESSAGE( "SALOME_Container_i::create_component_instance : NOT OK") return comp_iors, ret @@ -145,7 +154,10 @@ class SALOME_Container_i: def create_pyscriptnode(self,nodeName,code): try: - node=SALOME_PyNode.PyScriptNode_i(nodeName,code,self._poa,self) + logscript = None + if getSSLMode(): + logscript = self._log.addScript(nodeName,code) + node=SALOME_PyNode.PyScriptNode_i(nodeName,code,self._poa,self, logscript) id_o = self._poa.activate_object(node) comp_o = self._poa.id_to_reference(id_o) comp_iors = self._orb.object_to_string(comp_o) @@ -153,4 +165,20 @@ class SALOME_Container_i: except Exception: exc_typ,exc_val,exc_fr=sys.exc_info() l=traceback.format_exception(exc_typ,exc_val,exc_fr) + print("".join(l)) ; sys.stdout.flush() # print error also in logs of remote container return 1,"".join(l) + + def positionVerbosityOfLogger(self): + if VerbosityActivated(): + import salome_utils + salome_utils.positionVerbosityOfLoggerRegardingState() + + def monitoringtimeresms(self): + return self._timeIntervalInMs + + def setLogFileName(self, logFileName): + if getSSLMode(): + self._log = self.logm.declareContainer( self._containerName, logFileName ) + + def SetMonitoringtimeresms(self , value): + self._timeIntervalInMs = value diff --git a/src/Container/SALOME_ContainerHelper.py b/src/Container/SALOME_ContainerHelper.py new file mode 100644 index 000000000..7725f3f2c --- /dev/null +++ b/src/Container/SALOME_ContainerHelper.py @@ -0,0 +1,511 @@ +# -*- coding: iso-8859-1 -*- +# Copyright (C) 2023 CEA, EDF +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +from collections import defaultdict + +import pickle + +class ScriptExecInfo: + @classmethod + def GetRepresentationOfTimeDelta(cls,endTime, startTime): + if endTime is None and startTime is None: + return "not measured" + td = endTime - startTime + import time + ts_of_td = time.gmtime(td.total_seconds()) + return "{}.{:06d}".format(time.strftime("%H:%M:%S",ts_of_td),td.microseconds) + + @classmethod + def MemRepr(cls,memInByte): + m = memInByte + UNITS=["B","kB","MB","GB"] + remain = 0 + oss = "" + for i in range( len(UNITS) ): + if m<1024: + oss = "{:03d}".format( int( (remain/1024)*1000 ) ) + oss = "{}.{} {}".format(m,oss,UNITS[i]) + return oss + else: + if i!=3: + remain = m%1024 + m//=1024 + return "{} {}".format(m,UNITS[3]) + + def __init__(self): + self._measure_time_resolution_ms = None + self._cpu_mem_during_exec = None + self._start_exec_time = None + self._end_exec_time = None + self._start_input_time = None + self._end_input_time = None + self._start_output_time = None + self._end_output_time = None + self._input_mem = 0 + self._input_hdd_mem = None + self._output_mem = 0 + self._output_hdd_mem = None + self._start_pos_log = None + self._stop_pos_log = None + + @property + def measureTimeResolution(self): + return self._measure_time_resolution_ms + + @measureTimeResolution.setter + def measureTimeResolution(self, value): + self._measure_time_resolution_ms = value + + @property + def tracePosStart(self): + return self._start_pos_log + + @tracePosStart.setter + def tracePosStart(self,value): + self._start_pos_log = value + + @property + def tracePosStop(self): + return self._stop_pos_log + + @tracePosStop.setter + def tracePosStop(self,value): + self._stop_pos_log = value + + @property + def CPUMemDuringExec(self): + return self._cpu_mem_during_exec + + @CPUMemDuringExec.setter + def CPUMemDuringExec(self,value): + self._cpu_mem_during_exec = value + + @property + def CPUMemDuringExecStr(self): + cpu = self._cpu_mem_during_exec[::2] + mem_rss = self._cpu_mem_during_exec[1::2] + return [(a,ScriptExecInfo.MemRepr(b)) for a,b in self._cpu_mem_during_exec] + + @property + def inputMem(self): + return self._input_mem + + @inputMem.setter + def inputMem(self,value): + self._input_mem = value + + @property + def inputMemStr(self): + return ScriptExecInfo.MemRepr( self.inputMem ) + + @property + def outputMem(self): + return self._output_mem + + @outputMem.setter + def outputMem(self,value): + self._output_mem = value + + @property + def outputMemStr(self): + return ScriptExecInfo.MemRepr( self.outputMem ) + + @property + def inputHDDMem(self): + return self._input_hdd_mem + + @inputHDDMem.setter + def inputHDDMem(self,value): + self._input_hdd_mem = value + + @property + def inputHDDMemStr(self): + if self._input_hdd_mem is None: + return "not computed" + return " ".join( [ ScriptExecInfo.MemRepr( elt.getSizeOfFileRead() ) for elt in self._input_hdd_mem] ) + + @property + def outputHDDMem(self): + return self._output_hdd_mem + + @outputHDDMem.setter + def outputHDDMem(self,value): + self._output_hdd_mem = value + + @property + def outputHDDMemStr(self): + if self._output_hdd_mem is None: + return "not computed" + return " ".join( [ ScriptExecInfo.MemRepr( elt.getSizeOfFileRead() ) for elt in self._output_hdd_mem] ) + + @property + def startInputTime(self): + return self._start_input_time + + @startInputTime.setter + def startInputTime(self,value): + self._start_input_time = value + + @property + def endInputTime(self): + return self._end_input_time + + @endInputTime.setter + def endInputTime(self,value): + self._end_input_time = value + + @property + def startExecTime(self): + return self._start_exec_time + + @startExecTime.setter + def startExecTime(self,value): + self._start_exec_time = value + + @property + def endExecTime(self): + return self._end_exec_time + + @endExecTime.setter + def endExecTime(self,value): + self._end_exec_time = value + + @property + def startOutputTime(self): + return self._start_output_time + + @startOutputTime.setter + def startOutputTime(self,value): + self._start_output_time = value + + @property + def endOutputTime(self): + return self._end_output_time + + @endOutputTime.setter + def endOutputTime(self,value): + self._end_output_time = value + + @property + def execTimeStr(self): + return ScriptExecInfo.GetRepresentationOfTimeDelta(self.endExecTime,self.startExecTime) + + @property + def inputTimeStr(self): + return ScriptExecInfo.GetRepresentationOfTimeDelta(self.endInputTime,self.startInputTime) + + @property + def outputTimeStr(self): + return ScriptExecInfo.GetRepresentationOfTimeDelta(self.endOutputTime,self.startOutputTime) + + def __str__(self): + return """start exec time = {self.startExecTime} +end exec time = {self.endExecTime} +exec_time = {self.execTimeStr} +Measure time resolution = {self.measureTimeResolution} ms +CPU and mem monitoring = {self.CPUMemDuringExecStr} +input unpickling and ev load from disk time = {self.inputTimeStr} +output serialization and ev write to disk time = {self.outputTimeStr} +input memory size before exec (MemoryPeak 2x) = {self.inputMemStr} +input memory size from HDD = {self.inputHDDMemStr} +output memory size after exec (MemoryPeak 2x) = {self.outputMemStr} +output memory size from HDD = {self.outputHDDMemStr} +Start position in log = {self.tracePosStart} +End position in log = {self.tracePosStop}""".format(**locals()) + +class ScriptExecInfoDeco: + def __init__(self, eff, father): + self._eff = eff + self._father = father + @property + def father(self): + return self._father + def get(self): + return self._eff + def __getitem__(self,i): + return self._eff[i] + def __str__(self): + return self._eff.__str__() + def __repr__(self): + return self._eff.__repr__() + def log(self): + with open(self.father.father.logfile,"rb") as f: + cont = f.read() + return cont[self._eff.tracePosStart:self._eff.tracePosStop].decode() + +class ScriptInfoAbstract: + def __init__(self, scriptPtr): + self._node_name = scriptPtr.getName() + self._code = scriptPtr.getCode() + self._exec = [pickle.loads(elt.getObj()) for elt in scriptPtr.listOfExecs()] + + @property + def execs(self): + return self._exec + + @property + def nodeName(self): + return self._node_name + + @property + def code(self): + return self._code + + @code.setter + def code(self,value): + self._code = value + + def __len__(self): + return len( self._exec ) + + def __getitem__(self,i): + return self._exec[i] + + def __str__(self): + return """name = {self.nodeName}\ncode = {self.code}\nexecs = {self.execs}""".format(**locals()) + + def __repr__(self): + return """ScriptInfo \"{self.nodeName}\"""".format(**locals()) + +class ScriptInfoClt(ScriptInfoAbstract): + def __init__(self, scriptPtr): + self._node_name = scriptPtr.getName() + self._code = scriptPtr.getCode() + self._exec = [pickle.loads(elt.getObj()) for elt in scriptPtr.listOfExecs()] + +class ScriptInfo(ScriptInfoAbstract): + def __init__(self, nodeName, code, execs): + self._node_name = nodeName + self._code = code + self._exec = execs + +class ScriptInfoDeco: + def __init__(self, eff, father): + self._eff = eff + self._father = father + @property + def father(self): + return self._father + def get(self): + return self._eff + def __getitem__(self,i): + return ScriptExecInfoDeco( self._eff[i], self ) + def __len__(self): + return self._eff.__len__() + def __str__(self): + return self._eff.__str__() + def __repr__(self): + return self._eff.__repr__() + +class ContainerLogInfoAbstract: + + def log(self): + with open(self.logfile,"rb") as f: + cont = f.read() + return cont.decode() + + @property + def ns_entry(self): + return self._ns_entry + + @property + def logfile(self): + return self._log_file + + def __len__(self): + return len( self._scripts ) + + def __getitem__(self,i): + return ScriptInfoDeco( self._scripts[i], self) + + def __str__(self): + return """NS entry = {self.ns_entry} LogFile = {self.logfile}""".format(**locals()) + +class ContainerLogInfoClt(ContainerLogInfoAbstract): + def __init__(self,contLogPtr): + self._log_file = contLogPtr.getLogFile() + self._ns_entry = contLogPtr.getContainerEntryInNS() + self._scripts = [ScriptInfoClt(elt) for elt in contLogPtr.listOfScripts()] + +class ContainerLogInfo(ContainerLogInfoAbstract): + def __init__(self, nsEntry, logFile, scripts): + self._log_file = logFile + self._ns_entry = nsEntry + self._scripts = scripts + +from abc import ABC, abstractmethod + +class InOutputObjVisitorAbstract(ABC): + def __init__(self): + self._cur_obj = None + self._data = [] + + def enter(self): + self._cur_obj = ObjMemModel() + return self._cur_obj + + def leave(self): + self._data.append( self._cur_obj ) + self._cur_obj = None + + def getSizeOfFileRead(self): + return sum( [elt.getSizeOfFileRead() for elt in self._data] ) + + def visitor(self): + return self + + def setHDDMem(self, v): + pass + + def setFileName(self, fileName): + pass + + @abstractmethod + def getRepr(self): + pass + +class InOutputObjVisitorIter: + def __init__(self, visitor): + self._visitor = visitor + self._current = 0 + + def __next__(self): + if self._current >= len(self._visitor._data): + raise StopIteration + else: + ret = self._visitor._data[ self._current ] + self._current += 1 + return ret + +class InOutputObjVisitor(InOutputObjVisitorAbstract): + def __init__(self): + super().__init__() + + def getRepr(self): + return self.getSizeOfFileRead() + + def __iter__(self): + return InOutputObjVisitorIter(self) + +class ObjMemModel(InOutputObjVisitorAbstract): + def __init__(self): + super().__init__() + self._hdd_mem = 0 + self._file_name = None + + def setHDDMem(self, v): + self._hdd_mem = v + del self._data + + def setFileName(self, fileName): + self._file_name = fileName + pass + + def getSizeOfFileRead(self): + if hasattr(self,"_data"): + return super().getSizeOfFileRead() + else: + return self._hdd_mem + + def getRepr(self): + return self.getSizeOfFileRead() + +class FakeObjVisitor: + def setHDDMem(self, v): + pass + + def visitor(self): + return None + +class InOutputObjVisitorCM: + def __init__(self, visitor): + self._visitor = visitor + def __enter__(self): + if self._visitor: + r = self._visitor.enter() + return r + else: + return FakeObjVisitor() + def __exit__(self,exctype, exc, tb): + if self._visitor: + self._visitor.leave() + pass + +class OffsetType: + def __init__(self,i): + self._i = i + def __int__(self): + return self._i + def __iadd__(self,delta): + self._i += delta + return self + +def unserializeInt(structData, offset): + from ctypes import c_int + sz_of_cint = 4 + sz = c_int.from_buffer_copy( structData[int(offset):int(offset)+sz_of_cint] ).value + offset += sz_of_cint + return sz + +def unserializeString(structData,offset): + sz = unserializeInt(structData,offset) + ret = structData[int(offset):int(offset)+sz].decode() + offset += sz + return ret + +def unserializeContainerScriptExecPerfLog(structData, offset): + import pickle + sz = unserializeInt(structData,offset) + inst = None + if sz > 0: + inst = pickle.loads( structData[int(offset):int(offset)+sz] ) + offset += sz + return inst + +def unserializeContainerScriptPerfLog(structData, offset): + name = unserializeString(structData,offset) + code = unserializeString(structData,offset) + numberOfSessions = unserializeInt(structData,offset) + sessions = [] + for _ in range(numberOfSessions): + session = unserializeContainerScriptExecPerfLog(structData,offset) + sessions.append( session ) + return ScriptInfo(name,code,sessions) + +def unserializeContainerPerfLog(structData, offset): + nsEntry = unserializeString(structData,offset) + logFile = unserializeString(structData,offset) + scripts = [] + nbScripts = unserializeInt(structData,offset) + for _ in range(nbScripts): + script = unserializeContainerScriptPerfLog(structData,offset) + scripts.append( script ) + return ContainerLogInfo(nsEntry,logFile,scripts) + +def unserializeLogManager(structData): + offset = OffsetType(0) + numberOfScripts = unserializeInt(structData,offset) + logManagerInst = [] + for _ in range(numberOfScripts): + containerPerfLogInst = unserializeContainerPerfLog(structData,offset) + logManagerInst.append( containerPerfLogInst ) + if int(offset) != len(structData): + raise RuntimeError("Something went wrong during unserialization phase.") + return logManagerInst diff --git a/src/Container/SALOME_ContainerManager.cxx b/src/Container/SALOME_ContainerManager.cxx index 4fcfca574..506a44e51 100644 --- a/src/Container/SALOME_ContainerManager.cxx +++ b/src/Container/SALOME_ContainerManager.cxx @@ -24,6 +24,7 @@ #include "SALOME_ResourcesManager.hxx" #include "SALOME_LoadRateManager.hxx" #include "SALOME_NamingService.hxx" +#include "SALOME_Container_i.hxx" #include "SALOME_ResourcesManager_Client.hxx" #include "SALOME_Embedded_NamingService.hxx" #include "SALOME_ModuleCatalog.hh" @@ -88,9 +89,8 @@ Utils_Mutex SALOME_ContainerManager::_systemMutex; //============================================================================= SALOME_ContainerManager::SALOME_ContainerManager(CORBA::ORB_ptr orb, PortableServer::POA_var poa, SALOME_NamingService_Abstract *ns) - : _nbprocUsed(1),_delta_time_ns_lookup_in_ms(DFT_DELTA_TIME_NS_LOOKUP_IN_MS) + : _nbprocUsed(1),_delta_time_ns_lookup_in_ms(DFT_DELTA_TIME_NS_LOOKUP_IN_MS),_delta_time_measure_in_ms(Abstract_Engines_Container_i::DFT_TIME_INTERVAL_BTW_MEASURE) { - MESSAGE("constructor"); _NS = ns; _resManager = new SALOME_ResourcesManager_Client(ns); _time_out_in_second = GetTimeOutToLoaunchServer(); @@ -157,8 +157,6 @@ SALOME_ContainerManager::SALOME_ContainerManager(CORBA::ORB_ptr orb, PortableSer } #endif #endif - - MESSAGE("constructor end"); } //============================================================================= @@ -227,6 +225,16 @@ void SALOME_ContainerManager::SetDeltaTimeBetweenNSLookupAtLaunchTimeInMilliSeco this->_delta_time_ns_lookup_in_ms = timeInMS; } +CORBA::Long SALOME_ContainerManager::GetDeltaTimeBetweenCPUMemMeasureInMilliSecond() +{ + return this->_delta_time_measure_in_ms; +} + +void SALOME_ContainerManager::SetDeltaTimeBetweenCPUMemMeasureInMilliSecond(CORBA::Long timeInMS) +{ + this->_delta_time_measure_in_ms = timeInMS; +} + //============================================================================= //! Loop on all the containers listed in naming service, ask shutdown on each /*! CORBA Method: @@ -489,8 +497,10 @@ Engines::Container_ptr SALOME_ContainerManager::GiveContainer(const Engines::Con if (!CORBA::is_nil(cont)) { INFOS("[GiveContainer] container " << containerNameInNS << " launched"); + cont->monitoringtimeresms( this->_delta_time_measure_in_ms ); + INFOS("[GiveContainer] container " << containerNameInNS << " first CORBA invocation OK"); std::ostringstream envInfo; - std::for_each( _override_env.begin(), _override_env.end(), [&envInfo](const std::pair& p) { envInfo << p.first << " = " << p.second << std::endl; } ); + std::for_each( _override_env.begin(), _override_env.end(), [&envInfo](const std::pair& p) { envInfo << p.first << " = " << p.second << " "; } ); INFOS("[GiveContainer] container " << containerNameInNS << " override " << envInfo.str()); Engines::FieldsDict envCorba; { @@ -504,7 +514,12 @@ Engines::Container_ptr SALOME_ContainerManager::GiveContainer(const Engines::Con } cont->override_environment_python( envCorba ); if( !_code_to_exe_on_startup.empty() ) + { + INFOS("[GiveContainer] container " << containerNameInNS << " python code executed " << _code_to_exe_on_startup); cont->execute_python_code( _code_to_exe_on_startup.c_str() ); + } + INFOS("[GiveContainer] container " << containerNameInNS << " verbosity positionning Activation = " << SALOME::VerbosityActivated() << " Verbosity Level = " << SALOME::VerbosityLevelStr()); + cont->setVerbosity( SALOME::VerbosityActivated(), SALOME::VerbosityLevelStr().c_str() ); return cont._retn(); } else @@ -668,8 +683,10 @@ SALOME_ContainerManager::LaunchContainer(const Engines::ContainerParameters& par struct stat file_info; stat(val, &file_info); bool is_dir = S_ISDIR(file_info.st_mode); - if (is_dir)logFilename=val; - else std::cerr << "SALOME_TMP_DIR environment variable is not a directory use /tmp instead" << std::endl; + if (is_dir) + logFilename=val; + else + MESSAGE( "SALOME_TMP_DIR environment variable is not a directory use /tmp instead" << std::endl ); } logFilename += "/"; #endif @@ -720,6 +737,7 @@ SALOME_ContainerManager::LaunchContainer(const Engines::ContainerParameters& par else { // Setting log file name + ret->locallogfilename( logFilename.c_str() ); logFilename=":"+logFilename; logFilename="@"+Kernel_Utils::GetHostname()+logFilename;//threadsafe logFilename=user+logFilename; diff --git a/src/Container/SALOME_ContainerManager.hxx b/src/Container/SALOME_ContainerManager.hxx index b579d5c3d..1d517c237 100644 --- a/src/Container/SALOME_ContainerManager.hxx +++ b/src/Container/SALOME_ContainerManager.hxx @@ -71,6 +71,10 @@ public: void SetDeltaTimeBetweenNSLookupAtLaunchTimeInMilliSecond(CORBA::Long timeInMS) override; + CORBA::Long GetDeltaTimeBetweenCPUMemMeasureInMilliSecond() override; + + void SetDeltaTimeBetweenCPUMemMeasureInMilliSecond(CORBA::Long timeInMS) override; + static const char *_ContainerManagerNameInNS; protected: @@ -214,6 +218,7 @@ private: std::vector< std::pair > _override_env; int _time_out_in_second; int _delta_time_ns_lookup_in_ms; + int _delta_time_measure_in_ms; std::string _code_to_exe_on_startup; }; #endif diff --git a/src/Container/SALOME_Container_i.hxx b/src/Container/SALOME_Container_i.hxx index ca2fac0a3..2e0099b8d 100644 --- a/src/Container/SALOME_Container_i.hxx +++ b/src/Container/SALOME_Container_i.hxx @@ -115,8 +115,14 @@ public: CORBA::Long getTotalPhysicalMemoryInUseByMe(); char *name(); char *workingdir(); - char *logfilename(); - void logfilename(const char *name); + char *logfilename() override; + void logfilename(const char *name) override; + char *locallogfilename() override; + void locallogfilename(const char *name) override; + CORBA::Long monitoringtimeresms() override; + void monitoringtimeresms(CORBA::Long intervalInMs) override; + void verbosity(bool& activated, CORBA::String_out level) override; + void setVerbosity(bool activated, const char *level) override; virtual void Shutdown(); char *getHostName(); @@ -162,7 +168,8 @@ public: void unregisterTemporaryFile(const std::string &fileName); void clearTemporaryFiles(); PortableServer::ObjectId *getCORBAId() const { return _id; } - +public: + static const int DFT_TIME_INTERVAL_BTW_MEASURE; protected: static std::map _cntInstances_map; static std::map _library_map; // library names, loaded @@ -175,6 +182,7 @@ protected: std::string _library_path; std::string _containerName; std::string _logfilename; + std::string _localfilename; std::string _load_script; CORBA::ORB_var _orb; PortableServer::POA_var _poa; diff --git a/src/Container/SALOME_FileTransfer_i.cxx b/src/Container/SALOME_FileTransfer_i.cxx index ddcc40b88..85cceccac 100644 --- a/src/Container/SALOME_FileTransfer_i.cxx +++ b/src/Container/SALOME_FileTransfer_i.cxx @@ -41,7 +41,6 @@ fileTransfer_i::fileTransfer_i() { - MESSAGE("fileTransfer_i::fileTransfer_i"); _fileKey=1; } @@ -53,7 +52,6 @@ fileTransfer_i::fileTransfer_i() fileTransfer_i::~fileTransfer_i() { - MESSAGE("fileTransfer_i::~fileTransfer_i"); } @@ -69,7 +67,6 @@ fileTransfer_i::~fileTransfer_i() CORBA::Long fileTransfer_i::open(const char* fileName) { - MESSAGE(" fileTransfer_i::open " << fileName); int aKey = _fileKey++; _ctr=0; FILE* fp; @@ -92,7 +89,6 @@ CORBA::Long fileTransfer_i::open(const char* fileName) void fileTransfer_i::close(CORBA::Long fileId) { - MESSAGE("fileTransfer_i::close"); FILE* fp; if (! (fp = _fileAccess[fileId]) ) { @@ -119,7 +115,6 @@ void fileTransfer_i::close(CORBA::Long fileId) Engines::fileBlock* fileTransfer_i::getBlock(CORBA::Long fileId) { - //MESSAGE("fileTransfer_i::getBlock"); Engines::fileBlock* aBlock = new Engines::fileBlock; FILE* fp; @@ -149,7 +144,6 @@ Engines::fileBlock* fileTransfer_i::getBlock(CORBA::Long fileId) */ CORBA::Long fileTransfer_i::openW(const char* fileName) { - MESSAGE(" fileTransfer_i::openW " << fileName); int aKey = _fileKey++; _ctr=0; FILE* fp; @@ -171,7 +165,6 @@ CORBA::Long fileTransfer_i::openW(const char* fileName) */ void fileTransfer_i::putBlock(CORBA::Long fileId, const Engines::fileBlock& block) { - MESSAGE("fileTransfer_i::putBlock"); FILE* fp; if (! (fp = _fileAccess[fileId]) ) { diff --git a/src/Container/SALOME_PyNode.py b/src/Container/SALOME_PyNode.py index 1bb41877d..d6d46668f 100644 --- a/src/Container/SALOME_PyNode.py +++ b/src/Container/SALOME_PyNode.py @@ -29,6 +29,10 @@ import pickle import Engines__POA import SALOME__POA import SALOME +import logging +import os +import sys +from SALOME_ContainerHelper import ScriptExecInfo MY_CONTAINER_ENTRY_IN_GLBS = "my_container" @@ -129,10 +133,32 @@ TypeCounter = c_int def GetSizeOfTCnt(): return len( bytes(TypeCounter(0) ) ) -def GetObjectFromFile(fname): +def GetSizeOfBufferedReader(f): + """ + This method returns in bytes size of a file openned. + + Args: + ---- + f (io.IOBase): buffered reader returned by open + + Returns + ------- + int: number of bytes + """ + import io + pos = f.tell() + f.seek(0,io.SEEK_END) + pos2 = f.tell() + f.seek(pos,io.SEEK_SET) + return pos2-pos + +def GetObjectFromFile(fname, visitor = None): with open(fname,"rb") as f: cntb = f.read( GetSizeOfTCnt() ) cnt = TypeCounter.from_buffer_copy( cntb ).value + if visitor: + visitor.setHDDMem( GetSizeOfBufferedReader(f) ) + visitor.setFileName( fname ) obj = pickle.load(f) return obj,cnt @@ -241,8 +267,8 @@ class BigObjectOnDiskBase: def __dumpIntoFile(self, objSerialized): DumpInFile( objSerialized, self._filename ) - def get(self): - obj, _ = GetObjectFromFile( self._filename ) + def get(self, visitor = None): + obj, _ = GetObjectFromFile( self._filename, visitor ) return obj def __float__(self): @@ -269,8 +295,8 @@ class BigObjectOnDiskListElement(BigObjectOnDiskBase): self._pos = pos self._length = length - def get(self): - fullObj = BigObjectOnDiskBase.get(self) + def get(self, visitor = None): + fullObj = BigObjectOnDiskBase.get(self, visitor) return fullObj[ self._pos ] def __getitem__(self, i): @@ -298,36 +324,69 @@ class BigObjectOnDiskTuple(BigObjectOnDiskSequence): def __init__(self, length, fileName, objSerialized): BigObjectOnDiskSequence.__init__(self, length, fileName, objSerialized) -def SpoolPickleObject( obj ): - import pickle - pickleObjInit = pickle.dumps( obj , pickle.HIGHEST_PROTOCOL ) - if not ActivateProxyMecanismOrNot( len(pickleObjInit) ): - return pickleObjInit +def ProxyfyPickeled( obj, pickleObjInit = None, visitor = None ): + """ + This method return a proxy instance of pickled form of object given in input. + + Args: + ---- + obj (pickelable type) : object to be proxified + pickleObjInit (bytes) : Optionnal. Original pickeled form of object to be proxyfied if already computed. If not this method generate it + + Returns + ------- + BigObjectOnDiskBase: proxy instance + """ + pickleObj = pickleObjInit + if pickleObj is None: + pickleObj = pickle.dumps( obj , pickle.HIGHEST_PROTOCOL ) + fileName = GetBigObjectFileName() + if visitor: + visitor.setHDDMem( len(pickleObj) ) + visitor.setFileName(fileName) + if isinstance( obj, list): + proxyObj = BigObjectOnDiskList( len(obj), fileName, pickleObj ) + elif isinstance( obj, tuple): + proxyObj = BigObjectOnDiskTuple( len(obj), fileName , pickleObj ) else: - if isinstance( obj, list): - proxyObj = BigObjectOnDiskList( len(obj), GetBigObjectFileName() , pickleObjInit ) - elif isinstance( obj, tuple): - proxyObj = BigObjectOnDiskTuple( len(obj), GetBigObjectFileName() , pickleObjInit ) + proxyObj = BigObjectOnDisk( fileName , pickleObj ) + return proxyObj + +def SpoolPickleObject( obj, visitor = None ): + import pickle + with InOutputObjVisitorCM(visitor) as v: + pickleObjInit = pickle.dumps( obj , pickle.HIGHEST_PROTOCOL ) + if not ActivateProxyMecanismOrNot( len(pickleObjInit) ): + return pickleObjInit else: - proxyObj = BigObjectOnDisk( GetBigObjectFileName() , pickleObjInit ) - pickleProxy = pickle.dumps( proxyObj , pickle.HIGHEST_PROTOCOL ) - return pickleProxy + proxyObj = ProxyfyPickeled( obj, pickleObjInit, v.visitor() ) + pickleProxy = pickle.dumps( proxyObj , pickle.HIGHEST_PROTOCOL ) + return pickleProxy + +from SALOME_ContainerHelper import InOutputObjVisitorCM, InOutputObjVisitor -def UnProxyObjectSimple( obj ): +def UnProxyObjectSimple( obj, visitor = None ): """ Method to be called in Remote mode. Alterate the obj _status attribute. Because the slave process does not participate in the reference counting + + Args: + ---- + visitor (InOutputObjVisitor): A visitor to keep track of amount of memory on chip and those on HDD + """ - if isinstance(obj,BigObjectOnDiskBase): - obj.doNotTouchFile() - return obj.get() - elif isinstance( obj, list): - retObj = [] - for elt in obj: - retObj.append( UnProxyObjectSimple(elt) ) - return retObj - else: - return obj + with InOutputObjVisitorCM(visitor) as v: + logging.debug( "UnProxyObjectSimple {}".format(type(obj)) ) + if isinstance(obj,BigObjectOnDiskBase): + obj.doNotTouchFile() + return obj.get( v ) + elif isinstance( obj, list): + retObj = [] + for elt in obj: + retObj.append( UnProxyObjectSimple(elt,v.visitor()) ) + return retObj + else: + return obj def UnProxyObjectSimpleLocal( obj ): """ @@ -342,7 +401,64 @@ def UnProxyObjectSimpleLocal( obj ): return retObj else: return obj - + +class FileDeleter: + def __init__(self, fileName): + self._filename = fileName + @property + def filename(self): + return self._filename + def __del__(self): + import os + if os.path.exists( self._filename ): + os.unlink( self._filename ) + +def LaunchMonitoring( intervalInMs ): + """ + Launch a subprocess monitoring self process. + This monitoring subprocess is a python process lauching every intervalInMs ms evaluation of + CPU usage and RSS memory. + Communication between subprocess and self is done by file. + """ + import KernelBasis + def BuildPythonFileForCPUPercent( intervalInMs ): + import os + import tempfile + with tempfile.NamedTemporaryFile(prefix="htop_",suffix=".py") as f: + tempPyFile = f.name + tempOutFile = "{}.txt".format( os.path.splitext( tempPyFile )[0] ) + pid = os.getpid() + with open(tempPyFile,"w") as f: + f.write("""import psutil +pid = {} +process = psutil.Process( pid ) +import time +with open("{}","a") as f: + while True: + f.write( "{{}}\\n".format( str( process.cpu_percent() ) ) ) + f.write( "{{}}\\n".format( str( process.memory_info().rss ) ) ) + f.flush() + time.sleep( {} / 1000.0 ) +""".format(pid, tempOutFile, intervalInMs)) + return FileDeleter(tempPyFile), FileDeleter(tempOutFile) + pyFileName, outFileName = BuildPythonFileForCPUPercent( intervalInMs ) + KernelBasis.LaunchMonitoring(pyFileName.filename,outFileName.filename) + return pyFileName, outFileName + +def StopMonitoring( ): + """ + Retrieve data of monitoring and kill monitoring subprocess. + + Returns + ------- + list : list of pairs. First param of pair is CPU usage. Second param of pair is rss memory usage + """ + import KernelBasis + ret = KernelBasis.StopMonitoring() + cpu = ret[::2] + mem_rss = [ int(elt) for elt in ret[1::2]] + return [(a,b) for a,b in zip(cpu,mem_rss)] + class SeqByteReceiver: # 2GB limit to trigger split into chunks CHUNK_SIZE = 2000000000 @@ -372,19 +488,34 @@ class SeqByteReceiver: iStart = iEnd; iEnd = min(iStart + EFF_CHUNK_SIZE,size) return data_for_split_case +class LogOfCurrentExecutionSession: + def __init__(self, handleToCentralizedInst): + self._remote_handle = handleToCentralizedInst + self._current_instance = ScriptExecInfo() + + def addInfoOnLevel2(self, key, value): + setattr(self._current_instance,key,value) + + def finalizeAndPushToMaster(self): + self._remote_handle.assign( pickle.dumps( self._current_instance ) ) + class PyScriptNode_i (Engines__POA.PyScriptNode,Generic): """The implementation of the PyScriptNode CORBA IDL that executes a script""" - def __init__(self, nodeName,code,poa,my_container): + def __init__(self, nodeName,code,poa,my_container,logscript): """Initialize the node : compilation in the local context""" Generic.__init__(self,poa) self.nodeName=nodeName self.code=code + self.my_container_py = my_container self.my_container=my_container._container linecache.cache[nodeName]=0,None,code.split('\n'),nodeName self.ccode=compile(code,nodeName,'exec') self.context={} self.context[MY_CONTAINER_ENTRY_IN_GLBS] = self.my_container - + self._log_script = logscript + self._current_execution_session = None + sys.stdout.flush() ; sys.stderr.flush() # flush to correctly capture log per execution session + def __del__(self): # force removal of self.context. Don t know why it s not done by default self.removeAllVarsInContext() @@ -428,6 +559,7 @@ class PyScriptNode_i (Engines__POA.PyScriptNode,Generic): except Exception: exc_typ,exc_val,exc_fr=sys.exc_info() l=traceback.format_exception(exc_typ,exc_val,exc_fr) + print("".join(l)) ; sys.stdout.flush() # print error also in logs of remote container raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" % (self.nodeName),0)) def execute(self,outargsname,argsin): @@ -446,47 +578,73 @@ class PyScriptNode_i (Engines__POA.PyScriptNode,Generic): except Exception: exc_typ,exc_val,exc_fr=sys.exc_info() l=traceback.format_exception(exc_typ,exc_val,exc_fr) + print("".join(l)) ; sys.stdout.flush() # print error also in logs of remote container raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s, outargsname: %s" % (self.nodeName,outargsname),0)) def executeFirst(self,argsin): """ Same than first part of self.execute to reduce memory peak.""" - import time try: + self.beginOfCurrentExecutionSession() data = None + self.addTimeInfoOnLevel2("startInputTime") if True: # to force call of SeqByteReceiver's destructor argsInPy = SeqByteReceiver( argsin ) data = argsInPy.data() + self.addInfoOnLevel2("inputMem",len(data)) _,kws=pickle.loads(data) + vis = InOutputObjVisitor() for elt in kws: # fetch real data if necessary - kws[elt] = UnProxyObjectSimple( kws[elt] ) + kws[elt] = UnProxyObjectSimple( kws[elt],vis) + self.addInfoOnLevel2("inputHDDMem",vis) self.context.update(kws) + self.addTimeInfoOnLevel2("endInputTime") except Exception: exc_typ,exc_val,exc_fr=sys.exc_info() l=traceback.format_exception(exc_typ,exc_val,exc_fr) + print("".join(l)) ; sys.stdout.flush() # print error also in logs of remote container raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode:First %s" % (self.nodeName),0)) def executeSecond(self,outargsname): """ Same than second part of self.execute to reduce memory peak.""" + import sys try: + self.addTimeInfoOnLevel2("startExecTime") + ## + self.addInfoOnLevel2("measureTimeResolution",self.my_container_py.monitoringtimeresms()) + monitoringParams = LaunchMonitoring( self.my_container_py.monitoringtimeresms() ) exec(self.ccode, self.context) + cpumeminfo = StopMonitoring( ) + ## + self.addInfoOnLevel2("CPUMemDuringExec",cpumeminfo) + del monitoringParams + self.addTimeInfoOnLevel2("endExecTime") + self.addTimeInfoOnLevel2("startOutputTime") argsout=[] for arg in outargsname: if arg not in self.context: raise KeyError("There is no variable %s in context" % arg) argsout.append(self.context[arg]) ret = [ ] + outputMem = 0 + vis = InOutputObjVisitor() for arg in argsout: # the proxy mecanism is catched here - argPickle = SpoolPickleObject( arg ) + argPickle = SpoolPickleObject( arg, vis ) retArg = SenderByte_i( self.poa,argPickle ) id_o = self.poa.activate_object(retArg) retObj = self.poa.id_to_reference(id_o) ret.append( retObj._narrow( SALOME.SenderByte ) ) + outputMem += len(argPickle) + self.addInfoOnLevel2("outputMem",outputMem) + self.addInfoOnLevel2("outputHDDMem",vis) + self.addTimeInfoOnLevel2("endOutputTime") + self.endOfCurrentExecutionSession() return ret except Exception: exc_typ,exc_val,exc_fr=sys.exc_info() l=traceback.format_exception(exc_typ,exc_val,exc_fr) + print("".join(l)) ; sys.stdout.flush() # print error also in logs of remote container raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode:Second %s, outargsname: %s" % (self.nodeName,outargsname),0)) def listAllVarsInContext(self): @@ -524,3 +682,17 @@ class PyScriptNode_i (Engines__POA.PyScriptNode,Generic): l=traceback.format_exception(exc_typ,exc_val,exc_fr) raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" %self.nodeName,0)) pass + + def beginOfCurrentExecutionSession(self): + self._current_execution_session = LogOfCurrentExecutionSession( self._log_script.addExecutionSession() ) + + def endOfCurrentExecutionSession(self): + self._current_execution_session.finalizeAndPushToMaster() + self._current_execution_session = None + + def addInfoOnLevel2(self, key, value): + self._current_execution_session.addInfoOnLevel2(key, value) + + def addTimeInfoOnLevel2(self, key): + from datetime import datetime + self._current_execution_session.addInfoOnLevel2(key,datetime.now()) diff --git a/src/Container/ScriptsTemplate/script_parameters.py b/src/Container/ScriptsTemplate/script_parameters.py index ee1c6d3e7..cc1d3a8aa 100644 --- a/src/Container/ScriptsTemplate/script_parameters.py +++ b/src/Container/ScriptsTemplate/script_parameters.py @@ -21,15 +21,5 @@ class ScriptLocalParameters: return value def __str__(self): - str = [] - str.append("nb_proc: %s" % self.nb_proc) - str.append("workdir: %s" % self.workdir) - str.append("isTmpDir: %s" % self.isTmpDir) - str.append("name_server: %s" % self.name_server) - str.append("container: %s" % self.container) - str.append("container_name: %s" % self.container_name) - str.append("libbatch_nodefile: %s" % self.libbatch_nodefile) - str.append("machine_file: %s" % self.machine_file) - str.append("ompi_uri_file: %s" % self.ompi_uri_file) - str.append("--") - return "\n".join(str) \ No newline at end of file + str = """nb_proc: {self.nb_proc} workdir: {self.workdir} isTmpDir: {self.isTmpDir} name_server: {self.name_server} container: {self.container} container_name: {self.container_name} libbatch_nodefile: {self.libbatch_nodefile} machine_file: {self.machine_file} ompi_uri_file: {self.ompi_uri_file}""".format(**locals()) + return str diff --git a/src/KERNEL_PY/__init__.py b/src/KERNEL_PY/__init__.py index ee39881d4..cdc0ead26 100644 --- a/src/KERNEL_PY/__init__.py +++ b/src/KERNEL_PY/__init__.py @@ -161,7 +161,7 @@ if not flags: # sys.setdlopenflags(flags) # pass -orb, lcc, naming_service, cm, sg, esm, dsm, modulcat, rm = None,None,None,None,None,None,None,None,None +orb, lcc, naming_service, cm, sg, esm, dsm, logm, modulcat, rm = None,None,None,None,None,None,None,None,None,None myStudy, myStudyName = None,None salome_initial=True @@ -245,7 +245,7 @@ def salome_init_without_session(path=None, embedded=False, iorfakensfile=None): A Fake NamingService is created storing reference of all servants in the current process. """ salome_init_without_session_common(path,embedded) - global lcc,cm,dsm,esm,rm + global lcc,cm,dsm,esm,rm,logm import KernelLauncher cm = KernelLauncher.myContainerManager() type(cm).SetOverrideEnvForContainersSimple = ContainerManagerSetOverrideEnvForContainersSimple @@ -266,6 +266,12 @@ def salome_init_without_session(path=None, embedded=False, iorfakensfile=None): # esm inherits from SALOME_CPythonHelper singleton already initialized by GetDSMInstance # esm inherits also from SALOME_ResourcesManager creation/initialization (concerning SingleThreadPOA POA) when KernelLauncher.GetContainerManager() has been called esm = KernelLauncher.GetExternalServer() + # idem for logm + logm = KernelLauncher.myLogManager() + type(logm).NaiveFetch = LogManagerNaiveFetch + type(logm).Fetch = LogManagerFetch + type(logm).DumpInFile = LogManagerDumpInFile + type(logm).LoadFromFile = LogManagerLoadFromFile # import KernelLogger naming_service.Register(KernelLogger.myLogger(),"/Logger") @@ -282,7 +288,7 @@ def salome_init_without_session_attached(path=None, embedded=False): lcc is pointing to the FakeNamingService above. """ salome_init_without_session_common(path,embedded) - global lcc,cm,dsm,esm,rm + global lcc,cm,dsm,esm,rm,logm import CORBA orb=CORBA.ORB_init(['']) import Engines @@ -306,6 +312,10 @@ def salome_init_without_session_attached(path=None, embedded=False): ESM_NAME_IN_NS = "/ExternalServers" esm = orb.string_to_object( nsAbroad.Resolve(ESM_NAME_IN_NS).decode() ) naming_service.Register(esm,ESM_NAME_IN_NS) + # + LOGM_NAME_IN_NS = "/LogManager" + logm = orb.string_to_object( nsAbroad.Resolve(LOGM_NAME_IN_NS).decode() ) + naming_service.Register(logm,LOGM_NAME_IN_NS) def salome_init_with_session(path=None, embedded=False): """ @@ -444,5 +454,29 @@ def ContainerManagerSetOverrideEnvForContainersSimple(self,env): envEff = [ Engines.KeyValPairString(key=k,val=v) for k,v in env ] return self.SetOverrideEnvForContainers( envEff ) +def LogManagerNaiveFetch(self): + """ + Fetch data from server with multiple CORBA invokations. + """ + import SALOME_ContainerHelper + return [SALOME_ContainerHelper.ContainerLogInfoClt(elt) for elt in self.listOfContainerLogs()] + +def LogManagerFetch(self,clearMemory = False): + """ + Fetch data from server in one shot mode. + """ + from SALOME_ContainerHelper import unserializeLogManager + return unserializeLogManager( self.getAllStruct(clearMemory) ) + +def LogManagerDumpInFile(self,fileName,clearMemory = False): + with open(fileName,"wb") as f: + f.write( self.getAllStruct( clearMemory ) ) + +def LogManagerLoadFromFile(self,fileName): + from SALOME_ContainerHelper import unserializeLogManager + with open(fileName,"rb") as f: + data = f.read() + return unserializeLogManager( data ) + #to expose all objects to pydoc __all__=dir() diff --git a/src/KERNEL_PY/salome_study.py b/src/KERNEL_PY/salome_study.py index 7cd966f23..43f88003a 100644 --- a/src/KERNEL_PY/salome_study.py +++ b/src/KERNEL_PY/salome_study.py @@ -271,11 +271,9 @@ def FindFileInDataDir(filename): #-------------------------------------------------------------------------- def openStudy(theStudyPath): - if verbose(): print("openStudy (begin)") global myStudy, myStudyName myStudy.Open(theStudyPath) myStudyName = myStudy._get_Name() - if verbose(): print("openStudy (end):", theStudyPath, myStudy._get_Name()) #-------------------------------------------------------------------------- @@ -290,16 +288,13 @@ def salome_study_init_without_session(theStudyPath=None): global myStudy, myStudyName global orb, lcc, naming_service, cm - if verbose(): print("theStudyPath:", theStudyPath) if not myStudy: import CORBA orb = CORBA.ORB_init(['']) # get Study reference - if verbose(): print("looking for study...") import KernelDS myStudy = KernelDS.myStudy() - if verbose(): print("Study found") pass import types @@ -327,15 +322,12 @@ def salome_study_init(theStudyPath=None): global myStudy, myStudyName global orb, lcc, naming_service, cm - if verbose(): print("theStudyPath:", theStudyPath) if not myStudy: orb, lcc, naming_service, cm, _, _, _ = salome_kernel.salome_kernel_init() # get Study reference - if verbose(): print("looking for study...") obj = naming_service.Resolve('/Study') myStudy = obj._narrow(SALOMEDS.Study) - if verbose(): print("Study found") pass import types diff --git a/src/Launcher/CMakeLists.txt b/src/Launcher/CMakeLists.txt index 79d482119..caa851941 100644 --- a/src/Launcher/CMakeLists.txt +++ b/src/Launcher/CMakeLists.txt @@ -103,7 +103,7 @@ ADD_EXECUTABLE(TestLauncher TestLauncher.cxx) TARGET_LINK_LIBRARIES(TestLauncher ${TestLauncher_LIBS}) INSTALL(TARGETS TestLauncher DESTINATION ${SALOME_INSTALL_BINS}) -ADD_LIBRARY(SalomeLauncher BatchTest.cxx SALOME_Launcher.cxx SALOME_ExternalServerLauncher.cxx SALOME_LauncherException.cxx SALOME_ExternalServerHandler.cxx) +ADD_LIBRARY(SalomeLauncher BatchTest.cxx SALOME_Launcher.cxx SALOME_ExternalServerLauncher.cxx SALOME_LauncherException.cxx SALOME_ExternalServerHandler.cxx SALOME_LogManager.cxx) TARGET_LINK_LIBRARIES(SalomeLauncher Launcher ${COMMON_LIBS}) INSTALL(TARGETS SalomeLauncher EXPORT ${PROJECT_NAME}TargetGroup DESTINATION ${SALOME_INSTALL_LIBS}) @@ -133,6 +133,7 @@ SET(COMMON_HEADERS_HXX Launcher_Job_SALOME.hxx Launcher_Job_YACSFile.hxx Launcher_Utils.hxx + SALOME_LogManager.hxx SALOME_Launcher.hxx SALOME_Launcher_Parser.hxx SALOME_Launcher_defs.hxx @@ -143,6 +144,7 @@ SET(COMMON_HEADERS_HXX SET(LAUNCHER_PYTHON_SCRIPTS launcher_proxy.py + SALOME_LogManager.py ) SET(KernelLauncher_HEADERS KernelLauncher.hxx KernelLauncher.i) diff --git a/src/Launcher/KernelLauncher.cxx b/src/Launcher/KernelLauncher.cxx index 5ed46f83c..31297ad64 100644 --- a/src/Launcher/KernelLauncher.cxx +++ b/src/Launcher/KernelLauncher.cxx @@ -25,6 +25,7 @@ #include "SALOME_KernelServices.hxx" #include "SALOME_ResourcesManager.hxx" #include "SALOME_ExternalServerLauncher.hxx" +#include "SALOME_LogManager.hxx" #include "SALOME_CPythonHelper.hxx" #include @@ -82,3 +83,29 @@ std::string GetExternalServerInstance() CORBA::String_var ior = orb->object_to_string(esmCPtr); return std::string(ior.in()); } + +std::string GetLogManagerInstance() +{ + CORBA::ORB_ptr orb = KERNEL::getORB(); + CORBA::Object_var obj = orb->resolve_initial_references("RootPOA"); + PortableServer::POA_var root_poa = PortableServer::POA::_narrow(obj); + // + CORBA::PolicyList policies; + policies.length(1); + PortableServer::POAManager_var pman = root_poa->the_POAManager(); + PortableServer::ThreadPolicy_var threadPol(root_poa->create_thread_policy(PortableServer::SINGLE_THREAD_MODEL)); + policies[0] = PortableServer::ThreadPolicy::_duplicate(threadPol); + PortableServer::POA_var safePOA = root_poa->create_POA("SingleThreadPOAForLogManager",pman,policies); + threadPol->destroy(); + // + SALOME_CPythonHelper *cPyh(SALOME_CPythonHelper::Singleton()); + SALOME_Fake_NamingService *ns = new SALOME_Fake_NamingService; + SALOME_LogManager *esm(new SALOME_LogManager(orb,safePOA,ns)); + esm->_remove_ref(); + // + CORBA::Object_var esmPtr = safePOA->servant_to_reference(esm); + Engines::LogManager_var esmCPtr = Engines::LogManager::_narrow(esmPtr); + // + CORBA::String_var ior = orb->object_to_string(esmCPtr); + return std::string(ior.in()); +} diff --git a/src/Launcher/KernelLauncher.hxx b/src/Launcher/KernelLauncher.hxx index d7f8c880e..30dadfcf8 100644 --- a/src/Launcher/KernelLauncher.hxx +++ b/src/Launcher/KernelLauncher.hxx @@ -25,3 +25,4 @@ std::string RetrieveInternalInstanceOfLocalCppResourcesManager(); std::string GetContainerManagerInstance(); std::string GetResourcesManagerInstance(); std::string GetExternalServerInstance(); +std::string GetLogManagerInstance(); \ No newline at end of file diff --git a/src/Launcher/KernelLauncher.i b/src/Launcher/KernelLauncher.i index 7a97e9a11..5348795b2 100644 --- a/src/Launcher/KernelLauncher.i +++ b/src/Launcher/KernelLauncher.i @@ -31,6 +31,7 @@ std::string GetContainerManagerInstance(); std::string GetResourcesManagerInstance(); std::string GetExternalServerInstance(); + std::string GetLogManagerInstance(); } %pythoncode %{ @@ -51,4 +52,10 @@ def GetExternalServer(): import CORBA orb=CORBA.ORB_init(['']) return orb.string_to_object(GetExternalServerInstance()) + +def myLogManager(): + import Engines + import CORBA + orb=CORBA.ORB_init(['']) + return orb.string_to_object(GetLogManagerInstance()) %} diff --git a/src/Launcher/SALOME_LogManager.cxx b/src/Launcher/SALOME_LogManager.cxx new file mode 100644 index 000000000..b47772476 --- /dev/null +++ b/src/Launcher/SALOME_LogManager.cxx @@ -0,0 +1,382 @@ +// Copyright (C) 2024 CEA, EDF +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +#include "SALOME_LogManager.hxx" +#include "SALOME_Fake_NamingService.hxx" +#include "SALOME_ContainerManager.hxx" + +#ifndef WIN32 +#include +#else +#include +#include +#endif + +#include +#include +#include +#include +#include + +const char SALOME_LogManager::NAME_IN_NS[]="/LogManager"; + +static std::vector FromPyToCpp(PyObject *obj) +{ + std::vector ret; + char *buffer = nullptr; + Py_ssize_t length = 0; + PyBytes_AsStringAndSize(obj,&buffer,&length); + ret.resize(length); + for(auto i = 0 ; i < length ; ++i) + ret[i] = buffer[i]; + return ret; +} + +static SALOME::vectorOfByte *FromVectCharToCorba(const std::vector& data) +{ + SALOME::vectorOfByte_var ret = new SALOME::vectorOfByte; + auto length = data.size(); + ret->length(length); + for(auto i = 0 ; i < length ; ++i) + ret[i] = data[i]; + return ret._retn(); +} + +PortableServer::POA_var SALOME_ContainerScriptExecPerfLog::getPOA() +{ + return father()->getPOA(); +} + +void SALOME_ContainerScriptExecPerfLog::assign(const SALOME::vectorOfByte& value) +{ + auto sz = value.length(); + _data.resize( sz ); + for(auto i = 0 ; i < sz ; ++i) + _data[i] = value[i]; + // + AutoPyRef s( this->end( ) ); + // + _data = FromPyToCpp(s); +} + +SALOME::vectorOfByte *SALOME_ContainerScriptExecPerfLog::getObj() +{ + return FromVectCharToCorba(this->_data); +} + +void SALOME_ContainerScriptExecPerfLog::accept(SALOME_VisitorContainerLog &visitor) +{ + visitor.visitContainerScriptExecPerfLog( *this ); +} + +void SALOME_ContainerScriptExecPerfLog::start() +{ + AutoGIL gstate; + AutoPyRef result = PyObject_CallMethod(pyObj(),(char*)"start","",nullptr); + if (PyErr_Occurred()) + { + std::string error("can not start"); + PyErr_Print(); + THROW_SALOME_CORBA_EXCEPTION(error.c_str(),SALOME::INTERNAL_ERROR); + } +} + +AutoPyRef SALOME_ContainerScriptExecPerfLog::end() +{ + AutoGIL gstate; + //https://docs.python.org/3/c-api/arg.html#c.Py_BuildValue + AutoPyRef result(PyObject_CallMethod(pyObj(),(char*)"end","y#",_data.data(),_data.size(),nullptr) ) ; + if (PyErr_Occurred()) + { + std::string error("can not end"); + PyErr_Print(); + THROW_SALOME_CORBA_EXCEPTION(error.c_str(),SALOME::INTERNAL_ERROR); + } + return result; +} + +///// + +PortableServer::POA_var SALOME_ContainerScriptPerfLog::getPOA() +{ + return father()->getPOA(); +} + +char *SALOME_ContainerScriptPerfLog::getCode() +{ + return CORBA::string_dup( _code.c_str() ); +} + +char *SALOME_ContainerScriptPerfLog::getName() +{ + return CORBA::string_dup( _name.c_str() ); +} + +void SALOME_ContainerScriptPerfLog::accept(SALOME_VisitorContainerLog &visitor) +{ + visitor.enterContainerScriptPerfLog( *this ); + for(auto session : _sessions) + { + PortableServer::ServantBase *serv = getPOA()->reference_to_servant( session ); + serv->_remove_ref(); + SALOME_ContainerScriptExecPerfLog *servC = dynamic_cast(serv); + visitor.visitContainerScriptExecPerfLog( *servC ); + } + visitor.leaveContainerScriptPerfLog( *this ); +} + +Engines::ContainerScriptExecPerfLog_ptr SALOME_ContainerScriptPerfLog::addExecutionSession() +{ + SALOME_ContainerScriptExecPerfLog *execution = new SALOME_ContainerScriptExecPerfLog(this); + PortableServer::ObjectId_var id(getPOA()->activate_object(execution)); + execution->_remove_ref(); + CORBA::Object_var executionPtr(getPOA()->id_to_reference(id)); + Engines::ContainerScriptExecPerfLog_var executionPtr2 = Engines::ContainerScriptExecPerfLog::_narrow(executionPtr); + _sessions.push_back( executionPtr2 ); + { + AutoGIL gstate; + AutoPyRef result = PyObject_CallMethod(pyObj(),(char*)"addExecution","",nullptr); + if (PyErr_Occurred()) + { + std::string error("can not addExecution"); + PyErr_Print(); + THROW_SALOME_CORBA_EXCEPTION(error.c_str(),SALOME::INTERNAL_ERROR); + } + execution->setPyObj( result.retn() );//ownership of result is transfered to execution + } + execution->start(); + return executionPtr2._retn(); +} + +Engines::ListOfContainerScriptExecPerfLog *SALOME_ContainerScriptPerfLog::listOfExecs() +{ + Engines::ListOfContainerScriptExecPerfLog_var ret = new Engines::ListOfContainerScriptExecPerfLog; + auto sz = this->_sessions.size(); + ret->length(sz); + for(auto i = 0 ; i < sz ; ++i) + ret[i] = this->_sessions[i]; + return ret._retn(); +} + +///// + +PortableServer::POA_var SALOME_ContainerPerfLog::getPOA() +{ + return father()->getPOA(); + } + +Engines::ContainerScriptPerfLog_ptr SALOME_ContainerPerfLog::addScript(const char *name, const char *code) +{ + SALOME_ContainerScriptPerfLog *script = new SALOME_ContainerScriptPerfLog(this,name,code); + PortableServer::ObjectId_var id(getPOA()->activate_object(script)); + script->_remove_ref(); + CORBA::Object_var scriptPtr(getPOA()->id_to_reference(id)); + Engines::ContainerScriptPerfLog_var scriptPtr2 = Engines::ContainerScriptPerfLog::_narrow(scriptPtr); + _scripts.push_back( scriptPtr2 ); + { + AutoGIL gstate; + PyObject *result = PyObject_CallMethod(pyObj(),(char*)"addScript","",nullptr); + if (PyErr_Occurred()) + { + std::string error("can not addScript"); + PyErr_Print(); + THROW_SALOME_CORBA_EXCEPTION(error.c_str(),SALOME::INTERNAL_ERROR); + } + script->setPyObj( result ); + } + return scriptPtr2._retn(); +} + +Engines::ListOfContainerScriptPerfLog *SALOME_ContainerPerfLog::listOfScripts() +{ + Engines::ListOfContainerScriptPerfLog_var ret = new Engines::ListOfContainerScriptPerfLog; + std::size_t len( this->_scripts.size() ); + ret->length( len ); + for(std::size_t i = 0 ; i < len ; ++i) + ret[i] = this->_scripts[i]; + return ret._retn(); +} + +void SALOME_ContainerPerfLog::accept(SALOME_VisitorContainerLog &visitor) +{ + visitor.enterContainerPerfLog( *this ); + for(auto script : _scripts) + { + PortableServer::ServantBase *serv = getPOA()->reference_to_servant( script ); + serv->_remove_ref(); + SALOME_ContainerScriptPerfLog *servC = dynamic_cast(serv); + servC->accept(visitor); + } + visitor.leaveContainerPerfLog( *this ); +} + +char *SALOME_ContainerPerfLog::getLogFile() +{ + return CORBA::string_dup( this->_log_file.c_str() ); +} + +char *SALOME_ContainerPerfLog::getContainerEntryInNS() +{ + return CORBA::string_dup( this->_name_in_ns.c_str() ); +} + +///// + +SALOME_LogManager::SALOME_LogManager(CORBA::ORB_ptr orb, PortableServer::POA_var poa,SALOME_NamingService_Abstract *ns):_poa(poa) +{ + _NS.reset(ns); + PortableServer::ObjectId_var id(_poa->activate_object(this)); + CORBA::Object_var obj(_poa->id_to_reference(id)); + Engines::LogManager_var refPtr(Engines::LogManager::_narrow(obj)); + _NS->Register(refPtr,NAME_IN_NS); + { + AutoGIL gstate; + std::ostringstream myCommand; + myCommand << "mylogmanager = SALOME_LogManager.SALOME_LogManagerHelper()"; + PyRun_SimpleString("import SALOME_LogManager"); + PyRun_SimpleString((char*)myCommand.str().c_str()); + PyObject *mainmod = PyImport_AddModule("__main__"); + PyObject *globals = PyModule_GetDict(mainmod); + _pyLogManager = PyDict_GetItemString(globals, "mylogmanager"); + } +} + +Engines::ContainerPerfLog_ptr SALOME_LogManager::declareContainer(const char *contInNS, const char *logfile) +{ + SALOME_ContainerPerfLog *cont = new SALOME_ContainerPerfLog(this,contInNS,logfile); + PortableServer::ObjectId_var id(_poa->activate_object(cont)); + CORBA::Object_var contPtr(_poa->id_to_reference(id)); + cont->_remove_ref(); + Engines::ContainerPerfLog_var contPtr2 = Engines::ContainerPerfLog::_narrow(contPtr); + _containers.push_back( contPtr2 ); + { + AutoGIL gstate; + PyObject *result = PyObject_CallMethod(_pyLogManager,(char*)"declareContainer","ss",contInNS,logfile,nullptr); + if (PyErr_Occurred()) + { + std::string error("can not declareContainer"); + PyErr_Print(); + THROW_SALOME_CORBA_EXCEPTION(error.c_str(),SALOME::INTERNAL_ERROR); + } + cont->setPyObj( result ); + } + return contPtr2._retn(); +} + +Engines::ListOfContainerPerfLog *SALOME_LogManager::listOfContainerLogs() +{ + Engines::ListOfContainerPerfLog_var ret = new Engines::ListOfContainerPerfLog; + std::size_t len( this->_containers.size() ); + ret->length( len ); + for(std::size_t i = 0 ; i < len ; ++i) + { + ret[i] = this->_containers[i]; + } + return ret._retn(); +} + +void SALOME_LogManager::accept(SALOME_VisitorContainerLog &visitor) +{ + visitor.enterLogManager( *this ); + for(auto container : _containers) + { + PortableServer::ServantBase *serv = getPOA()->reference_to_servant( container ); + serv->_remove_ref(); + SALOME_ContainerPerfLog *servC = dynamic_cast(serv); + servC->accept(visitor); + } + visitor.leaveLogManager( *this ); +} + +/*! + \param [in] unloadMemory - specify if big part of struct data (SALOME_ContainerScriptExecPerfLog) is cleared after retrieving data + */ +SALOME::vectorOfByte *SALOME_LogManager::getAllStruct(bool clearMemory) +{ + std::vector data = this->dumpCppInternalFrmt(clearMemory); + return FromVectCharToCorba(data); +} + +/////////////////////// + + #include + +static void PushIntInVC(std::uint32_t val, std::vector& data) +{ + char *valPtr = reinterpret_cast(&val); + data.insert(data.end(),valPtr,valPtr+sizeof(std::uint32_t)); +} + +template +static void PushStringInVC(const T& str, std::vector& data) +{ + std::uint32_t sz = static_cast( str.size() ); + PushIntInVC(sz,data); + data.insert(data.end(),str.data(),str.data()+sz); +} + +class InternalFormatVisitorDump : public SALOME_VisitorContainerLog +{ +public: + InternalFormatVisitorDump(bool clearMemory, std::vector *data):_clear_memory(clearMemory),_data(data) { } + void enterLogManager(SALOME_LogManager& inst) override; + void leaveLogManager(SALOME_LogManager& inst) override { } + void enterContainerPerfLog(SALOME_ContainerPerfLog& inst) override; + void leaveContainerPerfLog(SALOME_ContainerPerfLog& inst) override { } + void enterContainerScriptPerfLog(SALOME_ContainerScriptPerfLog& inst) override; + void leaveContainerScriptPerfLog(SALOME_ContainerScriptPerfLog& inst) override { } + void visitContainerScriptExecPerfLog(SALOME_ContainerScriptExecPerfLog& inst) override; +private: + bool _clear_memory = false; + std::vector *_data = nullptr; +}; + +void InternalFormatVisitorDump::visitContainerScriptExecPerfLog(SALOME_ContainerScriptExecPerfLog& inst) +{ + PushStringInVC>(inst.data(),*_data); + if( _clear_memory ) + inst.clear(); +} + +void InternalFormatVisitorDump::enterContainerScriptPerfLog(SALOME_ContainerScriptPerfLog& inst) +{ + PushStringInVC(inst.name(),*_data); + PushStringInVC(inst.code(),*_data); + PushIntInVC((std::uint32_t)inst.getNumberOfSessions(),*_data); +} + +void InternalFormatVisitorDump::enterContainerPerfLog(SALOME_ContainerPerfLog& inst) +{ + PushStringInVC(inst.nameInNS(),*_data); + PushStringInVC(inst.logFile(),*_data); + PushIntInVC((std::uint32_t)inst.getNumberOfScripts(),*_data); +} + +void InternalFormatVisitorDump::enterLogManager(SALOME_LogManager& inst) +{ + PushIntInVC((std::uint32_t)inst.getNumberOfContainers(),*_data); +} + +std::vector SALOME_LogManager::dumpCppInternalFrmt(bool clearMemory) +{ + std::vector ret; + InternalFormatVisitorDump visitor(clearMemory,&ret); + this->accept( visitor ); + return ret; +} diff --git a/src/Launcher/SALOME_LogManager.hxx b/src/Launcher/SALOME_LogManager.hxx new file mode 100644 index 000000000..755e25268 --- /dev/null +++ b/src/Launcher/SALOME_LogManager.hxx @@ -0,0 +1,154 @@ +// Copyright (C) 2024 CEA, EDF +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +#pragma once + +#include "SALOME_Launcher_defs.hxx" + +#include "Python.h" + +#include "PythonCppUtils.hxx" +#include "Utils_CorbaException.hxx" +#include "SALOMEconfig.h" + +#include CORBA_SERVER_HEADER(SALOME_LogManager) + +#include +#include +#include + +class SALOME_NamingService_Abstract; +class SALOME_LogManager; +class SALOME_ContainerPerfLog; +class SALOME_ContainerScriptPerfLog; +class SALOME_ContainerScriptExecPerfLog; + +class SALOMELAUNCHER_EXPORT SALOME_VisitorContainerLog +{ +public: + virtual void enterLogManager(SALOME_LogManager& inst) = 0; + virtual void leaveLogManager(SALOME_LogManager& inst) = 0; + virtual void enterContainerPerfLog(SALOME_ContainerPerfLog& inst) = 0; + virtual void leaveContainerPerfLog(SALOME_ContainerPerfLog& inst) = 0; + virtual void enterContainerScriptPerfLog(SALOME_ContainerScriptPerfLog& inst) = 0; + virtual void leaveContainerScriptPerfLog(SALOME_ContainerScriptPerfLog& inst) = 0; + virtual void visitContainerScriptExecPerfLog(SALOME_ContainerScriptExecPerfLog& inst) = 0; +}; + +class SALOMELAUNCHER_EXPORT SALOME_ContainerScriptExecPerfLog : public POA_Engines::ContainerScriptExecPerfLog +{ +public: + SALOME_ContainerScriptExecPerfLog(SALOME_ContainerScriptPerfLog *father):_father(father) { } + SALOME_ContainerScriptPerfLog *father() const { return _father; } + void setPyObj(PyObject *obj) { _pyExecutionLog.set(obj); } + PyObject *pyObj() { return _pyExecutionLog.get(); } + PortableServer::POA_var getPOA(); + void assign(const SALOME::vectorOfByte& value) override; + SALOME::vectorOfByte *getObj() override; + const std::vector& data() const { return _data; } + void setData(std::vector&& data) { _data = std::move(data); } +public: + void accept(SALOME_VisitorContainerLog &visitor); +public: + void start(); + AutoPyRef end(); + void clear() { _data.clear(); } +private: + AutoPyRef _pyExecutionLog; + SALOME_ContainerScriptPerfLog *_father = nullptr; + std::vector _data; +}; + +class SALOMELAUNCHER_EXPORT SALOME_ContainerScriptPerfLog : public POA_Engines::ContainerScriptPerfLog +{ +public: + SALOME_ContainerScriptPerfLog(SALOME_ContainerPerfLog *father, const std::string& name, const std::string& code):_father(father),_name(name),_code(code) { } + SALOME_ContainerPerfLog *father() const { return _father; } + void setPyObj(PyObject *obj) { _pyScriptLog.set(obj); } + PyObject *pyObj() { return _pyScriptLog.get(); } + PortableServer::POA_var getPOA(); + Engines::ContainerScriptExecPerfLog_ptr addExecutionSession() override; + Engines::ListOfContainerScriptExecPerfLog *listOfExecs() override; + char *getCode() override; + char *getName() override; + const std::string& name() const { return _name; } + const std::string& code() const { return _code; } + void setName(const std::string& name) { _name = name; } + void setCode(const std::string& code) { _code = code; } + std::size_t getNumberOfSessions() const { return _sessions.size(); } +public: + void accept(SALOME_VisitorContainerLog &visitor); +private: + AutoPyRef _pyScriptLog; + SALOME_ContainerPerfLog *_father = nullptr; + std::string _name; + std::string _code; + std::vector< Engines::ContainerScriptExecPerfLog_var > _sessions; +}; + +class SALOMELAUNCHER_EXPORT SALOME_ContainerPerfLog : public POA_Engines::ContainerPerfLog +{ +public: + SALOME_ContainerPerfLog(SALOME_LogManager *father, const std::string& nameInNS, const std::string& logFile):_father(father),_name_in_ns(nameInNS),_log_file(logFile) { } + SALOME_LogManager *father() const { return _father; } + void setPyObj(PyObject *obj) { _pyContLog.set(obj); } + PyObject *pyObj() { return _pyContLog.get(); } + PortableServer::POA_var getPOA(); + char *getLogFile() override; + char *getContainerEntryInNS() override; + Engines::ContainerScriptPerfLog_ptr addScript(const char *name, const char *code) override; + Engines::ListOfContainerScriptPerfLog *listOfScripts() override; + const std::string& nameInNS() const { return _name_in_ns; } + const std::string& logFile() const { return _log_file; } + void setNameInNS(const std::string& name) { _name_in_ns = name; } + void setLogFile(const std::string& logFile) { _log_file = logFile; } + std::size_t getNumberOfScripts() const { return _scripts.size(); } +public: + void accept(SALOME_VisitorContainerLog &visitor); +private: + AutoPyRef _pyContLog; + SALOME_LogManager *_father = nullptr; + std::string _name_in_ns; + std::string _log_file; + std::vector< Engines::ContainerScriptPerfLog_var > _scripts; +}; + +class SALOMELAUNCHER_EXPORT SALOME_LogManager : public POA_Engines::LogManager +{ + public: + SALOME_LogManager(CORBA::ORB_ptr orb, PortableServer::POA_var poa, SALOME_NamingService_Abstract *ns = nullptr); + PortableServer::POA_var getPOA() { return _poa; } + PyObject *pyHelper() const { return _pyLogManager; } + virtual ~SALOME_LogManager() = default; + Engines::ContainerPerfLog_ptr declareContainer(const char *contInNS, const char *logfile) override; + Engines::ListOfContainerPerfLog *listOfContainerLogs() override; + SALOME::vectorOfByte *getAllStruct(bool clearMemory) override; + std::size_t getNumberOfContainers() const { return _containers.size(); } + public: + void accept(SALOME_VisitorContainerLog &visitor); + private: + std::vector dumpCppInternalFrmt(bool clearMemory); + private: + PyObject *_pyLogManager = nullptr; + std::unique_ptr _NS; + PortableServer::POA_var _poa; + std::vector _containers; + public: + static const char NAME_IN_NS[]; +}; diff --git a/src/Launcher/SALOME_LogManager.py b/src/Launcher/SALOME_LogManager.py new file mode 100644 index 000000000..6e30a380d --- /dev/null +++ b/src/Launcher/SALOME_LogManager.py @@ -0,0 +1,79 @@ +# -*- coding: iso-8859-1 -*- +# Copyright (C) 2024 CEA, EDF +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +import os +import sys +import pickle +from SALOME_ContainerHelper import ScriptExecInfo + +class SALOME_ContainerScriptExecPerfLog: + def __init__(self, father): + self._father = father + self._start_pos = None + self._stop_pos = None + + @property + def father(self): + return self._father + + def end(self,s): + obj = pickle.loads(s) + self._stop_pos = os.path.getsize( self.father.father.logfilename ) + setattr(obj,"tracePosStop",self._stop_pos) + setattr(obj,"tracePosStart",self._start_pos) + return pickle.dumps(obj) + + def start(self): + self._start_pos = os.path.getsize( self.father.father.logfilename ) + +class SALOME_ContainerScriptPerfLog: + def __init__(self, father): + self._father = father + + @property + def father(self): + return self._father + + def addExecution(self): + return SALOME_ContainerScriptExecPerfLog(self) + +class SALOME_ContainerPerfLog: + def __init__(self,contInNS,logfile): + self._container_in_ns = contInNS + self._log_filename = logfile + + @property + def logfilename(self): + return self._log_filename + + @property + def father(self): + return self._father + + def addScript(self): + return SALOME_ContainerScriptPerfLog(self) + +class SALOME_LogManagerHelper: + def __init__(self): + pass + + def declareContainer(self, contInNS,logfile): + inst = SALOME_ContainerPerfLog(contInNS,logfile) + return inst diff --git a/src/Launcher/Test/CMakeLists.txt b/src/Launcher/Test/CMakeLists.txt index f349063ea..41d0759fc 100644 --- a/src/Launcher/Test/CMakeLists.txt +++ b/src/Launcher/Test/CMakeLists.txt @@ -20,15 +20,9 @@ # --- rules --- IF(NOT WIN32) -# ** Now in CTestTestfileInstall.cmake ** -# ** In this file only remain unit tests (no SALOME session is needed) ** -# ADD_TEST(NAME SalomeLauncher -# COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/../../UnitTests/prepare_test.py -# ${CMAKE_CURRENT_SOURCE_DIR}/test_launcher.py -# -d KERNEL_ROOT_DIR=${CMAKE_INSTALL_PREFIX} -# ) + INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/test_launcher.py ${CMAKE_CURRENT_SOURCE_DIR}/TestSSLAttached.py - DESTINATION ${KERNEL_TEST_DIR}/Launcher) + ${CMAKE_CURRENT_SOURCE_DIR}/testPerfLogManager1.py DESTINATION ${KERNEL_TEST_DIR}/Launcher) INSTALL(FILES CTestTestfileInstall.cmake DESTINATION ${KERNEL_TEST_DIR}/Launcher diff --git a/src/Launcher/Test/CTestTestfileInstall.cmake b/src/Launcher/Test/CTestTestfileInstall.cmake index 040518a5c..07808c2ff 100644 --- a/src/Launcher/Test/CTestTestfileInstall.cmake +++ b/src/Launcher/Test/CTestTestfileInstall.cmake @@ -20,21 +20,19 @@ IF(NOT WIN32) SET(TEST_NAME ${COMPONENT_NAME}_Launcher) ADD_TEST(${TEST_NAME} ${PYTHON_TEST_DRIVER} 2000 test_launcher.py) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}" - # TIMEOUT 500 - ) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}") SET(TEST_NAME ${COMPONENT_NAME}_AttachedLauncher) ADD_TEST(${TEST_NAME} ${PYTHON_TEST_DRIVER} 2000 TestSSLAttached.py) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}" - # TIMEOUT 500 - ) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}") SET(TEST_NAME ${COMPONENT_NAME}_StressLauncher) ADD_TEST(${TEST_NAME} ${PYTHON_TEST_DRIVER} 2000 ./test_stress.sh) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}" - # TIMEOUT 500 - ) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}") + + SET(TEST_NAME ${COMPONENT_NAME}_PerfLogManager1) + ADD_TEST(${TEST_NAME} ${PYTHON_TEST_DRIVER} 2000 testPerfLogManager1.py) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}") # /!\ DO NOT SET TIMEOUT PROPERTY IF USING ${SALOME_TEST_DRIVER} # BUT PASS TIMEOUT VALUE TO THE DRIVER diff --git a/src/Launcher/Test/testPerfLogManager1.py b/src/Launcher/Test/testPerfLogManager1.py new file mode 100644 index 000000000..0279df2ed --- /dev/null +++ b/src/Launcher/Test/testPerfLogManager1.py @@ -0,0 +1,167 @@ +# -*- coding: iso-8859-1 -*- +# Copyright (C) 2024 CEA/DEN, EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +import unittest +import os +import salome +import Engines +import pylauncher +import SALOME_PyNode + +import pickle +import tempfile +import logging + +def flush(): + import sys + sys.stdout.flush() + sys.stderr.flush() + +def unProxyfy( objs ): + """ + objs is a list of SenderByte + """ + ret = [] + fileNamesProxyOut = [] + for ret2 in objs: + ret3 = pickle.loads( SALOME_PyNode.SeqByteReceiver(ret2).data() ) + ret4 = ret3 + fileNameProxyOut = None + if SALOME_PyNode.GetBigObjectOnDiskThreshold() != -1: + if isinstance( ret3, SALOME_PyNode.BigObjectOnDiskBase ): + ret4 = ret3.get() + else: + raise RuntimeError("Oooops") + ret3.unlinkOnDestructor() + fileNameProxyOut = ret3.getFileName() + logging.debug("Prxy file : {}".format( fileNameProxyOut )) + ret.append( ret3 ) + fileNamesProxyOut.append( fileNameProxyOut ) + return ret, fileNamesProxyOut + +class testPerfLogManager1(unittest.TestCase): + def testPerfLogManager0(self): + """ + [EDF29150] : This test checks measure performance methods + """ + hostname = "localhost" + cp = pylauncher.GetRequestForGiveContainer(hostname,"container_test") + #PROXY_THRES = "-1" + PROXY_THRES = "1" + with tempfile.TemporaryDirectory() as tmpdirname: + val_for_big_obj = str( tmpdirname ) + # Override environement for all containers launched + salome.cm.SetOverrideEnvForContainersSimple(env = [("SALOME_FILE_BIG_OBJ_DIR",val_for_big_obj),("SALOME_BIG_OBJ_ON_DISK_THRES",PROXY_THRES)]) + salome.cm.SetDeltaTimeBetweenCPUMemMeasureInMilliSecond( 250 ) + cont = salome.cm.GiveContainer(cp) + logging.debug("{} {}".format(40*"*",cont.getPID())) + script_st = """ +import logging +import sys +import KernelBasis +cst = KernelBasis.GetTimeAdjustmentCst() +logging.debug("constant = {}".format(cst)) +nbcore = 3 +print("coucou {} {}".format(len(zeinput0),len(zeinput1))) +logging.debug("debug or not debug") +ob = [ [ bytes(3000000) ] ] +pihm, ts = KernelBasis.HeatMarcel(1 * nbcore * cst,nbcore) +print("Time ellapse spent : {} s".format(ts)) +ob2 = [ [ bytes(100000) ] ] +pihm, ts = KernelBasis.HeatMarcel(1 * nbcore * cst,nbcore) +print("Time ellapse spent : {} s".format(ts)) +sys.stderr.write("fake error message\\n") +""" + poa = salome.orb.resolve_initial_references("RootPOA") + zeinput0 = [ bytes(100000000) ] + if SALOME_PyNode.GetBigObjectOnDiskThreshold() != -1: + zeinput0 = SALOME_PyNode.ProxyfyPickeled( zeinput0 ) + zeinput0.unlinkOnDestructor() + obj = SALOME_PyNode.SenderByte_i(poa,pickle.dumps( (["zeinput0"],{"zeinput0": [zeinput0], "zeinput1": [ [zeinput0], [zeinput0] ] }) )) + id_o = poa.activate_object(obj) + refPtr = poa.id_to_reference(id_o) + pyscript2 = cont.createPyScriptNode("testScript2",script_st) + pyscript2.executeFirst(refPtr) + ret2 = pyscript2.executeSecond(["ob","ob2"])# generate a DeprecationWarning: PY_SSIZE_T_CLEAN will be required for '#' formats on debian11 ? + ret3, fileNamesProxyOut = unProxyfy( ret2 ) + logging.getLogger().debug("test logging 1") + logging.debug("test logging 2") + logging.debug( salome.orb.object_to_string( salome.logm ) ) + a = salome.logm.NaiveFetch() + logging.debug(a) + logging.debug(a[0][1][0]) + logging.debug( a[0][1][0].get()._input_hdd_mem._data[0]._data[0]._hdd_mem ) # important + logging.debug( a[0][1][0].get()._input_hdd_mem._data[1]._data[0]._data[0]._hdd_mem ) # important + fileNameProxyIn = a[0][1][0].get()._input_hdd_mem._data[0]._data[0]._file_name + logging.debug( fileNameProxyIn ) + del zeinput0 + del ret3 + import gc ; gc.collect() + if fileNameProxyIn is not None: + if os.path.exists(fileNameProxyIn): + raise RuntimeError("Oooops 2") + for fileNameProxyOut in fileNamesProxyOut: + if fileNameProxyOut is not None: + if os.path.exists(fileNameProxyOut): + raise RuntimeError("Oooops 3") + + # execution #2 inside last + script_st2 = """ +import logging +b = 7+a +logging.debug("Execution 2") +import time +time.sleep(1) +""" + obj2 = SALOME_PyNode.SenderByte_i(poa,pickle.dumps((["a"],{"a":3}))) + id2_o = poa.activate_object(obj2) + refPtr2 = poa.id_to_reference(id2_o) + pyscript2.assignNewCompiledCode(script_st2) + pyscript2.executeFirst(refPtr2) + ret2_0 = pyscript2.executeSecond(["b"]) + ret2_1, fileNamesProxyOut2 = unProxyfy( ret2_0 ) + logging.debug( fileNamesProxyOut2 ) + a = salome.logm.NaiveFetch() + del ret2_1 + import gc ; gc.collect() + for fileNameProxyOut in fileNamesProxyOut2: + if fileNameProxyOut is not None: + if os.path.exists(fileNameProxyOut): + raise RuntimeError("Oooops 3") + # + fname = os.path.join(str( tmpdirname ),"perf.log") + salome.logm.DumpInFile( fname ) + logManagerInst0 = salome.logm.LoadFromFile( fname ) + logging.debug( logManagerInst0[0][1][0].get()._input_hdd_mem._data[1]._data[0]._data[0]._hdd_mem ) # important + logManagerInst = salome.logm.Fetch(True) + logManagerInst2 = salome.logm.Fetch(True) + logging.debug( logManagerInst[0][1][0].get()._input_hdd_mem._data[1]._data[0]._data[0]._hdd_mem ) # important + self.assertTrue( logManagerInst2[0][1][0].get() is None ) + self.assertTrue( logManagerInst[0][1][1].get()._output_hdd_mem._data[0]._file_name == fileNamesProxyOut2[0] ) + logging.debug( logManagerInst[0][1][1].log() ) + cont.Shutdown() + +if __name__ == '__main__': + from salome_utils import positionVerbosityOfLoggerRegardingState,setVerboseLevel,setVerbose + salome.standalone() + salome.salome_init() + setVerboseLevel(logging.DEBUG) + positionVerbosityOfLoggerRegardingState() + unittest.main() diff --git a/src/ModuleCatalog/SALOME_ModuleCatalog_Handler.cxx b/src/ModuleCatalog/SALOME_ModuleCatalog_Handler.cxx index 80fd83365..ee8a3c284 100644 --- a/src/ModuleCatalog/SALOME_ModuleCatalog_Handler.cxx +++ b/src/ModuleCatalog/SALOME_ModuleCatalog_Handler.cxx @@ -46,8 +46,6 @@ SALOME_ModuleCatalog_Handler::SALOME_ModuleCatalog_Handler(ParserPathPrefixes& p _typeMap(typeMap),_typeList(typeList) { - BEGIN_OF("SALOME_ModuleCatalog_Handler"); - // XML Tags initialisation // Used in the function endElement test_path_prefix_name = "path-prefix-name"; @@ -102,8 +100,6 @@ SALOME_ModuleCatalog_Handler::SALOME_ModuleCatalog_Handler(ParserPathPrefixes& p test_component_list = "component-list"; test_component="component"; - - END_OF("SALOME_ModuleCatalog_Handler"); } //---------------------------------------------------------------------- @@ -112,8 +108,6 @@ SALOME_ModuleCatalog_Handler::SALOME_ModuleCatalog_Handler(ParserPathPrefixes& p //---------------------------------------------------------------------- SALOME_ModuleCatalog_Handler::~SALOME_ModuleCatalog_Handler() { - BEGIN_OF("~SALOME_ModuleCatalog_Handler()"); - END_OF("~SALOME_ModuleCatalog_Handler()"); } @@ -125,7 +119,6 @@ SALOME_ModuleCatalog_Handler::~SALOME_ModuleCatalog_Handler() void SALOME_ModuleCatalog_Handler::ProcessXmlDocument(xmlDocPtr theDoc) { - MESSAGE("Begin parse document"); // Empty the private elements _pathList.resize(0); _pathPrefix.listOfComputer.resize(0); @@ -225,15 +218,14 @@ void SALOME_ModuleCatalog_Handler::ProcessXmlDocument(xmlDocPtr theDoc) { if ( _typeMap.find(aType.name) == _typeMap.end() ) { - MESSAGE("Registered basic type: " << aType.name << " " << aType.kind ); _typeMap[aType.name]=aType; _typeList.push_back(aType); } else - MESSAGE( "Warning: this type (" << aType.name << "," << aType.kind << ") already exists, it will be ignored." ); + INFO_MESSAGE( "Warning: this type (" << aType.name << "," << aType.kind << ") already exists, it will be ignored." ); } else - MESSAGE( "Warning: this type (" << aType.name << "," << aType.kind << ") has incorrect kind, it will be ignored." ); + INFO_MESSAGE( "Warning: this type (" << aType.name << "," << aType.kind << ") has incorrect kind, it will be ignored." ); } else if ( !xmlStrcmp(aTypeNode->name, (const xmlChar*)"sequence" )) { @@ -255,16 +247,15 @@ void SALOME_ModuleCatalog_Handler::ProcessXmlDocument(xmlDocPtr theDoc) { if ( _typeMap.find(aType.name) == _typeMap.end() ) { - MESSAGE("Registered sequence type: " << aType.name << " " << aType.content ); _typeMap[aType.name]=aType; _typeList.push_back(aType); } else - MESSAGE( "Warning: this type (" << aType.name << "," << aType.kind << ") already exists, it will be ignored." ); + INFO_MESSAGE( "Warning: this type (" << aType.name << "," << aType.kind << ") already exists, it will be ignored." ); } else { - MESSAGE( "Warning: this sequence type (" << aType.name << "," << aType.content << ") has unknown content type, it will be ignored." ); + INFO_MESSAGE( "Warning: this sequence type (" << aType.name << "," << aType.content << ") has unknown content type, it will be ignored." ); } } else if ( !xmlStrcmp(aTypeNode->name, (const xmlChar*)"objref" )) @@ -375,7 +366,6 @@ void SALOME_ModuleCatalog_Handler::ProcessXmlDocument(xmlDocPtr theDoc) { if ( _typeMap.find(aType.name) == _typeMap.end() ) { - MESSAGE("Registered struct type: " << aType.name << " " << aType.id ); _typeMap[aType.name]=aType; _typeList.push_back(aType); } diff --git a/src/ModuleCatalog/SALOME_ModuleCatalog_impl.cxx b/src/ModuleCatalog/SALOME_ModuleCatalog_impl.cxx index eeccfc096..fa6847222 100644 --- a/src/ModuleCatalog/SALOME_ModuleCatalog_impl.cxx +++ b/src/ModuleCatalog/SALOME_ModuleCatalog_impl.cxx @@ -208,7 +208,7 @@ public: SALOME_ModuleCatalogImpl::SALOME_ModuleCatalogImpl(int argc, char** argv, CORBA::ORB_ptr orb) : _orb(orb) { myPrivate = new Private; - MESSAGE("Catalog creation"); + DEBUG_MESSAGE("Catalog creation"); /* Init libxml */ xmlInitParser(); @@ -291,11 +291,11 @@ SALOME_ModuleCatalogImpl::SALOME_ModuleCatalogImpl(int argc, char** argv, CORBA: // Verification of _general_path_list content if (!myPrivate->_verify_path_prefix(myPrivate->_general_path_list)) { - MESSAGE( "Error while parsing the general path list, " + WARNING_MESSAGE( "Error while parsing the general path list, " "different paths are associated to the same computer," "the first one will be chosen"); } else { - MESSAGE("General path list OK"); + DEBUG_MESSAGE("General path list OK"); } if (myPrivate->_personal_path != NULL) { @@ -316,7 +316,7 @@ SALOME_ModuleCatalogImpl::SALOME_ModuleCatalogImpl(int argc, char** argv, CORBA: MESSAGE("Personal path list OK"); } }else - MESSAGE("No personal catalog indicated or error while " + INFO_MESSAGE("No personal catalog indicated or error while " "opening the personal catalog"); } } @@ -845,8 +845,7 @@ SALOME_ModuleCatalogImpl::Private::_parse_xml_file(const char* file, ParserTypes& typeMap, TypeList& typeList) { - BEGIN_OF("_parse_xml_file"); - SCRUTE(file); + DEBUG_MESSAGE("parsing of " << file); //Local path and module list for the file to parse ParserPathPrefixes _pathList; diff --git a/src/ResourcesManager/ResourcesManager.cxx b/src/ResourcesManager/ResourcesManager.cxx index fa528d523..22156a57b 100644 --- a/src/ResourcesManager/ResourcesManager.cxx +++ b/src/ResourcesManager/ResourcesManager.cxx @@ -25,6 +25,7 @@ #include "SALOME_ResourcesCatalog_Handler.hxx" #include #include +#include "utilities.h" #include #include @@ -118,18 +119,19 @@ ResourcesManager_cpp::ResourcesManager_cpp() std::ifstream ifile(user_file.c_str(), std::ifstream::in ); if (ifile) { // The file exists, and is open for input + DEBUG_MESSAGE("USER_CATALOG_RESOURCES_FILE positioned -> add it into resourcefiles list"); _path_resources.push_back(user_file); } else { default_catalog_resource = false; - RES_INFOS("Warning: USER_CATALOG_RESOURCES_FILE is set and file cannot be found.") - RES_INFOS("Warning: That's why we try to create a new one.") + WARNING_MESSAGE("Warning: USER_CATALOG_RESOURCES_FILE is set and file cannot be found.") + WARNING_MESSAGE("Warning: That's why we try to create a new one.") std::ofstream user_catalog_file; user_catalog_file.open(user_file.c_str()); if (user_catalog_file.fail()) { - RES_INFOS("Error: cannot write in the user catalog resources files"); - RES_INFOS("Error: using default CatalogResources.xml file"); + WARNING_MESSAGE("Error: cannot write in the user catalog resources files"); + WARNING_MESSAGE("Error: using default CatalogResources.xml file"); default_catalog_resource = true; } else @@ -155,6 +157,7 @@ ResourcesManager_cpp::ResourcesManager_cpp() std::ifstream ifile(default_file.c_str(), std::ifstream::in ); if (ifile) { // The file exists, and is open for input + DEBUG_MESSAGE("${APPLI}/CatalogResources.xml exists -> add it into resourcefiles list"); _path_resources.push_back(default_file); default_catalog_resource=false; } @@ -167,6 +170,7 @@ ResourcesManager_cpp::ResourcesManager_cpp() throw ResourcesException("you must define KERNEL_ROOT_DIR environment variable!! -> cannot load a CatalogResources.xml"); default_file = getenv("KERNEL_ROOT_DIR"); default_file += "/share/salome/resources/kernel/CatalogResources.xml"; + DEBUG_MESSAGE("${KERNEL_ROOT_DIR}/share/salome/resources/kernel/CatalogResources.xml -> add it into resourcefiles list"); _path_resources.push_back(default_file); } @@ -445,12 +449,13 @@ const MapOfParserResourcesType& ResourcesManager_cpp::ParseXmlFiles() int result = stat((*_path_resources_it).c_str(), &statinfo); if (result < 0) { - RES_MESSAGE("Resource file " << *_path_resources_it << " does not exist"); + WARNING_MESSAGE("Resource file " << *_path_resources_it << " does not exist -> no parsing"); return _resourcesList; } if(_lasttime == 0 || statinfo.st_mtime > _lasttime) { + DEBUG_MESSAGE("Resource file " << *_path_resources_it << " has been detected to be present and newer than Resources in memory"); to_parse = true; _lasttime = statinfo.st_mtime; } @@ -458,6 +463,7 @@ const MapOfParserResourcesType& ResourcesManager_cpp::ParseXmlFiles() if (to_parse) { + DEBUG_MESSAGE("After analyze of resoure files time meta data, a load of resources from scratch from files is necessary."); _resourcesList.clear(); AddDefaultResourceInCatalog(); // On parse tous les fichiers @@ -465,15 +471,15 @@ const MapOfParserResourcesType& ResourcesManager_cpp::ParseXmlFiles() { MapOfParserResourcesType _resourcesList_tmp; MapOfParserResourcesType _resourcesBatchList_tmp; - SALOME_ResourcesCatalog_Handler *handler( new SALOME_ResourcesCatalog_Handler(_resourcesList_tmp) ); + std::unique_ptr handler( new SALOME_ResourcesCatalog_Handler(_resourcesList_tmp) ); const char *aFilePath( (*_path_resources_it).c_str() ); FILE* aFile = fopen(aFilePath, "r"); - if (aFile != NULL) { xmlDocPtr aDoc = xmlReadFile(aFilePath, NULL, 0); if (aDoc != NULL) { + DEBUG_MESSAGE("XML parsing of Resource file \"" << aFilePath << "\""); handler->ProcessXmlDocument(aDoc); // adding new resources to the file @@ -482,39 +488,38 @@ const MapOfParserResourcesType& ResourcesManager_cpp::ParseXmlFiles() MapOfParserResourcesType_it j = _resourcesList.find(i->first); if (i->second.HostName == DEFAULT_RESOURCE_NAME || i->second.HostName == Kernel_Utils::GetHostname()) { + DEBUG_MESSAGE("Resource \"" << i->first << "\" in file \"" << aFilePath << "\" is detected as localhost"); MapOfParserResourcesType_it it0(_resourcesList.find(DEFAULT_RESOURCE_NAME)); if(it0!=_resourcesList.end()) { + DEBUG_MESSAGE("Resource \"" << i->first << "\" in file \"" << aFilePath << "\" detected as localhost is already in memory -> update resource with content in file ( for attributes : nbOfNodes, nbOfProcPerNode, CPUFreqMHz and memInMB)"); ParserResourcesType& localhostElt((*it0).second); localhostElt.DataForSort._nbOfNodes=(*i).second.DataForSort._nbOfNodes; localhostElt.DataForSort._nbOfProcPerNode=(*i).second.DataForSort._nbOfProcPerNode; localhostElt.DataForSort._CPUFreqMHz=(*i).second.DataForSort._CPUFreqMHz; localhostElt.DataForSort._memInMB=(*i).second.DataForSort._memInMB; } - RES_MESSAGE("Resource " << i->first << " is not added because it is the same " - "machine as default local resource \"" << DEFAULT_RESOURCE_NAME << "\""); + DEBUG_MESSAGE("Resource \"" << i->first << "\" is not added because it is the same machine as default local resource \"" << DEFAULT_RESOURCE_NAME << "\""); } else if (j != _resourcesList.end()) { - cerr << "ParseXmlFiles Warning, two resources with the same name were found, " - "taking the first declaration : " << i->first << endl; + WARNING_MESSAGE("ParseXmlFiles Warning, two resources with the same name were found, taking the first declaration : " << i->first ); } else { + DEBUG_MESSAGE("Resource \"" << i->first << "\" added"); _resourcesList[i->first] = i->second; } } } else - std::cerr << "ResourcesManager_cpp: could not parse file " << aFilePath << std::endl; + ERROR_MESSAGE( "ResourcesManager_cpp: could not parse file " << aFilePath ); // Free the document xmlFreeDoc(aDoc); fclose(aFile); } else - std::cerr << "ResourcesManager_cpp: file " << aFilePath << " is not readable." << std::endl; - - delete handler; + ERROR_MESSAGE( "ResourcesManager_cpp: file " << aFilePath << " is not readable." ); } } return _resourcesList; @@ -666,4 +671,5 @@ void ResourcesManager_cpp::AddDefaultResourceInCatalog() resource.can_launch_batch_jobs = true; resource.can_run_containers = true; _resourcesList[resource.Name] = resource; + DEBUG_MESSAGE("Put Ressource \"" << resource.Name << "\" in dictionary of resources. This resource will be present even if resource files define it later."); } diff --git a/src/ResourcesManager/SALOME_ResourcesCatalog_Handler.cxx b/src/ResourcesManager/SALOME_ResourcesCatalog_Handler.cxx index 1173574d9..7e898d2bc 100644 --- a/src/ResourcesManager/SALOME_ResourcesCatalog_Handler.cxx +++ b/src/ResourcesManager/SALOME_ResourcesCatalog_Handler.cxx @@ -156,18 +156,12 @@ void SALOME_ResourcesCatalog_Handler::ProcessXmlDocument(xmlDocPtr theDoc) } aCurNode = aCurNode->next; } - -#ifdef _DEBUG_ - for (std::map::const_iterator iter = _resources_list.begin(); - iter != _resources_list.end(); - iter++) + DEBUG_MESSAGE( "************ Resources in memory ************"); + for (std::map::const_iterator iter = _resources_list.begin(); iter != _resources_list.end(); iter++) { - MESSAGE( "************************************************" ); - MESSAGE( "Resource " << (*iter).first << " found:" ); - MESSAGE( (*iter).second ); - MESSAGE( "************************************************" ); + DEBUG_MESSAGE( "Resource \"" << (*iter).first << "\" -> " << (*iter).second.dump(' ')); } -#endif + DEBUG_MESSAGE( "************ Resources in memory ************"); } bool @@ -465,7 +459,7 @@ SALOME_ResourcesCatalog_Handler::ProcessMachine(xmlNodePtr machine_descr, Parser } catch (const ResourcesException &) { - MESSAGE( "Warning, invalid type \"" << (const char*)type << "\" for resource \"" << + INFO_MESSAGE( "Warning, invalid type \"" << (const char*)type << "\" for resource \"" << resource.Name << "\", using default value \"" << resource.getResourceTypeStr() << "\"" ) ; } @@ -473,7 +467,7 @@ SALOME_ResourcesCatalog_Handler::ProcessMachine(xmlNodePtr machine_descr, Parser } else { - MESSAGE( "Warning, no type found for resource \"" << resource.Name << + INFO_MESSAGE( "Warning, no type found for resource \"" << resource.Name << "\", using default value \"" << resource.getResourceTypeStr() << "\""); } diff --git a/src/ResourcesManager/SALOME_ResourcesCatalog_Parser.cxx b/src/ResourcesManager/SALOME_ResourcesCatalog_Parser.cxx index 037297f17..ec821320c 100644 --- a/src/ResourcesManager/SALOME_ResourcesCatalog_Parser.cxx +++ b/src/ResourcesManager/SALOME_ResourcesCatalog_Parser.cxx @@ -195,38 +195,45 @@ AccessProtocolType ParserResourcesType::stringToProtocol(const std::string & pro throw ResourcesException((string("Unknown protocol ") + protocolStr).c_str()); } -ostream & operator<<(ostream &os, const ParserResourcesType &prt) +std::string ParserResourcesType::dump(char sep) const { - os << "Name: " << prt.Name << endl << - "HostName: " << prt.HostName << endl << - "Type: " << prt.getResourceTypeStr() << endl << - "NbOfNodes: " << prt.DataForSort._nbOfNodes << endl << - "NbOfProcPerNode: " << prt.DataForSort._nbOfProcPerNode << endl << - "CPUFreqMHz: " << prt.DataForSort._CPUFreqMHz << endl << - "MemInMB: " << prt.DataForSort._memInMB << endl << - "Protocol: " << prt.getAccessProtocolTypeStr() << endl << - "ClusterInternalProtocol: " << prt.getClusterInternalProtocolStr() << endl << - "Batch: " << prt.getBatchTypeStr() << endl << - "mpi: " << prt.getMpiImplTypeStr() << endl << - "UserName: " << prt.UserName << endl << - "AppliPath: " << prt.AppliPath << endl << - "OS: " << prt.OS << endl << - "batchQueue: " << prt.batchQueue << endl << - "userCommands: " << prt.userCommands << endl << - "use: " << prt.use << endl << - "NbOfProc: " << prt.nbOfProc << endl << - "Can Launch Batch Jobs: " << prt.can_launch_batch_jobs << endl << - "Can Run Containers: " << prt.can_run_containers << endl << - "Working Directory: " << prt.working_directory << endl; - - for(unsigned int i=0 ; iName << sep << + "HostName: " << this->HostName << sep << + "Type: " << this->getResourceTypeStr() << sep << + "NbOfNodes: " << this->DataForSort._nbOfNodes << sep << + "NbOfProcPerNode: " << this->DataForSort._nbOfProcPerNode << sep << + "CPUFreqMHz: " << this->DataForSort._CPUFreqMHz << sep << + "MemInMB: " << this->DataForSort._memInMB << sep << + "Protocol: " << this->getAccessProtocolTypeStr() << sep << + "ClusterInternalProtocol: " << this->getClusterInternalProtocolStr() << sep << + "Batch: " << this->getBatchTypeStr() << sep << + "mpi: " << this->getMpiImplTypeStr() << sep << + "UserName: " << this->UserName << sep << + "AppliPath: " << this->AppliPath << sep << + "OS: " << this->OS << sep << + "batchQueue: " << this->batchQueue << sep << + "userCommands: " << this->userCommands << sep << + "use: " << this->use << sep << + "NbOfProc: " << this->nbOfProc << sep << + "Can Launch Batch Jobs: " << this->can_launch_batch_jobs << sep << + "Can Run Containers: " << this->can_run_containers << sep << + "Working Directory: " << this->working_directory << sep; + + for(unsigned int i=0 ; iComponentsList.size() ; i++) + oss << "Component " << i+1 << " called: " << this->ComponentsList[i] << sep; list::const_iterator it; - for(it = prt.ClusterMembersList.begin() ; it != prt.ClusterMembersList.end() ; it++) + for(it = this->ClusterMembersList.cbegin() ; it != this->ClusterMembersList.cend() ; it++) { - os << "Cluster member called: " << (*it).HostName << endl; + oss << "Cluster member called: " << (*it).HostName << sep; } + return oss.str(); +} + +ostream & operator<<(ostream &os, const ParserResourcesType &prt) +{ + os << prt.dump('\n'); return os; } diff --git a/src/ResourcesManager/SALOME_ResourcesCatalog_Parser.hxx b/src/ResourcesManager/SALOME_ResourcesCatalog_Parser.hxx index 1c3780758..2001d353c 100644 --- a/src/ResourcesManager/SALOME_ResourcesCatalog_Parser.hxx +++ b/src/ResourcesManager/SALOME_ResourcesCatalog_Parser.hxx @@ -107,6 +107,7 @@ public: void setClusterInternalProtocolStr(const std::string & internalProtocolTypeStr); void setCanLaunchBatchJobsStr(const std::string & canLaunchBatchJobsStr); void setCanRunContainersStr(const std::string & canRunContainersStr); + std::string dump(char sep) const; ResourceDataToSort DataForSort; std::string Name; diff --git a/src/ResourcesManager/SALOME_ResourcesManager.cxx b/src/ResourcesManager/SALOME_ResourcesManager.cxx index 632a326b6..4fe4b5fd5 100644 --- a/src/ResourcesManager/SALOME_ResourcesManager.cxx +++ b/src/ResourcesManager/SALOME_ResourcesManager.cxx @@ -96,7 +96,6 @@ SALOME_ResourcesManager::SALOME_ResourcesManager(CORBA::ORB_ptr orb, PortableServer::POA_var poa, SALOME_NamingService_Abstract *ns) : _rm(new ResourcesManager_cpp()) { - MESSAGE("SALOME_ResourcesManager constructor"); _NS = ns; _orb = CORBA::ORB::_duplicate(orb) ; // @@ -113,8 +112,6 @@ SALOME_ResourcesManager::SALOME_ResourcesManager(CORBA::ORB_ptr orb, Engines::ResourcesManager_var refContMan = Engines::ResourcesManager::_narrow(obj); if(_NS) _NS->Register(refContMan,_ResourcesManagerNameInNS); - - MESSAGE("SALOME_ResourcesManager constructor end"); } //============================================================================= diff --git a/src/SALOMEDS/SALOMEDS_Study_i.cxx b/src/SALOMEDS/SALOMEDS_Study_i.cxx index b91b25ea3..51d4b7ea6 100644 --- a/src/SALOMEDS/SALOMEDS_Study_i.cxx +++ b/src/SALOMEDS/SALOMEDS_Study_i.cxx @@ -483,7 +483,6 @@ void SALOMEDS_Study_i::Clear() PortableServer::POA_ptr SALOMEDS_Study_i::_default_POA() { PortableServer::POA_ptr poa = GetThePOA(); - MESSAGE("SALOMEDS_Study_i::_default_POA: " << poa); return PortableServer::POA::_duplicate(poa); } diff --git a/src/SALOMELocalTrace/LocalTraceCollector.cxx b/src/SALOMELocalTrace/LocalTraceCollector.cxx index e44c7a806..e348c8349 100644 --- a/src/SALOMELocalTrace/LocalTraceCollector.cxx +++ b/src/SALOMELocalTrace/LocalTraceCollector.cxx @@ -119,12 +119,7 @@ void* LocalTraceCollector::run(void* /*bid*/) if( SALOME::VerbosityActivated() ) { std::cout << std::flush ; -#ifndef WIN32 - std::cerr << "th. " << myTrace.threadId << " " << myTrace.trace; -#else - std::cerr << "th. " << (void*)(&myTrace.threadId) - << " " << myTrace.trace; -#endif + std::cerr << myTrace.trace; std::cerr << std::flush ; } } diff --git a/src/SALOMELocalTrace/utilities.h b/src/SALOMELocalTrace/utilities.h index e08694acc..72414f9d5 100644 --- a/src/SALOMELocalTrace/utilities.h +++ b/src/SALOMELocalTrace/utilities.h @@ -110,8 +110,12 @@ << " at " << __TIME__ << MESS_END }} -#define MESSAGE(msg) { if (SALOME::VerbosityActivated()) {MESS_BEGIN("- Trace ") << msg << MESS_END}} -#define SCRUTE(var) { if (SALOME::VerbosityActivated()) {MESS_BEGIN("- Trace ") << #var << "=" << var <retrieve(myTrace); - //if (!CORBA::is_nil(_orb)) - if (true) - { - if (myTrace.traceType == ABORT_MESS) - { - std::stringstream abortMessage(""); -#ifndef WIN32 - abortMessage << "INTERRUPTION from thread " - << myTrace.threadId << " : " << myTrace.trace; -#else - abortMessage << "INTERRUPTION from thread " - << (void*)&myTrace.threadId - << " : " << myTrace.trace; -#endif - CORBA::String_var LogMsg = - CORBA::string_dup(abortMessage.str().c_str()); - m_pInterfaceLogger->putMessage(LogMsg); - exit(1); - } - else - { - std::stringstream aMessage(""); -#ifndef WIN32 - aMessage << "th. " << myTrace.threadId -#else - aMessage << "th. " << (void*)&myTrace.threadId -#endif - << " " << myTrace.trace; - CORBA::String_var LogMsg = - CORBA::string_dup(aMessage.str().c_str()); - m_pInterfaceLogger->putMessage(LogMsg); - } - } + { + if (myTrace.traceType == ABORT_MESS) + { + std::ostringstream abortMessage; + abortMessage << "INTERRUPTION from thread : " << myTrace.trace; + CORBA::String_var LogMsg = + CORBA::string_dup(abortMessage.str().c_str()); + m_pInterfaceLogger->putMessage(LogMsg); + exit(1); + } + else + { + std::ostringstream aMessage; + aMessage << " " << myTrace.trace; + CORBA::String_var LogMsg = + CORBA::string_dup(aMessage.str().c_str()); + m_pInterfaceLogger->putMessage(LogMsg); + } + } } pthread_exit(NULL); return NULL; -- 2.39.2