NamingService
SALOMELocalTrace
LifeCycleCORBA
+ Container
Logger
SALOMETraceCollector
KernelHelpers
};
typedef sequence<KeyValuePair> FieldsDict;
+ typedef sequence<double> vectorOfDouble;
interface EngineComponent ;
interface fileRef ;
* Previous scripts created on container may have been stored in a map. This method removes them. It then clean all the contexts dict attached to them.
*/
void cleanAllPyScripts();
+
+ //! Return number of CPU cores in the calculation node.
+ long getNumberOfCPUCores();
+
+ //! Return a load of each CPU core.
+ vectorOfDouble loadOfCPUCores() raises(SALOME::SALOME_Exception);
+
+ //! Set custom script to calculate a load of each CPU core.
+ /*!
+ \param script Python script to execute
+ */
+ void setPyScriptForCPULoad(in string script);
+
+ //! Nullify custom script to calculate each CPU core's load.
+ void resetScriptForCPULoad();
+
+ //! Get total physical memory of calculation node, in megabytes.
+ long getTotalPhysicalMemory();
+
+ //! Get used physical memory of calculation node, in megabytes.
+ long getTotalPhysicalMemoryInUse();
+
+ //! Obtain physical memory, used by the current process, in megabytes.
+ long getTotalPhysicalMemoryInUseByMe();
};
/*! \brief Interface of the %component.
${PTHREAD_INCLUDE_DIR}
${HDF5_INCLUDE_DIRS}
${PYTHON_INCLUDE_DIRS}
+ ${Boost_INCLUDE_DIRS}
${PROJECT_BINARY_DIR}/salome_adm
${CMAKE_CURRENT_SOURCE_DIR}/../Basics
${CMAKE_CURRENT_SOURCE_DIR}/../SALOMELocalTrace
SALOME_ContainerPy.py
)
-ADD_DEFINITIONS(${HDF5_DEFINITIONS} ${OMNIORB_DEFINITIONS})
+ADD_DEFINITIONS(${HDF5_DEFINITIONS} ${OMNIORB_DEFINITIONS} ${BOOST_DEFINITIONS})
SET(COMMON_LIBS
Registry
SalomeNotification
SalomeIDLKernel
${OMNIORB_LIBRARIES}
${PYTHON_LIBRARIES}
+ ${Boost_PYTHON_LIBRARY}
)
IF(WITH_MPI_SEQ_CONTAINER)
SALOME_INSTALL_SCRIPTS("${SCRIPTS}" ${SALOME_INSTALL_SCRIPT_PYTHON})
FILE(GLOB COMMON_HEADERS_HXX "${CMAKE_CURRENT_SOURCE_DIR}/*.hxx")
INSTALL(FILES ${COMMON_HEADERS_HXX} DESTINATION ${SALOME_INSTALL_HEADERS})
+
+IF(SALOME_BUILD_TESTS)
+ ADD_SUBDIRECTORY(Test)
+ENDIF(SALOME_BUILD_TESTS)
#include <Python.h>
#include "Container_init_python.hxx"
+#include <boost/python.hpp>
bool _Sleeping = false ;
MESSAGE("Engines_Container_i::ping() pid "<< getpid());
}
+//=============================================================================
+//! Get number of CPU cores in the calculation node
+/*!
+* CORBA method: get number of CPU cores
+*/
+//=============================================================================
+
+CORBA::Long Abstract_Engines_Container_i::getNumberOfCPUCores()
+{
+ PyGILState_STATE gstate = PyGILState_Ensure();
+ PyObject *module = PyImport_ImportModuleNoBlock((char*)"salome_psutil");
+ PyObject *result = PyObject_CallMethod(module,
+ (char*)"getNumberOfCPUCores", NULL);
+ int n = PyLong_AsLong(result);
+ Py_DECREF(result);
+ PyGILState_Release(gstate);
+
+ return (CORBA::Long)n;
+}
+
+//=============================================================================
+//! Get a load of each CPU core in the calculation node
+/*!
+* CORBA method: get a load of each CPU core
+*/
+//=============================================================================
+namespace {
+ std::string parseException()
+ {
+ std::string error;
+ if (PyErr_Occurred())
+ {
+ PyObject *ptype = nullptr;
+ PyObject *pvalue = nullptr;
+ PyObject *ptraceback = nullptr;
+ PyErr_Fetch(&ptype, &pvalue, &ptraceback);
+ if (ptype == nullptr)
+ return std::string("Null exception type");
+ PyErr_NormalizeException(&ptype, &pvalue, &ptraceback);
+ if (ptraceback != nullptr)
+ PyException_SetTraceback(pvalue, ptraceback);
+ boost::python::handle<> htype(ptype);
+ boost::python::handle<> hvalue(boost::python::allow_null(pvalue));
+ boost::python::handle<> htraceback(boost::python::allow_null(ptraceback));
+ boost::python::object traceback = boost::python::import("traceback");
+ boost::python::object format_exc = traceback.attr("format_exception");
+ boost::python::object formatted = format_exc(htype, hvalue, htraceback);
+ error = boost::python::extract<std::string>(boost::python::str("\n").join(formatted));
+ }
+ return error;
+ }
+}
+
+Engines::vectorOfDouble* Abstract_Engines_Container_i::loadOfCPUCores()
+{
+ PyGILState_STATE gstate = PyGILState_Ensure();
+ PyObject *module = PyImport_ImportModuleNoBlock((char*)"salome_psutil");
+ PyObject *result = PyObject_CallMethod(module,
+ (char*)"loadOfCPUCores", "s",
+ _load_script.c_str());
+ if (PyErr_Occurred())
+ {
+ std::string error = parseException();
+ PyErr_Print();
+ PyGILState_Release(gstate);
+ SALOME::ExceptionStruct es;
+ es.type = SALOME::INTERNAL_ERROR;
+ es.text = CORBA::string_dup(error.c_str());
+ throw SALOME::SALOME_Exception(es);
+ }
+
+ int n = this->getNumberOfCPUCores();
+ if (!PyList_Check(result) || PyList_Size(result) != n) {
+ // bad number of cores
+ PyGILState_Release(gstate);
+ Py_DECREF(result);
+ SALOME::ExceptionStruct es;
+ es.type = SALOME::INTERNAL_ERROR;
+ es.text = "wrong number of cores";
+ throw SALOME::SALOME_Exception(es);
+ }
+
+ Engines::vectorOfDouble_var loads = new Engines::vectorOfDouble;
+ loads->length(n);
+ for (Py_ssize_t i = 0; i < PyList_Size(result); ++i) {
+ PyObject* item = PyList_GetItem(result, i);
+ double foo = PyFloat_AsDouble(item);
+ if (foo < 0.0 || foo > 1.0)
+ {
+ // value not in [0, 1] range
+ PyGILState_Release(gstate);
+ Py_DECREF(result);
+ SALOME::ExceptionStruct es;
+ es.type = SALOME::INTERNAL_ERROR;
+ es.text = "load not in [0, 1] range";
+ throw SALOME::SALOME_Exception(es);
+ }
+ loads[i] = foo;
+ }
+
+ Py_DECREF(result);
+ PyGILState_Release(gstate);
+
+ return loads._retn();
+}
+
+//=============================================================================
+//! Set custom script to calculate a load of each CPU core
+/*!
+* CORBA method: Set custom script to calculate CPU load
+* \param script Python script to execute
+*/
+//=============================================================================
+
+void Abstract_Engines_Container_i::setPyScriptForCPULoad(const char *script)
+{
+ _load_script = script;
+}
+
+//=============================================================================
+//! Nullify custom script to calculate each CPU core's load
+/*!
+* CORBA method: reset script for load calculation to default implementation
+*/
+//=============================================================================
+
+void Abstract_Engines_Container_i::resetScriptForCPULoad()
+{
+ _load_script = "";
+}
+
+//=============================================================================
+//! Get total physical memory of calculation node, in megabytes
+/*!
+* CORBA method: get total physical memory of calculation node
+*/
+//=============================================================================
+
+CORBA::Long Abstract_Engines_Container_i::getTotalPhysicalMemory()
+{
+ PyGILState_STATE gstate = PyGILState_Ensure();
+ PyObject *module = PyImport_ImportModuleNoBlock((char*)"salome_psutil");
+ PyObject *result = PyObject_CallMethod(module,
+ (char*)"getTotalPhysicalMemory", NULL);
+ int n = PyLong_AsLong(result);
+ Py_DECREF(result);
+ PyGILState_Release(gstate);
+
+ return (CORBA::Long)n;
+}
+
+//=============================================================================
+//! Get used physical memory of calculation node, in megabytes
+/*!
+* CORBA method: get used physical memory of calculation node
+*/
+//=============================================================================
+
+CORBA::Long Abstract_Engines_Container_i::getTotalPhysicalMemoryInUse()
+{
+ PyGILState_STATE gstate = PyGILState_Ensure();
+ PyObject *module = PyImport_ImportModuleNoBlock((char*)"salome_psutil");
+ PyObject *result = PyObject_CallMethod(module,
+ (char*)"getTotalPhysicalMemoryInUse", NULL);
+ int n = PyLong_AsLong(result);
+ Py_DECREF(result);
+ PyGILState_Release(gstate);
+
+ return (CORBA::Long)n;
+}
+
+//=============================================================================
+//! Obtain physical memory, used by the current process, in megabytes.
+/*!
+* CORBA method: get physical memory, used by the current process
+*/
+//=============================================================================
+
+CORBA::Long Abstract_Engines_Container_i::getTotalPhysicalMemoryInUseByMe()
+{
+ PyGILState_STATE gstate = PyGILState_Ensure();
+ PyObject *module = PyImport_ImportModuleNoBlock((char*)"salome_psutil");
+ PyObject *result = PyObject_CallMethod(module,
+ (char*)"getTotalPhysicalMemoryInUseByMe", NULL);
+ int n = PyLong_AsLong(result);
+ Py_DECREF(result);
+ PyGILState_Release(gstate);
+
+ return (CORBA::Long)n;
+}
+
//=============================================================================
//! Shutdown the container
/*!
from omniORB import CORBA, PortableServer
import SALOMEDS
import Engines, Engines__POA
+import salome_psutil
from SALOME_NamingServicePy import *
from SALOME_Embedded_NamingService import SALOME_Embedded_NamingService
from SALOME_ComponentPy import *
_poa = None
_numInstance = 0
_listInstances_map = {}
+ _script = ""
#-------------------------------------------------------------------------
MESSAGE( "SALOME_ContainerPy_i::__init__" )
self._orb = orb
self._poa = poa
+ self._load_script = None
myMachine=getShortHostName()
Container_path = "/Containers/" + myMachine + "/" + containerName
self._containerName = Container_path
#-------------------------------------------------------------------------
+ def getNumberOfCPUCores(self):
+ return salome_psutil.getNumberOfCPUCores()
+
+ #-------------------------------------------------------------------------
+
+ def loadOfCPUCores(self):
+ return salome_psutil.loadOfCPUCores(self._load_script)
+
+ #-------------------------------------------------------------------------
+
+ def setPyScriptForCPULoad(self, script):
+ self._load_script = script
+
+ #-------------------------------------------------------------------------
+
+ def resetScriptForCPULoad(self):
+ self._load_script = None
+
+ #-------------------------------------------------------------------------
+
+ def getTotalPhysicalMemory(self):
+ return salome_psutil.getTotalPhysicalMemory()
+
+ #-------------------------------------------------------------------------
+
+ def getTotalPhysicalMemoryInUse(self):
+ return salome_psutil.getTotalPhysicalMemoryInUse()
+
+ #-------------------------------------------------------------------------
+
+ def getTotalPhysicalMemoryInUseByMe(self):
+ return salome_psutil.getTotalPhysicalMemoryInUseByMe()
+
+ #-------------------------------------------------------------------------
+
def _get_name(self):
MESSAGE( "SALOME_ContainerPy_i::_get_name" )
return self._containerName
void finalize_removal();
virtual void ping();
+ CORBA::Long getNumberOfCPUCores();
+ Engines::vectorOfDouble* loadOfCPUCores();
+ void setPyScriptForCPULoad(const char *script);
+ void resetScriptForCPULoad();
+ CORBA::Long getTotalPhysicalMemory();
+ CORBA::Long getTotalPhysicalMemoryInUse();
+ CORBA::Long getTotalPhysicalMemoryInUseByMe();
char *name();
char *workingdir();
char *logfilename();
std::string _library_path;
std::string _containerName;
std::string _logfilename;
+ std::string _load_script;
CORBA::ORB_var _orb;
PortableServer::POA_var _poa;
PortableServer::ObjectId *_id;
--- /dev/null
+# Copyright (C) 2012-2021 CEA/DEN, EDF R&D, OPEN CASCADE
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
+#
+
+SET(LOCAL_TEST_DIR ${KERNEL_TEST_DIR}/Container)
+INSTALL(FILES testcontainer.py DESTINATION ${LOCAL_TEST_DIR})
+
+INSTALL(FILES CTestTestfileInstall.cmake
+ DESTINATION ${LOCAL_TEST_DIR}
+ RENAME CTestTestfile.cmake)
--- /dev/null
+# Copyright (C) 2015-2021 CEA/DEN, EDF R&D
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
+#
+
+IF(NOT WIN32)
+ SET(TEST_NAME ${COMPONENT_NAME}_testcontainer)
+ ADD_TEST(${TEST_NAME} ${PYTHON_TEST_DRIVER} ${TIMEOUT} testcontainer.py)
+ SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES
+ LABELS "${COMPONENT_NAME}"
+ ENVIRONMENT "LD_LIBRARY_PATH=${KERNEL_TEST_LIB}:$ENV{LD_LIBRARY_PATH}"
+ )
+ENDIF()
--- /dev/null
+# -*- coding: iso-8859-1 -*-
+# Copyright (C) 2007-2021 CEA/DEN, EDF R&D, OPEN CASCADE
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
+#
+
+import unittest
+from os import getcwd
+from Engines import ContainerParameters, ResourceParameters
+import SALOME
+import salome
+
+from time import sleep
+
+class TestResourceManager(unittest.TestCase):
+ def getContainer(self, name):
+ rp = ResourceParameters(name="localhost",
+ hostname="localhost",
+ can_launch_batch_jobs=False,
+ can_run_containers=True,
+ OS="Linux",
+ componentList=[],
+ nb_proc=1,
+ mem_mb=1000,
+ cpu_clock=1000,
+ nb_node=1,
+ nb_proc_per_node=1,
+ policy="first",
+ resList=[])
+ cp = ContainerParameters(container_name=name,
+ mode="start",
+ workingdir=getcwd(),
+ nb_proc=1,
+ isMPI=False,
+ parallelLib="",
+ resource_params=rp)
+ cm = salome.naming_service.Resolve("/ContainerManager")
+ return cm.GiveContainer(cp)
+
+ def checkLoads(self, cont, loads):
+ self.assertEqual(len(loads), cont.getNumberOfCPUCores())
+ for load in loads:
+ self.assertTrue(0.0 <= load <= 1.0)
+
+ def test1(self):
+ # Check loadOfCPUCores
+ cont = self.getContainer("test_container_1")
+ loads1 = cont.loadOfCPUCores()
+ self.checkLoads(cont, loads1)
+ sleep(1)
+ loads2 = cont.loadOfCPUCores()
+ self.checkLoads(cont, loads2)
+ self.assertNotEqual(loads1, loads2)
+ cont.Shutdown()
+
+ def test2(self):
+ # Check custom script
+ cont = self.getContainer("test_container_2")
+ import multiprocessing as mp
+ ref_load = [max(0.1*(i+1),1.0) for i in range(mp.cpu_count())]
+ cont.setPyScriptForCPULoad('cpu_loads = {}'.format(ref_load))
+ loads1 = cont.loadOfCPUCores()
+ self.assertEqual(loads1, ref_load)
+ cont.resetScriptForCPULoad()
+ loads2 = cont.loadOfCPUCores()
+ self.checkLoads(cont, loads2)
+ cont.Shutdown()
+
+ def test3(self):
+ # Check bad script
+ cont = self.getContainer("test_container_3")
+ cont.setPyScriptForCPULoad("bla-bla-bla")
+ self.assertRaises(Exception, cont.loadOfCPUCores)
+ cont.Shutdown()
+
+ def test4(self):
+ # check memory sizes
+ cont = self.getContainer("test_container_4")
+ memory_total = cont.getTotalPhysicalMemory()
+ memory_in_use = cont.getTotalPhysicalMemoryInUse()
+ memory_by_me = cont.getTotalPhysicalMemoryInUseByMe()
+ self.assertGreater(memory_total, memory_in_use)
+ self.assertGreater(memory_in_use, memory_by_me)
+ cont.Shutdown()
+
+ def test5(self):
+ """
+ Test checking memory consumption of container
+ """
+ cont = self.getContainer("test_container_5")
+ memory_by_me_start = cont.getTotalPhysicalMemoryInUseByMe()
+ import pickle
+ psn = cont.createPyScriptNode("n","""b = bytearray(10485760)""")# create 10MB byte array abroad
+ psn.execute([],pickle.dumps(((),{})))
+ memory_by_me_end = cont.getTotalPhysicalMemoryInUseByMe()
+ self.assertGreater(memory_by_me_end,memory_by_me_start)
+ self.assertIn(memory_by_me_end-memory_by_me_start,[10,11,12])# test elevation of memory
+ cont.Shutdown()
+
+if __name__ == '__main__':
+ salome.standalone()
+ salome.salome_init()
+ unittest.main()
salome_notebook.py
salome_pynode.py
salome_genericobj.py
+ salome_psutil.py
)
SALOME_INSTALL_SCRIPTS("${salomepython_PYTHON}" ${SALOME_INSTALL_PYTHON})
--- /dev/null
+# -*- coding: utf-8 -*-
+# Copyright (C) 2007-2021 CEA/DEN, EDF R&D, OPEN CASCADE
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
+#
+
+'''
+SALOME utilities for process management.
+'''
+
+import os
+import psutil
+
+
+def getNumberOfCPUCores(): # pragma pylint: disable=invalid-name
+ '''
+ Get number of CPU cores in the calculation node.
+ :return Number of cores
+ '''
+ return psutil.cpu_count(logical=True)
+
+
+def loadOfCPUCores(script=None): # pragma pylint: disable=invalid-name
+ '''
+ Get a load of each CPU core in the calculation node.
+
+ A script to compute loads can be customized via `script` parameter.
+ In that case, the script must either set `cpu_loads` variable (which
+ should be of list type), or specify `getCPULoads()` function returning
+ list as result. In both cases, the list's size must be equal to the value
+ returning by method `getNumberOfCPUCores()`, and each value in this list
+ must be in range [0, 1], otherwise exception is raised.
+
+ If `script` is not specified, default implementation is used.
+
+ :param script Custom script to calculate loads
+ :return List that contains loads of each CPU core.
+ '''
+ if not script:
+ return [x/100 for x in psutil.cpu_percent(interval=None, percpu=True)]
+ cpu_loads, loc = None, {}
+ exec(script, globals(), loc) # pragma pylint: disable=exec-used
+ cpu_loads = loc['getCPULoads']() if 'getCPULoads' in loc else loc.get('cpu_loads')
+ if cpu_loads is None:
+ raise ValueError('Bad script. Specify `getCPULoads` function or `cpu_loads` variable')
+ if not isinstance(cpu_loads, (list, tuple)):
+ raise TypeError('Bad script. Result must be list or tuple.')
+ size = getNumberOfCPUCores()
+ if len(cpu_loads) != size:
+ raise ValueError('Bad script. Result is of incorrect length (must be {})'.format(size))
+ values = [i for i in cpu_loads if 0 <= i <= 1]
+ if len(values) != size:
+ raise ValueError('Bad script. Values are not in [0, 1] range')
+ return [i for i in cpu_loads]
+
+
+def getTotalPhysicalMemory(): # pragma pylint: disable=invalid-name
+ '''
+ Get total physical memory of the calculation node.
+ :return Size of physical memory, in megabytes.
+ '''
+ return int(psutil.virtual_memory().total/1024/1024)
+
+
+def getTotalPhysicalMemoryInUse(): # pragma pylint: disable=invalid-name
+ '''
+ Get used physical memory of the calculation node.
+ :return Size of used physical memory, in megabytes.
+ '''
+ return int(psutil.virtual_memory().used/1024/1024)
+
+
+def getTotalPhysicalMemoryInUseByMe(): # pragma pylint: disable=invalid-name
+ '''
+ Get physical memory occupied by the current process.
+ :return Size of physical memory, used by current process, in megabytes.
+ '''
+ process = psutil.Process(os.getpid())
+ memory_in_mb = int(process.memory_info().rss/1024/1024)
+ return memory_in_mb
