From 02ccddf4c317b8bce954b4f713ff37643d2ae9bf Mon Sep 17 00:00:00 2001 From: Anthony Geay Date: Wed, 26 Jun 2024 18:06:39 +0200 Subject: [PATCH] [EDF30382] : ease access of resources --- SalomeKERNELConfig.cmake.in | 1 + src/Container/SALOME_PyNode.py | 2 +- src/Container/Test/testProxy.py | 51 +++++++ src/Launcher_SWIG/CMakeLists.txt | 9 +- src/Launcher_SWIG/Launcher.i | 127 ++++-------------- .../LauncherResourceDefinition.cxx | 65 +++++++++ .../LauncherResourceDefinition.hxx | 52 +++++++ src/Launcher_SWIG/SALOME_Launcher_SWIG.hxx | 30 +++++ 8 files changed, 235 insertions(+), 102 deletions(-) create mode 100644 src/Launcher_SWIG/LauncherResourceDefinition.cxx create mode 100644 src/Launcher_SWIG/LauncherResourceDefinition.hxx create mode 100644 src/Launcher_SWIG/SALOME_Launcher_SWIG.hxx diff --git a/SalomeKERNELConfig.cmake.in b/SalomeKERNELConfig.cmake.in index cc9aa9850..56b04e3f1 100644 --- a/SalomeKERNELConfig.cmake.in +++ b/SalomeKERNELConfig.cmake.in @@ -154,6 +154,7 @@ SET(SALOME_INSTALL_AMCONFIG_LOCAL "@SALOME_INSTALL_AMCONFIG_LOCAL@") # to be rem SET(KERNEL_CalciumC CalciumC) SET(KERNEL_DF DF) SET(KERNEL_Launcher Launcher) +SET(KERNEL_LauncherWrapHelper LauncherWrapHelper) SET(KERNEL_LifeCycleCORBATest LifeCycleCORBATest) SET(KERNEL_NamingServiceTest NamingServiceTest) SET(KERNEL_OpUtil OpUtil) diff --git a/src/Container/SALOME_PyNode.py b/src/Container/SALOME_PyNode.py index d0622a7ca..b5a947975 100644 --- a/src/Container/SALOME_PyNode.py +++ b/src/Container/SALOME_PyNode.py @@ -1007,7 +1007,7 @@ sys.stderr.flush()""".format( MY_KEY_TO_DETECT_FINISH ) ) resFileName = os.path.join( dirForReplayFiles, "outcontextsafe_{}.pckl".format( RetrieveUniquePartFromPfx( codeFileName ) ) ) mainExecFileName = os.path.join( dirForReplayFiles, "mainexecsafe_{}.py".format( RetrieveUniquePartFromPfx( codeFileName ) ) ) with open(mainExecFileName,"w") as f: - f.write( FinalCode.format( os.getcwd(), dirForReplayFiles, codeFileName, contextFileName, resFileName, outargsname, iorScriptLog ) ) + f.write( FinalCode.format( os.getcwd(), dirForReplayFiles, codeFileName, os.path.basename( contextFileName ), os.path.basename( resFileName ), outargsname, iorScriptLog ) ) for iTry in range( KernelBasis.GetNumberOfRetry() ): if iTry > 0: print( "WARNING : Retry # {}. Following code has generated non zero return code ( {} ). Trying again ... \n{}".format( iTry, returnCode, code ) ) diff --git a/src/Container/Test/testProxy.py b/src/Container/Test/testProxy.py index 1be8344fe..4637e895a 100644 --- a/src/Container/Test/testProxy.py +++ b/src/Container/Test/testProxy.py @@ -164,6 +164,57 @@ ret = os.getcwd() # cont.Shutdown() + def testAccessOfResourcesAcrossWorkers(self): + """ + [EDF30157] To ease testing advanced configuration. This test checks that + """ + ### start of catalog manipulation in memory + rmcpp = pylauncher.RetrieveRMCppSingleton() + res0 = pylauncher.CreateContainerResource("zFakeHost","/home/appli/fakeappli","ssh") + rmcpp.AddResourceInCatalogNoQuestion(res0)#<- key point is here + res1 = pylauncher.CreateContainerResource("zzFakeHost","/home/appli/zzfakeappli","ssh") + rmcpp.AddResourceInCatalogNoQuestion(res1)#<- key point is here + ### end of catalog manipulation in memory + + ### start to check effectivity of manipulation locally + machines = salome.rm.ListAllResourceEntriesInCatalog() + localStructure = { machine : salome.rm.GetResourceDefinition( machine ) for machine in machines } + ### end of check effectivity of manipulation locally + + cp = pylauncher.GetRequestForGiveContainer("localhost","gg") + cont = salome.cm.GiveContainer(cp) + pyscript = cont.createPyScriptNode("testScript",""" +import salome +salome.salome_init() +machines = salome.rm.ListAllResourceEntriesInCatalog() +structure = { machine : salome.rm.GetResourceDefinition( machine ) for machine in machines } +""") # retrieve the content remotely and then return it back to current process + import SALOME_PyNode + import pickle + poa = salome.orb.resolve_initial_references("RootPOA") + obj = SALOME_PyNode.SenderByte_i(poa,pickle.dumps( ([],{}) )) + id_o = poa.activate_object(obj) + refPtr = poa.id_to_reference(id_o) + # + pyscript.executeFirst(refPtr) + ret = pyscript.executeSecond(["structure"]) + ret = ret[0] + retPy = pickle.loads( SALOME_PyNode.SeqByteReceiver(ret).data() ) + + assert( len(localStructure) == len(retPy) ) + for k in localStructure: + a = pylauncher.FromEngineResourceDefinitionToCPP( localStructure[k] ) + self.assertTrue( isinstance(a,pylauncher.ResourceDefinition_cpp) ) + b = pylauncher.FromEngineResourceDefinitionToCPP( retPy[k] ) + self.assertTrue( isinstance(b,pylauncher.ResourceDefinition_cpp) ) + self.assertTrue( a==b ) #<- key point is here + a1 = pylauncher.ToEngineResourceDefinitionFromCPP( a ) + b1 = pylauncher.ToEngineResourceDefinitionFromCPP( b ) + a2 = pylauncher.FromEngineResourceDefinitionToCPP( a1 ) + b2 = pylauncher.FromEngineResourceDefinitionToCPP( b1 ) + self.assertTrue( a2==b2 ) + cont.Shutdown() + if __name__ == '__main__': salome.standalone() salome.salome_init() diff --git a/src/Launcher_SWIG/CMakeLists.txt b/src/Launcher_SWIG/CMakeLists.txt index 226116fdf..357aef6b0 100644 --- a/src/Launcher_SWIG/CMakeLists.txt +++ b/src/Launcher_SWIG/CMakeLists.txt @@ -35,17 +35,22 @@ SET_SOURCE_FILES_PROPERTIES(Launcher.i PROPERTIES SWIG_FLAGS "-py3") INCLUDE_DIRECTORIES( ${PYTHON_INCLUDE_DIRS} ${LIBBATCH_INCLUDE_DIRS} + ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ResourcesManager ${CMAKE_CURRENT_SOURCE_DIR}/../Launcher ) +ADD_LIBRARY(LauncherWrapHelper LauncherResourceDefinition.cxx) +TARGET_LINK_LIBRARIES(LauncherWrapHelper Launcher) +INSTALL(TARGETS LauncherWrapHelper EXPORT ${PROJECT_NAME}TargetGroup DESTINATION ${SALOME_INSTALL_LIBS}) + SWIG_ADD_LIBRARY(${Launcher_target_name} LANGUAGE python SOURCES Launcher.i) IF(${MACHINE} STREQUAL WINDOWS) SET_TARGET_PROPERTIES(_${Launcher_target_name} PROPERTIES DEBUG_OUTPUT_NAME _${Launcher_target_name}_d) ENDIF(${MACHINE} STREQUAL WINDOWS) -SWIG_LINK_LIBRARIES(${Launcher_target_name} ${PYTHON_LIBRARIES} ${PLATFORM_LIBS} Launcher) +SWIG_LINK_LIBRARIES(${Launcher_target_name} ${PYTHON_LIBRARIES} ${PLATFORM_LIBS} LauncherWrapHelper) SWIG_CHECK_GENERATION(${Launcher_target_name}) @@ -55,6 +60,8 @@ SET(PYFILES_TO_INSTALL ${PYFILES_TO_INSTALL} ${CMAKE_CURRENT_BINARY_DIR}/pylaunc SALOME_INSTALL_SCRIPTS("${PYFILES_TO_INSTALL}" ${SALOME_INSTALL_PYTHON} EXTRA_DPYS "${SWIG_MODULE_${Launcher_target_name}_REAL_NAME}") +install(FILES LauncherResourceDefinition.hxx SALOME_Launcher_SWIG.hxx DESTINATION ${SALOME_INSTALL_HEADERS}) + IF(SALOME_BUILD_TESTS) ADD_SUBDIRECTORY(Test) ENDIF(SALOME_BUILD_TESTS) diff --git a/src/Launcher_SWIG/Launcher.i b/src/Launcher_SWIG/Launcher.i index e472ede51..03e7d5fd4 100644 --- a/src/Launcher_SWIG/Launcher.i +++ b/src/Launcher_SWIG/Launcher.i @@ -22,31 +22,10 @@ %{ #include "Launcher.hxx" #include "ResourcesManager.hxx" +#include "LauncherResourceDefinition.hxx" #include -struct ResourceDefinition_cpp -{ -public: - std::string name; - std::string hostname; - std::string type; - std::string protocol; - std::string username; - std::string applipath; - std::string OS; - int mem_mb; - int cpu_clock; - int nb_node; - int nb_proc_per_node; - std::string batch; - std::string mpiImpl; - std::string iprotocol; - bool can_launch_batch_jobs; - bool can_run_containers; - std::string working_directory; -}; - std::shared_ptr HandleToLocalInstance(const std::string& ptrInStringFrmt) { std::istringstream iss(ptrInStringFrmt); @@ -126,27 +105,7 @@ public: // see ResourceDefinition from SALOME_ResourcesManager.idl // no other c++ equivalent. Convertion from ParserResourcesType -struct ResourceDefinition_cpp -{ -public: - std::string name; - std::string hostname; - std::string type; - std::string protocol; - std::string username; - std::string applipath; - std::string OS; - int mem_mb; - int cpu_clock; - int nb_node; - int nb_proc_per_node; - std::string batch; - std::string mpiImpl; - std::string iprotocol; - bool can_launch_batch_jobs; - bool can_run_containers; - std::string working_directory; -}; +%include %exception { @@ -180,25 +139,7 @@ public: { ResourceDefinition_cpp swig_result; ParserResourcesType cpp_result = $self->GetResourcesDescr(name); - - swig_result.name = cpp_result.Name; - swig_result.hostname = cpp_result.HostName; - swig_result.type = cpp_result.getResourceTypeStr(); - swig_result.protocol = cpp_result.getAccessProtocolTypeStr(); - swig_result.username = cpp_result.UserName; - swig_result.applipath = cpp_result.AppliPath; - swig_result.OS = cpp_result.OS; - swig_result.mem_mb = cpp_result.DataForSort._memInMB; - swig_result.cpu_clock = cpp_result.DataForSort._CPUFreqMHz; - swig_result.nb_node = cpp_result.DataForSort._nbOfNodes; - swig_result.nb_proc_per_node = cpp_result.DataForSort._nbOfProcPerNode; - swig_result.batch = cpp_result.getBatchTypeStr(); - swig_result.mpiImpl = cpp_result.getMpiImplTypeStr(); - swig_result.iprotocol = cpp_result.getClusterInternalProtocolStr(); - swig_result.can_launch_batch_jobs = cpp_result.can_launch_batch_jobs; - swig_result.can_run_containers = cpp_result.can_run_containers; - swig_result.working_directory = cpp_result.working_directory; - + swig_result.fromPRT( cpp_result ); return swig_result; } @@ -209,49 +150,13 @@ public: void AddResourceInCatalog (const ResourceDefinition_cpp& new_resource) { - ParserResourcesType new_resource_cpp; - new_resource_cpp.Name = new_resource.name; - new_resource_cpp.HostName = new_resource.hostname; - new_resource_cpp.setResourceTypeStr( new_resource.type ); - new_resource_cpp.setAccessProtocolTypeStr( new_resource.protocol ); - new_resource_cpp.UserName = new_resource.username; - new_resource_cpp.AppliPath = new_resource.applipath; - new_resource_cpp.OS = new_resource.OS; - new_resource_cpp.DataForSort._Name = new_resource.name; - new_resource_cpp.DataForSort._memInMB = new_resource.mem_mb; - new_resource_cpp.DataForSort._CPUFreqMHz = new_resource.cpu_clock; - new_resource_cpp.DataForSort._nbOfNodes = new_resource.nb_node; - new_resource_cpp.DataForSort._nbOfProcPerNode = new_resource.nb_proc_per_node; - new_resource_cpp.setBatchTypeStr(new_resource.batch); - new_resource_cpp.setMpiImplTypeStr(new_resource.mpiImpl); - new_resource_cpp.setClusterInternalProtocolStr(new_resource.iprotocol); - new_resource_cpp.can_launch_batch_jobs = new_resource.can_launch_batch_jobs; - new_resource_cpp.can_run_containers = new_resource.can_run_containers; - new_resource_cpp.working_directory = new_resource.working_directory; + ParserResourcesType new_resource_cpp( new_resource.toPRT() ); $self->AddResourceInCatalog(new_resource_cpp); } void AddResourceInCatalogNoQuestion (const ResourceDefinition_cpp& new_resource) { - ParserResourcesType new_resource_cpp; - new_resource_cpp.Name = new_resource.name; - new_resource_cpp.HostName = new_resource.hostname; - new_resource_cpp.setResourceTypeStr( new_resource.type ); - new_resource_cpp.setAccessProtocolTypeStr( new_resource.protocol ); - new_resource_cpp.UserName = new_resource.username; - new_resource_cpp.AppliPath = new_resource.applipath; - new_resource_cpp.OS = new_resource.OS; - new_resource_cpp.DataForSort._Name = new_resource.name; - new_resource_cpp.DataForSort._memInMB = new_resource.mem_mb; - new_resource_cpp.DataForSort._CPUFreqMHz = new_resource.cpu_clock; - new_resource_cpp.DataForSort._nbOfNodes = new_resource.nb_node; - new_resource_cpp.DataForSort._nbOfProcPerNode = new_resource.nb_proc_per_node; - new_resource_cpp.setBatchTypeStr(new_resource.batch); - new_resource_cpp.setMpiImplTypeStr(new_resource.mpiImpl); - new_resource_cpp.setClusterInternalProtocolStr(new_resource.iprotocol); - new_resource_cpp.can_launch_batch_jobs = new_resource.can_launch_batch_jobs; - new_resource_cpp.can_run_containers = new_resource.can_run_containers; - new_resource_cpp.working_directory = new_resource.working_directory; + ParserResourcesType new_resource_cpp( new_resource.toPRT() ); $self->AddResourceInCatalogNoQuestion(new_resource_cpp); } @@ -368,6 +273,11 @@ def ResourcesManager_cpp___getitem__(self,name): def ResourcesManager_cpp___repr__(self): return str( self.GetList() ) +ListOfAttrCommon = ['name', 'OS', 'applipath', 'batch', 'can_launch_batch_jobs', 'can_run_containers', 'componentList', 'cpu_clock', 'hostname', 'iprotocol', 'mem_mb', 'mpiImpl', 'nb_node', 'nb_proc_per_node', 'protocol', 'type', 'username', 'working_directory'] + +def ResourceDefinition_cpp_isEqual(self,other): + return all( [getattr(self,k) == getattr(other,k) for k in ListOfAttrCommon] ) + def RetrieveRMCppSingleton(): import KernelLauncher return HandleToLocalInstance( KernelLauncher.RetrieveInternalInstanceOfLocalCppResourcesManager() ) @@ -417,9 +327,26 @@ def GetRequestForGiveContainer(hostname, contName): resource_params=rp) return cp + +def FromEngineResourceDefinitionToCPP( corbaInstance ): + """ + Convert Engine.ResourceDefinition to ResourceDefinition_cpp instance + """ + ret = ResourceDefinition_cpp() + for k in ListOfAttrCommon: + setattr(ret,k,getattr(corbaInstance,k)) + return ret + +def ToEngineResourceDefinitionFromCPP( cppInstance ): + import Engines + return Engines.ResourceDefinition(**{k:getattr(cppInstance,k) for k in ListOfAttrCommon}) + ResourceDefinition_cpp.repr = ResourceDefinition_cpp_repr ResourceDefinition_cpp.__repr__ = ResourceDefinition_cpp_repr +ResourceDefinition_cpp.__eq__ = ResourceDefinition_cpp_isEqual + ResourcesManager_cpp.GetList = ResourcesManager_cpp_GetList ResourcesManager_cpp.__getitem__ = ResourcesManager_cpp___getitem__ ResourcesManager_cpp.__repr__ = ResourcesManager_cpp___repr__ %} + diff --git a/src/Launcher_SWIG/LauncherResourceDefinition.cxx b/src/Launcher_SWIG/LauncherResourceDefinition.cxx new file mode 100644 index 000000000..0d94a3ecf --- /dev/null +++ b/src/Launcher_SWIG/LauncherResourceDefinition.cxx @@ -0,0 +1,65 @@ +// Copyright (C) 2024 CEA, EDF +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +#include "LauncherResourceDefinition.hxx" + +void ResourceDefinition_cpp::fromPRT(const ParserResourcesType& other) +{ + this->name = other.Name; + this->hostname = other.HostName; + this->type = other.getResourceTypeStr(); + this->protocol = other.getAccessProtocolTypeStr(); + this->username = other.UserName; + this->applipath = other.AppliPath; + this->OS = other.OS; + this->mem_mb = other.DataForSort._memInMB; + this->cpu_clock = other.DataForSort._CPUFreqMHz; + this->nb_node = other.DataForSort._nbOfNodes; + this->nb_proc_per_node = other.DataForSort._nbOfProcPerNode; + this->batch = other.getBatchTypeStr(); + this->mpiImpl = other.getMpiImplTypeStr(); + this->iprotocol = other.getClusterInternalProtocolStr(); + this->can_launch_batch_jobs = other.can_launch_batch_jobs; + this->can_run_containers = other.can_run_containers; + this->working_directory = other.working_directory; +} + +ParserResourcesType ResourceDefinition_cpp::toPRT() const +{ + ParserResourcesType new_resource_cpp; + new_resource_cpp.Name = this->name; + new_resource_cpp.HostName = this->hostname; + new_resource_cpp.setResourceTypeStr( this->type ); + new_resource_cpp.setAccessProtocolTypeStr( this->protocol ); + new_resource_cpp.UserName = this->username; + new_resource_cpp.AppliPath = this->applipath; + new_resource_cpp.OS = this->OS; + new_resource_cpp.DataForSort._Name = this->name; + new_resource_cpp.DataForSort._memInMB = this->mem_mb; + new_resource_cpp.DataForSort._CPUFreqMHz = this->cpu_clock; + new_resource_cpp.DataForSort._nbOfNodes = this->nb_node; + new_resource_cpp.DataForSort._nbOfProcPerNode = this->nb_proc_per_node; + new_resource_cpp.setBatchTypeStr(this->batch); + new_resource_cpp.setMpiImplTypeStr(this->mpiImpl); + new_resource_cpp.setClusterInternalProtocolStr(this->iprotocol); + new_resource_cpp.can_launch_batch_jobs = this->can_launch_batch_jobs; + new_resource_cpp.can_run_containers = this->can_run_containers; + new_resource_cpp.working_directory = this->working_directory; + return new_resource_cpp; +} diff --git a/src/Launcher_SWIG/LauncherResourceDefinition.hxx b/src/Launcher_SWIG/LauncherResourceDefinition.hxx new file mode 100644 index 000000000..9b32bb8e5 --- /dev/null +++ b/src/Launcher_SWIG/LauncherResourceDefinition.hxx @@ -0,0 +1,52 @@ +// Copyright (C) 2024 CEA, EDF +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +#pragma once + +#include "SALOME_Launcher_SWIG.hxx" +#include "SALOME_ResourcesCatalog_Parser.hxx" + +#ifndef SWIG +struct LAUNCHER_SWIG_EXPORT ResourceDefinition_cpp +#else +struct ResourceDefinition_cpp +#endif +{ +public: + std::string name; + std::string hostname; + std::string type; + std::string protocol; + std::string username; + std::string applipath; + std::string OS; + int mem_mb; + int cpu_clock; + int nb_node; + int nb_proc_per_node; + std::string batch; + std::string mpiImpl; + std::string iprotocol; + bool can_launch_batch_jobs; + bool can_run_containers; + std::string working_directory; +public: + void fromPRT(const ParserResourcesType& other); + ParserResourcesType toPRT() const; +}; diff --git a/src/Launcher_SWIG/SALOME_Launcher_SWIG.hxx b/src/Launcher_SWIG/SALOME_Launcher_SWIG.hxx new file mode 100644 index 000000000..b283ae2e9 --- /dev/null +++ b/src/Launcher_SWIG/SALOME_Launcher_SWIG.hxx @@ -0,0 +1,30 @@ +// Copyright (C) 2024 CEA, EDF +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +#pragma once + +#ifdef WIN32 +# if defined LAUNCHER_SWIG_EXPORTS || defined SalomeLAUNCHER_SWIG_EXPORTS +# define LAUNCHER_SWIG_EXPORT __declspec( dllexport ) +# else +# define LAUNCHER_SWIG_EXPORT __declspec( dllimport ) +# endif +#else +# define LAUNCHER_SWIG_EXPORT +#endif -- 2.39.2