src/Launcher_SWIG/Launcher.i

   1 // Copyright (C) 2019-2024  CEA, EDF
   2 //
   3 // This library is free software; you can redistribute it and/or
   4 // modify it under the terms of the GNU Lesser General Public
   5 // License as published by the Free Software Foundation; either
   6 // version 2.1 of the License, or (at your option) any later version.
   7 //
   8 // This library is distributed in the hope that it will be useful,
   9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11 // Lesser General Public License for more details.
  12 //
  13 // You should have received a copy of the GNU Lesser General Public
  14 // License along with this library; if not, write to the Free Software
  15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  16 //
  17 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
  18 //
  19
  20 %module pylauncher
  21
  22 %{
  23 #include "Launcher.hxx"
  24 #include "ResourcesManager.hxx"
  25
  26 #include <sstream>
  27
  28 struct ResourceDefinition_cpp
  29 {
  30 public:
  31   std::string name;
  32   std::string hostname;
  33   std::string type;
  34   std::string protocol;
  35   std::string username;
  36   std::string applipath;
  37   std::string OS;
  38   int  mem_mb;
  39   int  cpu_clock;
  40   int  nb_node;
  41   int  nb_proc_per_node;
  42   std::string batch;
  43   std::string mpiImpl;
  44   std::string iprotocol;
  45   bool can_launch_batch_jobs;
  46   bool can_run_containers;
  47   std::string working_directory;
  48 };
  49
  50 std::shared_ptr<ResourcesManager_cpp> HandleToLocalInstance(const std::string& ptrInStringFrmt)
  51 {
  52   std::istringstream iss(ptrInStringFrmt);
  53   void *zePtr(nullptr);
  54   iss >> zePtr;
  55   std::shared_ptr<ResourcesManager_cpp> *effPtr = reinterpret_cast<std::shared_ptr<ResourcesManager_cpp> *>(zePtr);
  56   std::shared_ptr<ResourcesManager_cpp> ret(*effPtr);
  57   delete effPtr;
  58   return ret;
  59 }
  60 %}
  61
  62 %include "std_string.i"
  63 %include "std_vector.i"
  64 %include "std_list.i"
  65 %include "std_map.i"
  66
  67 namespace std {
  68   %template(list_int) list<int>;
  69   %template(list_str) list<string>;
  70   %template(vector_str) vector<string>;
  71   %template(map_ss) map<string,string>;
  72 };
  73
  74 // see ResourceParameters from SALOME_ResourcesManager.idl
  75 // see resourceParams from ResourcesManager.hxx
  76 %naturalvar JobParameters_cpp::componentList;
  77 %naturalvar JobParameters_cpp::resourceList;
  78 struct resourceParams
  79 {
  80   resourceParams();
  81
  82   std::string name;
  83   std::string hostname;
  84   bool can_launch_batch_jobs;
  85   bool can_run_containers;
  86   std::string OS;
  87   long nb_proc;
  88   long nb_node;
  89   long nb_proc_per_node;
  90   long cpu_clock;
  91   long mem_mb;
  92   std::vector<std::string> componentList;
  93   std::vector<std::string> resourceList;
  94 };
  95
  96 // see JobParameters from SALOME_Launcher.idl
  97 // see JobParameters_cpp from Launcher.hxx
  98 %naturalvar JobParameters_cpp::in_files;
  99 %naturalvar JobParameters_cpp::out_files;
 100 %naturalvar JobParameters_cpp::specific_parameters;
 101 struct JobParameters_cpp
 102 {
 103 public:
 104   std::string job_name;
 105   std::string job_type;
 106   std::string job_file;
 107   std::string pre_command;
 108   std::string env_file;
 109   std::list<std::string> in_files;
 110   std::list<std::string> out_files;
 111   std::string work_directory;
 112   std::string local_directory;
 113   std::string result_directory;
 114   std::string maximum_duration;
 115   resourceParams resource_required;
 116   std::string queue;
 117   std::string partition;
 118   bool exclusive;
 119   unsigned int mem_per_cpu;
 120   std::string wckey;
 121   std::string extra_params;
 122   std::map<std::string, std::string> specific_parameters;
 123   std::string launcher_file;
 124   std::string launcher_args;
 125 };
 126
 127 // see ResourceDefinition from SALOME_ResourcesManager.idl
 128 // no other c++ equivalent. Convertion from ParserResourcesType
 129 struct ResourceDefinition_cpp
 130 {
 131 public:
 132   std::string name;
 133   std::string hostname;
 134   std::string type;
 135   std::string protocol;
 136   std::string username;
 137   std::string applipath;
 138   std::string OS;
 139   int  mem_mb;
 140   int  cpu_clock;
 141   int  nb_node;
 142   int  nb_proc_per_node;
 143   std::string batch;
 144   std::string mpiImpl;
 145   std::string iprotocol;
 146   bool can_launch_batch_jobs;
 147   bool can_run_containers;
 148   std::string working_directory;
 149 };
 150
 151 %exception
 152 {
 153   try
 154   {
 155     $function
 156   }
 157   catch (ResourcesException& e)
 158   {
 159     SWIG_exception_fail(SWIG_RuntimeError, e.msg.c_str());
 160   }
 161   catch(...)
 162   {
 163     SWIG_exception_fail(SWIG_RuntimeError,"Unknown exception");
 164   }
 165 }
 166
 167 %include <std_shared_ptr.i>
 168 %shared_ptr(ResourcesManager_cpp)
 169
 170 class ResourcesManager_cpp
 171 {
 172 public:
 173   ResourcesManager_cpp(const char *xmlFilePath);
 174   std::vector<std::string> GetFittingResources(const resourceParams& params);
 175   void WriteInXmlFile(std::string xml_file);
 176   void DeleteAllResourcesInCatalog();
 177 %extend
 178 {
 179   ResourceDefinition_cpp GetResourceDefinition(const std::string& name)
 180   {
 181     ResourceDefinition_cpp swig_result;
 182     ParserResourcesType cpp_result = $self->GetResourcesDescr(name);
 183
 184     swig_result.name = cpp_result.Name;
 185     swig_result.hostname = cpp_result.HostName;
 186     swig_result.type = cpp_result.getResourceTypeStr();
 187     swig_result.protocol = cpp_result.getAccessProtocolTypeStr();
 188     swig_result.username = cpp_result.UserName;
 189     swig_result.applipath = cpp_result.AppliPath;
 190     swig_result.OS = cpp_result.OS;
 191     swig_result.mem_mb = cpp_result.DataForSort._memInMB;
 192     swig_result.cpu_clock = cpp_result.DataForSort._CPUFreqMHz;
 193     swig_result.nb_node = cpp_result.DataForSort._nbOfNodes;
 194     swig_result.nb_proc_per_node = cpp_result.DataForSort._nbOfProcPerNode;
 195     swig_result.batch = cpp_result.getBatchTypeStr();
 196     swig_result.mpiImpl = cpp_result.getMpiImplTypeStr();
 197     swig_result.iprotocol = cpp_result.getClusterInternalProtocolStr();
 198     swig_result.can_launch_batch_jobs = cpp_result.can_launch_batch_jobs;
 199     swig_result.can_run_containers = cpp_result.can_run_containers;
 200     swig_result.working_directory = cpp_result.working_directory;
 201
 202     return swig_result;
 203   }
 204
 205   void DeleteResourceInCatalog(const std::string& name)
 206   {
 207     $self->DeleteResourceInCatalog(name.c_str());
 208   }
 209
 210   void AddResourceInCatalog (const ResourceDefinition_cpp& new_resource)
 211   {
 212     ParserResourcesType new_resource_cpp;
 213     new_resource_cpp.Name = new_resource.name;
 214     new_resource_cpp.HostName = new_resource.hostname;
 215     new_resource_cpp.setResourceTypeStr( new_resource.type );
 216     new_resource_cpp.setAccessProtocolTypeStr( new_resource.protocol );
 217     new_resource_cpp.UserName = new_resource.username;
 218     new_resource_cpp.AppliPath = new_resource.applipath;
 219     new_resource_cpp.OS = new_resource.OS;
 220     new_resource_cpp.DataForSort._Name = new_resource.name;
 221     new_resource_cpp.DataForSort._memInMB = new_resource.mem_mb;
 222     new_resource_cpp.DataForSort._CPUFreqMHz = new_resource.cpu_clock;
 223     new_resource_cpp.DataForSort._nbOfNodes = new_resource.nb_node;
 224     new_resource_cpp.DataForSort._nbOfProcPerNode = new_resource.nb_proc_per_node;
 225     new_resource_cpp.setBatchTypeStr(new_resource.batch);
 226     new_resource_cpp.setMpiImplTypeStr(new_resource.mpiImpl);
 227     new_resource_cpp.setClusterInternalProtocolStr(new_resource.iprotocol);
 228     new_resource_cpp.can_launch_batch_jobs = new_resource.can_launch_batch_jobs;
 229     new_resource_cpp.can_run_containers = new_resource.can_run_containers;
 230     new_resource_cpp.working_directory = new_resource.working_directory;
 231     $self->AddResourceInCatalog(new_resource_cpp);
 232   }
 233
 234   void ParseXmlFiles()
 235   {
 236     $self->ParseXmlFiles();
 237   }
 238
 239   std::vector<std::string> GetListOfEntries() const
 240   {
 241     const MapOfParserResourcesType& allRes = $self->GetList();
 242     std::vector<std::string> ret;
 243     for(auto it : allRes)
 244       ret.push_back(it.first);
 245     return ret;
 246   }
 247 }
 248 };
 249
 250 %inline
 251 {
 252   std::shared_ptr<ResourcesManager_cpp> HandleToLocalInstance(const std::string& ptrInStringFrmt);
 253 }
 254
 255 %exception
 256 {
 257   try
 258   {
 259     $function
 260   }
 261   catch (LauncherException& e)
 262   {
 263     SWIG_exception_fail(SWIG_RuntimeError, e.msg.c_str());
 264   }
 265   catch(...)
 266   {
 267     SWIG_exception_fail(SWIG_RuntimeError,"Unknown exception");
 268   }
 269 }
 270
 271 class Launcher_cpp
 272 {
 273 public:
 274   Launcher_cpp();
 275   virtual ~Launcher_cpp();
 276   int          createJob(const JobParameters_cpp& job_parameters);
 277   void         launchJob(int job_id);
 278   std::string  getJobState(int job_id);
 279   std::string  getAssignedHostnames(int job_id); // Get names or ids of hosts assigned to the job
 280   void         exportInputFiles(int job_id);
 281   void         getJobResults(int job_id, std::string directory);
 282   void         clearJobWorkingDir(int job_id);
 283   bool         getJobDumpState(int job_id, std::string directory);
 284   bool         getJobWorkFile(int job_id, std::string work_file, std::string directory);
 285   void         stopJob(int job_id);
 286   void         removeJob(int job_id);
 287   std::string  dumpJob(int job_id);
 288   int restoreJob(const std::string& dumpedJob);
 289   JobParameters_cpp getJobParameters(int job_id);
 290   std::list<int> loadJobs(const char* jobs_file);
 291   void saveJobs(const char* jobs_file);
 292   long createJobWithFile(std::string xmlExecuteFile, std::string clusterName);
 293   void SetResourcesManager(std::shared_ptr<ResourcesManager_cpp>& rm );
 294 };
 295
 296 %pythoncode %{
 297 def CreateSSHContainerResource(hostname,applipath,nbOfNodes=1):
 298   return CreateContainerResource(hostname,applipath,"ssh",nbOfNodes)
 299
 300 def CreateSRUNContainerResource(hostname,applipath,nbOfNodes=1):
 301   return CreateContainerResource(hostname,applipath,"srun",nbOfNodes)
 302
 303 def CreateContainerResource(hostname,applipath,protocol,nbOfNodes=1):
 304   import getpass
 305   ret = ResourceDefinition_cpp()
 306   ret.name = hostname.split(".")[0]
 307   ret.hostname = ret.name
 308   ret.protocol = protocol
 309   ret.applipath = applipath
 310   ret.nb_node = nbOfNodes
 311   ret.nb_proc_per_node = 1
 312   ret.can_run_containers = True
 313   ret.can_launch_batch_jobs = False
 314   ret.mpiImpl = "no mpi"
 315   ret.iprotocol = protocol
 316   ret.type = "single_machine"
 317   ret.username = getpass.getuser()
 318   return ret
 319
 320 def ResourceDefinition_cpp_repr(self):
 321   pat0 = "{} = {}"
 322   pat1 = "{} = \"{}\""
 323   data = [("name","name",pat0),
 324   ("hostname","hostname",pat0),
 325   ("type","type",pat0),
 326   ("protocol","protocol",pat0),
 327   ("userName","username",pat0),
 328   ("appliPath","applipath",pat1),
 329   ("mpi","mpiImpl",pat0),
 330   ("nbOfNodes","nb_node",pat0),
 331   ("nbOfProcPerNode","nb_proc_per_node",pat0),
 332   ("canRunContainer","can_run_containers",pat0)
 333   ]
 334   ret = [c.format(a,getattr(self,b)) for a,b,c in data]
 335   return "\n".join( ret )
 336
 337 def ResourcesManager_cpp_GetList(self):
 338   return {name:self.GetResourceDefinition(name) for name in self.GetListOfEntries()}
 339
 340 def ResourcesManager_cpp___getitem__(self,name):
 341   return self.GetResourceDefinition(name)
 342
 343 def ResourcesManager_cpp___repr__(self):
 344   return str( self.GetList() )
 345
 346 def RetrieveRMCppSingleton():
 347   import KernelLauncher
 348   return HandleToLocalInstance( KernelLauncher.RetrieveInternalInstanceOfLocalCppResourcesManager() )
 349
 350 def GetPlayGroundInsideASlurmJob():
 351   import subprocess as sp
 352   cont = sp.check_output(["srun","hostname"])
 353   nodesMul = [elt for elt in cont.decode().split("\n") if elt != ""]
 354   from collections import defaultdict
 355   d = defaultdict(int)
 356   for elt in nodesMul:
 357       d[elt]+=1
 358   return d
 359
 360 def BuildCatalogFromScratch(protocol):
 361   import os
 362   d = GetPlayGroundInsideASlurmJob()
 363   rmcpp = RetrieveRMCppSingleton()
 364   rmcpp.DeleteAllResourcesInCatalog()
 365   for k,v in d.items():
 366       contRes = CreateContainerResource(hostname=k,applipath=os.environ["APPLI"],protocol=protocol,nbOfNodes=v)
 367       rmcpp.AddResourceInCatalog(contRes)
 368
 369 def GetRequestForGiveContainer(hostname, contName):
 370   import Engines
 371   import os
 372   rp=Engines.ResourceParameters(name=hostname,
 373                                 hostname=hostname,
 374                                 can_launch_batch_jobs=False,
 375                                 can_run_containers=True,
 376                                 OS="Linux",
 377                                 componentList=[],
 378                                 nb_proc=1,
 379                                 mem_mb=1000,
 380                                 cpu_clock=1000,
 381                                 nb_node=1,
 382                                 nb_proc_per_node=1,
 383                                 policy="first",
 384                                 resList=[])
 385
 386   cp=Engines.ContainerParameters(container_name=contName,
 387                                   mode="start",
 388                                   workingdir=os.path.expanduser("~"),
 389                                   nb_proc=1,
 390                                   isMPI=False,
 391                                   parallelLib="",
 392                                   resource_params=rp)
 393   return cp
 394
 395 ResourceDefinition_cpp.repr = ResourceDefinition_cpp_repr
 396 ResourceDefinition_cpp.__repr__ = ResourceDefinition_cpp_repr
 397 ResourcesManager_cpp.GetList = ResourcesManager_cpp_GetList
 398 ResourcesManager_cpp.__getitem__ = ResourcesManager_cpp___getitem__
 399 ResourcesManager_cpp.__repr__ = ResourcesManager_cpp___repr__
 400 %}