1 // Copyright (C) 2007-2022 CEA/DEN, EDF R&D, OPEN CASCADE
3 // Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN,
4 // CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS
6 // This library is free software; you can redistribute it and/or
7 // modify it under the terms of the GNU Lesser General Public
8 // License as published by the Free Software Foundation; either
9 // version 2.1 of the License, or (at your option) any later version.
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 // Lesser General Public License for more details.
16 // You should have received a copy of the GNU Lesser General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
23 #include "SALOME_ContainerManager.hxx"
24 #include "SALOME_ResourcesManager.hxx"
25 #include "SALOME_LoadRateManager.hxx"
26 #include "SALOME_NamingService.hxx"
27 #include "SALOME_ResourcesManager_Client.hxx"
28 #include "SALOME_Embedded_NamingService.hxx"
29 #include "SALOME_ModuleCatalog.hh"
30 #include "Basics_Utils.hxx"
31 #include "Basics_DirUtils.hxx"
32 #include "PythonCppUtils.hxx"
33 #include <sys/types.h>
40 #include "Utils_CorbaException.hxx"
45 #include <SALOMEconfig.h>
46 #include CORBA_CLIENT_HEADER(SALOME_Session)
55 #define getpid _getpid
58 #define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG)
63 #ifdef WITH_PACO_PARALLEL
67 const int SALOME_ContainerManager::TIME_OUT_TO_LAUNCH_CONT=60;
69 const char *SALOME_ContainerManager::_ContainerManagerNameInNS =
72 omni_mutex SALOME_ContainerManager::_numInstanceMutex;
74 Utils_Mutex SALOME_ContainerManager::_getenvMutex;
76 Utils_Mutex SALOME_ContainerManager::_systemMutex;
78 //=============================================================================
82 * Define a CORBA single thread policy for the server, which avoid to deal
83 * with non thread-safe usage like Change_Directory in SALOME naming service
85 //=============================================================================
87 SALOME_ContainerManager::SALOME_ContainerManager(CORBA::ORB_ptr orb, PortableServer::POA_var poa, SALOME_NamingService_Abstract *ns)
90 MESSAGE("constructor");
92 _resManager = new SALOME_ResourcesManager_Client(ns);
94 PortableServer::POAManager_var pman = poa->the_POAManager();
95 _orb = CORBA::ORB::_duplicate(orb) ;
96 CORBA::PolicyList policies;
98 PortableServer::ThreadPolicy_var threadPol(poa->create_thread_policy(PortableServer::ORB_CTRL_MODEL));
99 policies[0] = PortableServer::ThreadPolicy::_duplicate(threadPol);
101 _poa = poa->create_POA("MThreadPOA",pman,policies);
102 threadPol->destroy();
103 PortableServer::ObjectId_var id = _poa->activate_object(this);
104 CORBA::Object_var obj = _poa->id_to_reference(id);
105 Engines::ContainerManager_var refContMan = Engines::ContainerManager::_narrow(obj);
107 _NS->Register(refContMan,_ContainerManagerNameInNS);
108 _isAppliSalomeDefined = (GetenvThreadSafe("APPLI") != 0);
113 // the urifile name depends on pid of the process
114 std::stringstream urifile;
115 urifile << GetenvThreadSafeAsString("HOME") << "/.urifile_" << getpid();
116 setenv("OMPI_URI_FILE",urifile.str().c_str(),1);
117 if( GetenvThreadSafe("OMPI_URI_FILE") != NULL ){
118 // Linux specific code
119 pid_t pid = fork(); // spawn a child process, following code is executed in both processes
120 if ( pid == 0 ) // I'm a child, replace myself with a new ompi-server
122 std::string uriarg = GetenvThreadSafeAsString("OMPI_URI_FILE");
123 execlp( "ompi-server", "ompi-server", "-r", uriarg.c_str(), NULL );
124 throw SALOME_Exception("Error when launching ompi-server"); // execlp failed
128 throw SALOME_Exception("fork() failed");
132 //wait(NULL); // wait(?) for a child end
133 _pid_mpiServer = pid;
138 // Linux specific code
139 pid_t pid = fork(); // spawn a child process, following code is executed in both processes
140 if ( pid == 0 ) // I'm a child, replace myself with a new hydra_nameserver
142 execlp( "hydra_nameserver", "hydra_nameserver", NULL );
143 throw SALOME_Exception("Error when launching hydra_nameserver"); // execlp failed
147 throw SALOME_Exception("fork() failed");
152 _pid_mpiServer = pid;
157 MESSAGE("constructor end");
160 //=============================================================================
164 //=============================================================================
166 SALOME_ContainerManager::~SALOME_ContainerManager()
168 MESSAGE("destructor");
172 if( GetenvThreadSafe("OMPI_URI_FILE") != NULL ){
173 // kill my ompi-server
174 if( kill(_pid_mpiServer,SIGTERM) != 0 )
175 throw SALOME_Exception("Error when killing ompi-server");
177 int status=SystemThreadSafe("rm -f ${OMPI_URI_FILE}");
179 throw SALOME_Exception("Error when removing urifile");
182 // kill my hydra_nameserver
183 if(_pid_mpiServer > -1)
184 if( kill(_pid_mpiServer,SIGTERM) != 0 )
185 throw SALOME_Exception("Error when killing hydra_nameserver");
190 //=============================================================================
191 //! shutdown all the containers, then the ContainerManager servant
194 //=============================================================================
196 void SALOME_ContainerManager::Shutdown()
199 ShutdownContainers();
201 _NS->Destroy_Name(_ContainerManagerNameInNS);
202 PortableServer::ObjectId_var oid = _poa->servant_to_id(this);
203 _poa->deactivate_object(oid);
206 //=============================================================================
207 //! Loop on all the containers listed in naming service, ask shutdown on each
210 //=============================================================================
212 void SALOME_ContainerManager::ShutdownContainers()
214 MESSAGE("ShutdownContainers");
217 SALOME::Session_var session = SALOME::Session::_nil();
219 CORBA::Object_var objS = _NS->Resolve("/Kernel/Session");
220 if (!CORBA::is_nil(objS))
222 session = SALOME::Session::_narrow(objS);
223 if (!CORBA::is_nil(session))
224 pid = session->getPID();
228 isOK = _NS->Change_Directory("/Containers");
230 std::vector<std::string> vec = _NS->list_directory_recurs();
231 std::list<std::string> lstCont;
232 for(std::vector<std::string>::iterator iter = vec.begin();iter!=vec.end();iter++)
235 CORBA::Object_var obj=_NS->Resolve((*iter).c_str());
238 Engines::Container_var cont=Engines::Container::_narrow(obj);
239 if(!CORBA::is_nil(cont) && pid != cont->getPID())
240 lstCont.push_back((*iter));
242 catch(const CORBA::Exception&)
244 // ignore this entry and continue
247 MESSAGE("Container list: ");
248 for(std::list<std::string>::iterator iter=lstCont.begin();iter!=lstCont.end();iter++){
251 for(std::list<std::string>::iterator iter=lstCont.begin();iter!=lstCont.end();iter++)
256 CORBA::Object_var obj=_NS->Resolve((*iter).c_str());
257 Engines::Container_var cont=Engines::Container::_narrow(obj);
258 if(!CORBA::is_nil(cont))
260 MESSAGE("ShutdownContainers: " << (*iter));
264 MESSAGE("ShutdownContainers: no container ref for " << (*iter));
266 catch(CORBA::SystemException& e)
268 INFOS("CORBA::SystemException ignored : " << e);
270 catch(CORBA::Exception&)
272 INFOS("CORBA::Exception ignored.");
276 INFOS("Unknown exception ignored.");
282 //=============================================================================
283 //! Give a suitable Container given constraints
285 * \param params Container Parameters required for the container
286 * \return the container or nil
288 //=============================================================================
289 Engines::Container_ptr SALOME_ContainerManager::GiveContainer(const Engines::ContainerParameters& params)
291 std::string machFile;
292 Engines::Container_ptr ret(Engines::Container::_nil());
294 // Step 0: Default mode is start
295 Engines::ContainerParameters local_params(params);
296 if (std::string(local_params.mode.in()) == "")
297 local_params.mode = CORBA::string_dup("start");
298 std::string mode = local_params.mode.in();
299 MESSAGE("[GiveContainer] starting with mode: " << mode);
301 // Step 1: Find Container for find and findorstart mode
302 if (mode == "find" || mode == "findorstart")
304 ret = FindContainer(params, params.resource_params.resList);
305 if(!CORBA::is_nil(ret))
311 MESSAGE("[GiveContainer] no container found");
321 // Step 2: Get all possibleResources from the parameters
322 // Consider only resources that can run containers
323 resourceParams resource_params = resourceParameters_CORBAtoCPP(local_params.resource_params);
324 resource_params.can_run_containers = true;
325 std::vector<std::string> possibleResources = _resManager->GetFittingResources(resource_params);
326 MESSAGE("[GiveContainer] - length of possible resources " << possibleResources.size());
327 std::vector<std::string> local_resources;
329 // Step 3: if mode is "get" keep only machines with existing containers
332 for(unsigned int i=0; i < possibleResources.size(); i++)
334 Engines::Container_ptr cont = FindContainer(params, possibleResources[i]);
337 if(!cont->_non_existent())
338 local_resources.push_back(possibleResources[i]);
340 catch(CORBA::Exception&) {}
343 // if local_resources is empty, we cannot give a container
344 if (local_resources.size() == 0)
346 MESSAGE("[GiveContainer] cannot find a container for mode get");
351 local_resources = possibleResources;
353 // Step 4: select the resource where to get/start the container
354 bool resource_available = true;
355 std::string resource_selected;
356 std::vector<std::string> resources = local_resources;
357 while (resource_available)
359 if (resources.size() == 0)
360 resource_available = false;
365 resource_selected = _resManager->Find(params.resource_params.policy.in(), resources);
366 // Remove resource_selected from vector
367 std::vector<std::string>::iterator it;
368 for (it=resources.begin() ; it < resources.end(); it++ )
369 if (*it == resource_selected)
375 catch(const SALOME_Exception &ex) //!< TODO: unused variable
377 MESSAGE("[GiveContainer] Exception in ResourceManager find !: " << ex.what());
380 MESSAGE("[GiveContainer] Resource selected is: " << resource_selected);
382 // Step 5: Create container name
383 ParserResourcesType resource_definition = _resManager->GetResourceDefinition(resource_selected);
384 std::string hostname(resource_definition.HostName);
385 std::string containerNameInNS;
387 int nbproc = params.nb_proc <= 0 ? 1 : params.nb_proc;
390 if( GetenvThreadSafe("LIBBATCH_NODEFILE") != NULL )
391 machFile = machinesFile(nbproc);
393 catch(const SALOME_Exception & ex)
395 std::string err_msg = ex.what();
396 err_msg += params.container_name;
397 INFOS(err_msg.c_str());
400 // A mpi parallel container register on zero node in NS
401 std::string mpiZeroNode = GetMPIZeroNode(resource_selected,machFile).c_str();
402 containerNameInNS = _NS->BuildContainerNameForNS(params, mpiZeroNode.c_str());
405 containerNameInNS = _NS->BuildContainerNameForNS(params, hostname.c_str());
406 MESSAGE("[GiveContainer] Container name in the naming service: " << containerNameInNS);
408 // Step 6: check if the name exists in naming service
409 //if params.mode == "getorstart" or "get" use the existing container
410 //if params.mode == "start" shutdown the existing container before launching a new one with that name
412 { // critical section
413 Utils_Locker lock (&_giveContainerMutex1);
414 CORBA::Object_var obj = _NS->Resolve(containerNameInNS.c_str());
415 if (!CORBA::is_nil(obj))
419 Engines::Container_var cont=Engines::Container::_narrow(obj);
420 if(!cont->_non_existent())
422 if(std::string(params.mode.in())=="getorstart" || std::string(params.mode.in())=="get"){
423 return cont._retn(); /* the container exists and params.mode is getorstart or get use it*/
427 INFOS("[GiveContainer] A container is already registered with the name: " << containerNameInNS << ", shutdown the existing container");
428 cont->Shutdown(); // shutdown the registered container if it exists
432 catch(CORBA::Exception&)
434 INFOS("[GiveContainer] CORBA::Exception ignored when trying to get the container - we start a new one");
437 } // end critical section
438 Engines::Container_var cont = LaunchContainer(params, resource_selected, hostname, machFile, containerNameInNS);
439 if (!CORBA::is_nil(cont))
441 INFOS("[GiveContainer] container " << containerNameInNS << " launched");
446 INFOS("[GiveContainer] Failed to launch container on resource " << resource_selected);
451 // We were not able to launch the container
452 INFOS("[GiveContainer] Cannot launch the container on the following selected resources:")
453 std::vector<std::string>::iterator it;
454 for (it=local_resources.begin() ; it < local_resources.end(); it++ )
455 INFOS("[GiveContainer] " << *it)
459 std::string SALOME_ContainerManager::GetCppBinaryOfKernelContainer() const
461 std::string ret = this->_isSSL ? "SALOME_Container_No_NS_Serv" : "SALOME_Container";
465 std::string SALOME_ContainerManager::GetRunRemoteExecutableScript() const
467 std::string ret = this->_isSSL ? "runRemoteSSL.sh" : "runRemote.sh";
471 Engines::Container_ptr
472 SALOME_ContainerManager::LaunchContainer(const Engines::ContainerParameters& params,
473 const std::string & resource_selected,
474 const std::string & hostname,
475 const std::string & machFile,
476 const std::string & containerNameInNS)
478 std::string user,command,logFilename,tmpFileName;
480 Engines::Container_ptr ret(Engines::Container::_nil());
481 {//start of critical section
482 Utils_Locker lock (&_giveContainerMutex1);
483 // Step 1: type of container: PaCO, Exe, Mpi or Classic
484 // Mpi already tested in step 5, specific code on BuildCommandToLaunch Local/Remote Container methods
485 // TODO -> separates Mpi from Classic/Exe
487 std::string container_exe = this->GetCppBinaryOfKernelContainer();
488 Engines::ContainerParameters local_params(params);
492 CORBA::String_var container_exe_tmp;
493 CORBA::Object_var obj = _NS->Resolve("/Kernel/ModulCatalog");
494 SALOME_ModuleCatalog::ModuleCatalog_var Catalog = SALOME_ModuleCatalog::ModuleCatalog::_narrow(obj) ;
495 if (CORBA::is_nil (Catalog))
497 INFOS("[GiveContainer] Module Catalog is not found -> cannot launch a container");
500 // Loop through component list
501 for(unsigned int i=0; i < local_params.resource_params.componentList.length(); i++)
503 const char* compoi = local_params.resource_params.componentList[i];
504 SALOME_ModuleCatalog::Acomponent_var compoInfo = Catalog->GetComponent(compoi);
505 if (CORBA::is_nil (compoInfo))
509 SALOME_ModuleCatalog::ImplType impl=compoInfo->implementation_type();
510 container_exe_tmp=compoInfo->implementation_name();
511 if(impl==SALOME_ModuleCatalog::CEXE)
515 INFOS("ContainerManager Error: you can't have 2 CEXE component in the same container" );
516 return Engines::Container::_nil();
518 MESSAGE("[GiveContainer] Exe container found !: " << container_exe_tmp);
519 container_exe = container_exe_tmp.in();
524 catch (ServiceUnreachable&)
526 INFOS("Caught exception: Naming Service Unreachable");
531 INFOS("Caught unknown exception.");
535 // Step 2: test resource
536 // Only if an application directory is set
537 if(hostname != Kernel_Utils::GetHostname() && _isAppliSalomeDefined)
540 const ParserResourcesType resInfo(_resManager->GetResourceDefinition(resource_selected));
541 std::string command = getCommandToRunRemoteProcess(resInfo.Protocol, resInfo.HostName,
542 resInfo.UserName, resInfo.AppliPath);
544 // Launch remote command
545 command += " \"ls /tmp >/dev/null 2>&1\"";
546 // Anthony : command is NO MORE launched to improve dramatically time to launch containers
550 // Error on resource - cannot launch commands
551 INFOS("[LaunchContainer] Cannot launch commands on machine " << hostname);
552 INFOS("[LaunchContainer] Command was " << command);
554 INFOS("[LaunchContainer] Command status is " << WEXITSTATUS(status));
556 return Engines::Container::_nil();
560 // Step 3: start a new container
561 // Check if a PaCO container
563 if (std::string(local_params.parallelLib.in()) != "")
565 ret = StartPaCOPPContainer(params, resource_selected);
568 // Other type of containers...
569 MESSAGE("[GiveContainer] Try to launch a new container on " << resource_selected);
570 // if a parallel container is launched in batch job, command is: "mpirun -np nbproc -machinefile nodesfile SALOME_MPIContainer"
571 if( GetenvThreadSafe("LIBBATCH_NODEFILE") != NULL && params.isMPI )
572 command = BuildCommandToLaunchLocalContainer(params, machFile, container_exe, tmpFileName);
573 // if a container is launched on localhost, command is "SALOME_Container" or "mpirun -np nbproc SALOME_MPIContainer"
574 else if(hostname == Kernel_Utils::GetHostname())
575 command = BuildCommandToLaunchLocalContainer(params, machFile, container_exe, tmpFileName);
576 // if a container is launched in remote mode, command is "ssh resource_selected SALOME_Container" or "ssh resource_selected mpirun -np nbproc SALOME_MPIContainer"
578 command = BuildCommandToLaunchRemoteContainer(resource_selected, params, container_exe);
580 //redirect stdout and stderr in a file
582 logFilename=GetenvThreadSafeAsString("TEMP");
584 user = GetenvThreadSafeAsString( "USERNAME" );
586 user = GetenvThreadSafeAsString( "USER" );
588 user = GetenvThreadSafeAsString( "LOGNAME" );
590 char* val = GetenvThreadSafe("SALOME_TMP_DIR");
593 struct stat file_info;
594 stat(val, &file_info);
595 bool is_dir = S_ISDIR(file_info.st_mode);
596 if (is_dir)logFilename=val;
597 else std::cerr << "SALOME_TMP_DIR environment variable is not a directory use /tmp instead" << std::endl;
601 logFilename += _NS->ContainerName(params)+"_"+ resource_selected +"_"+user;
602 std::ostringstream tmp;
603 tmp << "_" << getpid();
604 logFilename += tmp.str();
605 logFilename += ".log" ;
606 command += " > " + logFilename + " 2>&1";
607 MakeTheCommandToBeLaunchedASync(command);
609 // launch container with a system call
610 status=SystemThreadSafe(command.c_str());
611 }//end of critical of section
615 INFOS("[LaunchContainer] command failed (system command status -1): " << command);
616 RmTmpFile(tmpFileName); // command file can be removed here
617 return Engines::Container::_nil();
619 else if (status == 217)
621 INFOS("[LaunchContainer] command failed (system command status 217): " << command);
622 RmTmpFile(tmpFileName); // command file can be removed here
623 return Engines::Container::_nil();
627 // Step 4: Wait for the container
628 int count(GetTimeOutToLoaunchServer());
629 INFOS("[GiveContainer] waiting " << count << " second steps container " << containerNameInNS);
630 while (CORBA::is_nil(ret) && count)
634 MESSAGE("[GiveContainer] step " << count << " Waiting for container on " << resource_selected);
635 CORBA::Object_var obj(_NS->Resolve(containerNameInNS.c_str()));
636 ret=Engines::Container::_narrow(obj);
638 if (CORBA::is_nil(ret))
640 INFOS("[GiveContainer] was not able to launch container " << containerNameInNS);
644 // Setting log file name
645 logFilename=":"+logFilename;
646 logFilename="@"+Kernel_Utils::GetHostname()+logFilename;//threadsafe
647 logFilename=user+logFilename;
648 ret->logfilename(logFilename.c_str());
649 RmTmpFile(tmpFileName); // command file can be removed here
655 //=============================================================================
656 //! Find a container given constraints (params) on a list of machines (possibleComputers)
657 //! agy : this method is ThreadSafe
661 //=============================================================================
663 Engines::Container_ptr SALOME_ContainerManager::FindContainer(const Engines::ContainerParameters& params, const Engines::ResourceList& possibleResources)
665 MESSAGE("[FindContainer] FindContainer on " << possibleResources.length() << " resources");
666 for(unsigned int i=0; i < possibleResources.length();i++)
668 Engines::Container_ptr cont = FindContainer(params, possibleResources[i].in());
669 if(!CORBA::is_nil(cont))
672 MESSAGE("[FindContainer] no container found");
673 return Engines::Container::_nil();
676 //=============================================================================
677 //! Find a container given constraints (params) on a machine (theMachine)
678 //! agy : this method is ThreadSafe
682 //=============================================================================
684 Engines::Container_ptr
685 SALOME_ContainerManager::FindContainer(const Engines::ContainerParameters& params, const std::string& resource)
687 ParserResourcesType resource_definition = _resManager->GetResourceDefinition(resource);
688 std::string hostname(resource_definition.HostName);
689 std::string containerNameInNS(_NS->BuildContainerNameForNS(params, hostname.c_str()));
690 MESSAGE("[FindContainer] Try to find a container " << containerNameInNS << " on resource " << resource);
691 CORBA::Object_var obj = _NS->Resolve(containerNameInNS.c_str());
694 if(obj->_non_existent())
695 return Engines::Container::_nil();
697 return Engines::Container::_narrow(obj);
699 catch(const CORBA::Exception&)
701 return Engines::Container::_nil();
706 bool isPythonContainer(const char* ContainerName);
708 //=============================================================================
710 * This is no longer valid (C++ container are also python containers)
712 //=============================================================================
713 bool isPythonContainer(const char* ContainerName)
715 return false; // VSR 02/08/2013: Python containers are no more supported
717 size_t len = strlen(ContainerName);
720 if (strcmp(ContainerName + len - 2, "Py") == 0)
726 //=============================================================================
728 * Builds the script to be launched
730 * If SALOME Application not defined ($APPLI),
731 * see BuildTempFileToLaunchRemoteContainer()
733 * Else rely on distant configuration. Command is under the form (example):
734 * ssh user@machine distantPath/runRemote.sh hostNS portNS WORKINGDIR workingdir \
735 * SALOME_Container containerName &"
737 * - where user is omitted if not specified in CatalogResources,
738 * - where distant path is always relative to user@machine $HOME, and
739 * equal to $APPLI if not specified in CatalogResources,
740 * - where hostNS is the hostname of CORBA naming server (set by scripts to
741 * use to launch SALOME and servers in $APPLI: runAppli.sh, runRemote.sh)
742 * - where portNS is the port used by CORBA naming server (set by scripts to
743 * use to launch SALOME and servers in $APPLI: runAppli.sh, runRemote.sh)
744 * - where workingdir is the requested working directory for the container.
745 * If WORKINGDIR (and workingdir) is not present the working dir will be $HOME
747 //=============================================================================
750 SALOME_ContainerManager::BuildCommandToLaunchRemoteContainer(const std::string& resource_name, const Engines::ContainerParameters& params, const std::string& container_exe) const
752 std::string command,tmpFileName;
753 const ParserResourcesType resInfo(_resManager->GetResourceDefinition(resource_name));
754 std::string wdir = params.workingdir.in();
755 if (!_isAppliSalomeDefined)
757 command = getCommandToRunRemoteProcessNoAppli(resInfo.Protocol, resInfo.HostName,
758 resInfo.UserName, resInfo.AppliPath,
763 // "ssh -l user machine distantPath/runRemote.sh hostNS portNS WORKINGDIR workingdir
764 // SALOME_Container containerName -ORBInitRef NameService=IOR:01000..."
766 // "ssh -l user machine distantLauncher remote -p hostNS -m portNS -d dir
767 // -- SALOME_Container contName -ORBInitRef NameService=IOR:01000..."
768 command = getCommandToRunRemoteProcess(resInfo.Protocol, resInfo.HostName,
769 resInfo.UserName, resInfo.AppliPath,
774 int nbproc = params.nb_proc <= 0 ? 1 : params.nb_proc;
775 command += " mpirun -np ";
776 std::ostringstream o;
780 command += "-x PATH,LD_LIBRARY_PATH,OMNIORB_CONFIG,SALOME_trace ";
781 #elif defined(OPEN_MPI)
782 if( GetenvThreadSafe("OMPI_URI_FILE") == NULL )
783 command += "-x PATH -x LD_LIBRARY_PATH -x OMNIORB_CONFIG -x SALOME_trace";
785 command += "-x PATH -x LD_LIBRARY_PATH -x OMNIORB_CONFIG -x SALOME_trace -ompi-server file:";
786 command += GetenvThreadSafeAsString("OMPI_URI_FILE");
789 command += "-nameserver " + Kernel_Utils::GetHostname();
791 command += " SALOME_MPIContainer ";
794 command += " " +container_exe+ " ";
796 command += _NS->ContainerName(params) + " ";
799 Engines::EmbeddedNamingService_var ns = GetEmbeddedNamingService();
800 CORBA::String_var iorNS = _orb->object_to_string(ns);
801 command += std::string(iorNS);
803 else //if(!this->_isSSL)
806 AddOmninamesParams(command);
808 MESSAGE("command =" << command);
813 //=============================================================================
814 //! Return a path to the directory with scripts templates
816 * \return the path pointed by SALOME_KERNEL_SCRIPTS_DIR environment variable, if it is defined,
817 * ${KERNEL_ROOT_DIR}/share/salome/resources/separator/kernel/ScriptsTemplate - otherwise
819 //=============================================================================
820 std::string getScriptTemplateFilePath()
822 auto parseScriptTemplateFilePath = []() -> std::string
824 std::string scriptTemplateFilePath = SALOME_ContainerManager::GetenvThreadSafeAsString("SALOME_KERNEL_SCRIPTS_DIR");
825 if (!scriptTemplateFilePath.empty())
827 return scriptTemplateFilePath;
830 return SALOME_ContainerManager::GetenvThreadSafeAsString("KERNEL_ROOT_DIR") +
831 "/share/salome/resources/kernel/ScriptsTemplate";
835 static const std::string scriptTemplateFilePath = parseScriptTemplateFilePath();
836 return scriptTemplateFilePath;
839 //=============================================================================
840 //! Return a command line constructed based on Python scripts templates
842 * \param theScriptName the name of Python script template
843 * \param theScriptParameters the queue of parameter values
844 * \return the command line constructed according to the given parameters
846 //=============================================================================
847 std::string GetCommandFromTemplate(const std::string& theScriptName,
848 std::queue<std::string>& theScriptParameters)
854 PyObject* mod(PyImport_ImportModule(theScriptName.c_str()));
857 PyObject* sys = PyImport_ImportModule("sys");
858 PyObject* sys_path = PyObject_GetAttrString(sys, "path");
859 PyObject* folder_path = PyUnicode_FromString(getScriptTemplateFilePath().c_str());
860 PyList_Append(sys_path, folder_path);
862 mod = PyImport_ImportModule(theScriptName.c_str());
864 Py_XDECREF(folder_path);
865 Py_XDECREF(sys_path);
871 PyObject* meth(PyObject_GetAttrString(mod, "command"));
879 PyObject* tuple(PyTuple_New(theScriptParameters.size()));
881 auto insert_parameter = [&tuple, &theScriptParameters, &id]()
883 if (!theScriptParameters.empty())
885 PyTuple_SetItem(tuple, ++id, PyUnicode_FromString(theScriptParameters.front().c_str()));
886 theScriptParameters.pop();
890 while (!theScriptParameters.empty())
895 PyObject *args(PyTuple_New(1));
896 PyTuple_SetItem(args, 0, tuple);
898 PyObject *res(PyObject_CallObject(meth, args));
901 command = PyUnicode_AsUTF8(res);
912 MESSAGE("Command from template is ... " << command << std::endl);
915 //=============================================================================
917 //=============================================================================
919 * builds the command to be launched.
921 //=============================================================================
922 std::string SALOME_ContainerManager::BuildCommandToLaunchLocalContainer(const Engines::ContainerParameters& params, const std::string& machinesFile, const std::string& container_exe, std::string& tmpFileName) const
924 // Prepare name of the script to be used
925 std::string script_name = "SALOME_CM_LOCAL_NO_MPI";
929 script_name = "SALOME_CM_LOCAL_MPI_LAN";
930 #elif defined(OPEN_MPI)
931 script_name = "SALOME_CM_LOCAL_MPI_OPENMPI";
933 script_name = "SALOME_CM_LOCAL_MPI_MPICH";
937 // Prepare parameters to use in the Python script:
938 // 1. All parameters are strings.
939 // 2. For some booleans use "1" = True, "0" = False.
940 // 3. If a parameter is NULL, then its value is "NULL".
942 std::queue<std::string> script_parameters;
944 // ===== Number of processes (key = "nb_proc")
945 script_parameters.push(params.isMPI ? std::to_string(params.nb_proc <= 0 ? 1 : params.nb_proc) : "NULL");
947 // ===== Working directory (key = "workdir") and temporary directory flag (key = "isTmpDir")
948 // A working directory is requested
949 std::string workdir = params.workingdir.in();
950 std::string isTmpDir = std::to_string(0);
951 if (workdir == "$TEMPDIR")
953 // A new temporary directory is requested
954 isTmpDir = std::to_string(1);
955 workdir = Kernel_Utils::GetTmpDir();
957 script_parameters.push(workdir);
958 script_parameters.push(isTmpDir);
960 // ===== Server name (key = "name_server")
961 script_parameters.push(Kernel_Utils::GetHostname());
963 // ===== Container (key = "container")
964 std::string container;
967 container = isPythonContainer(params.container_name) ? "pyMPI SALOME_ContainerPy.py" : "SALOME_MPIContainer";
971 container = isPythonContainer(params.container_name) ? "SALOME_ContainerPy.py" : container_exe;
973 script_parameters.push(container);
975 // ===== Container name (key = "container_name")
976 script_parameters.push(_NS->ContainerName(params));
978 // ===== LIBBATCH node file (key = "libbatch_nodefile")
979 script_parameters.push(std::to_string(GetenvThreadSafe("LIBBATCH_NODEFILE") != NULL ? 1 : 0));
981 // ===== Machine file (key = "machine_file")
982 script_parameters.push(machinesFile.empty() ? "NULL" : machinesFile);
984 // ===== OMPI uri file (key = "ompi_uri_file")
985 std::string ompi_uri_file = GetenvThreadSafeAsString("OMPI_URI_FILE");
986 script_parameters.push(ompi_uri_file.empty() ? "NULL" : ompi_uri_file);
988 std::string command_from_template = GetCommandFromTemplate(script_name, script_parameters);
990 std::ostringstream o;
991 o << command_from_template << " ";
993 //==================================================================================== */
997 Engines::EmbeddedNamingService_var ns = GetEmbeddedNamingService();
998 CORBA::String_var iorNS = _orb->object_to_string(ns);
1004 AddOmninamesParams(o);
1007 tmpFileName = BuildTemporaryFileName();
1008 std::ofstream command_file( tmpFileName.c_str() );
1009 command_file << o.str();
1010 command_file.close();
1013 chmod(tmpFileName.c_str(), 0x1ED);
1016 std::string command = tmpFileName;
1017 MESSAGE("Command is file ... " << command);
1018 MESSAGE("Command is ... " << o.str());
1023 //=============================================================================
1025 * removes the generated temporary file in case of a remote launch.
1026 * This method is thread safe
1028 //=============================================================================
1030 void SALOME_ContainerManager::RmTmpFile(std::string& tmpFileName)
1032 size_t length = tmpFileName.size();
1036 std::string command = "del /F ";
1038 std::string command = "rm ";
1041 command += tmpFileName.substr(0, length - 3 );
1043 command += tmpFileName;
1045 SystemThreadSafe(command.c_str());
1046 //if dir is empty - remove it
1047 std::string tmp_dir = Kernel_Utils::GetDirByPath( tmpFileName );
1048 if ( Kernel_Utils::IsEmptyDir( tmp_dir ) )
1051 command = "del /F " + tmp_dir;
1053 command = "rmdir " + tmp_dir;
1055 SystemThreadSafe(command.c_str());
1060 //=============================================================================
1062 * add to command all options relative to naming service.
1064 //=============================================================================
1066 void SALOME_ContainerManager::AddOmninamesParams(std::string& command) const
1068 std::ostringstream oss;
1069 AddOmninamesParams(oss);
1073 //=============================================================================
1075 * add to command all options relative to naming service.
1077 //=============================================================================
1079 void SALOME_ContainerManager::AddOmninamesParams(std::ostream& fileStream) const
1081 AddOmninamesParams(fileStream,_NS);
1084 //=============================================================================
1086 * add to command all options relative to naming service.
1088 //=============================================================================
1090 void SALOME_ContainerManager::AddOmninamesParams(std::ostream& fileStream, SALOME_NamingService_Abstract *ns)
1092 SALOME_NamingService *nsTrad(dynamic_cast<SALOME_NamingService *>(ns));
1095 CORBA::String_var iorstr(nsTrad->getIORaddr());
1096 fileStream << "ORBInitRef NameService=";
1097 fileStream << iorstr;
1101 void SALOME_ContainerManager::MakeTheCommandToBeLaunchedASync(std::string& command)
1104 command = "%PYTHONBIN% -c \"import subprocess ; subprocess.Popen(r'" + command + "').pid\"";
1110 int SALOME_ContainerManager::GetTimeOutToLoaunchServer()
1112 int count(TIME_OUT_TO_LAUNCH_CONT);
1113 if (GetenvThreadSafe("TIMEOUT_TO_LAUNCH_CONTAINER") != 0)
1115 std::string new_count_str(GetenvThreadSafeAsString("TIMEOUT_TO_LAUNCH_CONTAINER"));
1117 std::istringstream ss(new_count_str);
1118 if (!(ss >> new_count))
1120 INFOS("[LaunchContainer] TIMEOUT_TO_LAUNCH_CONTAINER should be an int");
1128 void SALOME_ContainerManager::SleepInSecond(int ellapseTimeInSecond)
1131 sleep( ellapseTimeInSecond ) ;
1133 int timeInMS(1000*ellapseTimeInSecond);
1138 //=============================================================================
1140 * generate a file name in /tmp directory
1142 //=============================================================================
1144 std::string SALOME_ContainerManager::BuildTemporaryFileName()
1146 //build more complex file name to support multiple salome session
1147 std::string aFileName = Kernel_Utils::GetTmpFileName();
1148 std::ostringstream str_pid;
1149 str_pid << ::getpid();
1150 aFileName = aFileName + "-" + str_pid.str();
1154 aFileName += ".bat";
1159 //=============================================================================
1161 * Builds in a temporary file the script to be launched.
1163 * Used if SALOME Application ($APPLI) is not defined.
1164 * The command is build with data from CatalogResources, in which every path
1165 * used on remote computer must be defined.
1167 //=============================================================================
1169 std::string SALOME_ContainerManager::BuildTempFileToLaunchRemoteContainer (const std::string& resource_name, const Engines::ContainerParameters& params, std::string& tmpFileName) const
1173 tmpFileName = BuildTemporaryFileName();
1174 std::ofstream tempOutputFile;
1175 tempOutputFile.open(tmpFileName.c_str(), std::ofstream::out );
1176 const ParserResourcesType resInfo(_resManager->GetResourceDefinition(resource_name));
1177 tempOutputFile << "#! /bin/sh" << std::endl;
1181 tempOutputFile << "export SALOME_trace=local" << std::endl; // mkr : 27.11.2006 : PAL13967 - Distributed supervision graphs - Problem with "SALOME_trace"
1182 //tempOutputFile << "source " << resInfo.PreReqFilePath << endl;
1188 int nbproc = params.nb_proc <= 0 ? 1 : params.nb_proc;
1190 tempOutputFile << "mpirun -np ";
1192 tempOutputFile << nbproc << " ";
1194 tempOutputFile << "-x PATH,LD_LIBRARY_PATH,OMNIORB_CONFIG,SALOME_trace ";
1195 #elif defined(OPEN_MPI)
1196 if( GetenvThreadSafe("OMPI_URI_FILE") == NULL )
1197 tempOutputFile << "-x PATH -x LD_LIBRARY_PATH -x OMNIORB_CONFIG -x SALOME_trace";
1199 tempOutputFile << "-x PATH -x LD_LIBRARY_PATH -x OMNIORB_CONFIG -x SALOME_trace -ompi-server file:";
1200 tempOutputFile << GetenvThreadSafeAsString("OMPI_URI_FILE");
1202 #elif defined(MPICH)
1203 tempOutputFile << "-nameserver " + Kernel_Utils::GetHostname();
1207 tempOutputFile << GetenvThreadSafeAsString("KERNEL_ROOT_DIR") << "/bin/salome/";
1211 if (isPythonContainer(params.container_name))
1212 tempOutputFile << " pyMPI SALOME_ContainerPy.py ";
1214 tempOutputFile << " SALOME_MPIContainer ";
1219 if (isPythonContainer(params.container_name))
1220 tempOutputFile << "SALOME_ContainerPy.py ";
1222 tempOutputFile << "SALOME_Container ";
1225 tempOutputFile << _NS->ContainerName(params) << " -";
1226 AddOmninamesParams(tempOutputFile);
1227 tempOutputFile << " &" << std::endl;
1228 tempOutputFile.flush();
1229 tempOutputFile.close();
1231 chmod(tmpFileName.c_str(), 0x1ED);
1234 // --- Build command
1236 std::string command;
1238 if (resInfo.Protocol == rsh)
1241 std::string commandRcp = "rcp ";
1242 commandRcp += tmpFileName;
1244 commandRcp += resInfo.HostName;
1246 commandRcp += tmpFileName;
1247 status = SystemThreadSafe(commandRcp.c_str());
1250 else if (resInfo.Protocol == ssh)
1253 std::string commandRcp = "scp ";
1254 commandRcp += tmpFileName;
1256 commandRcp += resInfo.HostName;
1258 commandRcp += tmpFileName;
1259 status = SystemThreadSafe(commandRcp.c_str());
1262 else if (resInfo.Protocol == srun)
1264 command = "srun -n 1 -N 1 -s --mem-per-cpu=0 --cpu-bind=none --nodelist=";
1265 std::string commandRcp = "rcp ";
1266 commandRcp += tmpFileName;
1268 commandRcp += resInfo.HostName;
1270 commandRcp += tmpFileName;
1271 status = SystemThreadSafe(commandRcp.c_str());
1274 throw SALOME_Exception("Unknown protocol");
1277 throw SALOME_Exception("Error of connection on remote host");
1279 command += resInfo.HostName;
1281 command += tmpFileName;
1289 std::string SALOME_ContainerManager::GetMPIZeroNode(const std::string machine, const std::string machinesFile) const
1292 std::string zeronode;
1293 std::string command;
1294 std::string tmpFile = BuildTemporaryFileName();
1295 const ParserResourcesType resInfo(_resManager->GetResourceDefinition(machine));
1297 if(resInfo.Protocol == sh)
1299 return resInfo.HostName;
1302 if( GetenvThreadSafe("LIBBATCH_NODEFILE") == NULL )
1304 if (_isAppliSalomeDefined)
1306 command = getCommandToRunRemoteProcess(resInfo.Protocol, resInfo.HostName,
1307 resInfo.UserName, resInfo.AppliPath);
1308 command += " mpirun -np 1 hostname -s > " + tmpFile;
1311 command = "mpirun -np 1 hostname -s > " + tmpFile;
1314 command = "mpirun -np 1 -machinefile " + machinesFile + " hostname -s > " + tmpFile;
1316 status = SystemThreadSafe(command.c_str());
1318 std::ifstream fp(tmpFile.c_str(),std::ios::in);
1319 while(fp >> zeronode);
1327 std::string SALOME_ContainerManager::machinesFile(const int nbproc)
1330 std::string nodesFile = GetenvThreadSafeAsString("LIBBATCH_NODEFILE");
1331 std::string machinesFile = Kernel_Utils::GetTmpFileName();
1332 std::ifstream fpi(nodesFile.c_str(),std::ios::in);
1333 std::ofstream fpo(machinesFile.c_str(),std::ios::out);
1335 _numInstanceMutex.lock();
1337 for(int i=0;i<_nbprocUsed;i++)
1340 for(int i=0;i<nbproc;i++)
1342 fpo << tmp << std::endl;
1344 throw SALOME_Exception("You need more processors than batch session have allocated for you! Unable to launch the mpi container: ");
1346 _nbprocUsed += nbproc;
1350 _numInstanceMutex.unlock();
1352 return machinesFile;
1356 std::string SALOME_ContainerManager::getCommandToRunRemoteProcessNoAppli(AccessProtocolType protocol, const std::string & hostname, const std::string & username, const std::string & applipath, const std::string & workdir) const
1358 return getCommandToRunRemoteProcessCommon("SALOME_CM_REMOTE","salome shell --",protocol,hostname,username,applipath,workdir);
1361 std::string SALOME_ContainerManager::getCommandToRunRemoteProcess(AccessProtocolType protocol, const std::string & hostname, const std::string & username, const std::string & applipath, const std::string & workdir) const
1363 return getCommandToRunRemoteProcessCommon("SALOME_CM_REMOTE_OLD",this->GetRunRemoteExecutableScript(),protocol,hostname,username,applipath,workdir);
1366 std::string SALOME_ContainerManager::getCommandToRunRemoteProcessCommon(const std::string& templateName,
1367 const std::string& remoteScript,
1368 AccessProtocolType protocol,
1369 const std::string & hostname,
1370 const std::string & username,
1371 const std::string & applipath,
1372 const std::string & workdir) const
1374 std::ostringstream command;
1376 // Prepare parameters to use in the Python script:
1377 // 1. All parameters are strings.
1378 // 2. For some booleans use "1" = True, "0" = False.
1379 // 3. If a parameter is NULL, then its value is "NULL".
1381 std::queue<std::string> script_parameters;
1383 // ===== Protocol (key = "protocol")
1384 std::string strProtocol;
1387 case rsh: strProtocol = "rsh"; break;
1388 case ssh: strProtocol = "ssh"; break;
1389 case srun: strProtocol = "srun"; break;
1390 case pbsdsh: strProtocol = "pbsdsh"; break;
1391 case blaunch: strProtocol = "blaunch"; break;
1393 throw SALOME_Exception("Unknown protocol");
1395 script_parameters.push(strProtocol);
1397 // ===== User name (key = "user")
1398 script_parameters.push(username.empty() ? "NULL" : username);
1400 // ===== Host name (key = "host")
1401 script_parameters.push(hostname.empty() ? "NULL" : hostname);
1404 // ===== Remote APPLI path (key = "appli")
1405 script_parameters.push(applipath.empty() ? GetenvThreadSafeAsString("APPLI") : applipath);
1409 ASSERT(GetenvThreadSafe("NSHOST"));
1410 ASSERT(GetenvThreadSafe("NSPORT"));
1413 struct stat statbuf;
1414 std::string appli_mode = (stat(GetenvThreadSafe("APPLI"), &statbuf) == 0 && S_ISREG(statbuf.st_mode)) ? "launcher" : "dir";
1416 // ===== Working directory (key = "workdir")
1417 script_parameters.push(workdir == "$TEMPDIR" ? "\\$TEMPDIR" : workdir);
1419 // ===== SSL (key = "ssl")
1420 script_parameters.push(this->_isSSL ? "1" : "0");
1422 // ===== Hostname of CORBA name server (key = "nshost")
1423 std::string nshost = GetenvThreadSafeAsString("NSHOST");
1424 script_parameters.push(nshost.empty() ? "NULL" : nshost);
1426 // ===== Port of CORBA name server (key = "nsport")
1427 std::string nsport = GetenvThreadSafeAsString("NSPORT");
1428 script_parameters.push(nsport.empty() ? "NULL" : nsport);
1430 // ===== Remote script (key = "remote_script")
1431 script_parameters.push(remoteScript.empty() ? "NONE" : remoteScript);
1433 // ===== Naming service (key = "naming_service")
1434 std::string namingService = "NONE";
1437 Engines::EmbeddedNamingService_var ns = GetEmbeddedNamingService();
1438 CORBA::String_var iorNS = _orb->object_to_string(ns);
1439 namingService = iorNS;
1441 script_parameters.push(namingService);
1443 // ===== APPLI mode (key = "appli_mode")
1444 // $APPLI points either to an application directory, or to a salome launcher file
1445 // we prepare the remote command according to the case
1446 script_parameters.push(appli_mode);
1448 command << GetCommandFromTemplate(templateName, script_parameters);
1450 return command.str();
1454 SALOME_ContainerManager::checkPaCOParameters(Engines::ContainerParameters & params, std::string resource_selected)
1458 // Step 1 : check ContainerParameters
1459 // Check container_name, has to be defined
1460 if (std::string(params.container_name.in()) == "")
1462 INFOS("[checkPaCOParameters] You must define a container_name to launch a PaCO++ container");
1465 // Check parallelLib
1466 std::string parallelLib = params.parallelLib.in();
1467 if (parallelLib != "Mpi" && parallelLib != "Dummy")
1469 INFOS("[checkPaCOParameters] parallelLib is not correctly defined");
1470 INFOS("[checkPaCOParameters] you can chosse between: Mpi and Dummy");
1471 INFOS("[checkPaCOParameters] you entered: " << parallelLib);
1475 if (params.nb_proc <= 0)
1477 INFOS("[checkPaCOParameters] You must define a nb_proc > 0");
1481 // Step 2 : check resource_selected
1482 const ParserResourcesType resource_definition = _resManager->GetResourceDefinition(resource_selected);
1483 //std::string protocol = resource_definition->protocol.in();
1484 std::string username = resource_definition.UserName;
1485 std::string applipath = resource_definition.AppliPath;
1487 //if (protocol == "" || username == "" || applipath == "")
1488 if (username == "" || applipath == "")
1490 INFOS("[checkPaCOParameters] resource selected is not well defined");
1491 INFOS("[checkPaCOParameters] resource name: " << resource_definition.Name);
1492 INFOS("[checkPaCOParameters] resource hostname: " << resource_definition.HostName);
1493 INFOS("[checkPaCOParameters] resource protocol: " << resource_definition.getAccessProtocolTypeStr());
1494 INFOS("[checkPaCOParameters] resource username: " << username);
1495 INFOS("[checkPaCOParameters] resource applipath: " << applipath);
1503 * :WARNING: Do not directly convert returned value to std::string
1504 * This function may return NULL if env variable is not defined.
1505 * And std::string(NULL) causes undefined behavior.
1506 * Use GetenvThreadSafeAsString to properly get a std::string.
1508 char *SALOME_ContainerManager::GetenvThreadSafe(const char *name)
1509 {// getenv is not thread safe. See man 7 pthread.
1510 Utils_Locker lock (&_getenvMutex);
1511 return getenv(name);
1515 * Return env variable as a std::string.
1516 * Return empty string if env variable is not set.
1518 std::string SALOME_ContainerManager::GetenvThreadSafeAsString(const char *name)
1520 char* var = GetenvThreadSafe(name);
1521 return var ? std::string(var) : std::string();
1524 int SALOME_ContainerManager::SystemThreadSafe(const char *command)
1526 Utils_Locker lock (&_systemMutex);
1527 return system(command);
1530 long SALOME_ContainerManager::SystemWithPIDThreadSafe(const std::vector<std::string>& command)
1532 Utils_Locker lock(&_systemMutex);
1533 if(command.size()<1)
1534 throw SALOME_Exception("SystemWithPIDThreadSafe : command is expected to have a length of size 1 at least !");
1536 pid_t pid ( fork() ) ; // spawn a child process, following code is executed in both processes
1538 pid_t pid = -1; //Throw SALOME_Exception on Windows
1540 if ( pid == 0 ) // I'm a child, replace myself with a new ompi-server
1542 std::size_t sz(command.size());
1543 char **args = new char *[sz+1];
1544 for(std::size_t i=0;i<sz;i++)
1545 args[i] = strdup(command[i].c_str());
1547 execvp( command[0].c_str() , args );
1548 std::ostringstream oss;
1549 oss << "Error when launching " << command[0];
1550 throw SALOME_Exception(oss.str().c_str()); // execvp failed
1554 throw SALOME_Exception("fork() failed");
1556 else // I'm a parent
1562 #ifdef WITH_PACO_PARALLEL
1564 //=============================================================================
1566 * Start a suitable PaCO++ Parallel Container in a list of machines.
1567 * \param params Container Parameters required for the container
1568 * \return CORBA container reference.
1570 //=============================================================================
1571 Engines::Container_ptr
1572 SALOME_ContainerManager::StartPaCOPPContainer(const Engines::ContainerParameters& params_const,
1573 std::string resource_selected)
1575 CORBA::Object_var obj;
1576 PaCO::InterfaceManager_var container_proxy;
1577 Engines::Container_ptr ret = Engines::Container::_nil();
1578 Engines::ContainerParameters params(params_const);
1579 params.resource_params.name = CORBA::string_dup(resource_selected.c_str());
1581 // Step 0 : Check parameters
1582 if (!checkPaCOParameters(params, resource_selected))
1584 INFOS("[StartPaCOPPContainer] check parameters failed ! see logs...");
1588 // Step 1 : Starting a new parallel container !
1589 INFOS("[StartPaCOPPContainer] Starting a PaCO++ parallel container");
1590 INFOS("[StartPaCOPPContainer] on resource : " << resource_selected);
1592 // Step 2 : Get a MachineFile for the parallel container
1593 std::string machine_file_name = _resManager->getMachineFile(resource_selected,
1595 params.parallelLib.in());
1597 if (machine_file_name == "")
1599 INFOS("[StartPaCOPPContainer] Machine file generation failed");
1603 // Step 3 : starting parallel container proxy
1604 std::string command_proxy("");
1605 std::string proxy_machine;
1608 command_proxy = BuildCommandToLaunchPaCOProxyContainer(params, machine_file_name, proxy_machine);
1610 catch(const SALOME_Exception & ex)
1612 INFOS("[StartPaCOPPContainer] Exception in BuildCommandToLaunchPaCOContainer");
1616 obj = LaunchPaCOProxyContainer(command_proxy, params, proxy_machine);
1617 if (CORBA::is_nil(obj))
1619 INFOS("[StartPaCOPPContainer] LaunchPaCOContainer for proxy returns NIL !");
1622 container_proxy = PaCO::InterfaceManager::_narrow(obj);
1623 MESSAGE("[StartPaCOPPContainer] PaCO container proxy is launched");
1625 // Step 4 : starting parallel container nodes
1626 std::string command_nodes("");
1627 SALOME_ContainerManager::actual_launch_machine_t nodes_machines;
1630 command_nodes = BuildCommandToLaunchPaCONodeContainer(params, machine_file_name, nodes_machines, proxy_machine);
1632 catch(const SALOME_Exception & ex)
1634 INFOS("[StarPaCOPPContainer] Exception in BuildCommandToLaunchPaCONodeContainer");
1639 std::string container_generic_node_name = std::string(params.container_name.in()) + std::string("Node");
1640 bool result = LaunchPaCONodeContainer(command_nodes, params, container_generic_node_name, nodes_machines);
1643 INFOS("[StarPaCOPPContainer] LaunchPaCONodeContainer failed !");
1644 // Il faut tuer le proxy
1647 Engines::Container_var proxy = Engines::Container::_narrow(container_proxy);
1652 INFOS("[StarPaCOPPContainer] Exception caught from proxy Shutdown...");
1657 // Step 4 : connecting nodes and the proxy to actually create a parallel container
1658 for (int i = 0; i < params.nb_proc; i++)
1660 std::ostringstream tmp;
1662 std::string proc_number = tmp.str();
1663 std::string container_node_name = container_generic_node_name + proc_number;
1665 std::string theNodeMachine(nodes_machines[i]);
1666 std::string containerNameInNS = _NS->BuildContainerNameForNS(container_node_name.c_str(), theNodeMachine.c_str());
1667 obj = _NS->Resolve(containerNameInNS.c_str());
1668 if (CORBA::is_nil(obj))
1670 INFOS("[StarPaCOPPContainer] CONNECTION FAILED From Naming Service !");
1671 INFOS("[StarPaCOPPContainer] Container name is " << containerNameInNS);
1676 MESSAGE("[StarPaCOPPContainer] Deploying node : " << container_node_name);
1677 PaCO::InterfaceParallel_var node = PaCO::InterfaceParallel::_narrow(obj);
1679 MESSAGE("[StarPaCOPPContainer] node " << container_node_name << " is deployed");
1681 catch(CORBA::SystemException& e)
1683 INFOS("[StarPaCOPPContainer] Exception in deploying node : " << containerNameInNS);
1684 INFOS("CORBA::SystemException : " << e);
1687 catch(CORBA::Exception& e)
1689 INFOS("[StarPaCOPPContainer] Exception in deploying node : " << containerNameInNS);
1690 INFOS("CORBA::Exception" << e);
1695 INFOS("[StarPaCOPPContainer] Exception in deploying node : " << containerNameInNS);
1696 INFOS("Unknown exception !");
1701 // Step 5 : starting parallel container
1704 MESSAGE ("[StarPaCOPPContainer] Starting parallel object");
1705 container_proxy->start();
1706 MESSAGE ("[StarPaCOPPContainer] Parallel object is started");
1707 ret = Engines::Container::_narrow(container_proxy);
1709 catch(CORBA::SystemException& e)
1711 INFOS("Caught CORBA::SystemException. : " << e);
1713 catch(PortableServer::POA::ServantAlreadyActive&)
1715 INFOS("Caught CORBA::ServantAlreadyActiveException");
1717 catch(CORBA::Exception&)
1719 INFOS("Caught CORBA::Exception.");
1721 catch(std::exception& exc)
1723 INFOS("Caught std::exception - "<<exc.what());
1727 INFOS("Caught unknown exception.");
1733 SALOME_ContainerManager::BuildCommandToLaunchPaCOProxyContainer(const Engines::ContainerParameters& params,
1734 std::string machine_file_name,
1735 std::string & proxy_hostname)
1737 // In the proxy case, we always launch a Dummy Proxy
1738 std::string exe_name = "SALOME_ParallelContainerProxyDummy";
1739 std::string container_name = params.container_name.in();
1741 // Convert nb_proc in string
1742 std::ostringstream tmp_string;
1743 tmp_string << params.nb_proc;
1744 std::string nb_proc_str = tmp_string.str();
1746 // Get resource definition
1747 ParserResourcesType resource_definition =
1748 _resManager->GetResourceDefinition(params.resource_params.name.in());
1751 std::string hostname;
1752 std::ifstream machine_file(machine_file_name.c_str());
1753 std::getline(machine_file, hostname, ' ');
1754 size_t found = hostname.find('\n');
1755 if (found!=std::string::npos)
1756 hostname.erase(found, 1); // Remove \n
1757 proxy_hostname = hostname;
1758 MESSAGE("[BuildCommandToLaunchPaCOProxyContainer] machine file name extracted is " << hostname);
1761 bool remote_execution = false;
1762 if (hostname != std::string(Kernel_Utils::GetHostname()))
1764 MESSAGE("[BuildCommandToLaunchPaCOProxyContainer] remote machine case detected !");
1765 remote_execution = true;
1769 std::string log_type("");
1770 char * get_val = GetenvThreadSafe("PARALLEL_LOG");
1774 // Generating the command
1775 std::string command_begin("");
1776 std::string command_end("");
1777 std::ostringstream command;
1779 LogConfiguration(log_type, "proxy", container_name, hostname, command_begin, command_end);
1780 command << command_begin;
1782 // Adding connection command
1783 // We can only have a remote execution with
1784 // a SALOME application
1785 if (remote_execution)
1787 ASSERT(GetenvThreadSafe("NSHOST"));
1788 ASSERT(GetenvThreadSafe("NSPORT"));
1790 command << resource_definition.getAccessProtocolTypeStr();
1792 command << resource_definition.UserName;
1793 command << " " << hostname;
1794 command << " " << resource_definition.AppliPath;
1795 command << "/runRemote.sh ";
1796 command << GetenvThreadSafeAsString("NSHOST") << " "; // hostname of CORBA name server
1797 command << GetenvThreadSafeAsString("NSPORT") << " "; // port of CORBA name server
1800 command << exe_name;
1801 command << " " << container_name;
1802 command << " Dummy";
1803 command << " " << hostname;
1804 command << " " << nb_proc_str;
1806 AddOmninamesParams(command);
1809 command << command_end;
1810 MESSAGE("[BuildCommandToLaunchPaCOProxyContainer] Command is: " << command.str());
1812 return command.str();
1816 SALOME_ContainerManager::BuildCommandToLaunchPaCONodeContainer(const Engines::ContainerParameters& params,
1817 const std::string & machine_file_name,
1818 SALOME_ContainerManager::actual_launch_machine_t & vect_machine,
1819 const std::string & proxy_hostname)
1822 std::string exe_name = "SALOME_ParallelContainerNode";
1823 exe_name += params.parallelLib.in();
1824 std::string container_name = params.container_name.in();
1826 // Convert nb_proc in string
1827 std::ostringstream nb_proc_stream;
1828 nb_proc_stream << params.nb_proc;
1830 // Get resource definition
1831 ParserResourcesType resource_definition =
1832 _resManager->GetResourceDefinition(params.resource_params.name.in());
1835 std::string log_type("");
1836 char * get_val = GetenvThreadSafe("PARALLEL_LOG");
1840 // Now the command is different according to paralleLib
1841 std::ostringstream command_nodes;
1842 std::ifstream machine_file(machine_file_name.c_str());
1843 if (std::string(params.parallelLib.in()) == "Dummy")
1845 for (int i= 0; i < params.nb_proc; i++)
1848 std::string hostname;
1849 std::getline(machine_file, hostname);
1850 MESSAGE("[BuildCommandToLaunchPaCONodeContainer] machine file name extracted is " << hostname);
1853 bool remote_execution = false;
1854 if (hostname != std::string(Kernel_Utils::GetHostname()))
1856 MESSAGE("[BuildCommandToLaunchPaCONodeContainer] remote machine case detected !");
1857 remote_execution = true;
1860 // For each node we have a new command
1861 // Generating the command
1862 std::ostringstream command_node_stream;
1863 std::string command_node_begin("");
1864 std::string command_node_end("");
1865 std::ostringstream node_number;
1867 std::string container_node_name = container_name + node_number.str();
1868 LogConfiguration(log_type, "node", container_node_name, hostname, command_node_begin, command_node_end);
1870 // Adding connection command
1871 // We can only have a remote execution with
1872 // a SALOME application
1873 if (remote_execution)
1875 ASSERT(GetenvThreadSafe("NSHOST"));
1876 ASSERT(GetenvThreadSafe("NSPORT"));
1878 command_node_stream << resource_definition.getAccessProtocolTypeStr();
1879 command_node_stream << " -l ";
1880 command_node_stream << resource_definition.UserName;
1881 command_node_stream << " " << hostname;
1882 command_node_stream << " " << resource_definition.AppliPath;
1883 command_node_stream << "/runRemote.sh ";
1884 command_node_stream << GetenvThreadSafeAsString("NSHOST") << " "; // hostname of CORBA name server
1885 command_node_stream << GetenvThreadSafeAsString("NSPORT") << " "; // port of CORBA name server
1888 command_node_stream << exe_name;
1889 command_node_stream << " " << container_name;
1890 command_node_stream << " " << params.parallelLib.in();
1891 command_node_stream << " " << proxy_hostname;
1892 command_node_stream << " " << node_number.str();
1893 command_node_stream << " -";
1894 AddOmninamesParams(command_node_stream);
1896 command_nodes << command_node_begin << command_node_stream.str() << command_node_end;
1897 vect_machine.push_back(hostname);
1901 else if (std::string(params.parallelLib.in()) == "Mpi")
1904 std::string hostname;
1905 std::getline(machine_file, hostname, ' ');
1906 MESSAGE("[BuildCommandToLaunchPaCONodeContainer] machine file name extracted is " << hostname);
1909 bool remote_execution = false;
1910 if (hostname != std::string(Kernel_Utils::GetHostname()))
1912 MESSAGE("[BuildCommandToLaunchPaCONodeContainer] remote machine case detected !");
1913 remote_execution = true;
1916 // In case of Mpi and Remote, we copy machine_file in the applipath
1917 // scp mpi_machine_file user@machine:Path
1918 std::ostringstream command_remote_stream;
1919 std::string::size_type last = machine_file_name.find_last_of("/");
1920 if (last == std::string::npos)
1923 if (resource_definition.Protocol == rsh)
1924 command_remote_stream << "rcp ";
1926 command_remote_stream << "scp ";
1927 command_remote_stream << machine_file_name << " ";
1928 command_remote_stream << resource_definition.UserName << "@";
1929 command_remote_stream << hostname << ":" << resource_definition.AppliPath;
1930 command_remote_stream << "/" << machine_file_name.substr(last+1);
1932 int status = SystemThreadSafe(command_remote_stream.str().c_str());
1935 INFOS("copy of the MPI machine file failed ! - sorry !");
1939 // Generating the command
1940 std::string command_begin("");
1941 std::string command_end("");
1943 LogConfiguration(log_type, "nodes", container_name, hostname, command_begin, command_end);
1944 command_nodes << command_begin;
1946 // Adding connection command
1947 // We can only have a remote execution with
1948 // a SALOME application
1949 if (remote_execution)
1951 ASSERT(GetenvThreadSafe("NSHOST"));
1952 ASSERT(GetenvThreadSafe("NSPORT"));
1954 command_nodes << resource_definition.getAccessProtocolTypeStr();
1955 command_nodes << " -l ";
1956 command_nodes << resource_definition.UserName;
1957 command_nodes << " " << hostname;
1958 command_nodes << " " << resource_definition.AppliPath;
1959 command_nodes << "/runRemote.sh ";
1960 command_nodes << GetenvThreadSafeAsString("NSHOST") << " "; // hostname of CORBA name server
1961 command_nodes << GetenvThreadSafeAsString("NSPORT") << " "; // port of CORBA name server
1964 if (resource_definition.mpi == lam)
1966 command_nodes << "mpiexec -ssi boot ";
1967 command_nodes << "-machinefile " << machine_file_name << " ";
1968 command_nodes << "-n " << params.nb_proc;
1972 command_nodes << "mpirun -np " << params.nb_proc;
1974 command_nodes << " " << exe_name;
1975 command_nodes << " " << container_name;
1976 command_nodes << " " << params.parallelLib.in();
1977 command_nodes << " " << proxy_hostname;
1978 command_nodes << " -";
1979 AddOmninamesParams(command_nodes);
1981 // We don't put hostname, because nodes are registered in the resource of the proxy
1982 for (int i= 0; i < params.nb_proc; i++)
1983 vect_machine.push_back(proxy_hostname);
1985 command_nodes << command_end;
1987 return command_nodes.str();
1991 SALOME_ContainerManager::LogConfiguration(const std::string & log_type,
1992 const std::string & exe_type,
1993 const std::string & container_name,
1994 const std::string & hostname,
1995 std::string & begin,
1998 if(log_type == "xterm")
2000 begin = "xterm -e \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH; export PATH=$PATH;";
2003 else if(log_type == "xterm_debug")
2005 begin = "xterm -e \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH; export PATH=$PATH;";
2010 // default into a file...
2011 std::string logFilename = "/tmp/" + container_name + "_" + hostname + "_" + exe_type + "_";
2012 std::string user = GetenvThreadSafeAsString("USER");
2014 user = GetenvThreadSafeAsString("LOGNAME");
2015 logFilename += user + ".log";
2016 end = " > " + logFilename + " 2>&1 & ";
2021 SALOME_ContainerManager::LaunchPaCOProxyContainer(const std::string& command,
2022 const Engines::ContainerParameters& params,
2023 const std::string & hostname)
2025 PaCO::InterfaceManager_ptr container_proxy = PaCO::InterfaceManager::_nil();
2027 MESSAGE("[LaunchPaCOProxyContainer] Launch command");
2028 int status = SystemThreadSafe(command.c_str());
2030 INFOS("[LaunchPaCOProxyContainer] failed : system command status -1");
2031 return container_proxy;
2033 else if (status == 217) {
2034 INFOS("[LaunchPaCOProxyContainer] failed : system command status 217");
2035 return container_proxy;
2038 int count(GetTimeOutToLoaunchServer());
2039 CORBA::Object_var obj = CORBA::Object::_nil();
2040 std::string containerNameInNS = _NS->BuildContainerNameForNS(params.container_name.in(),
2042 MESSAGE("[LaunchParallelContainer] Waiting for Parallel Container proxy : " << containerNameInNS);
2044 while (CORBA::is_nil(obj) && count)
2048 obj = _NS->Resolve(containerNameInNS.c_str());
2053 container_proxy = PaCO::InterfaceManager::_narrow(obj);
2055 catch(CORBA::SystemException& e)
2057 INFOS("[StarPaCOPPContainer] Exception in _narrow after LaunchParallelContainer for proxy !");
2058 INFOS("CORBA::SystemException : " << e);
2059 return container_proxy;
2061 catch(CORBA::Exception& e)
2063 INFOS("[StarPaCOPPContainer] Exception in _narrow after LaunchParallelContainer for proxy !");
2064 INFOS("CORBA::Exception" << e);
2065 return container_proxy;
2069 INFOS("[StarPaCOPPContainer] Exception in _narrow after LaunchParallelContainer for proxy !");
2070 INFOS("Unknown exception !");
2071 return container_proxy;
2073 if (CORBA::is_nil(container_proxy))
2075 INFOS("[StarPaCOPPContainer] PaCO::InterfaceManager::_narrow returns NIL !");
2076 return container_proxy;
2081 //=============================================================================
2082 /*! This method launches the parallel container.
2083 * It will may be placed on the resources manager.
2085 * \param command to launch
2086 * \param container's parameters
2087 * \param name of the container
2089 * \return CORBA container reference
2091 //=============================================================================
2093 SALOME_ContainerManager::LaunchPaCONodeContainer(const std::string& command,
2094 const Engines::ContainerParameters& params,
2095 const std::string& name,
2096 SALOME_ContainerManager::actual_launch_machine_t & vect_machine)
2098 INFOS("[LaunchPaCONodeContainer] Launch command");
2099 int status = SystemThreadSafe(command.c_str());
2101 INFOS("[LaunchPaCONodeContainer] failed : system command status -1");
2104 else if (status == 217) {
2105 INFOS("[LaunchPaCONodeContainer] failed : system command status 217");
2109 INFOS("[LaunchPaCONodeContainer] Waiting for the nodes of the parallel container");
2110 // We are waiting all the nodes
2111 for (int i = 0; i < params.nb_proc; i++)
2113 CORBA::Object_var obj = CORBA::Object::_nil();
2114 std::string theMachine(vect_machine[i]);
2116 std::ostringstream tmp;
2118 std::string proc_number = tmp.str();
2119 std::string container_node_name = name + proc_number;
2120 std::string containerNameInNS = _NS->BuildContainerNameForNS((char*) container_node_name.c_str(), theMachine.c_str());
2121 INFOS("[LaunchPaCONodeContainer] Waiting for Parallel Container node " << containerNameInNS << " on " << theMachine);
2122 int count(GetTimeOutToLoaunchServer());
2123 while (CORBA::is_nil(obj) && count) {
2126 obj = _NS->Resolve(containerNameInNS.c_str());
2128 if (CORBA::is_nil(obj))
2130 INFOS("[LaunchPaCONodeContainer] Launch of node failed (or not found) !");
2139 Engines::Container_ptr
2140 SALOME_ContainerManager::StartPaCOPPContainer(const Engines::ContainerParameters& /*params*/,
2141 std::string /*resource_selected*/)
2143 Engines::Container_ptr ret = Engines::Container::_nil();
2144 INFOS("[StarPaCOPPContainer] is disabled !");
2145 INFOS("[StarPaCOPPContainer] recompile SALOME Kernel to enable PaCO++ parallel extension");
2150 SALOME_ContainerManager::BuildCommandToLaunchPaCOProxyContainer(const Engines::ContainerParameters& /*params*/,
2151 std::string /*machine_file_name*/,
2152 std::string & /*proxy_hostname*/)
2158 SALOME_ContainerManager::BuildCommandToLaunchPaCONodeContainer(const Engines::ContainerParameters& /*params*/,
2159 const std::string & /*machine_file_name*/,
2160 SALOME_ContainerManager::actual_launch_machine_t & /*vect_machine*/,
2161 const std::string & /*proxy_hostname*/)
2166 SALOME_ContainerManager::LogConfiguration(const std::string & /*log_type*/,
2167 const std::string & /*exe_type*/,
2168 const std::string & /*container_name*/,
2169 const std::string & /*hostname*/,
2170 std::string & /*begin*/,
2171 std::string & /*end*/)
2176 SALOME_ContainerManager::LaunchPaCOProxyContainer(const std::string& /*command*/,
2177 const Engines::ContainerParameters& /*params*/,
2178 const std::string& /*hostname*/)
2180 CORBA::Object_ptr ret = CORBA::Object::_nil();
2185 SALOME_ContainerManager::LaunchPaCONodeContainer(const std::string& /*command*/,
2186 const Engines::ContainerParameters& /*params*/,
2187 const std::string& /*name*/,
2188 SALOME_ContainerManager::actual_launch_machine_t & /*vect_machine*/)