From e7aaafb029041c14e63461225845c820dc7abccb Mon Sep 17 00:00:00 2001 From: caremoli Date: Wed, 15 Apr 2009 13:37:35 +0000 Subject: [PATCH] CCAR: 1- add mode member to MachineParameters. This string member can be "start" or "get" or "getorstart" if mode=="start" a new container is launched on each request. if mode=="get" an old container with the requested parameters is used if it exists. if mode=="getorstart" an old container is used or a new one if the old one does not exist. 2- add policy member to MachineParameters. This string member replaces the enum policy argument of GiveContainer and StartContainer. This argument is left for compatibility but will be removed soon. 3- replace LoadRateManager methods by three objects. These objects are referenced in a map (_resourceManagerMap) that can be extended without changing IDL interface. 4- add a Find method to ResourceManager IDL for testing purpose. --- idl/SALOME_ContainerManager.idl | 22 ++- src/Container/SALOME_ContainerManager.cxx | 159 ++++++++++-------- src/LifeCycleCORBA/SALOME_LifeCycleCORBA.cxx | 4 + src/LifeCycleCORBA_SWIG/LifeCycleCORBA.py | 8 +- .../libSALOME_LifeCycleCORBA.i | 4 + src/ResourcesManager/ResourcesManager.cxx | 44 ++--- src/ResourcesManager/ResourcesManager.hxx | 7 +- .../SALOME_LoadRateManager.cxx | 41 ++++- .../SALOME_LoadRateManager.hxx | 34 +++- .../SALOME_ResourcesManager.cxx | 11 +- .../SALOME_ResourcesManager.hxx | 1 + 11 files changed, 207 insertions(+), 128 deletions(-) diff --git a/idl/SALOME_ContainerManager.idl b/idl/SALOME_ContainerManager.idl index 392f56519..b7c4ace4a 100644 --- a/idl/SALOME_ContainerManager.idl +++ b/idl/SALOME_ContainerManager.idl @@ -55,7 +55,11 @@ struct MachineParameters string username; //! salome application to use to start a remote container string applipath; - ModulesList modList; + //! if given list of components that could be loaded on the container + CompoList componentList; + //! if given restricted list of machines to search in + MachineList computerList; + //! required operating system string OS; //! required memory size long mem_mb; @@ -73,8 +77,16 @@ struct MachineParameters string batch; //! container working directory string workingdir; - - // PaCO specific informations + //! creation mode for GiveContainer. + /*!start creates a new container + * get try to find an existing container + * getorstart use an existing container if it exists or creates a new one + */ + string mode; + //! resource management policy : first, cycl, altcycl or best (can be extended) + string policy; + + //! PaCO specific informations string parallelLib; long nb_component_nodes; }; @@ -212,9 +224,11 @@ struct BatchParameters //! Find first available computer in a computers list string FindFirst(in MachineList possibleComputers); + //! Find best available computer according to policy in a computers list + string Find(in string policy, in MachineList possibleComputers); + //! Get a list of computers that are best suited to launch a container given constraints /*! - The constraints are resource constraints (params) and components constraints (componentList) */ MachineList GetFittingResources( in MachineParameters params, diff --git a/src/Container/SALOME_ContainerManager.cxx b/src/Container/SALOME_ContainerManager.cxx index d00618fd2..e2acfc461 100644 --- a/src/Container/SALOME_ContainerManager.cxx +++ b/src/Container/SALOME_ContainerManager.cxx @@ -108,8 +108,6 @@ void SALOME_ContainerManager::Shutdown() _NS->Destroy_Name(_ContainerManagerNameInNS); PortableServer::ObjectId_var oid = _poa->servant_to_id(this); _poa->deactivate_object(oid); - //_remove_ref() has already been done at creation - //_remove_ref(); } //============================================================================= @@ -134,7 +132,7 @@ void SALOME_ContainerManager::ShutdownContainers() { Engines::Container_var cont=Engines::Container::_narrow(obj); if(!CORBA::is_nil(cont)) - lstCont.push_back((*iter)); + lstCont.push_back((*iter)); } catch(const CORBA::Exception& e) { @@ -151,7 +149,7 @@ void SALOME_ContainerManager::ShutdownContainers() Engines::Container_var cont=Engines::Container::_narrow(obj); if(!CORBA::is_nil(cont)) { - MESSAGE("ShutdownContainers: " << (*iter)); + MESSAGE("ShutdownContainers: " << (*iter)); try { cont->Shutdown(); @@ -186,7 +184,7 @@ void SALOME_ContainerManager::ShutdownContainers() Engines::Container_ptr SALOME_ContainerManager:: FindOrStartContainer(const Engines::MachineParameters& params, - const Engines::MachineList& possibleComputers) + const Engines::MachineList& possibleComputers) { Engines::Container_ptr ret = FindContainer(params,possibleComputers); if(!CORBA::is_nil(ret)) @@ -211,8 +209,8 @@ FindOrStartContainer(const Engines::MachineParameters& params, Engines::Container_ptr SALOME_ContainerManager:: StartContainer(const Engines::MachineParameters& params, - const Engines::MachineList& possibleComputers, - Engines::ResPolicy policy,const std::string& container_exe) + const Engines::MachineList& possibleComputers, + Engines::ResPolicy policy,const std::string& container_exe) { #ifdef WITH_PACO_PARALLEL std::string parallelLib(params.parallelLib); @@ -222,55 +220,51 @@ StartContainer(const Engines::MachineParameters& params, string containerNameInNS; Engines::Container_ptr ret = Engines::Container::_nil(); - MESSAGE("SALOME_ContainerManager::StartContainer " << - possibleComputers.length()); + MESSAGE("SALOME_ContainerManager::StartContainer " << possibleComputers.length()); vector lm; - for(unsigned int i=0;i_non_existent()) + lm.push_back(string(possibleComputers[i])); + } + catch(CORBA::Exception&) + { + // CORBA::Exception ignored. + } + } + } + else + { + for(unsigned int i=0;iGetImpl()->FindFirst(lm); - break; - case Engines::P_CYCL: - theMachine=_ResManager->GetImpl()->FindNext(lm); - break; - case Engines::P_BEST: - theMachine=_ResManager->GetImpl()->FindBest(lm); - break; + try + { + theMachine=_ResManager->GetImpl()->Find(params.policy.in(),lm); + } + catch( const SALOME_Exception &ex ) + { + MESSAGE(ex.what()); + return Engines::Container::_nil(); } - } - catch( const SALOME_Exception &ex ){ - MESSAGE(ex.what()); - return Engines::Container::_nil(); - } //If the machine name is localhost use the real name if(theMachine == "localhost") theMachine=Kernel_Utils::GetHostname(); - MESSAGE("try to launch it on " << theMachine); - - string command; - if(theMachine==""){ - MESSAGE("SALOME_ContainerManager::StartContainer : " << - "no possible computer"); - return Engines::Container::_nil(); - } - else if(theMachine==Kernel_Utils::GetHostname()) - command = BuildCommandToLaunchLocalContainer(params,container_exe); - else - command = BuildCommandToLaunchRemoteContainer(theMachine,params,container_exe); - //check if an entry exists in Naming service - if(params.isMPI) - // A parallel container register on zero node in NS - containerNameInNS = _NS->BuildContainerNameForNS(params,GetMPIZeroNode(theMachine).c_str()); - else - containerNameInNS = _NS->BuildContainerNameForNS(params,theMachine.c_str()); + //if params.mode == "start" or "" shutdown the existing container before launching a new one with that name + //if params.mode == "getorstart" or "get" use the existing container + containerNameInNS = _NS->BuildContainerNameForNS(params,theMachine.c_str()); SCRUTE(containerNameInNS); CORBA::Object_var obj = _NS->Resolve(containerNameInNS.c_str()); @@ -278,10 +272,14 @@ StartContainer(const Engines::MachineParameters& params, { try { - // shutdown the registered container if it exists Engines::Container_var cont=Engines::Container::_narrow(obj); - if(!CORBA::is_nil(cont)) - cont->Shutdown(); + if(!cont->_non_existent()) + { + if(std::string(params.mode.in())=="getorstart"||std::string(params.mode.in())=="get") + return cont._retn(); /* the container exists and params.mode is getorstart or get use it*/ + else + cont->Shutdown(); // shutdown the registered container if it exists + } } catch(CORBA::Exception&) { @@ -289,6 +287,19 @@ StartContainer(const Engines::MachineParameters& params, } } + //try to launch a new container + MESSAGE("try to launch it on " << theMachine); + + string command; + if(theMachine==""){ + MESSAGE("SALOME_ContainerManager::StartContainer : no possible computer"); + return Engines::Container::_nil(); + } + else if(theMachine==Kernel_Utils::GetHostname()) + command = BuildCommandToLaunchLocalContainer(params,container_exe); + else + command = BuildCommandToLaunchRemoteContainer(theMachine,params,container_exe); + //redirect stdout and stderr in a file string logFilename="/tmp/"+_NS->ContainerName(params)+"_"+ theMachine +"_"+getenv( "USER" )+".log" ; command += " > " + logFilename + " 2>&1 &"; @@ -296,16 +307,13 @@ StartContainer(const Engines::MachineParameters& params, // launch container with a system call int status=system(command.c_str()); - if (status == -1){ - MESSAGE("SALOME_LifeCycleCORBA::StartOrFindContainer rsh failed " << - "(system command status -1)"); + MESSAGE("SALOME_ContainerManager::StartContainer rsh failed (system command status -1)"); RmTmpFile(_TmpFileName); // command file can be removed here return Engines::Container::_nil(); } else if (status == 217){ - MESSAGE("SALOME_LifeCycleCORBA::StartOrFindContainer rsh failed " << - "(system command status 217)"); + MESSAGE("SALOME_ContainerManager::StartContainer rsh failed (system command status 217)"); RmTmpFile(_TmpFileName); // command file can be removed here return Engines::Container::_nil(); } @@ -320,7 +328,7 @@ StartContainer(const Engines::MachineParameters& params, #endif count-- ; if ( count != 10 ) - MESSAGE( count << ". Waiting for container on " << theMachine); + MESSAGE( count << ". Waiting for container on " << theMachine); CORBA::Object_var obj = _NS->Resolve(containerNameInNS.c_str()); ret=Engines::Container::_narrow(obj); @@ -328,7 +336,7 @@ StartContainer(const Engines::MachineParameters& params, if ( CORBA::is_nil(ret) ) { - MESSAGE("SALOME_LifeCycleCORBA::StartOrFindContainer rsh failed"); + MESSAGE("SALOME_ContainerManager::StartContainer rsh failed"); } else { @@ -355,8 +363,8 @@ StartContainer(const Engines::MachineParameters& params, Engines::Container_ptr SALOME_ContainerManager:: StartContainer(const Engines::MachineParameters& params, - Engines::ResPolicy policy, - const Engines::CompoList& componentList) + Engines::ResPolicy policy, + const Engines::CompoList& componentList) { Engines::MachineList_var possibleComputers = _ResManager->GetFittingResources(params,componentList); @@ -421,7 +429,7 @@ StartContainer(const Engines::MachineParameters& params, Engines::Container_ptr SALOME_ContainerManager:: FindOrStartParallelContainer(const Engines::MachineParameters& params_const, - const Engines::MachineList& possibleComputers) + const Engines::MachineList& possibleComputers) { CORBA::Object_var obj; PaCO::InterfaceManager_var proxy; @@ -449,16 +457,16 @@ FindOrStartParallelContainer(const Engines::MachineParameters& params_const, INFOS("[FindOrStartParallelContainer] on machine : " << theMachine); string command; if(theMachine == Kernel_Utils::GetHostname()) { - // Step 3 : starting parallel container proxy - params.hostname = CORBA::string_dup(theMachine.c_str()); - Engines::MachineParameters params_proxy(params); - try { - command = BuildCommandToLaunchLocalParallelContainer("SALOME_ParallelContainerProxy", params_proxy, "xterm"); - } - catch(const SALOME_Exception & ex){ - MESSAGE(ex.what()); - return Engines::Container::_nil(); - } + // Step 3 : starting parallel container proxy + params.hostname = CORBA::string_dup(theMachine.c_str()); + Engines::MachineParameters params_proxy(params); + try { + command = BuildCommandToLaunchLocalParallelContainer("SALOME_ParallelContainerProxy", params_proxy, "xterm"); + } + catch(const SALOME_Exception & ex){ + MESSAGE(ex.what()); + return Engines::Container::_nil(); + } // LaunchParallelContainer uses this value to know if it launches the proxy or the nodes params_proxy.nb_component_nodes = 0; obj = LaunchParallelContainer(command, params_proxy, _NS->ContainerName(params)); @@ -598,10 +606,17 @@ FindContainer(const Engines::MachineParameters& params, { string containerNameInNS(_NS->BuildContainerNameForNS(params,theMachine)); CORBA::Object_var obj = _NS->Resolve(containerNameInNS.c_str()); - if( !CORBA::is_nil(obj) ) - return Engines::Container::_narrow(obj); - else - return Engines::Container::_nil(); + try + { + if(obj->_non_existent()) + return Engines::Container::_nil(); + else + return Engines::Container::_narrow(obj); + } + catch(const CORBA::Exception& e) + { + return Engines::Container::_nil(); + } } //============================================================================= diff --git a/src/LifeCycleCORBA/SALOME_LifeCycleCORBA.cxx b/src/LifeCycleCORBA/SALOME_LifeCycleCORBA.cxx index 3d68dcee7..a96f9fac7 100644 --- a/src/LifeCycleCORBA/SALOME_LifeCycleCORBA.cxx +++ b/src/LifeCycleCORBA/SALOME_LifeCycleCORBA.cxx @@ -373,6 +373,8 @@ void SALOME_LifeCycleCORBA::preSet( Engines::MachineParameters& params) params.protocol = ""; params.username = ""; params.applipath = ""; + //param.componentList = 0; + //param.computerList = 0; params.OS = ""; params.mem_mb = 0; params.cpu_clock = 0; @@ -382,6 +384,8 @@ void SALOME_LifeCycleCORBA::preSet( Engines::MachineParameters& params) params.mpiImpl=""; params.batch=""; params.workingdir = ""; + params.mode = ""; + params.policy = ""; params.parallelLib = ""; params.nb_component_nodes = 0; } diff --git a/src/LifeCycleCORBA_SWIG/LifeCycleCORBA.py b/src/LifeCycleCORBA_SWIG/LifeCycleCORBA.py index d5e0c3a4d..a2422f8b5 100644 --- a/src/LifeCycleCORBA_SWIG/LifeCycleCORBA.py +++ b/src/LifeCycleCORBA_SWIG/LifeCycleCORBA.py @@ -40,11 +40,11 @@ class LifeCycleCORBA (SALOME_LifeCycleCORBA): class MachineParameters (Engines.MachineParameters): def __init__(self, container_name='', hostname='', alias='', protocol='', username='', - applipath='', componentList=[], OS='', mem_mb=0, cpu_clock=0, + applipath='', componentList=[], computerList=[], OS='', mem_mb=0, cpu_clock=0, nb_proc_per_node=0, nb_node=0, isMPI=False, mpiImpl='', batch='', workingdir='', - parallelLib='', nb_component_nodes=0): + mode='start', policy='altcycl', parallelLib='', nb_component_nodes=0): Engines.MachineParameters.__init__(self,container_name, hostname, alias, protocol, username, - applipath, componentList, OS, mem_mb, cpu_clock, + applipath, componentList, computerList, OS, mem_mb, cpu_clock, nb_proc_per_node, nb_node, isMPI, mpiImpl, batch, workingdir, - parallelLib, nb_component_nodes) + mode, policy, parallelLib, nb_component_nodes) diff --git a/src/LifeCycleCORBA_SWIG/libSALOME_LifeCycleCORBA.i b/src/LifeCycleCORBA_SWIG/libSALOME_LifeCycleCORBA.i index 4482c92e8..3da83ad0d 100644 --- a/src/LifeCycleCORBA_SWIG/libSALOME_LifeCycleCORBA.i +++ b/src/LifeCycleCORBA_SWIG/libSALOME_LifeCycleCORBA.i @@ -187,6 +187,10 @@ using namespace std; param->batch = CORBA::string_dup(PyString_AsString(value)); else if (strcmp(keystr,"workingdir")==0) param->workingdir = CORBA::string_dup(PyString_AsString(value)); + else if (strcmp(keystr,"mode")==0) + param->mode = CORBA::string_dup(PyString_AsString(value)); + else if (strcmp(keystr,"policy")==0) + param->policy = CORBA::string_dup(PyString_AsString(value)); else if (strcmp(keystr,"parallelLib")==0) { param->parallelLib = CORBA::string_dup(PyString_AsString(value)); diff --git a/src/ResourcesManager/ResourcesManager.cxx b/src/ResourcesManager/ResourcesManager.cxx index 7a68eb2bc..6bdfdce25 100644 --- a/src/ResourcesManager/ResourcesManager.cxx +++ b/src/ResourcesManager/ResourcesManager.cxx @@ -38,6 +38,9 @@ using namespace std; +static LoadRateManagerFirst first; +static LoadRateManagerCycl cycl; +static LoadRateManagerAltCycl altcycl; //============================================================================= /*! * just for test @@ -51,6 +54,10 @@ ResourcesManager_cpp(const char *xmlFilePath) : #if defined(_DEBUG_) || defined(_DEBUG) cerr << "ResourcesManager_cpp constructor" << endl; #endif + _resourceManagerMap["first"]=&first; + _resourceManagerMap["cycl"]=&cycl; + _resourceManagerMap["altcycl"]=&altcycl; + _resourceManagerMap["best"]=&altcycl; } //============================================================================= @@ -69,6 +76,10 @@ ResourcesManager_cpp::ResourcesManager_cpp() throw(ResourcesException) #if defined(_DEBUG_) || defined(_DEBUG) cerr << "ResourcesManager_cpp constructor" << endl; #endif + _resourceManagerMap["first"]=&first; + _resourceManagerMap["cycl"]=&cycl; + _resourceManagerMap["altcycl"]=&altcycl; + _resourceManagerMap["best"]=&altcycl; _isAppliSalomeDefined = (getenv("APPLI") != 0); if(!getenv("KERNEL_ROOT_DIR")) throw ResourcesException("you must define KERNEL_ROOT_DIR environment variable!!"); @@ -373,37 +384,10 @@ const MapOfParserResourcesType& ResourcesManager_cpp::GetList() const return _resourcesList; } - -//============================================================================= -/*! - * dynamically obtains the first machines - */ -//============================================================================= - -string ResourcesManager_cpp::FindFirst(const std::vector& listOfMachines) -{ - return _dynamicResourcesSelecter.FindFirst(listOfMachines); -} - -//============================================================================= -/*! - * dynamically obtains the best machines - */ -//============================================================================= - -string ResourcesManager_cpp::FindNext(const std::vector& listOfMachines) -{ - return _dynamicResourcesSelecter.FindNext(listOfMachines,_resourcesList); -} -//============================================================================= -/*! - * dynamically obtains the best machines - */ -//============================================================================= - -string ResourcesManager_cpp::FindBest(const std::vector& listOfMachines) +string ResourcesManager_cpp::Find(const std::string& policy, const std::vector& listOfMachines) { - return _dynamicResourcesSelecter.FindBest(listOfMachines); + if(_resourceManagerMap.count(policy)==0)return ""; + return _resourceManagerMap[policy]->Find(listOfMachines,_resourcesList); } //============================================================================= diff --git a/src/ResourcesManager/ResourcesManager.hxx b/src/ResourcesManager/ResourcesManager.hxx index 839faccf0..608ce750b 100644 --- a/src/ResourcesManager/ResourcesManager.hxx +++ b/src/ResourcesManager/ResourcesManager.hxx @@ -68,9 +68,7 @@ class RESOURCESMANAGER_EXPORT ResourcesManager_cpp GetFittingResources(const machineParams& params, const std::vector& componentList) throw(ResourcesException); - std::string FindFirst(const std::vector& listOfMachines); - std::string FindNext(const std::vector& listOfMachines); - std::string FindBest(const std::vector& listOfMachines); + std::string Find(const std::string& policy, const std::vector& listOfMachines); int AddResourceInCatalog (const machineParams& paramsOfNewResources, @@ -109,7 +107,8 @@ class RESOURCESMANAGER_EXPORT ResourcesManager_cpp //! will contain the informations on the data type catalog(after parsing) MapOfParserResourcesType _resourcesBatchList; - SALOME_LoadRateManager _dynamicResourcesSelecter; + //! a map that contains all the available load rate managers (the key is the name) + std::map _resourceManagerMap; //! different behaviour if $APPLI exists (SALOME Application) bool _isAppliSalomeDefined; diff --git a/src/ResourcesManager/SALOME_LoadRateManager.cxx b/src/ResourcesManager/SALOME_LoadRateManager.cxx index 5865483a6..c422a3fd0 100644 --- a/src/ResourcesManager/SALOME_LoadRateManager.cxx +++ b/src/ResourcesManager/SALOME_LoadRateManager.cxx @@ -25,7 +25,8 @@ using namespace std; -string SALOME_LoadRateManager::FindFirst(const vector& hosts) +string LoadRateManagerFirst::Find(const vector& hosts, + MapOfParserResourcesType& resList) { if (hosts.size() == 0) return string(""); @@ -33,7 +34,8 @@ string SALOME_LoadRateManager::FindFirst(const vector& hosts) return string(hosts[0]); } -string SALOME_LoadRateManager::FindNext(const vector& hosts,MapOfParserResourcesType& resList) +string LoadRateManagerCycl::Find(const vector& hosts, + MapOfParserResourcesType& resList) { static int imachine = 0; static int iproc = 0; @@ -52,15 +54,40 @@ string SALOME_LoadRateManager::FindNext(const vector& hosts,MapOfParserR else{ iproc = 1; imachine++; - if(imachine == hosts.size()) - imachine = 0; + if(imachine >= hosts.size()) + imachine = 0; return string(hosts[imachine]); } } } -string SALOME_LoadRateManager::FindBest(const vector& hosts) +string LoadRateManagerAltCycl::Find(const vector& hosts, + MapOfParserResourcesType& resList) { - // for the moment then "maui" will be used for dynamic selection ... - return FindFirst(hosts); + if (hosts.size() == 0) + return string(""); + + std::string selected=hosts[0]; + int uses=0; + if(_numberOfUses.count(selected) != 0) + uses=_numberOfUses[selected]; + else + uses=0; + + for (std::vector::const_iterator iter = hosts.begin(); iter != hosts.end(); iter++) + { + std::string machine=*iter; + if(_numberOfUses.count(machine) == 0) + _numberOfUses[machine]=0; + if(_numberOfUses[machine] < uses) + { + selected=machine; + uses=_numberOfUses[machine]; + } + } + + _numberOfUses[selected]=_numberOfUses[selected]+1; + std::cerr << "selected: " << selected << " " << _numberOfUses[selected] << std::endl; + return selected; } + diff --git a/src/ResourcesManager/SALOME_LoadRateManager.hxx b/src/ResourcesManager/SALOME_LoadRateManager.hxx index 4a43bd7e3..af167bef8 100644 --- a/src/ResourcesManager/SALOME_LoadRateManager.hxx +++ b/src/ResourcesManager/SALOME_LoadRateManager.hxx @@ -24,15 +24,37 @@ #include "ResourcesManager_Defs.hxx" #include +#include #include "SALOME_ResourcesCatalog_Parser.hxx" -class RESOURCESMANAGER_EXPORT SALOME_LoadRateManager - { +class RESOURCESMANAGER_EXPORT LoadRateManager +{ + public: + virtual std::string Find(const std::vector& hosts, + MapOfParserResourcesType& resList){return "";}; +}; + +class RESOURCESMANAGER_EXPORT LoadRateManagerFirst:public LoadRateManager +{ + public: + virtual std::string Find(const std::vector& hosts, + MapOfParserResourcesType& resList); +}; + +class RESOURCESMANAGER_EXPORT LoadRateManagerCycl :public LoadRateManager +{ + public: + virtual std::string Find(const std::vector& hosts, + MapOfParserResourcesType& resList); +}; +class RESOURCESMANAGER_EXPORT LoadRateManagerAltCycl :public LoadRateManager +{ public: - std::string FindFirst(const std::vector& hosts); - std::string FindNext(const std::vector& hosts,MapOfParserResourcesType& resList); - std::string FindBest(const std::vector& hosts); - }; + virtual std::string Find(const std::vector& hosts, + MapOfParserResourcesType& resList); + protected: + std::map _numberOfUses; +}; #endif diff --git a/src/ResourcesManager/SALOME_ResourcesManager.cxx b/src/ResourcesManager/SALOME_ResourcesManager.cxx index 1817edc86..874b79ab6 100644 --- a/src/ResourcesManager/SALOME_ResourcesManager.cxx +++ b/src/ResourcesManager/SALOME_ResourcesManager.cxx @@ -183,7 +183,16 @@ SALOME_ResourcesManager::FindFirst(const Engines::MachineList& listOfMachines) for(unsigned int i=0;i ml; + for(unsigned int i=0;i