1 // Copyright (C) 2012-2020 CEA/DEN, EDF R&D
3 // This library is free software; you can redistribute it and/or
4 // modify it under the terms of the GNU Lesser General Public
5 // License as published by the Free Software Foundation; either
6 // version 2.1 of the License, or (at your option) any later version.
8 // This library is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 // Lesser General Public License for more details.
13 // You should have received a copy of the GNU Lesser General Public
14 // License along with this library; if not, write to the Free Software
15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
19 // Author : Anthony Geay (EDF R&D)
21 #include "YACSEvalYFXPattern.hxx"
22 #include "YACSEvalResource.hxx"
23 #include "YACSEvalSeqAny.hxx"
24 #include "YACSEvalSession.hxx"
25 #include "YACSEvalObserver.hxx"
26 #include "YACSEvalSessionInternal.hxx"
27 #include "YACSEvalAutoPtr.hxx"
28 #include "YACSEvalExecParams.hxx"
30 #include "ElementaryNode.hxx"
31 #include "RuntimeSALOME.hxx"
32 #include "Dispatcher.hxx"
33 #include "Executor.hxx"
34 #include "InputPort.hxx"
35 #include "LinkInfo.hxx"
36 #include "TypeCode.hxx"
38 #include "Dispatcher.hxx"
40 #include "PythonPorts.hxx"
41 #include "ForEachLoop.hxx"
42 #include "PythonNode.hxx"
43 #include "InlineNode.hxx"
44 #include "ServiceNode.hxx"
45 #include "PyStdout.hxx"
46 #include "AutoGIL.hxx"
48 #include "ResourcesManager.hxx"
59 const char YACSEvalYFXPattern::ST_OK[]="ALL_OK";
61 const char YACSEvalYFXPattern::ST_FAILED[]="SOME_SAMPLES_FAILED_AND_ALL_OF_THEM_FAILED_DETERMINISTICALLY";
63 const char YACSEvalYFXPattern::ST_ERROR[]="SOME_SAMPLES_FAILED_BUT_IMPOSSIBLE_TO_CONCLUDE_ON_THEM";
65 const std::size_t YACSEvalYFXPattern::MAX_LGTH_OF_INP_DUMP=10000;
67 const char YACSEvalYFXGraphGen::DFT_PROC_NAME[]="YFX";
69 const char YACSEvalYFXGraphGen::FIRST_FE_SUBNODE_NAME[]="Bloc";
71 const char YACSEvalYFXGraphGen::GATHER_NODE_NAME[]="__gather__";
73 const char YACSEvalYFXGraphGen::HIDDEN_INDEX_VAR[]="___idx___";
75 class MyAutoThreadSaver
78 MyAutoThreadSaver(bool isToSave):_isToSave(isToSave),_save(0)
82 _save = PyEval_SaveThread();
85 ~MyAutoThreadSaver() { if(_isToSave) PyEval_RestoreThread(_save); }
91 std::vector< YACSEvalInputPort *> YACSEvalYFXPattern::getFreeInputPorts() const
93 std::size_t sz(_inputs.size());
94 std::vector< YACSEvalInputPort *> ret;
95 std::vector< YACSEvalInputPort >::const_iterator it(_inputs.begin());
96 for(std::size_t i=0;i<sz;i++,it++)
97 ret.push_back(const_cast<YACSEvalInputPort *>(&(*it)));
101 std::vector< YACSEvalOutputPort *> YACSEvalYFXPattern::getFreeOutputPorts() const
103 std::size_t sz(_outputs.size());
104 std::vector< YACSEvalOutputPort *> ret;
105 std::vector< YACSEvalOutputPort >::const_iterator it(_outputs.begin());
106 for(std::size_t i=0;i<sz;i++,it++)
107 ret.push_back(const_cast<YACSEvalOutputPort *>(&(*it)));
111 YACSEvalYFXPattern *YACSEvalYFXPattern::FindPatternFrom(YACSEvalYFX *boss, YACS::ENGINE::Proc *scheme, bool ownScheme)
114 throw YACS::Exception("YACSEvalYFXPattern::FindPatternFrom : input scheme must be not null !");
116 YACS::ENGINE::ComposedNode *zeRunNode(0);
117 bool isMatchingRunOnlyPattern(YACSEvalYFXRunOnlyPattern::IsMatching(scheme,zeRunNode));
118 if(isMatchingRunOnlyPattern)
119 return new YACSEvalYFXRunOnlyPattern(boss,scheme,ownScheme,zeRunNode);
121 throw YACS::Exception("YACSEvalYFXPattern::FindPatternFrom : no pattern found for the input scheme !");
124 bool YACSEvalYFXPattern::isAlreadyComputedResources() const
129 void YACSEvalYFXPattern::checkNonAlreadyComputedResources() const
131 if(isAlreadyComputedResources())
132 throw YACS::Exception("checkNonAlreadyComputedResources : instance of computed resources already computed !");
135 void YACSEvalYFXPattern::checkAlreadyComputedResources() const
137 if(!isAlreadyComputedResources())
138 throw YACS::Exception("checkAlreadyComputedResources : instance of computed resources not already computed !");
141 void YACSEvalYFXPattern::checkLocked() const
144 throw YACS::Exception("YACSEvalYFXPattern::checkLocked : Pattern is not locked !");
147 void YACSEvalYFXPattern::checkNonLocked() const
150 throw YACS::Exception("YACSEvalYFXPattern::checkNonLocked : Pattern is locked !");
153 void YACSEvalYFXPattern::CheckNodeIsOK(YACS::ENGINE::ComposedNode *node)
155 /*YACS::ENGINE::LinkInfo info(YACS::ENGINE::LinkInfo::WARN_ONLY_DONT_STOP);
158 node->checkConsistency(info);
160 catch(YACS::Exception& e)
163 if(info.getNumberOfErrLinks(YACS::ENGINE::E_ALL)!=0)
164 throw YACS::Exception("YACSEvalYFXPattern::CheckNodeIsOK : found node is not OK !");
165 std::list<YACS::ENGINE::ElementaryNode *> allNodes(node->getRecursiveConstituents());
166 for(std::list<YACS::ENGINE::ElementaryNode *>::const_iterator it=allNodes.begin();it!=allNodes.end();it++)
168 YACS::ENGINE::ServiceNode *node0(dynamic_cast<YACS::ENGINE::ServiceNode *>(*it));
169 YACS::ENGINE::InlineNode *node1(dynamic_cast<YACS::ENGINE::InlineNode *>(*it));
172 YACS::ENGINE::Container *cont(node0->getContainer());
173 YACS::ENGINE::ComponentInstance *comp(node0->getComponent());
176 std::ostringstream oss; oss << "YACSEvalYFXPattern::CheckNodeIsOK : ServiceNode called \"" << node0->getName() << "\" is not correctly defined !";
177 throw YACS::Exception(oss.str());
182 YACS::ENGINE::Container *cont(node1->getContainer());
183 if(!cont && node1->getExecutionMode()==YACS::ENGINE::InlineNode::REMOTE_STR)
185 std::ostringstream oss; oss << "YACSEvalYFXPattern::CheckNodeIsOK : InlineNode called \"" << node1->getName() << "\" is not correctly defined !";
186 throw YACS::Exception(oss.str());
192 void YACSEvalYFXPattern::registerObserver(YACSEvalObserver *observer)
194 if(_observer==observer)
197 _observer->decrRef();
200 _observer->incrRef();
203 YACSEvalYFXPattern::YACSEvalYFXPattern(YACSEvalYFX *boss, YACS::ENGINE::Proc *scheme, bool ownScheme):_boss(boss),_scheme(scheme),_ownScheme(ownScheme),_parallelizeStatus(true),_rm(new ResourcesManager_cpp),_res(0),_observer(0)
207 YACS::ENGINE::TypeCode *YACSEvalYFXPattern::CreateSeqTypeCodeFrom(YACS::ENGINE::Proc *scheme, const std::string& zeType)
209 std::ostringstream oss; oss << "list[" << zeType << "]";
210 YACS::ENGINE::TypeCode *tc(scheme->getTypeCode(zeType));
211 return scheme->createSequenceTc(oss.str(),oss.str(),tc);
214 void YACSEvalYFXPattern::setResources(YACSEvalListOfResources *res)
216 checkNonAlreadyComputedResources();
222 void YACSEvalYFXPattern::resetResources()
228 YACSEvalSeqAny *YACSEvalYFXPattern::BuildValueInPort(YACS::ENGINE::InputPyPort *port)
231 throw YACS::Exception("YACSEvalYFXPattern::GetValueInPort : null input port !");
232 PyObject *obj(port->getPyObj());
233 YACS::ENGINE::TypeCode *tc(port->edGetType());
234 YACS::ENGINE::TypeCodeSeq *tcc(dynamic_cast<YACS::ENGINE::TypeCodeSeq *>(tc));
237 std::ostringstream oss; oss << "YACSEvalYFXPattern::GetValueInPort : internal error for tc of input \"" << port->getName() << "\"";
238 throw YACS::Exception(oss.str());
240 const YACS::ENGINE::TypeCode *tcct(tcc->contentType());
241 if(!PyList_Check(obj))
242 throw YACS::Exception("YACSEvalYFXPattern::GetValueInPort : internal error 2 !");
243 std::size_t sz(PyList_Size(obj));
244 if(tcct->kind()==YACS::ENGINE::Double)
246 std::vector<double> eltCpp(sz);
247 for(std::size_t i=0;i<sz;i++)
249 PyObject *elt(PyList_GetItem(obj,i));
250 eltCpp[i]=PyFloat_AsDouble(elt);
252 YACS::AutoCppPtr<YACSEvalSeqAnyDouble> elt(new YACSEvalSeqAnyDouble(eltCpp));
253 return elt.dettach();
255 else if(tcct->kind()==YACS::ENGINE::Int)
257 std::vector<int> eltCpp(sz);
258 for(std::size_t i=0;i<sz;i++)
260 PyObject *elt(PyList_GetItem(obj,i));
261 eltCpp[i]=PyLong_AsLong(elt);
263 YACS::AutoCppPtr<YACSEvalSeqAnyInt> elt(new YACSEvalSeqAnyInt(eltCpp));
264 return elt.dettach();
267 throw YACS::Exception("YACSEvalYFXPattern::GetValueInPort : not implemented yet for other than Double and Int !");
270 YACSEvalSeqAny *YACSEvalYFXPattern::BuildValueFromEngineFrmt(YACS::ENGINE::SequenceAny *data)
272 unsigned int sz(data->size());
273 std::vector<double> eltCpp(sz);
274 for(unsigned int ii=0;ii<sz;ii++)
276 YACS::ENGINE::AnyPtr elt((*data)[ii]);
277 YACS::ENGINE::Any *eltPtr((YACS::ENGINE::Any *)elt);
278 YACS::ENGINE::AtomAny *eltPtr2(dynamic_cast<YACS::ENGINE::AtomAny *>(eltPtr));
281 std::ostringstream oss; oss << "YACSEvalYFXPattern::BuildValueFromEngineFrmt : error at pos #" << ii << " ! It is not an AtomAny !";
282 throw YACS::Exception(oss.str());
284 eltCpp[ii]=eltPtr2->getDoubleValue();
286 return new YACSEvalSeqAnyDouble(eltCpp);
289 void YACSEvalYFXPattern::cleanScheme()
296 YACSEvalYFXPattern::~YACSEvalYFXPattern()
299 _observer->decrRef();
304 /////////////////////
306 class YACSEvalYFXRunOnlyPatternInternalObserver : public YACS::ENGINE::Observer
309 YACSEvalYFXRunOnlyPatternInternalObserver(YACSEvalYFXRunOnlyPattern *boss):_boss(boss) { if(!_boss) throw YACS::Exception("YACSEvalYFXRunOnlyPatternInternalObserver constructor : null boss not supported :)"); }
310 void notifyObserver2(YACS::ENGINE::Node *object, const std::string& event, void *something);
312 YACSEvalYFXRunOnlyPattern *_boss;
315 void YACSEvalYFXRunOnlyPatternInternalObserver::notifyObserver2(YACS::ENGINE::Node *object, const std::string& event, void *something)
317 YACS::ENGINE::ForEachLoop *object2(_boss->getUndergroundForEach());
318 YACSEvalObserver *obs(_boss->getObserver());
321 if(event=="progress_ok" && object2==object)
323 int *casted(reinterpret_cast<int *>(something));
324 obs->notifySampleOK(_boss->getBoss(),*casted);
327 if(event=="progress_ko" && object2==object)
329 int *casted(reinterpret_cast<int *>(something));
330 obs->notifySampleKO(_boss->getBoss(),*casted);
335 /////////////////////
337 YACSEvalYFXRunOnlyPattern::YACSEvalYFXRunOnlyPattern(YACSEvalYFX *boss, YACS::ENGINE::Proc *scheme, bool ownScheme, YACS::ENGINE::ComposedNode *runNode):YACSEvalYFXPattern(boss,scheme,ownScheme),_lockedStatus(false),_runNode(runNode),_gen(0),_obs(new YACSEvalYFXRunOnlyPatternInternalObserver(this))
340 throw YACS::Exception("YACSEvalYFXRunOnlyPattern : internal run node must be not null !");
345 YACSEvalYFXRunOnlyPattern::~YACSEvalYFXRunOnlyPattern()
351 void YACSEvalYFXRunOnlyPattern::setOutPortsOfInterestForEvaluation(const std::vector<YACSEvalOutputPort *>& outputsOfInterest)
354 _outputsOfInterest=outputsOfInterest;
358 void YACSEvalYFXRunOnlyPattern::resetOutputsOfInterest()
361 _outputsOfInterest.clear();
365 void YACSEvalYFXRunOnlyPattern::generateGraph()
368 if(getResourcesInternal()->isInteractive())
369 _gen=new YACSEvalYFXGraphGenInteractive(this);
371 _gen=new YACSEvalYFXGraphGenCluster(this);
372 _gen->generateGraph();
375 void YACSEvalYFXRunOnlyPattern::resetGeneratedGraph()
378 _gen->resetGeneratedGraph();
381 int YACSEvalYFXRunOnlyPattern::assignNbOfBranches()
383 checkAlreadyComputedResources();
385 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::assignNbOfBranches : generator is NULL ! Please invoke generateGraph before !");
386 return _gen->assignNbOfBranches();
389 void YACSEvalYFXRunOnlyPattern::assignRandomVarsInputs()
391 std::size_t sz(std::numeric_limits<std::size_t>::max());
392 for(std::vector< YACSEvalInputPort >::const_iterator it=_inputs.begin();it!=_inputs.end();it++)
393 if((*it).isRandomVar())
395 std::size_t locSize((*it).initializeUndergroundWithSeq());
396 if(sz==std::numeric_limits<std::size_t>::max())
400 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::assignRandomVarsInputs : length of sequences in random vars must be the same !");
404 bool YACSEvalYFXRunOnlyPattern::isLocked() const
406 return _lockedStatus;
409 YACSEvalListOfResources *YACSEvalYFXRunOnlyPattern::giveResources()
412 if(!isAlreadyComputedResources())
414 YACS::ENGINE::DeploymentTree dt(_runNode->getDeploymentTree());
415 _runNode->removeRecursivelyRedundantCL();
416 YACSEvalListOfResources *res(new YACSEvalListOfResources(_runNode->getMaxLevelOfParallelism(),getCatalogInAppli(),dt));
419 return getResourcesInternal();
422 YACS::ENGINE::Proc *YACSEvalYFXRunOnlyPattern::getUndergroundGeneratedGraph() const
424 return getGenerator()->getUndergroundGeneratedGraph();
427 std::string YACSEvalYFXRunOnlyPattern::getErrorDetailsInCaseOfFailure() const
429 std::string generatorErrors = getGenerator()->getErrors();
430 if(generatorErrors.size() > 0)
432 return generatorErrors;
434 std::string st(getStatusOfRunStr());//test if a run has occurred.
436 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::getErrorDetailsInCaseOfFailure : The execution of scheme has been carried out to the end without any problem !");
437 // All the problem can only comes from foreach -> scan it
438 YACS::ENGINE::ForEachLoop *fe(getUndergroundForEach());
439 YACS::ENGINE::NodeStateNameMap nsm;
440 unsigned nbB(fe->getNumberOfBranchesCreatedDyn());
441 std::ostringstream oss;
442 for(unsigned j=0;j<nbB;j++)
444 YACS::ENGINE::Node *nn(fe->getChildByNameExec(YACSEvalYFXGraphGen::FIRST_FE_SUBNODE_NAME,j));
445 YACS::ENGINE::Bloc *nnc(dynamic_cast<YACS::ENGINE::Bloc *>(nn));
447 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::getErrorDetailsInCaseOfFailure : internal error 1 ! The direct son of main foreach is expected to be a Bloc !");
448 if(nnc->getState()==YACS::DONE)
450 std::list< YACS::ENGINE::ElementaryNode *> fec(nnc->getRecursiveConstituents());
451 for(std::list< YACS::ENGINE::ElementaryNode *>::reverse_iterator it1=fec.rbegin();it1!=fec.rend();it1++)
453 YACS::StatesForNode st0((*it1)->getState());
456 oss << "NODE = " << nnc->getChildName(*it1) << std::endl;
457 oss << "STATUS = " << nsm[st0] << std::endl;
458 oss << "BRANCH ID = " << j << std::endl;
459 oss << "DETAILS = " << std::endl;
460 oss << (*it1)->getErrorDetails();
467 std::string YACSEvalYFXRunOnlyPattern::getStatusOfRunStr() const
469 YACS::StatesForNode st(getUndergroundGeneratedGraph()->getState());
475 case YACS::TOACTIVATE:
476 case YACS::ACTIVATED:
477 case YACS::SUSPENDED:
480 case YACS::DESACTIVATED:
482 std::ostringstream oss; oss << "YACSEvalYFXRunOnlyPattern::getStatusOfRunStr : Unexpected state \"" << YACS::ENGINE::Node::getStateName(st) << "\" ! Did you invoke run ?";
483 throw YACS::Exception(oss.str());
485 case YACS::LOADFAILED:
486 case YACS::EXECFAILED:
488 case YACS::INTERNALERR:
489 return std::string(ST_ERROR);
491 return std::string(ST_FAILED);
493 return std::string(ST_OK);
496 std::ostringstream oss; oss << "YACSEvalYFXRunOnlyPattern::getStatusOfRunStr : unrecognized and managed state \"" << YACS::ENGINE::Node::getStateName(st) << "\" !";
497 throw YACS::Exception(oss.str());
502 std::vector<YACSEvalSeqAny *> YACSEvalYFXRunOnlyPattern::getResults() const
504 return _gen->getResults();
508 * This method works if run succeeded (true return) and also if graph has failed. Graph failed means soft error of evaluation due to error in evaluation (example 1/0 or a normal throw from one node)
509 * If a more serious error occured (SIGSEGV of a server or internal error in YACS engine, cluster error, loose of connection...) this method will throw an exception to warn the caller that the results may be
511 std::vector<YACSEvalSeqAny *> YACSEvalYFXRunOnlyPattern::getResultsInCaseOfFailure(std::vector<unsigned int>& passedIds) const
513 YACS::StatesForNode st(getUndergroundGeneratedGraph()->getState());
517 std::vector<YACSEvalSeqAny *> ret(getResults());
521 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::getResultsInCaseOfFailure : internal error ! The returned vector has a null pointer at pos #0 !");
522 std::size_t sz(ret[0]->size());
523 passedIds.resize(sz);
524 for(std::size_t i=0;i<sz;i++)
529 getStatusOfRunStr();// To check that the status is recognized.
530 std::list<YACS::ENGINE::Node *> lns(getUndergroundGeneratedGraph()->edGetDirectDescendants());
531 YACS::ENGINE::ForEachLoop *fe(getUndergroundForEach());
533 YACS::ENGINE::Executor exe;
534 std::vector<YACS::ENGINE::SequenceAny *> outputs;
535 std::vector<std::string> nameOfOutputs;
536 passedIds=fe->getPassedResults(&exe,outputs,nameOfOutputs);//<- the key invokation is here.
537 std::size_t sz(passedIds.size()),ii(0);
538 std::vector<YACSEvalSeqAny *> ret(_outputsOfInterest.size());
539 for(std::vector<YACSEvalOutputPort *>::const_iterator it1=_outputsOfInterest.begin();it1!=_outputsOfInterest.end();it1++,ii++)
541 YACS::ENGINE::OutputPort *p((*it1)->getUndergroundPtr());
542 std::string st(_runNode->getOutPortName(p));
543 std::ostringstream oss; oss << YACSEvalYFXGraphGen::FIRST_FE_SUBNODE_NAME << '.' << _runNode->getName() << '.' << st;
545 YACS::ENGINE::ForEachLoop::InterceptorizeNameOfPort(st);
546 std::vector<std::string>::iterator it2(std::find(nameOfOutputs.begin(),nameOfOutputs.end(),st));
547 if(it2==nameOfOutputs.end())
549 std::ostringstream oss; oss << "YACSEvalYFXRunOnlyPattern::getResultsInCaseOfFailure : internal error 3 ! Unable to locate interceptor with name " << st << " ! Possibilities are : ";
550 std::copy(nameOfOutputs.begin(),nameOfOutputs.end(),std::ostream_iterator<std::string>(oss," "));
552 throw YACS::Exception(oss.str());
554 std::size_t pos(std::distance(nameOfOutputs.begin(),it2));
555 ret[ii]=BuildValueFromEngineFrmt(outputs[pos]);
560 void YACSEvalYFXRunOnlyPattern::emitStart() const
562 YACSEvalObserver *obs(getObserver());
565 obs->startComputation(getBoss());
568 bool YACSEvalYFXRunOnlyPattern::go(const YACSEvalExecParams& params, YACSEvalSession *session) const
571 YACS::ENGINE::Dispatcher *disp(YACS::ENGINE::Dispatcher::getDispatcher());
572 disp->addObserver(_obs,getUndergroundForEach(),"progress_ok");
573 disp->addObserver(_obs,getUndergroundForEach(),"progress_ko");
574 bool ret(getGenerator()->go(params,session));
575 disp->removeObserver(_obs,getUndergroundForEach(),"progress_ok");
576 disp->removeObserver(_obs,getUndergroundForEach(),"progress_ko");
580 YACS::ENGINE::ForEachLoop *YACSEvalYFXRunOnlyPattern::getUndergroundForEach() const
582 return getGenerator()->getUndergroundForEach();
585 bool YACSEvalYFXRunOnlyPattern::IsMatching(YACS::ENGINE::Proc *scheme, YACS::ENGINE::ComposedNode *& runNode)
587 std::list<YACS::ENGINE::Node *> nodes(scheme->getChildren());
590 bool areAllElementary(true);
591 for(std::list<YACS::ENGINE::Node *>::const_iterator it=nodes.begin();it!=nodes.end() && areAllElementary;it++)
592 if(!dynamic_cast<YACS::ENGINE::ElementaryNode *>(*it))
593 areAllElementary=false;
597 CheckNodeIsOK(scheme);
603 YACS::ENGINE::ComposedNode *candidate(dynamic_cast<YACS::ENGINE::ComposedNode *>(nodes.front()));
606 CheckNodeIsOK(candidate);
610 void YACSEvalYFXRunOnlyPattern::buildInputPorts()
613 std::list< YACS::ENGINE::InputPort *> allInputPorts(_runNode->getSetOfInputPort());
614 std::vector<std::string> allNames;
615 for(std::list< YACS::ENGINE::InputPort *>::const_iterator it=allInputPorts.begin();it!=allInputPorts.end();it++)
617 YACS::ENGINE::InputPort *elt(*it);
619 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::buildInputPorts : presence of null input !");
620 std::set<YACS::ENGINE::OutPort *> bls(elt->edSetOutPort());
623 if(YACSEvalPort::IsInputPortPublishable(elt))
625 std::string inpName(elt->getName());
627 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::buildInputPorts : an input has empty name ! Should not !");
628 _inputs.push_back(YACSEvalInputPort(elt));
629 if(std::find(allNames.begin(),allNames.end(),inpName)!=allNames.end())
631 std::ostringstream oss; oss << "YACSEvalYFXRunOnlyPattern::buildInputPorts : input name \"" << inpName << "\" appears more than once !";
632 throw YACS::Exception(oss.str());
634 allNames.push_back(inpName);
640 void YACSEvalYFXRunOnlyPattern::buildOutputPorts()
643 std::list< YACS::ENGINE::OutputPort *> allOutputPorts(_runNode->getSetOfOutputPort());
644 std::vector<std::string> allNames;
645 for(std::list< YACS::ENGINE::OutputPort *>::const_iterator it=allOutputPorts.begin();it!=allOutputPorts.end();it++)
647 YACS::ENGINE::OutputPort *elt(*it);
648 if(YACSEvalPort::IsOutputPortPublishable(elt))
651 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::buildOutputPorts : presence of null output !");
652 std::string outpName(elt->getName());
654 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::buildOutputPorts : an output has empty name ! Should not !");
655 if(std::find(allNames.begin(),allNames.end(),outpName)!=allNames.end())
657 std::ostringstream oss; oss << "YACSEvalYFXRunOnlyPattern::buildOutputPorts : output name \"" << outpName << "\" appears more than once !";
658 throw YACS::Exception(oss.str());
660 _outputs.push_back(YACSEvalOutputPort(*it));
665 YACSEvalYFXGraphGen *YACSEvalYFXRunOnlyPattern::getGenerator() const
668 throw YACS::Exception("getGenerator : generator is NULL !");
672 /////////////////////
674 YACSEvalYFXGraphGen::YACSEvalYFXGraphGen(YACSEvalYFXRunOnlyPattern *boss):_boss(boss),_generatedGraph(0),_FEInGeneratedGraph(0)
677 throw YACS::Exception("YACSEvalYFXGraphGen constructor : boss is NULL !");
680 YACSEvalYFXGraphGen::~YACSEvalYFXGraphGen()
682 //delete _generatedGraph;// -> TODO : AGY why ?
685 void YACSEvalYFXGraphGen::resetGeneratedGraph()
687 delete _generatedGraph;
688 _generatedGraph=0; _FEInGeneratedGraph=0;
691 void YACSEvalYFXGraphGen::generateGraphCommon(CustomPatcher& patcher)
694 { delete _generatedGraph; _generatedGraph=0; _FEInGeneratedGraph=0; }
695 static const char LISTPYOBJ_STR[]="list[pyobj]";
696 if(getBoss()->getOutputsOfInterest().empty())
698 YACS::ENGINE::RuntimeSALOME::setRuntime();
699 YACS::ENGINE::RuntimeSALOME *r(YACS::ENGINE::getSALOMERuntime());
700 _generatedGraph=r->createProc(DFT_PROC_NAME);
701 YACS::ENGINE::TypeCode *pyobjTC(_generatedGraph->createInterfaceTc("python:obj:1.0","pyobj",std::list<YACS::ENGINE::TypeCodeObjref *>()));
702 std::ostringstream oss; oss << "Loop_" << getBoss()->getRunNode()->getName();
703 _generatedGraph->createType(YACSEvalAnyDouble::TYPE_REPR,"double");
704 YACS::ENGINE::TypeCode *tcInt(_generatedGraph->createType(YACSEvalAnyInt::TYPE_REPR,"int"));
706 YACS::ENGINE::InlineNode *n0(r->createScriptNode(YACS::ENGINE::PythonNode::KIND,"__initializer__"));
707 _generatedGraph->edAddChild(n0);
708 YACS::ENGINE::TypeCode *listPyobjTC(_generatedGraph->createSequenceTc(LISTPYOBJ_STR,LISTPYOBJ_STR,pyobjTC));
709 YACS::ENGINE::OutputPort *sender(n0->edAddOutputPort("sender",listPyobjTC));
710 std::ostringstream var0;
711 const std::vector< YACSEvalInputPort >& inputs(getBoss()->getInputs());
712 for(std::vector< YACSEvalInputPort >::const_iterator it=inputs.begin();it!=inputs.end();it++)
714 if((*it).isRandomVar())
716 var0 << (*it).getName() << ",";
717 YACS::ENGINE::TypeCode *tc(YACSEvalYFXPattern::CreateSeqTypeCodeFrom(_generatedGraph,(*it).getTypeOfData()));
718 YACS::ENGINE::InputPort *inp(n0->edAddInputPort((*it).getName(),tc));
719 YACS::ENGINE::InputPyPort *inpc(dynamic_cast<YACS::ENGINE::InputPyPort *>(inp));
721 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::generateGraph : internal error 1 !");
722 (*it).setUndergroundPortToBeSet(inpc);
725 std::ostringstream n0Script; n0Script << "sender=[tuple([__p9Sq]+list(__p9Sw)) for __p9Sq,__p9Sw in enumerate(zip(" << var0.str() << "))]\n";
726 n0->setScript(n0Script.str());
728 YACS::ENGINE::ForEachLoop *n1(r->createForEachLoop(oss.str(),pyobjTC));
729 _FEInGeneratedGraph=n1;
730 _generatedGraph->edAddChild(n1);
731 _generatedGraph->edAddCFLink(n0,n1);
732 _generatedGraph->edAddDFLink(sender,n1->edGetSeqOfSamplesPort());
733 YACS::ENGINE::InlineNode *n2(r->createScriptNode(YACS::ENGINE::PythonNode::KIND,GATHER_NODE_NAME));
734 _generatedGraph->edAddChild(n2);
735 _generatedGraph->edAddCFLink(n1,n2);
737 YACS::ENGINE::Bloc *n10(r->createBloc(FIRST_FE_SUBNODE_NAME));
739 YACS::ENGINE::InlineNode *n100(r->createScriptNode(YACS::ENGINE::PythonNode::KIND,"__dispatch__"));
740 YACS::ENGINE::ComposedNode *runNode(getBoss()->getRunNode());
741 YACS::ENGINE::Node *n101(runNode->cloneWithoutCompAndContDeepCpy(0,true));
742 n10->edAddChild(n100);
743 n10->edAddChild(n101);
744 YACS::ENGINE::InputPort *dispatchIn(n100->edAddInputPort("i0",pyobjTC));
745 n10->edAddCFLink(n100,n101);
746 n1->edAddDFLink(n1->edGetSamplePort(),dispatchIn);
747 std::ostringstream var1;
748 YACS::ENGINE::OutputPort *n100_output(n100->edAddOutputPort(HIDDEN_INDEX_VAR,tcInt));
749 var1 << HIDDEN_INDEX_VAR << ",";
750 for(std::vector< YACSEvalInputPort >::const_iterator it=inputs.begin();it!=inputs.end();it++)
752 if((*it).isRandomVar())
754 var1 << (*it).getName() << ",";
755 YACS::ENGINE::OutputPort *myOut(n100->edAddOutputPort((*it).getName(),_generatedGraph->getTypeCode((*it).getTypeOfData())));
756 std::string tmpPortName(runNode->getInPortName((*it).getUndergroundPtr()));
757 YACS::ENGINE::InputPort *myIn(n101->getInputPort(tmpPortName));
758 n10->edAddDFLink(myOut,myIn);
761 std::ostringstream n100Script; n100Script << var1.str() << "=i0\n";
762 n100->setScript(n100Script.str());
763 const std::vector<YACSEvalOutputPort *>& outputsOfInt(getBoss()->getOutputsOfInterest());
765 std::list<YACS::ENGINE::ElementaryNode *> n101_constit(n101->getRecursiveConstituents());
766 for(std::list<YACS::ENGINE::ElementaryNode *>::const_iterator it=n101_constit.begin();it!=n101_constit.end();it++)
768 YACS::ENGINE::InlineNode *eltc(dynamic_cast<YACS::ENGINE::InlineNode *>(*it));
771 YACS::ENGINE::InputPort *n101_input(eltc->edAddInputPort(HIDDEN_INDEX_VAR,tcInt));
772 _generatedGraph->edAddDFLink(n100_output,n101_input);
776 for(std::vector< YACSEvalOutputPort * >::const_iterator it=outputsOfInt.begin();it!=outputsOfInt.end();it++)
778 patcher.addOutputVar((*it)->getName());
779 YACS::ENGINE::TypeCode *tc(YACSEvalYFXPattern::CreateSeqTypeCodeFrom(_generatedGraph,(*it)->getTypeOfData()));
780 YACS::ENGINE::InputPort *myIn(n2->edAddInputPort((*it)->getName(),tc));
781 std::string tmpPortName(runNode->getOutPortName((*it)->getUndergroundPtr()));
782 YACS::ENGINE::OutputPort *myOut(n101->getOutputPort(tmpPortName));
783 _generatedGraph->edAddDFLink(myOut,myIn);
785 patcher.assignOutput(n2);
786 _generatedGraph->updateContainersAndComponents();
789 bool YACSEvalYFXGraphGen::isLocked() const
791 return _generatedGraph!=0;
794 int YACSEvalYFXGraphGen::assignNbOfBranches()
797 throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : the generated graph has not been created !");
798 std::list<YACS::ENGINE::Node *> nodes(_generatedGraph->getChildren());
799 YACS::ENGINE::ForEachLoop *zeMainNode(0);
800 for(std::list<YACS::ENGINE::Node *>::const_iterator it=nodes.begin();it!=nodes.end();it++)
802 YACS::ENGINE::ForEachLoop *isZeMainNode(dynamic_cast<YACS::ENGINE::ForEachLoop *>(*it));
806 zeMainNode=isZeMainNode;
808 throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : internal error 1 !");
812 throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : internal error 2 !");
813 unsigned int nbProcsDeclared(getBoss()->getResourcesInternal()->getNumberOfProcsDeclared());
814 nbProcsDeclared=std::max(nbProcsDeclared,4u);
816 if(getBoss()->getParallelizeStatus())
818 nbOfBranch=(nbProcsDeclared/getBoss()->getResourcesInternal()->getMaxLevelOfParallelism());
819 nbOfBranch=std::max(nbOfBranch,1);
821 YACS::ENGINE::InputPort *zeInputToSet(zeMainNode->edGetNbOfBranchesPort());
822 YACS::ENGINE::AnyInputPort *zeInputToSetC(dynamic_cast<YACS::ENGINE::AnyInputPort *>(zeInputToSet));
824 throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : internal error 3 !");
825 YACS::ENGINE::Any *a(YACS::ENGINE::AtomAny::New(nbOfBranch));
826 zeInputToSetC->put(a);
827 zeInputToSetC->exSaveInit();
832 void YACSEvalYFXGraphGenInteractive::generateGraph()
834 class LocalPatcher : public YACSEvalYFXGraphGen::CustomPatcher
837 void addOutputVar(const std::string& name) { }
838 void assignOutput(YACS::ENGINE::InlineNode *node) { }
841 this->generateGraphCommon(lp);
844 bool YACSEvalYFXGraphGenInteractive::go(const YACSEvalExecParams& params, YACSEvalSession *session) const
846 YACS::ENGINE::Executor exe;
847 exe.setKeepGoingProperty(!params.getStopASAPAfterErrorStatus());
849 MyAutoThreadSaver locker(!session->isAlreadyPyThreadSaved());
850 exe.RunW(getUndergroundGeneratedGraph());
852 return getUndergroundGeneratedGraph()->getState()==YACS::DONE;
855 std::string YACSEvalYFXGraphGenInteractive::getErrors()const
860 std::vector<YACSEvalSeqAny *> YACSEvalYFXGraphGenInteractive::getResults() const
862 if(getUndergroundGeneratedGraph()->getState()!=YACS::DONE)
863 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::getResults : the execution did not finished correctly ! getResults should not be called !");
864 const std::vector<YACSEvalOutputPort *>& outputsOfInt(getBoss()->getOutputsOfInterest());
865 std::vector<YACSEvalSeqAny *> ret(outputsOfInt.size());
866 YACS::ENGINE::Node *node(getUndergroundGeneratedGraph()->getChildByName(YACSEvalYFXGraphGen::GATHER_NODE_NAME));
867 YACS::ENGINE::PythonNode *nodeC(dynamic_cast<YACS::ENGINE::PythonNode *>(node));
869 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::getResults : internal error !");
871 for(std::vector< YACSEvalOutputPort * >::const_iterator it=outputsOfInt.begin();it!=outputsOfInt.end();it++,ii++)
873 YACS::ENGINE::InPort *input(nodeC->getInPort((*it)->getName()));
874 YACS::ENGINE::InputPyPort *inputC(dynamic_cast<YACS::ENGINE::InputPyPort *>(input));
877 std::ostringstream oss; oss << "YACSEvalYFXRunOnlyPattern::getResults : internal error for input \"" << (*it)->getName() << "\"";
878 throw YACS::Exception(oss.str());
880 ret[ii]=YACSEvalYFXPattern::BuildValueInPort(inputC);
887 void YACSEvalYFXGraphGenCluster::generateGraph()
889 YACS::ENGINE::AutoGIL agil;
891 const char EFXGenFileName[]="EFXGenFileName";
892 const char EFXGenContent[]="import getpass,datetime,os\nn=datetime.datetime.now()\nreturn os.path.join(os.path.sep,\"tmp\",\"EvalYFX_%s_%s_%s.xml\"%(getpass.getuser(),n.strftime(\"%d%m%y\"),n.strftime(\"%H%M%S\")))";
893 const char EFXGenContent2[]="import getpass,datetime\nn=datetime.datetime.now()\nreturn \"EvalYFX_%s_%s_%s\"%(getpass.getuser(),n.strftime(\"%d%m%y\"),n.strftime(\"%H%M%S\"))";
895 YACS::ENGINE::AutoPyRef func(YACS::ENGINE::evalPy(EFXGenFileName,EFXGenContent));
896 YACS::ENGINE::AutoPyRef val(YACS::ENGINE::evalFuncPyWithNoParams(func));
897 if (PyUnicode_Check(val))
898 _locSchemaFile = PyUnicode_AsUTF8(val);
900 throw YACS::Exception("YACSEvalYFXGraphGenCluster::generateGraph: python call error. ");
902 func=YACS::ENGINE::evalPy(EFXGenFileName,EFXGenContent2);
903 val=YACS::ENGINE::evalFuncPyWithNoParams(func);
904 if (PyUnicode_Check(val))
905 _jobName = PyUnicode_AsUTF8(val);
907 throw YACS::Exception("YACSEvalYFXGraphGenCluster::generateGraph: python call error. ");
909 class ClusterPatcher : public YACSEvalYFXGraphGen::CustomPatcher
912 ClusterPatcher(const std::string& jobName):_jobName(jobName) { n2Script << "zeRes=["; }
913 void addOutputVar(const std::string& name) { n2Script<< name << ", "; }
914 void assignOutput(YACS::ENGINE::InlineNode *node) {
915 n2Script << "]\nwith open(\"" << _jobName << "\",\"w\") as f:" << std::endl;
916 n2Script << " f.write(repr(zeRes))" << std::endl;
917 node->setScript(n2Script.str());
920 std::ostringstream n2Script;
921 std::string _jobName;
923 ClusterPatcher cp(_jobName);
925 this->generateGraphCommon(cp);
928 bool YACSEvalYFXGraphGenCluster::go(const YACSEvalExecParams& params, YACSEvalSession *session) const
930 YACS::ENGINE::AutoGIL agil;
932 getUndergroundGeneratedGraph()->saveSchema(_locSchemaFile);
933 YACSEvalListOfResources *rss(getBoss()->getResourcesInternal());
934 const YACSEvalParamsForCluster& cli(rss->getAddParamsForCluster());
935 std::vector<std::string> machines(rss->getAllChosenMachines());
936 if(machines.size()!=1)
937 throw YACS::Exception("YACSEvalYFXGraphGenCluster::go : internal error ! In batch mode and not exactly one machine !");
938 Engines::SalomeLauncher_var sl(session->getInternal()->goFetchingSalomeLauncherInNS());
939 Engines::ResourceParameters rr;
940 rr.name=CORBA::string_dup(machines[0].c_str());
941 rr.hostname=CORBA::string_dup("");
942 rr.can_launch_batch_jobs=true;
943 rr.can_run_containers=true;
944 rr.OS=CORBA::string_dup("Linux");
945 rr.componentList.length(0);
946 rr.nb_proc=rss->getNumberOfProcsDeclared();// <- important
947 rr.mem_mb=0; // use default value
948 rr.cpu_clock=0; // use default value
949 rr.nb_node=0;// the number of nodes is not explicitly set
950 rr.nb_proc_per_node=1;// useless only nb_proc used.
951 rr.policy=CORBA::string_dup("cycl");
952 rr.resList.length(0);
953 Engines::JobParameters jp;
954 jp.job_name=CORBA::string_dup(_jobName.c_str());
955 jp.job_type=CORBA::string_dup("yacs_file");
956 jp.job_file=CORBA::string_dup(_locSchemaFile.c_str());
957 jp.env_file=CORBA::string_dup("");
958 jp.in_files.length(cli.getInFiles().size());
959 std::list<std::string>::const_iterator it;
961 for (it = cli.getInFiles().begin(), i=0 ;
962 it != cli.getInFiles().end();
965 jp.in_files[i] = CORBA::string_dup((*it).c_str());
967 jp.out_files.length(1);
968 jp.out_files[0]=CORBA::string_dup(_jobName.c_str());
969 jp.work_directory=CORBA::string_dup(cli.getRemoteWorkingDir().c_str());
970 jp.local_directory=CORBA::string_dup(cli.getLocalWorkingDir().c_str());
971 jp.result_directory=CORBA::string_dup(cli.getLocalWorkingDir().c_str());
972 jp.maximum_duration=CORBA::string_dup(cli.getMaxDuration().c_str());
973 jp.resource_required=rr;
974 jp.queue=CORBA::string_dup("");
976 jp.mem_per_cpu=rr.mem_mb;
977 jp.wckey=CORBA::string_dup(cli.getWCKey().c_str());
978 jp.extra_params=CORBA::string_dup("");
979 jp.specific_parameters.length(0);
980 jp.launcher_file=CORBA::string_dup("");
981 jp.launcher_args=CORBA::string_dup("");
982 _jobid=sl->createJob(jp);
985 sl->launchJob(_jobid);
987 catch (const SALOME::SALOME_Exception & ex)
989 _errors = ex.details.text.in();
992 catch (const CORBA::SystemException& ex)
994 _errors = "Receive CORBA SystemException.";
1001 PyRun_SimpleString("import time ; time.sleep(10)");
1002 char *state(sl->getJobState(_jobid));//"CREATED", "IN_PROCESS", "QUEUED", "RUNNING", "PAUSED", "FINISHED" or "FAILED"
1003 std::string sstate(state);
1004 CORBA::string_free(state);
1005 if(sstate=="FINISHED" || sstate=="FAILED")
1007 ret=sstate=="FINISHED";
1011 sl->getJobResults(_jobid,cli.getLocalWorkingDir().c_str());
1015 std::ostringstream oss; oss << "import os" << std::endl << "p=os.path.join(\"" << cli.getLocalWorkingDir() << "\",\"" << _jobName << "\")" << std::endl;
1016 oss << "if not os.path.exists(p):\n return None\n";
1017 oss << "f=open(p,\"r\")" << std::endl;
1018 oss << "return eval(f.read())";
1019 std::string zeInput(oss.str());
1020 YACS::ENGINE::AutoPyRef func(YACS::ENGINE::evalPy("fetch",zeInput));
1021 YACS::ENGINE::AutoPyRef val(YACS::ENGINE::evalFuncPyWithNoParams(func));
1022 if(!PyList_Check(val))
1023 throw YACS::Exception("Fetched file does not contain a list !");
1024 Py_ssize_t sz(PyList_Size(val));
1026 for(Py_ssize_t i=0;i<sz;i++)
1028 std::vector<double>& res0(_res[i]);
1029 PyObject *elt0(PyList_GetItem(val,i));
1030 if(!PyList_Check(elt0))
1031 throw YACS::Exception("Fetched file does contain a list of list !");
1032 Py_ssize_t sz0(PyList_Size(elt0)); res0.resize(sz0);
1033 for(Py_ssize_t j=0;j<sz0;j++)
1035 PyObject *elt1(PyList_GetItem(elt0,j));
1036 res0[j]=PyFloat_AsDouble(elt1);
1040 std::ostringstream oss1; oss1 << "import os" << std::endl << "p=os.path.join(\"" << cli.getLocalWorkingDir() << "\",\"" << _jobName << "\") ; os.remove(p)" << std::endl;
1041 std::string s1(oss1.str());
1042 PyRun_SimpleString(s1.c_str());
1043 if(!params.getFetchRemoteDirForClusterStatus())
1045 std::ostringstream oss2; oss2 << "import os,shutil" << std::endl << "p=os.path.join(\"" << cli.getLocalWorkingDir() << "\",\"logs\") ; shutil.rmtree(p)" << std::endl;
1046 std::string s2(oss2.str());
1047 PyRun_SimpleString(s2.c_str());
1050 catch(YACS::Exception& e)
1059 std::string YACSEvalYFXGraphGenCluster::getErrors()const
1064 std::vector<YACSEvalSeqAny *> YACSEvalYFXGraphGenCluster::getResults() const
1066 std::size_t sz(_res.size());
1067 std::vector<YACSEvalSeqAny *> ret(sz);
1068 for(std::size_t i=0;i<sz;i++)
1070 YACS::AutoCppPtr<YACSEvalSeqAnyDouble> elt(new YACSEvalSeqAnyDouble(_res[i]));
1071 ret[i]=elt.dettach();