1 // Copyright (C) 2012-2016 CEA/DEN, EDF R&D
3 // This library is free software; you can redistribute it and/or
4 // modify it under the terms of the GNU Lesser General Public
5 // License as published by the Free Software Foundation; either
6 // version 2.1 of the License, or (at your option) any later version.
8 // This library is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 // Lesser General Public License for more details.
13 // You should have received a copy of the GNU Lesser General Public
14 // License along with this library; if not, write to the Free Software
15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
19 // Author : Anthony Geay (EDF R&D)
21 #include "YACSEvalYFXPattern.hxx"
22 #include "YACSEvalResource.hxx"
23 #include "YACSEvalSeqAny.hxx"
24 #include "YACSEvalSession.hxx"
25 #include "YACSEvalObserver.hxx"
26 #include "YACSEvalSessionInternal.hxx"
27 #include "YACSEvalAutoPtr.hxx"
28 #include "YACSEvalExecParams.hxx"
30 #include "ElementaryNode.hxx"
31 #include "RuntimeSALOME.hxx"
32 #include "Dispatcher.hxx"
33 #include "Executor.hxx"
34 #include "InputPort.hxx"
35 #include "LinkInfo.hxx"
36 #include "TypeCode.hxx"
38 #include "Dispatcher.hxx"
40 #include "PythonPorts.hxx"
41 #include "ForEachLoop.hxx"
42 #include "PythonNode.hxx"
43 #include "InlineNode.hxx"
44 #include "ServiceNode.hxx"
45 #include "PyStdout.hxx"
46 #include "AutoGIL.hxx"
48 #include "ResourcesManager.hxx"
59 const char YACSEvalYFXPattern::ST_OK[]="ALL_OK";
61 const char YACSEvalYFXPattern::ST_FAILED[]="SOME_SAMPLES_FAILED_AND_ALL_OF_THEM_FAILED_DETERMINISTICALLY";
63 const char YACSEvalYFXPattern::ST_ERROR[]="SOME_SAMPLES_FAILED_BUT_IMPOSSIBLE_TO_CONCLUDE_ON_THEM";
65 const std::size_t YACSEvalYFXPattern::MAX_LGTH_OF_INP_DUMP=10000;
67 const char YACSEvalYFXGraphGen::DFT_PROC_NAME[]="YFX";
69 const char YACSEvalYFXGraphGen::FIRST_FE_SUBNODE_NAME[]="Bloc";
71 const char YACSEvalYFXGraphGen::GATHER_NODE_NAME[]="__gather__";
73 const char YACSEvalYFXGraphGen::HIDDEN_INDEX_VAR[]="___idx___";
75 class MyAutoThreadSaver
78 MyAutoThreadSaver(bool isToSave):_isToSave(isToSave),_save(0)
82 PyThreadState *save(PyThreadState_Swap(NULL));// safe call of PyEval_SaveThread()
90 ~MyAutoThreadSaver() { if(_isToSave) if(_save) { PyEval_AcquireLock(); PyThreadState_Swap(_save); /*safe call of PyEval_RestoreThread*/ } }
96 std::vector< YACSEvalInputPort *> YACSEvalYFXPattern::getFreeInputPorts() const
98 std::size_t sz(_inputs.size());
99 std::vector< YACSEvalInputPort *> ret;
100 std::vector< YACSEvalInputPort >::const_iterator it(_inputs.begin());
101 for(std::size_t i=0;i<sz;i++,it++)
102 ret.push_back(const_cast<YACSEvalInputPort *>(&(*it)));
106 std::vector< YACSEvalOutputPort *> YACSEvalYFXPattern::getFreeOutputPorts() const
108 std::size_t sz(_outputs.size());
109 std::vector< YACSEvalOutputPort *> ret;
110 std::vector< YACSEvalOutputPort >::const_iterator it(_outputs.begin());
111 for(std::size_t i=0;i<sz;i++,it++)
112 ret.push_back(const_cast<YACSEvalOutputPort *>(&(*it)));
116 YACSEvalYFXPattern *YACSEvalYFXPattern::FindPatternFrom(YACSEvalYFX *boss, YACS::ENGINE::Proc *scheme, bool ownScheme)
119 throw YACS::Exception("YACSEvalYFXPattern::FindPatternFrom : input scheme must be not null !");
121 YACS::ENGINE::ComposedNode *zeRunNode(0);
122 bool isMatchingRunOnlyPattern(YACSEvalYFXRunOnlyPattern::IsMatching(scheme,zeRunNode));
123 if(isMatchingRunOnlyPattern)
124 return new YACSEvalYFXRunOnlyPattern(boss,scheme,ownScheme,zeRunNode);
126 throw YACS::Exception("YACSEvalYFXPattern::FindPatternFrom : no pattern found for the input scheme !");
129 bool YACSEvalYFXPattern::isAlreadyComputedResources() const
134 void YACSEvalYFXPattern::checkNonAlreadyComputedResources() const
136 if(isAlreadyComputedResources())
137 throw YACS::Exception("checkNonAlreadyComputedResources : instance of computed resources already computed !");
140 void YACSEvalYFXPattern::checkAlreadyComputedResources() const
142 if(!isAlreadyComputedResources())
143 throw YACS::Exception("checkAlreadyComputedResources : instance of computed resources not already computed !");
146 void YACSEvalYFXPattern::checkLocked() const
149 throw YACS::Exception("YACSEvalYFXPattern::checkLocked : Pattern is not locked !");
152 void YACSEvalYFXPattern::checkNonLocked() const
155 throw YACS::Exception("YACSEvalYFXPattern::checkNonLocked : Pattern is locked !");
158 void YACSEvalYFXPattern::CheckNodeIsOK(YACS::ENGINE::ComposedNode *node)
160 /*YACS::ENGINE::LinkInfo info(YACS::ENGINE::LinkInfo::WARN_ONLY_DONT_STOP);
163 node->checkConsistency(info);
165 catch(YACS::Exception& e)
168 if(info.getNumberOfErrLinks(YACS::ENGINE::E_ALL)!=0)
169 throw YACS::Exception("YACSEvalYFXPattern::CheckNodeIsOK : found node is not OK !");
170 std::list<YACS::ENGINE::ElementaryNode *> allNodes(node->getRecursiveConstituents());
171 for(std::list<YACS::ENGINE::ElementaryNode *>::const_iterator it=allNodes.begin();it!=allNodes.end();it++)
173 YACS::ENGINE::ServiceNode *node0(dynamic_cast<YACS::ENGINE::ServiceNode *>(*it));
174 YACS::ENGINE::InlineNode *node1(dynamic_cast<YACS::ENGINE::InlineNode *>(*it));
177 YACS::ENGINE::Container *cont(node0->getContainer());
178 YACS::ENGINE::ComponentInstance *comp(node0->getComponent());
181 std::ostringstream oss; oss << "YACSEvalYFXPattern::CheckNodeIsOK : ServiceNode called \"" << node0->getName() << "\" is not correctly defined !";
182 throw YACS::Exception(oss.str());
187 YACS::ENGINE::Container *cont(node1->getContainer());
188 if(!cont && node1->getExecutionMode()==YACS::ENGINE::InlineNode::REMOTE_STR)
190 std::ostringstream oss; oss << "YACSEvalYFXPattern::CheckNodeIsOK : InlineNode called \"" << node1->getName() << "\" is not correctly defined !";
191 throw YACS::Exception(oss.str());
197 void YACSEvalYFXPattern::registerObserver(YACSEvalObserver *observer)
199 if(_observer==observer)
202 _observer->decrRef();
205 _observer->incrRef();
208 YACSEvalYFXPattern::YACSEvalYFXPattern(YACSEvalYFX *boss, YACS::ENGINE::Proc *scheme, bool ownScheme):_boss(boss),_scheme(scheme),_ownScheme(ownScheme),_parallelizeStatus(true),_rm(new ResourcesManager_cpp),_res(0),_observer(0)
212 YACS::ENGINE::TypeCode *YACSEvalYFXPattern::CreateSeqTypeCodeFrom(YACS::ENGINE::Proc *scheme, const std::string& zeType)
214 std::ostringstream oss; oss << "list[" << zeType << "]";
215 YACS::ENGINE::TypeCode *tc(scheme->getTypeCode(zeType));
216 return scheme->createSequenceTc(oss.str(),oss.str(),tc);
219 void YACSEvalYFXPattern::setResources(YACSEvalListOfResources *res)
221 checkNonAlreadyComputedResources();
227 void YACSEvalYFXPattern::resetResources()
233 YACSEvalSeqAny *YACSEvalYFXPattern::BuildValueInPort(YACS::ENGINE::InputPyPort *port)
236 throw YACS::Exception("YACSEvalYFXPattern::GetValueInPort : null input port !");
237 PyObject *obj(port->getPyObj());
238 YACS::ENGINE::TypeCode *tc(port->edGetType());
239 YACS::ENGINE::TypeCodeSeq *tcc(dynamic_cast<YACS::ENGINE::TypeCodeSeq *>(tc));
242 std::ostringstream oss; oss << "YACSEvalYFXPattern::GetValueInPort : internal error for tc of input \"" << port->getName() << "\"";
243 throw YACS::Exception(oss.str());
245 const YACS::ENGINE::TypeCode *tcct(tcc->contentType());
246 if(!PyList_Check(obj))
247 throw YACS::Exception("YACSEvalYFXPattern::GetValueInPort : internal error 2 !");
248 std::size_t sz(PyList_Size(obj));
249 if(tcct->kind()==YACS::ENGINE::Double)
251 std::vector<double> eltCpp(sz);
252 for(std::size_t i=0;i<sz;i++)
254 PyObject *elt(PyList_GetItem(obj,i));
255 eltCpp[i]=PyFloat_AsDouble(elt);
257 YACS::AutoCppPtr<YACSEvalSeqAnyDouble> elt(new YACSEvalSeqAnyDouble(eltCpp));
258 return elt.dettach();
260 else if(tcct->kind()==YACS::ENGINE::Int)
262 std::vector<int> eltCpp(sz);
263 for(std::size_t i=0;i<sz;i++)
265 PyObject *elt(PyList_GetItem(obj,i));
266 eltCpp[i]=PyLong_AsLong(elt);
268 YACS::AutoCppPtr<YACSEvalSeqAnyInt> elt(new YACSEvalSeqAnyInt(eltCpp));
269 return elt.dettach();
272 throw YACS::Exception("YACSEvalYFXPattern::GetValueInPort : not implemented yet for other than Double and Int !");
275 YACSEvalSeqAny *YACSEvalYFXPattern::BuildValueFromEngineFrmt(YACS::ENGINE::SequenceAny *data)
277 unsigned int sz(data->size());
278 std::vector<double> eltCpp(sz);
279 for(unsigned int ii=0;ii<sz;ii++)
281 YACS::ENGINE::AnyPtr elt((*data)[ii]);
282 YACS::ENGINE::Any *eltPtr((YACS::ENGINE::Any *)elt);
283 YACS::ENGINE::AtomAny *eltPtr2(dynamic_cast<YACS::ENGINE::AtomAny *>(eltPtr));
286 std::ostringstream oss; oss << "YACSEvalYFXPattern::BuildValueFromEngineFrmt : error at pos #" << ii << " ! It is not an AtomAny !";
287 throw YACS::Exception(oss.str());
289 eltCpp[ii]=eltPtr2->getDoubleValue();
291 return new YACSEvalSeqAnyDouble(eltCpp);
294 void YACSEvalYFXPattern::cleanScheme()
301 YACSEvalYFXPattern::~YACSEvalYFXPattern()
304 _observer->decrRef();
309 /////////////////////
311 class YACSEvalYFXRunOnlyPatternInternalObserver : public YACS::ENGINE::Observer
314 YACSEvalYFXRunOnlyPatternInternalObserver(YACSEvalYFXRunOnlyPattern *boss):_boss(boss) { if(!_boss) throw YACS::Exception("YACSEvalYFXRunOnlyPatternInternalObserver constructor : null boss not supported :)"); }
315 void notifyObserver2(YACS::ENGINE::Node *object, const std::string& event, void *something);
317 YACSEvalYFXRunOnlyPattern *_boss;
320 void YACSEvalYFXRunOnlyPatternInternalObserver::notifyObserver2(YACS::ENGINE::Node *object, const std::string& event, void *something)
322 YACS::ENGINE::ForEachLoop *object2(_boss->getUndergroundForEach());
323 YACSEvalObserver *obs(_boss->getObserver());
326 if(event=="progress_ok" && object2==object)
328 int *casted(reinterpret_cast<int *>(something));
329 obs->notifySampleOK(_boss->getBoss(),*casted);
332 if(event=="progress_ko" && object2==object)
334 int *casted(reinterpret_cast<int *>(something));
335 obs->notifySampleKO(_boss->getBoss(),*casted);
340 /////////////////////
342 YACSEvalYFXRunOnlyPattern::YACSEvalYFXRunOnlyPattern(YACSEvalYFX *boss, YACS::ENGINE::Proc *scheme, bool ownScheme, YACS::ENGINE::ComposedNode *runNode):YACSEvalYFXPattern(boss,scheme,ownScheme),_lockedStatus(false),_runNode(runNode),_gen(0),_obs(new YACSEvalYFXRunOnlyPatternInternalObserver(this))
345 throw YACS::Exception("YACSEvalYFXRunOnlyPattern : internal run node must be not null !");
350 YACSEvalYFXRunOnlyPattern::~YACSEvalYFXRunOnlyPattern()
356 void YACSEvalYFXRunOnlyPattern::setOutPortsOfInterestForEvaluation(const std::vector<YACSEvalOutputPort *>& outputsOfInterest)
359 _outputsOfInterest=outputsOfInterest;
363 void YACSEvalYFXRunOnlyPattern::resetOutputsOfInterest()
366 _outputsOfInterest.clear();
370 void YACSEvalYFXRunOnlyPattern::generateGraph()
373 if(getResourcesInternal()->isInteractive())
374 _gen=new YACSEvalYFXGraphGenInteractive(this);
376 _gen=new YACSEvalYFXGraphGenCluster(this);
377 _gen->generateGraph();
380 void YACSEvalYFXRunOnlyPattern::resetGeneratedGraph()
383 _gen->resetGeneratedGraph();
386 int YACSEvalYFXRunOnlyPattern::assignNbOfBranches()
388 checkAlreadyComputedResources();
390 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::assignNbOfBranches : generator is NULL ! Please invoke generateGraph before !");
391 return _gen->assignNbOfBranches();
394 void YACSEvalYFXRunOnlyPattern::assignRandomVarsInputs()
396 std::size_t sz(std::numeric_limits<std::size_t>::max());
397 for(std::vector< YACSEvalInputPort >::const_iterator it=_inputs.begin();it!=_inputs.end();it++)
398 if((*it).isRandomVar())
400 std::size_t locSize((*it).initializeUndergroundWithSeq());
401 if(sz==std::numeric_limits<std::size_t>::max())
405 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::assignRandomVarsInputs : length of sequences in random vars must be the same !");
409 bool YACSEvalYFXRunOnlyPattern::isLocked() const
411 return _lockedStatus;
414 YACSEvalListOfResources *YACSEvalYFXRunOnlyPattern::giveResources()
417 if(!isAlreadyComputedResources())
419 YACS::ENGINE::DeploymentTree dt(_runNode->getDeploymentTree());
420 _runNode->removeRecursivelyRedundantCL();
421 YACSEvalListOfResources *res(new YACSEvalListOfResources(_runNode->getMaxLevelOfParallelism(),getCatalogInAppli(),dt));
424 return getResourcesInternal();
427 YACS::ENGINE::Proc *YACSEvalYFXRunOnlyPattern::getUndergroundGeneratedGraph() const
429 return getGenerator()->getUndergroundGeneratedGraph();
432 std::string YACSEvalYFXRunOnlyPattern::getErrorDetailsInCaseOfFailure() const
434 std::string generatorErrors = getGenerator()->getErrors();
435 if(generatorErrors.size() > 0)
437 return generatorErrors;
439 std::string st(getStatusOfRunStr());//test if a run has occurred.
441 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::getErrorDetailsInCaseOfFailure : The execution of scheme has been carried out to the end without any problem !");
442 // All the problem can only comes from foreach -> scan it
443 YACS::ENGINE::ForEachLoop *fe(getUndergroundForEach());
444 YACS::ENGINE::NodeStateNameMap nsm;
445 unsigned nbB(fe->getNumberOfBranchesCreatedDyn());
446 std::ostringstream oss;
447 for(unsigned j=0;j<nbB;j++)
449 YACS::ENGINE::Node *nn(fe->getChildByNameExec(YACSEvalYFXGraphGen::FIRST_FE_SUBNODE_NAME,j));
450 YACS::ENGINE::Bloc *nnc(dynamic_cast<YACS::ENGINE::Bloc *>(nn));
452 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::getErrorDetailsInCaseOfFailure : internal error 1 ! The direct son of main foreach is expected to be a Bloc !");
453 if(nnc->getState()==YACS::DONE)
455 std::list< YACS::ENGINE::ElementaryNode *> fec(nnc->getRecursiveConstituents());
456 for(std::list< YACS::ENGINE::ElementaryNode *>::reverse_iterator it1=fec.rbegin();it1!=fec.rend();it1++)
458 YACS::StatesForNode st0((*it1)->getState());
461 oss << "NODE = " << nnc->getChildName(*it1) << std::endl;
462 oss << "STATUS = " << nsm[st0] << std::endl;
463 oss << "BRANCH ID = " << j << std::endl;
464 oss << "DETAILS = " << std::endl;
465 oss << (*it1)->getErrorDetails();
472 std::string YACSEvalYFXRunOnlyPattern::getStatusOfRunStr() const
474 YACS::StatesForNode st(getUndergroundGeneratedGraph()->getState());
480 case YACS::TOACTIVATE:
481 case YACS::ACTIVATED:
482 case YACS::SUSPENDED:
485 case YACS::DESACTIVATED:
487 std::ostringstream oss; oss << "YACSEvalYFXRunOnlyPattern::getStatusOfRunStr : Unexpected state \"" << YACS::ENGINE::Node::getStateName(st) << "\" ! Did you invoke run ?";
488 throw YACS::Exception(oss.str());
490 case YACS::LOADFAILED:
491 case YACS::EXECFAILED:
493 case YACS::INTERNALERR:
494 return std::string(ST_ERROR);
496 return std::string(ST_FAILED);
498 return std::string(ST_OK);
501 std::ostringstream oss; oss << "YACSEvalYFXRunOnlyPattern::getStatusOfRunStr : unrecognized and managed state \"" << YACS::ENGINE::Node::getStateName(st) << "\" !";
502 throw YACS::Exception(oss.str());
507 std::vector<YACSEvalSeqAny *> YACSEvalYFXRunOnlyPattern::getResults() const
509 return _gen->getResults();
513 * This method works if run succeeded (true return) and also if graph has failed. Graph failed means soft error of evaluation due to error in evaluation (example 1/0 or a normal throw from one node)
514 * If a more serious error occured (SIGSEGV of a server or internal error in YACS engine, cluster error, loose of connection...) this method will throw an exception to warn the caller that the results may be
516 std::vector<YACSEvalSeqAny *> YACSEvalYFXRunOnlyPattern::getResultsInCaseOfFailure(std::vector<unsigned int>& passedIds) const
518 YACS::StatesForNode st(getUndergroundGeneratedGraph()->getState());
522 std::vector<YACSEvalSeqAny *> ret(getResults());
526 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::getResultsInCaseOfFailure : internal error ! The returned vector has a null pointer at pos #0 !");
527 std::size_t sz(ret[0]->size());
528 passedIds.resize(sz);
529 for(std::size_t i=0;i<sz;i++)
534 getStatusOfRunStr();// To check that the status is recognized.
535 std::list<YACS::ENGINE::Node *> lns(getUndergroundGeneratedGraph()->edGetDirectDescendants());
536 YACS::ENGINE::ForEachLoop *fe(getUndergroundForEach());
538 YACS::ENGINE::Executor exe;
539 std::vector<YACS::ENGINE::SequenceAny *> outputs;
540 std::vector<std::string> nameOfOutputs;
541 passedIds=fe->getPassedResults(&exe,outputs,nameOfOutputs);//<- the key invokation is here.
542 std::size_t sz(passedIds.size()),ii(0);
543 std::vector<YACSEvalSeqAny *> ret(_outputsOfInterest.size());
544 for(std::vector<YACSEvalOutputPort *>::const_iterator it1=_outputsOfInterest.begin();it1!=_outputsOfInterest.end();it1++,ii++)
546 YACS::ENGINE::OutputPort *p((*it1)->getUndergroundPtr());
547 std::string st(_runNode->getOutPortName(p));
548 std::ostringstream oss; oss << YACSEvalYFXGraphGen::FIRST_FE_SUBNODE_NAME << '.' << _runNode->getName() << '.' << st;
550 YACS::ENGINE::ForEachLoop::InterceptorizeNameOfPort(st);
551 std::vector<std::string>::iterator it2(std::find(nameOfOutputs.begin(),nameOfOutputs.end(),st));
552 if(it2==nameOfOutputs.end())
554 std::ostringstream oss; oss << "YACSEvalYFXRunOnlyPattern::getResultsInCaseOfFailure : internal error 3 ! Unable to locate interceptor with name " << st << " ! Possibilities are : ";
555 std::copy(nameOfOutputs.begin(),nameOfOutputs.end(),std::ostream_iterator<std::string>(oss," "));
557 throw YACS::Exception(oss.str());
559 std::size_t pos(std::distance(nameOfOutputs.begin(),it2));
560 ret[ii]=BuildValueFromEngineFrmt(outputs[pos]);
565 void YACSEvalYFXRunOnlyPattern::emitStart() const
567 YACSEvalObserver *obs(getObserver());
570 obs->startComputation(getBoss());
573 bool YACSEvalYFXRunOnlyPattern::go(const YACSEvalExecParams& params, YACSEvalSession *session) const
576 YACS::ENGINE::Dispatcher *disp(YACS::ENGINE::Dispatcher::getDispatcher());
577 disp->addObserver(_obs,getUndergroundForEach(),"progress_ok");
578 disp->addObserver(_obs,getUndergroundForEach(),"progress_ko");
579 bool ret(getGenerator()->go(params,session));
580 disp->removeObserver(_obs,getUndergroundForEach(),"progress_ok");
581 disp->removeObserver(_obs,getUndergroundForEach(),"progress_ko");
585 YACS::ENGINE::ForEachLoop *YACSEvalYFXRunOnlyPattern::getUndergroundForEach() const
587 return getGenerator()->getUndergroundForEach();
590 bool YACSEvalYFXRunOnlyPattern::IsMatching(YACS::ENGINE::Proc *scheme, YACS::ENGINE::ComposedNode *& runNode)
592 std::list<YACS::ENGINE::Node *> nodes(scheme->getChildren());
595 bool areAllElementary(true);
596 for(std::list<YACS::ENGINE::Node *>::const_iterator it=nodes.begin();it!=nodes.end() && areAllElementary;it++)
597 if(!dynamic_cast<YACS::ENGINE::ElementaryNode *>(*it))
598 areAllElementary=false;
602 CheckNodeIsOK(scheme);
608 YACS::ENGINE::ComposedNode *candidate(dynamic_cast<YACS::ENGINE::ComposedNode *>(nodes.front()));
611 CheckNodeIsOK(candidate);
615 void YACSEvalYFXRunOnlyPattern::buildInputPorts()
618 std::list< YACS::ENGINE::InputPort *> allInputPorts(_runNode->getSetOfInputPort());
619 std::vector<std::string> allNames;
620 for(std::list< YACS::ENGINE::InputPort *>::const_iterator it=allInputPorts.begin();it!=allInputPorts.end();it++)
622 YACS::ENGINE::InputPort *elt(*it);
624 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::buildInputPorts : presence of null input !");
625 std::set<YACS::ENGINE::OutPort *> bls(elt->edSetOutPort());
628 if(YACSEvalPort::IsInputPortPublishable(elt))
630 std::string inpName(elt->getName());
632 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::buildInputPorts : an input has empty name ! Should not !");
633 _inputs.push_back(YACSEvalInputPort(elt));
634 if(std::find(allNames.begin(),allNames.end(),inpName)!=allNames.end())
636 std::ostringstream oss; oss << "YACSEvalYFXRunOnlyPattern::buildInputPorts : input name \"" << inpName << "\" appears more than once !";
637 throw YACS::Exception(oss.str());
639 allNames.push_back(inpName);
645 void YACSEvalYFXRunOnlyPattern::buildOutputPorts()
648 std::list< YACS::ENGINE::OutputPort *> allOutputPorts(_runNode->getSetOfOutputPort());
649 std::vector<std::string> allNames;
650 for(std::list< YACS::ENGINE::OutputPort *>::const_iterator it=allOutputPorts.begin();it!=allOutputPorts.end();it++)
652 YACS::ENGINE::OutputPort *elt(*it);
653 if(YACSEvalPort::IsOutputPortPublishable(elt))
656 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::buildOutputPorts : presence of null output !");
657 std::string outpName(elt->getName());
659 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::buildOutputPorts : an output has empty name ! Should not !");
660 if(std::find(allNames.begin(),allNames.end(),outpName)!=allNames.end())
662 std::ostringstream oss; oss << "YACSEvalYFXRunOnlyPattern::buildOutputPorts : output name \"" << outpName << "\" appears more than once !";
663 throw YACS::Exception(oss.str());
665 _outputs.push_back(YACSEvalOutputPort(*it));
670 YACSEvalYFXGraphGen *YACSEvalYFXRunOnlyPattern::getGenerator() const
673 throw YACS::Exception("getGenerator : generator is NULL !");
677 /////////////////////
679 YACSEvalYFXGraphGen::YACSEvalYFXGraphGen(YACSEvalYFXRunOnlyPattern *boss):_boss(boss),_generatedGraph(0),_FEInGeneratedGraph(0)
682 throw YACS::Exception("YACSEvalYFXGraphGen constructor : boss is NULL !");
685 YACSEvalYFXGraphGen::~YACSEvalYFXGraphGen()
687 //delete _generatedGraph;// -> TODO : AGY why ?
690 void YACSEvalYFXGraphGen::resetGeneratedGraph()
692 delete _generatedGraph;
693 _generatedGraph=0; _FEInGeneratedGraph=0;
696 void YACSEvalYFXGraphGen::generateGraphCommon(CustomPatcher& patcher)
699 { delete _generatedGraph; _generatedGraph=0; _FEInGeneratedGraph=0; }
700 static const char LISTPYOBJ_STR[]="list[pyobj]";
701 if(getBoss()->getOutputsOfInterest().empty())
703 YACS::ENGINE::RuntimeSALOME::setRuntime();
704 YACS::ENGINE::RuntimeSALOME *r(YACS::ENGINE::getSALOMERuntime());
705 _generatedGraph=r->createProc(DFT_PROC_NAME);
706 YACS::ENGINE::TypeCode *pyobjTC(_generatedGraph->createInterfaceTc("python:obj:1.0","pyobj",std::list<YACS::ENGINE::TypeCodeObjref *>()));
707 std::ostringstream oss; oss << "Loop_" << getBoss()->getRunNode()->getName();
708 _generatedGraph->createType(YACSEvalAnyDouble::TYPE_REPR,"double");
709 YACS::ENGINE::TypeCode *tcInt(_generatedGraph->createType(YACSEvalAnyInt::TYPE_REPR,"int"));
711 YACS::ENGINE::InlineNode *n0(r->createScriptNode(YACS::ENGINE::PythonNode::KIND,"__initializer__"));
712 _generatedGraph->edAddChild(n0);
713 YACS::ENGINE::TypeCode *listPyobjTC(_generatedGraph->createSequenceTc(LISTPYOBJ_STR,LISTPYOBJ_STR,pyobjTC));
714 YACS::ENGINE::OutputPort *sender(n0->edAddOutputPort("sender",listPyobjTC));
715 std::ostringstream var0;
716 const std::vector< YACSEvalInputPort >& inputs(getBoss()->getInputs());
717 for(std::vector< YACSEvalInputPort >::const_iterator it=inputs.begin();it!=inputs.end();it++)
719 if((*it).isRandomVar())
721 var0 << (*it).getName() << ",";
722 YACS::ENGINE::TypeCode *tc(YACSEvalYFXPattern::CreateSeqTypeCodeFrom(_generatedGraph,(*it).getTypeOfData()));
723 YACS::ENGINE::InputPort *inp(n0->edAddInputPort((*it).getName(),tc));
724 YACS::ENGINE::InputPyPort *inpc(dynamic_cast<YACS::ENGINE::InputPyPort *>(inp));
726 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::generateGraph : internal error 1 !");
727 (*it).setUndergroundPortToBeSet(inpc);
730 std::ostringstream n0Script; n0Script << "sender=[tuple([__p9Sq]+list(__p9Sw)) for __p9Sq,__p9Sw in enumerate(zip(" << var0.str() << "))]\n";
731 n0->setScript(n0Script.str());
733 YACS::ENGINE::ForEachLoop *n1(r->createForEachLoop(oss.str(),pyobjTC));
734 _FEInGeneratedGraph=n1;
735 _generatedGraph->edAddChild(n1);
736 _generatedGraph->edAddCFLink(n0,n1);
737 _generatedGraph->edAddDFLink(sender,n1->edGetSeqOfSamplesPort());
738 YACS::ENGINE::InlineNode *n2(r->createScriptNode(YACS::ENGINE::PythonNode::KIND,GATHER_NODE_NAME));
739 _generatedGraph->edAddChild(n2);
740 _generatedGraph->edAddCFLink(n1,n2);
742 YACS::ENGINE::Bloc *n10(r->createBloc(FIRST_FE_SUBNODE_NAME));
744 YACS::ENGINE::InlineNode *n100(r->createScriptNode(YACS::ENGINE::PythonNode::KIND,"__dispatch__"));
745 YACS::ENGINE::ComposedNode *runNode(getBoss()->getRunNode());
746 YACS::ENGINE::Node *n101(runNode->cloneWithoutCompAndContDeepCpy(0,true));
747 n10->edAddChild(n100);
748 n10->edAddChild(n101);
749 YACS::ENGINE::InputPort *dispatchIn(n100->edAddInputPort("i0",pyobjTC));
750 n10->edAddCFLink(n100,n101);
751 n1->edAddDFLink(n1->edGetSamplePort(),dispatchIn);
752 std::ostringstream var1;
753 YACS::ENGINE::OutputPort *n100_output(n100->edAddOutputPort(HIDDEN_INDEX_VAR,tcInt));
754 var1 << HIDDEN_INDEX_VAR << ",";
755 for(std::vector< YACSEvalInputPort >::const_iterator it=inputs.begin();it!=inputs.end();it++)
757 if((*it).isRandomVar())
759 var1 << (*it).getName() << ",";
760 YACS::ENGINE::OutputPort *myOut(n100->edAddOutputPort((*it).getName(),_generatedGraph->getTypeCode((*it).getTypeOfData())));
761 std::string tmpPortName(runNode->getInPortName((*it).getUndergroundPtr()));
762 YACS::ENGINE::InputPort *myIn(n101->getInputPort(tmpPortName));
763 n10->edAddDFLink(myOut,myIn);
766 std::ostringstream n100Script; n100Script << var1.str() << "=i0\n";
767 n100->setScript(n100Script.str());
768 const std::vector<YACSEvalOutputPort *>& outputsOfInt(getBoss()->getOutputsOfInterest());
770 std::list<YACS::ENGINE::ElementaryNode *> n101_constit(n101->getRecursiveConstituents());
771 for(std::list<YACS::ENGINE::ElementaryNode *>::const_iterator it=n101_constit.begin();it!=n101_constit.end();it++)
773 YACS::ENGINE::InlineNode *eltc(dynamic_cast<YACS::ENGINE::InlineNode *>(*it));
776 YACS::ENGINE::InputPort *n101_input(eltc->edAddInputPort(HIDDEN_INDEX_VAR,tcInt));
777 _generatedGraph->edAddDFLink(n100_output,n101_input);
781 for(std::vector< YACSEvalOutputPort * >::const_iterator it=outputsOfInt.begin();it!=outputsOfInt.end();it++)
783 patcher.addOutputVar((*it)->getName());
784 YACS::ENGINE::TypeCode *tc(YACSEvalYFXPattern::CreateSeqTypeCodeFrom(_generatedGraph,(*it)->getTypeOfData()));
785 YACS::ENGINE::InputPort *myIn(n2->edAddInputPort((*it)->getName(),tc));
786 std::string tmpPortName(runNode->getOutPortName((*it)->getUndergroundPtr()));
787 YACS::ENGINE::OutputPort *myOut(n101->getOutputPort(tmpPortName));
788 _generatedGraph->edAddDFLink(myOut,myIn);
790 patcher.assignOutput(n2);
791 _generatedGraph->updateContainersAndComponents();
794 bool YACSEvalYFXGraphGen::isLocked() const
796 return _generatedGraph!=0;
799 int YACSEvalYFXGraphGen::assignNbOfBranches()
802 throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : the generated graph has not been created !");
803 std::list<YACS::ENGINE::Node *> nodes(_generatedGraph->getChildren());
804 YACS::ENGINE::ForEachLoop *zeMainNode(0);
805 for(std::list<YACS::ENGINE::Node *>::const_iterator it=nodes.begin();it!=nodes.end();it++)
807 YACS::ENGINE::ForEachLoop *isZeMainNode(dynamic_cast<YACS::ENGINE::ForEachLoop *>(*it));
811 zeMainNode=isZeMainNode;
813 throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : internal error 1 !");
817 throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : internal error 2 !");
818 unsigned int nbProcsDeclared(getBoss()->getResourcesInternal()->getNumberOfProcsDeclared());
819 nbProcsDeclared=std::max(nbProcsDeclared,4u);
821 if(getBoss()->getParallelizeStatus())
823 nbOfBranch=(nbProcsDeclared/getBoss()->getResourcesInternal()->getMaxLevelOfParallelism());
824 nbOfBranch=std::max(nbOfBranch,1);
826 YACS::ENGINE::InputPort *zeInputToSet(zeMainNode->edGetNbOfBranchesPort());
827 YACS::ENGINE::AnyInputPort *zeInputToSetC(dynamic_cast<YACS::ENGINE::AnyInputPort *>(zeInputToSet));
829 throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : internal error 3 !");
830 YACS::ENGINE::Any *a(YACS::ENGINE::AtomAny::New(nbOfBranch));
831 zeInputToSetC->put(a);
832 zeInputToSetC->exSaveInit();
837 void YACSEvalYFXGraphGenInteractive::generateGraph()
839 class LocalPatcher : public YACSEvalYFXGraphGen::CustomPatcher
842 void addOutputVar(const std::string& name) { }
843 void assignOutput(YACS::ENGINE::InlineNode *node) { }
846 this->generateGraphCommon(lp);
849 bool YACSEvalYFXGraphGenInteractive::go(const YACSEvalExecParams& params, YACSEvalSession *session) const
851 YACS::ENGINE::Executor exe;
852 exe.setKeepGoingProperty(!params.getStopASAPAfterErrorStatus());
854 MyAutoThreadSaver locker(!session->isAlreadyPyThreadSaved());
855 exe.RunW(getUndergroundGeneratedGraph());
857 return getUndergroundGeneratedGraph()->getState()==YACS::DONE;
860 std::string YACSEvalYFXGraphGenInteractive::getErrors()const
865 std::vector<YACSEvalSeqAny *> YACSEvalYFXGraphGenInteractive::getResults() const
867 if(getUndergroundGeneratedGraph()->getState()!=YACS::DONE)
868 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::getResults : the execution did not finished correctly ! getResults should not be called !");
869 const std::vector<YACSEvalOutputPort *>& outputsOfInt(getBoss()->getOutputsOfInterest());
870 std::vector<YACSEvalSeqAny *> ret(outputsOfInt.size());
871 YACS::ENGINE::Node *node(getUndergroundGeneratedGraph()->getChildByName(YACSEvalYFXGraphGen::GATHER_NODE_NAME));
872 YACS::ENGINE::PythonNode *nodeC(dynamic_cast<YACS::ENGINE::PythonNode *>(node));
874 throw YACS::Exception("YACSEvalYFXRunOnlyPattern::getResults : internal error !");
876 for(std::vector< YACSEvalOutputPort * >::const_iterator it=outputsOfInt.begin();it!=outputsOfInt.end();it++,ii++)
878 YACS::ENGINE::InPort *input(nodeC->getInPort((*it)->getName()));
879 YACS::ENGINE::InputPyPort *inputC(dynamic_cast<YACS::ENGINE::InputPyPort *>(input));
882 std::ostringstream oss; oss << "YACSEvalYFXRunOnlyPattern::getResults : internal error for input \"" << (*it)->getName() << "\"";
883 throw YACS::Exception(oss.str());
885 ret[ii]=YACSEvalYFXPattern::BuildValueInPort(inputC);
892 void YACSEvalYFXGraphGenCluster::generateGraph()
894 YACS::ENGINE::AutoGIL agil;
896 const char EFXGenFileName[]="EFXGenFileName";
897 const char EFXGenContent[]="import getpass,datetime,os\nn=datetime.datetime.now()\nreturn os.path.join(os.path.sep,\"tmp\",\"EvalYFX_%s_%s_%s.xml\"%(getpass.getuser(),n.strftime(\"%d%m%y\"),n.strftime(\"%H%M%S\")))";
898 const char EFXGenContent2[]="import getpass,datetime\nn=datetime.datetime.now()\nreturn \"EvalYFX_%s_%s_%s\"%(getpass.getuser(),n.strftime(\"%d%m%y\"),n.strftime(\"%H%M%S\"))";
900 YACS::ENGINE::AutoPyRef func(YACS::ENGINE::evalPy(EFXGenFileName,EFXGenContent));
901 YACS::ENGINE::AutoPyRef val(YACS::ENGINE::evalFuncPyWithNoParams(func));
902 _locSchemaFile=PyBytes_AsString(val);
903 func=YACS::ENGINE::evalPy(EFXGenFileName,EFXGenContent2);
904 val=YACS::ENGINE::evalFuncPyWithNoParams(func);
905 _jobName=PyBytes_AsString(val);
907 class ClusterPatcher : public YACSEvalYFXGraphGen::CustomPatcher
910 ClusterPatcher(const std::string& jobName):_jobName(jobName) { n2Script << "zeRes=["; }
911 void addOutputVar(const std::string& name) { n2Script<< name << ", "; }
912 void assignOutput(YACS::ENGINE::InlineNode *node) {
913 n2Script << "]\nwith open(\"" << _jobName << "\",\"w\") as f:" << std::endl;
914 n2Script << " f.write(str(zeRes))" << std::endl;
915 node->setScript(n2Script.str());
918 std::ostringstream n2Script;
919 std::string _jobName;
921 ClusterPatcher cp(_jobName);
923 this->generateGraphCommon(cp);
926 bool YACSEvalYFXGraphGenCluster::go(const YACSEvalExecParams& params, YACSEvalSession *session) const
928 YACS::ENGINE::AutoGIL agil;
930 getUndergroundGeneratedGraph()->saveSchema(_locSchemaFile);
931 YACSEvalListOfResources *rss(getBoss()->getResourcesInternal());
932 const YACSEvalParamsForCluster& cli(rss->getAddParamsForCluster());
933 std::vector<std::string> machines(rss->getAllChosenMachines());
934 if(machines.size()!=1)
935 throw YACS::Exception("YACSEvalYFXGraphGenCluster::go : internal error ! In batch mode and not exactly one machine !");
936 Engines::SalomeLauncher_var sl(session->getInternal()->goFetchingSalomeLauncherInNS());
937 Engines::ResourceParameters rr;
938 rr.name=CORBA::string_dup(machines[0].c_str());
939 rr.hostname=CORBA::string_dup("");
940 rr.can_launch_batch_jobs=true;
941 rr.can_run_containers=true;
942 rr.OS=CORBA::string_dup("Linux");
943 rr.componentList.length(0);
944 rr.nb_proc=rss->getNumberOfProcsDeclared();// <- important
947 rr.nb_node=1;// useless only nb_proc used.
948 rr.nb_proc_per_node=1;// useless only nb_proc used.
949 rr.policy=CORBA::string_dup("cycl");
950 rr.resList.length(0);
951 Engines::JobParameters jp;
952 jp.job_name=CORBA::string_dup(_jobName.c_str());
953 jp.job_type=CORBA::string_dup("yacs_file");
954 jp.job_file=CORBA::string_dup(_locSchemaFile.c_str());
955 jp.env_file=CORBA::string_dup("");
956 jp.in_files.length(cli.getInFiles().size());
957 std::list<std::string>::const_iterator it;
959 for (it = cli.getInFiles().begin(), i=0 ;
960 it != cli.getInFiles().end();
963 jp.in_files[i] = CORBA::string_dup((*it).c_str());
965 jp.out_files.length(1);
966 jp.out_files[0]=CORBA::string_dup(_jobName.c_str());
967 jp.work_directory=CORBA::string_dup(cli.getRemoteWorkingDir().c_str());
968 jp.local_directory=CORBA::string_dup(cli.getLocalWorkingDir().c_str());
969 jp.result_directory=CORBA::string_dup(cli.getLocalWorkingDir().c_str());
970 jp.maximum_duration=CORBA::string_dup(cli.getMaxDuration().c_str());
971 jp.resource_required=rr;
972 jp.queue=CORBA::string_dup("");
974 jp.mem_per_cpu=rr.mem_mb;
975 jp.wckey=CORBA::string_dup(cli.getWCKey().c_str());
976 jp.extra_params=CORBA::string_dup("");
977 jp.specific_parameters.length(0);
978 jp.launcher_file=CORBA::string_dup("");
979 jp.launcher_args=CORBA::string_dup("");
980 _jobid=sl->createJob(jp);
983 sl->launchJob(_jobid);
985 catch (const SALOME::SALOME_Exception & ex)
987 _errors = ex.details.text.in();
990 catch (const CORBA::SystemException& ex)
992 _errors = "Receive CORBA SystemException.";
999 PyRun_SimpleString("import time ; time.sleep(10)");
1000 char *state(sl->getJobState(_jobid));//"CREATED", "IN_PROCESS", "QUEUED", "RUNNING", "PAUSED", "FINISHED" or "FAILED"
1001 std::string sstate(state);
1002 CORBA::string_free(state);
1003 if(sstate=="FINISHED" || sstate=="FAILED")
1005 ret=sstate=="FINISHED";
1009 sl->getJobResults(_jobid,cli.getLocalWorkingDir().c_str());
1013 std::ostringstream oss; oss << "import os" << std::endl << "p=os.path.join(\"" << cli.getLocalWorkingDir() << "\",\"" << _jobName << "\")" << std::endl;
1014 oss << "if not os.path.exists(p):\n return None\n";
1015 oss << "f=file(p,\"r\")" << std::endl;
1016 oss << "return eval(f.read())";
1017 std::string zeInput(oss.str());
1018 YACS::ENGINE::AutoPyRef func(YACS::ENGINE::evalPy("fetch",zeInput));
1019 YACS::ENGINE::AutoPyRef val(YACS::ENGINE::evalFuncPyWithNoParams(func));
1020 if(!PyList_Check(val))
1021 throw YACS::Exception("Fetched file does not contain a list !");
1022 Py_ssize_t sz(PyList_Size(val));
1024 for(Py_ssize_t i=0;i<sz;i++)
1026 std::vector<double>& res0(_res[i]);
1027 PyObject *elt0(PyList_GetItem(val,i));
1028 if(!PyList_Check(elt0))
1029 throw YACS::Exception("Fetched file does contain a list of list !");
1030 Py_ssize_t sz0(PyList_Size(elt0)); res0.resize(sz0);
1031 for(Py_ssize_t j=0;j<sz0;j++)
1033 PyObject *elt1(PyList_GetItem(elt0,j));
1034 res0[j]=PyFloat_AsDouble(elt1);
1038 std::ostringstream oss1; oss1 << "import os" << std::endl << "p=os.path.join(\"" << cli.getLocalWorkingDir() << "\",\"" << _jobName << "\") ; os.remove(p)" << std::endl;
1039 std::string s1(oss1.str());
1040 PyRun_SimpleString(s1.c_str());
1041 if(!params.getFetchRemoteDirForClusterStatus())
1043 std::ostringstream oss2; oss2 << "import os,shutil" << std::endl << "p=os.path.join(\"" << cli.getLocalWorkingDir() << "\",\"logs\") ; shutil.rmtree(p)" << std::endl;
1044 std::string s2(oss2.str());
1045 PyRun_SimpleString(s2.c_str());
1048 catch(YACS::Exception& e)
1057 std::string YACSEvalYFXGraphGenCluster::getErrors()const
1062 std::vector<YACSEvalSeqAny *> YACSEvalYFXGraphGenCluster::getResults() const
1064 std::size_t sz(_res.size());
1065 std::vector<YACSEvalSeqAny *> ret(sz);
1066 for(std::size_t i=0;i<sz;i++)
1068 YACS::AutoCppPtr<YACSEvalSeqAnyDouble> elt(new YACSEvalSeqAnyDouble(_res[i]));
1069 ret[i]=elt.dettach();