YDFX gives to the pyscript nodes into study function the ___idx___ var to identify...

diff --git a/src/evalyfx/YACSEvalYFXPattern.cxx b/src/evalyfx/YACSEvalYFXPattern.cxx
index 1ab3d00d2f880f2ce985541dc61f6f8d46bc40c3..db435ffa3fbb4f8eb40b433f164109523fd6a403 100644 (file)
--- a/src/evalyfx/YACSEvalYFXPattern.cxx
+++ b/src/evalyfx/YACSEvalYFXPattern.cxx
@@ -70,6 +70,8 @@ const char YACSEvalYFXGraphGen::FIRST_FE_SUBNODE_NAME[]="Bloc";
 
 const char YACSEvalYFXGraphGen::GATHER_NODE_NAME[]="__gather__";
 
+const char YACSEvalYFXGraphGen::HIDDEN_INDEX_VAR[]="___idx___";
+
 class MyAutoThreadSaver
 {
 public:
@@ -691,50 +693,7 @@ void YACSEvalYFXGraphGen::resetGeneratedGraph()
   _generatedGraph=0; _FEInGeneratedGraph=0;
 }
 
-bool YACSEvalYFXGraphGen::isLocked() const
-{
-  return _generatedGraph!=0;
-}
-
-int YACSEvalYFXGraphGen::assignNbOfBranches()
-{
-  if(!_generatedGraph)
-    throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : the generated graph has not been created !");
-  std::list<YACS::ENGINE::Node *> nodes(_generatedGraph->getChildren());
-  YACS::ENGINE::ForEachLoop *zeMainNode(0);
-  for(std::list<YACS::ENGINE::Node *>::const_iterator it=nodes.begin();it!=nodes.end();it++)
-    {
-      YACS::ENGINE::ForEachLoop *isZeMainNode(dynamic_cast<YACS::ENGINE::ForEachLoop *>(*it));
-      if(isZeMainNode)
-        {
-          if(!zeMainNode)
-            zeMainNode=isZeMainNode;
-          else
-            throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : internal error 1 !");
-        }
-    }
-  if(!zeMainNode)
-    throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : internal error 2 !");
-  unsigned int nbProcsDeclared(getBoss()->getResourcesInternal()->getNumberOfProcsDeclared());
-  nbProcsDeclared=std::max(nbProcsDeclared,4u);
-  int nbOfBranch=1;
-  if(getBoss()->getParallelizeStatus())
-    {
-      nbOfBranch=(nbProcsDeclared/getBoss()->getResourcesInternal()->getMaxLevelOfParallelism());
-      nbOfBranch=std::max(nbOfBranch,1);
-    }
-  YACS::ENGINE::InputPort *zeInputToSet(zeMainNode->edGetNbOfBranchesPort());
-  YACS::ENGINE::AnyInputPort *zeInputToSetC(dynamic_cast<YACS::ENGINE::AnyInputPort *>(zeInputToSet));
-  if(!zeInputToSetC)
-    throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : internal error 3 !");
-  YACS::ENGINE::Any *a(YACS::ENGINE::AtomAny::New(nbOfBranch));
-  zeInputToSetC->put(a);
-  zeInputToSetC->exSaveInit();
-  a->decrRef();
-  return nbOfBranch;
-}
-
-void YACSEvalYFXGraphGenInteractive::generateGraph()
+void YACSEvalYFXGraphGen::generateGraphCommon(CustomPatcher& patcher)
 {
   if(_generatedGraph)
     { delete _generatedGraph; _generatedGraph=0; _FEInGeneratedGraph=0; }
@@ -747,7 +706,7 @@ void YACSEvalYFXGraphGenInteractive::generateGraph()
   YACS::ENGINE::TypeCode *pyobjTC(_generatedGraph->createInterfaceTc("python:obj:1.0","pyobj",std::list<YACS::ENGINE::TypeCodeObjref *>()));
   std::ostringstream oss; oss << "Loop_" << getBoss()->getRunNode()->getName();
   _generatedGraph->createType(YACSEvalAnyDouble::TYPE_REPR,"double");
-  _generatedGraph->createType(YACSEvalAnyInt::TYPE_REPR,"int");
+  YACS::ENGINE::TypeCode *tcInt(_generatedGraph->createType(YACSEvalAnyInt::TYPE_REPR,"int"));
   //
   YACS::ENGINE::InlineNode *n0(r->createScriptNode(YACS::ENGINE::PythonNode::KIND,"__initializer__"));
   _generatedGraph->edAddChild(n0);
@@ -768,7 +727,7 @@ void YACSEvalYFXGraphGenInteractive::generateGraph()
           (*it).setUndergroundPortToBeSet(inpc);
         }
     }
-  std::ostringstream n0Script; n0Script << "sender=zip(" << var0.str() << ")\n";
+  std::ostringstream n0Script; n0Script << "sender=[tuple([__p9Sq]+list(__p9Sw)) for __p9Sq,__p9Sw in enumerate(zip(" << var0.str() << "))]\n";
   n0->setScript(n0Script.str());
   //
   YACS::ENGINE::ForEachLoop *n1(r->createForEachLoop(oss.str(),pyobjTC));
@@ -791,6 +750,8 @@ void YACSEvalYFXGraphGenInteractive::generateGraph()
   n10->edAddCFLink(n100,n101);
   n1->edAddDFLink(n1->edGetSamplePort(),dispatchIn);
   std::ostringstream var1;
+  YACS::ENGINE::OutputPort *n100_output(n100->edAddOutputPort(HIDDEN_INDEX_VAR,tcInt));
+  var1 << HIDDEN_INDEX_VAR << ",";
   for(std::vector< YACSEvalInputPort >::const_iterator it=inputs.begin();it!=inputs.end();it++)
     {
       if((*it).isRandomVar())
@@ -805,17 +766,86 @@ void YACSEvalYFXGraphGenInteractive::generateGraph()
   std::ostringstream n100Script;  n100Script << var1.str() << "=i0\n";
   n100->setScript(n100Script.str());
   const std::vector<YACSEvalOutputPort *>& outputsOfInt(getBoss()->getOutputsOfInterest());
+  {
+    std::list<YACS::ENGINE::ElementaryNode *> n101_constit(n101->getRecursiveConstituents());
+    for(std::list<YACS::ENGINE::ElementaryNode *>::const_iterator it=n101_constit.begin();it!=n101_constit.end();it++)
+      {
+        YACS::ENGINE::InlineNode *eltc(dynamic_cast<YACS::ENGINE::InlineNode *>(*it));
+        if(eltc)
+          {
+            YACS::ENGINE::InputPort *n101_input(eltc->edAddInputPort(HIDDEN_INDEX_VAR,tcInt));
+            _generatedGraph->edAddDFLink(n100_output,n101_input);
+          }
+      }
+  }
   for(std::vector< YACSEvalOutputPort * >::const_iterator it=outputsOfInt.begin();it!=outputsOfInt.end();it++)
     {
+      patcher.addOutputVar((*it)->getName());
       YACS::ENGINE::TypeCode *tc(YACSEvalYFXPattern::CreateSeqTypeCodeFrom(_generatedGraph,(*it)->getTypeOfData()));
       YACS::ENGINE::InputPort *myIn(n2->edAddInputPort((*it)->getName(),tc));
       std::string tmpPortName(runNode->getOutPortName((*it)->getUndergroundPtr()));
       YACS::ENGINE::OutputPort *myOut(n101->getOutputPort(tmpPortName));
       _generatedGraph->edAddDFLink(myOut,myIn);
     }
+  patcher.assignOutput(n2);
   _generatedGraph->updateContainersAndComponents();
 }
 
+bool YACSEvalYFXGraphGen::isLocked() const
+{
+  return _generatedGraph!=0;
+}
+
+int YACSEvalYFXGraphGen::assignNbOfBranches()
+{
+  if(!_generatedGraph)
+    throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : the generated graph has not been created !");
+  std::list<YACS::ENGINE::Node *> nodes(_generatedGraph->getChildren());
+  YACS::ENGINE::ForEachLoop *zeMainNode(0);
+  for(std::list<YACS::ENGINE::Node *>::const_iterator it=nodes.begin();it!=nodes.end();it++)
+    {
+      YACS::ENGINE::ForEachLoop *isZeMainNode(dynamic_cast<YACS::ENGINE::ForEachLoop *>(*it));
+      if(isZeMainNode)
+        {
+          if(!zeMainNode)
+            zeMainNode=isZeMainNode;
+          else
+            throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : internal error 1 !");
+        }
+    }
+  if(!zeMainNode)
+    throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : internal error 2 !");
+  unsigned int nbProcsDeclared(getBoss()->getResourcesInternal()->getNumberOfProcsDeclared());
+  nbProcsDeclared=std::max(nbProcsDeclared,4u);
+  int nbOfBranch=1;
+  if(getBoss()->getParallelizeStatus())
+    {
+      nbOfBranch=(nbProcsDeclared/getBoss()->getResourcesInternal()->getMaxLevelOfParallelism());
+      nbOfBranch=std::max(nbOfBranch,1);
+    }
+  YACS::ENGINE::InputPort *zeInputToSet(zeMainNode->edGetNbOfBranchesPort());
+  YACS::ENGINE::AnyInputPort *zeInputToSetC(dynamic_cast<YACS::ENGINE::AnyInputPort *>(zeInputToSet));
+  if(!zeInputToSetC)
+    throw YACS::Exception("YACSEvalYFXGraphGen::assignNbOfBranches : internal error 3 !");
+  YACS::ENGINE::Any *a(YACS::ENGINE::AtomAny::New(nbOfBranch));
+  zeInputToSetC->put(a);
+  zeInputToSetC->exSaveInit();
+  a->decrRef();
+  return nbOfBranch;
+}
+
+void YACSEvalYFXGraphGenInteractive::generateGraph()
+{
+  class LocalPatcher : public YACSEvalYFXGraphGen::CustomPatcher
+  {
+  public:
+    void addOutputVar(const std::string& name) { }
+    void assignOutput(YACS::ENGINE::InlineNode *node) { }
+  };
+  LocalPatcher lp;
+  this->generateGraphCommon(lp);
+}
+
 bool YACSEvalYFXGraphGenInteractive::go(const YACSEvalExecParams& params, YACSEvalSession *session) const
 {
   YACS::ENGINE::Executor exe;
@@ -862,8 +892,6 @@ std::vector<YACSEvalSeqAny *> YACSEvalYFXGraphGenInteractive::getResults() const
 void YACSEvalYFXGraphGenCluster::generateGraph()
 {
   YACS::ENGINE::AutoGIL agil;
-  if(_generatedGraph)
-    { delete _generatedGraph; _generatedGraph=0; _FEInGeneratedGraph=0; }
   //
   const char EFXGenFileName[]="EFXGenFileName";
   const char EFXGenContent[]="import getpass,datetime,os\nn=datetime.datetime.now()\nreturn os.path.join(os.path.sep,\"tmp\",\"EvalYFX_%s_%s_%s.xml\"%(getpass.getuser(),n.strftime(\"%d%m%y\"),n.strftime(\"%H%M%S\")))";
@@ -876,86 +904,23 @@ void YACSEvalYFXGraphGenCluster::generateGraph()
   val=YACS::ENGINE::evalFuncPyWithNoParams(func);
   _jobName=PyString_AsString(val);
   //
-  static const char LISTPYOBJ_STR[]="list[pyobj]";
-  if(getBoss()->getOutputsOfInterest().empty())
-    return ;
-  YACS::ENGINE::RuntimeSALOME::setRuntime();
-  YACS::ENGINE::RuntimeSALOME *r(YACS::ENGINE::getSALOMERuntime());
-  _generatedGraph=r->createProc(DFT_PROC_NAME);
-  YACS::ENGINE::TypeCode *pyobjTC(_generatedGraph->createInterfaceTc("python:obj:1.0","pyobj",std::list<YACS::ENGINE::TypeCodeObjref *>()));
-  std::ostringstream oss; oss << "Loop_" << getBoss()->getRunNode()->getName();
-  _generatedGraph->createType(YACSEvalAnyDouble::TYPE_REPR,"double");
-  _generatedGraph->createType(YACSEvalAnyInt::TYPE_REPR,"int");
-  //
-  YACS::ENGINE::InlineNode *n0(r->createScriptNode(YACS::ENGINE::PythonNode::KIND,"__initializer__"));
-  _generatedGraph->edAddChild(n0);
-  YACS::ENGINE::TypeCode *listPyobjTC(_generatedGraph->createSequenceTc(LISTPYOBJ_STR,LISTPYOBJ_STR,pyobjTC));
-  YACS::ENGINE::OutputPort *sender(n0->edAddOutputPort("sender",listPyobjTC));
-  std::ostringstream var0;
-  const std::vector< YACSEvalInputPort >& inputs(getBoss()->getInputs());
-  for(std::vector< YACSEvalInputPort >::const_iterator it=inputs.begin();it!=inputs.end();it++)
-    {
-      if((*it).isRandomVar())
-        {
-          var0 << (*it).getName() << ",";
-          YACS::ENGINE::TypeCode *tc(YACSEvalYFXPattern::CreateSeqTypeCodeFrom(_generatedGraph,(*it).getTypeOfData()));
-          YACS::ENGINE::InputPort *inp(n0->edAddInputPort((*it).getName(),tc));
-          YACS::ENGINE::InputPyPort *inpc(dynamic_cast<YACS::ENGINE::InputPyPort *>(inp));
-          if(!inpc)
-            throw YACS::Exception("YACSEvalYFXRunOnlyPattern::generateGraph : internal error 1 !");
-          (*it).setUndergroundPortToBeSet(inpc);
-        }
+  class ClusterPatcher : public YACSEvalYFXGraphGen::CustomPatcher
+  {
+  public:
+    ClusterPatcher(const std::string& jobName):_jobName(jobName) { n2Script << "zeRes=["; }
+    void addOutputVar(const std::string& name) { n2Script<< name << ", "; }
+    void assignOutput(YACS::ENGINE::InlineNode *node) {
+      n2Script << "]\nwith open(\"" << _jobName << "\",\"w\") as f:" << std::endl;
+      n2Script << "  f.write(str(zeRes))" << std::endl;
+      node->setScript(n2Script.str());
     }
-  std::ostringstream n0Script; n0Script << "sender=zip(" << var0.str() << ")\n";
-  n0->setScript(n0Script.str());
+  private:
+    std::ostringstream n2Script;
+    std::string _jobName;
+  };
+  ClusterPatcher cp(_jobName);
   //
-  YACS::ENGINE::ForEachLoop *n1(r->createForEachLoop(oss.str(),pyobjTC));
-  _FEInGeneratedGraph=n1;
-  _generatedGraph->edAddChild(n1);
-  _generatedGraph->edAddCFLink(n0,n1);
-  _generatedGraph->edAddDFLink(sender,n1->edGetSeqOfSamplesPort());
-  YACS::ENGINE::InlineNode *n2(r->createScriptNode(YACS::ENGINE::PythonNode::KIND,GATHER_NODE_NAME));
-  _generatedGraph->edAddChild(n2);
-  _generatedGraph->edAddCFLink(n1,n2);
-  //
-  YACS::ENGINE::Bloc *n10(r->createBloc(FIRST_FE_SUBNODE_NAME));
-  n1->edAddChild(n10);
-  YACS::ENGINE::InlineNode *n100(r->createScriptNode(YACS::ENGINE::PythonNode::KIND,"__dispatch__"));
-  YACS::ENGINE::ComposedNode *runNode(getBoss()->getRunNode());
-  YACS::ENGINE::Node *n101(runNode->cloneWithoutCompAndContDeepCpy(0,true));
-  n10->edAddChild(n100);
-  n10->edAddChild(n101);
-  YACS::ENGINE::InputPort *dispatchIn(n100->edAddInputPort("i0",pyobjTC));
-  n10->edAddCFLink(n100,n101);
-  n1->edAddDFLink(n1->edGetSamplePort(),dispatchIn);
-  std::ostringstream var1;
-  for(std::vector< YACSEvalInputPort >::const_iterator it=inputs.begin();it!=inputs.end();it++)
-    {
-      if((*it).isRandomVar())
-        {
-          var1 << (*it).getName() << ",";
-          YACS::ENGINE::OutputPort *myOut(n100->edAddOutputPort((*it).getName(),_generatedGraph->getTypeCode((*it).getTypeOfData())));
-          std::string tmpPortName(runNode->getInPortName((*it).getUndergroundPtr()));
-          YACS::ENGINE::InputPort *myIn(n101->getInputPort(tmpPortName));
-          n10->edAddDFLink(myOut,myIn);
-        }
-    }
-  std::ostringstream n100Script;  n100Script << var1.str() << "=i0\n";
-  n100->setScript(n100Script.str());
-  const std::vector<YACSEvalOutputPort *>& outputsOfInt(getBoss()->getOutputsOfInterest());
-  std::ostringstream n2Script; n2Script << "zeRes=[";
-  for(std::vector< YACSEvalOutputPort * >::const_iterator it=outputsOfInt.begin();it!=outputsOfInt.end();it++)
-    {
-      YACS::ENGINE::TypeCode *tc(YACSEvalYFXPattern::CreateSeqTypeCodeFrom(_generatedGraph,(*it)->getTypeOfData()));
-      YACS::ENGINE::InputPort *myIn(n2->edAddInputPort((*it)->getName(),tc));
-      n2Script << (*it)->getName() << ", ";
-      std::string tmpPortName(runNode->getOutPortName((*it)->getUndergroundPtr()));
-      YACS::ENGINE::OutputPort *myOut(n101->getOutputPort(tmpPortName));
-      _generatedGraph->edAddDFLink(myOut,myIn);
-    }
-  n2Script << "]\nf=file(\"" << _jobName << "\",\"w\") ; f.write(str(zeRes)) ; del f";
-  n2->setScript(n2Script.str());
-  _generatedGraph->updateContainersAndComponents();
+  this->generateGraphCommon(cp);
 }
 
 bool YACSEvalYFXGraphGenCluster::go(const YACSEvalExecParams& params, YACSEvalSession *session) const

diff --git a/src/evalyfx/YACSEvalYFXPattern.hxx b/src/evalyfx/YACSEvalYFXPattern.hxx
index 39a56ff45c51df8cac7d6975e91843e9b56612b2..c9ed3787db1cbc0536febac56a5717818d4c3bad 100644 (file)
--- a/src/evalyfx/YACSEvalYFXPattern.hxx
+++ b/src/evalyfx/YACSEvalYFXPattern.hxx
@@ -37,6 +37,7 @@ namespace YACS
     class TypeCode;
     class ForEachLoop;
     class ComposedNode;
+    class InlineNode;
     class InputPyPort;
     class SequenceAny;
   }
@@ -155,6 +156,14 @@ private:
 
 class YACSEvalYFXGraphGen
 {
+public:
+  class CustomPatcher
+  {
+  public:
+    virtual ~CustomPatcher() { }
+    virtual void addOutputVar(const std::string& name) = 0;
+    virtual void assignOutput(YACS::ENGINE::InlineNode *node) = 0;
+  };
 protected:
   YACSEvalYFXGraphGen(YACSEvalYFXRunOnlyPattern *boss);
   YACSEvalYFXRunOnlyPattern *getBoss() const { return _boss; }
@@ -169,6 +178,8 @@ public:
   void resetGeneratedGraph();
   YACS::ENGINE::Proc *getUndergroundGeneratedGraph() const { return _generatedGraph; }
   YACS::ENGINE::ForEachLoop *getUndergroundForEach() const { return _FEInGeneratedGraph; }
+protected:
+  void generateGraphCommon(CustomPatcher& patcher);
 private:
   YACSEvalYFXRunOnlyPattern *_boss;
 protected:
@@ -178,6 +189,7 @@ public:
   static const char DFT_PROC_NAME[];
   static const char FIRST_FE_SUBNODE_NAME[];
   static const char GATHER_NODE_NAME[];
+  static const char HIDDEN_INDEX_VAR[];
 };
 
 class YACSEvalYFXGraphGenInteractive : public YACSEvalYFXGraphGen
@@ -193,7 +205,7 @@ public:
 class YACSEvalYFXGraphGenCluster : public YACSEvalYFXGraphGen
 {
 public:
-  YACSEvalYFXGraphGenCluster(YACSEvalYFXRunOnlyPattern *boss):YACSEvalYFXGraphGen(boss) { }
+  YACSEvalYFXGraphGenCluster(YACSEvalYFXRunOnlyPattern *boss):YACSEvalYFXGraphGen(boss),_jobid(0) { }
   void generateGraph();
   bool go(const YACSEvalExecParams& params, YACSEvalSession *session) const;
   std::vector<YACSEvalSeqAny *> getResults() const;

diff --git a/src/evalyfx_swig/test3.py b/src/evalyfx_swig/test3.py
index 9fd10cc45564e6cd715831476d45ef94e5a7922f..8b9f3c12e7c74e5a27cf88959a1653a7bfeaa139 100644 (file)
--- a/src/evalyfx_swig/test3.py
+++ b/src/evalyfx_swig/test3.py
@@ -24,7 +24,7 @@ import os,getpass
 CatalogResources="""<?xml version="1.0"?>
 <resources>
   <machine name="localhost" hostname="dsp0698184" type="single_machine" appliPath="" batchQueue="" userCommands="" protocol="ssh" iprotocol="ssh" workingDirectory="" canLaunchBatchJobs="false" canRunContainers="true" batch="none" mpi="no mpi" userName="" OS="" memInMB="0" CPUFreqMHz="0" nbOfNodes="1" nbOfProcPerNode="4"/>
-  <machine name="athos" hostname="athos" type="cluster" appliPath="/home/H87074/ATHOS_V771/appli_V7_7_1" batchQueue="" userCommands="" protocol="ssh" iprotocol="ssh" workingDirectory="" canLaunchBatchJobs="true" canRunContainers="false" batch="slurm" mpi="no mpi" userName="" OS="" memInMB="0" CPUFreqMHz="0" nbOfNodes="250" nbOfProcPerNode="28"/>
+  <machine name="athos" hostname="athos" type="cluster" appliPath="/home/H87074/ATHOS_V830/appli_V8_3_0" batchQueue="" userCommands="" protocol="ssh" iprotocol="ssh" workingDirectory="" canLaunchBatchJobs="true" canRunContainers="false" batch="slurm" mpi="no mpi" userName="" OS="" memInMB="0" CPUFreqMHz="0" nbOfNodes="250" nbOfProcPerNode="28"/>
 </resources>"""
 
 def buildScheme(fname):
@@ -43,7 +43,8 @@ def buildScheme(fname):
     p.edAddChild(n0)
     q=n0.edAddInputPort("q",td)
     ep=n0.edAddOutputPort("ep",td)
-    n0.setScript("ep=1./(4.-q)") # <- force division by 0
+    #n0.setScript("ep=1./(4.-q)") # <- force division by 0
+    n0.setScript("ep=2.*float(___idx___)")
     n0.setExecutionMode("remote")
     n0.setContainer(cont)
     p0.saveSchema(fname)