const char Container::AOC_ENTRY[]="attached_on_cloning";
-const char Container::STORE_CONTEXT_PROPERTY[]="store_context";
+const char Container::USE_PYCACHE_PROPERTY[]="use_py_cache";
Container::Container():_isAttachedOnCloning(false),_proc(0)
{
setProperty((*it).first,(*it).second);
}
-bool Container::storeContext()
+bool Container::isUsingPythonCache()
{
bool found = false;
std::string str_value;
- str_value = getProperty(STORE_CONTEXT_PROPERTY);
+ str_value = getProperty(USE_PYCACHE_PROPERTY);
const char* yes_values[] = {"YES", "Yes", "yes", "TRUE", "True", "true", "1",
"ON", "on", "On"};
for(const char* v : yes_values)
return found;
}
-void Container::setStoreContext(bool v)
+void Container::usePythonCache(bool v)
{
if(v)
- setProperty(STORE_CONTEXT_PROPERTY, "1");
+ setProperty(USE_PYCACHE_PROPERTY, "1");
else
- setProperty(STORE_CONTEXT_PROPERTY, "0");
+ setProperty(USE_PYCACHE_PROPERTY, "0");
}
const std::string& resource_name,
const std::string& container_name);
virtual bool canAcceptImposedResource();
- virtual bool storeContext();
- virtual void setStoreContext(bool v);
+ virtual bool isUsingPythonCache();
+ virtual void usePythonCache(bool v);
virtual std::string getPlacementId(const Task *askingNode) const = 0;
virtual std::string getFullPlacementId(const Task *askingNode) const = 0;
//Edition only methods
virtual void shutdown(int level) = 0;
static const char KIND_ENTRY[];
static const char AOC_ENTRY[];
- static const char STORE_CONTEXT_PROPERTY[];
+ static const char USE_PYCACHE_PROPERTY[];
protected:
std::string _name;
mutable bool _isAttachedOnCloning;
try
{
std::ostringstream container_name;
- container_name << runInfo.resource.name << "-"
- << runInfo.type.name << "-" << runInfo.index;
+ container_name << runInfo.type.name << "-" << runInfo.index;
task->imposeResource(runInfo.resource.name, container_name.str());
traceExec(task, "load", ComputePlacement(task));
task->load();
connect(sb_procNode, SIGNAL(valueChanged(const QString&)), this, SLOT(onModifyProcs(const QString&)));
connect(sb_nbprocpar, SIGNAL(valueChanged(const QString&)), this, SLOT(onModifyProcPar(const QString&)));
connect(sb_nbproc, SIGNAL(valueChanged(const QString&)), this, SLOT(onModifyProcRes(const QString&)));
+ connect(ch_pycache,SIGNAL(stateChanged(int)),this,SLOT(onModifyUsePyCache(int)));
}
FormAdvParamContainer::~FormAdvParamContainer()
else
le_workdir->setText("");
+ if(_container)
+ ch_pycache->setCheckState(_container->isUsingPythonCache()?Qt::Checked:Qt::Unchecked);
+ else
+ ch_pycache->setCheckState(Qt::Unchecked);
+
if(_properties.count("container_name"))
le_contname->setText(_properties["container_name"].c_str());
else
if (properties["resource_list"] != text.toStdString())
onModified();
}
+
+void FormAdvParamContainer::onModifyUsePyCache(int val)
+{
+ if (!_container)
+ return;
+ bool val2(false);
+ if(val==Qt::Unchecked)
+ val2=false;
+ if(val==Qt::Checked)
+ val2=true;
+ bool prop = _container->isUsingPythonCache();
+ int prop2((int)val2);
+ std::ostringstream oss; oss << prop2;
+ _container->usePythonCache(val2);
+ if(prop!=val2)
+ onModified();
+}
void onModifyProcRes(const QString &text);
void onModifyCompoList(const QString &text);
void onModifyResourceList(const QString &text);
-
+ void onModifyUsePyCache(int val);
protected:
bool _advanced;
<widget class="QLineEdit" name="le_workdir"/>
</item>
<item row="3" column="0" colspan="3">
+ <widget class="QCheckBox" name="ch_pycache">
+ <property name="text">
+ <string>Use python cache</string>
+ </property>
+ </widget>
+ </item>
+ <item row="4" column="0" colspan="3">
<widget class="QGroupBox" name="groupBox_2">
<property name="title">
<string>Parallel parameters</string>
</layout>
</widget>
</item>
- <item row="4" column="1">
+ <item row="5" column="1">
<spacer name="verticalSpacer">
<property name="orientation">
<enum>Qt::Vertical</enum>
FormContainer::FillPanel(0); // --- set widgets before signal connexion to avoid false modif detection
connect(cb_type, SIGNAL(activated(const QString&)),this, SLOT(onModifyType(const QString&)));
connect(ch_aoc,SIGNAL(stateChanged(int)),this,SLOT(onModifyAOC(int)));
- connect(ch_pycache,SIGNAL(stateChanged(int)),this,SLOT(onModifyStorePyCache(int)));
}
FormContainer::~FormContainer()
if(container)
{
ch_aoc->setCheckState(container->isAttachedOnCloning()?Qt::Checked:Qt::Unchecked);
- ch_pycache->setCheckState(container->storeContext()?Qt::Checked:Qt::Unchecked);
}
cb_type->clear();
cb_type->addItem("mono");
if(prop!=val2)
onModified();
}
-
-void FormContainer::onModifyStorePyCache(int val)
-{
- if (!_container)
- return;
- bool val2(false);
- if(val==Qt::Unchecked)
- val2=false;
- if(val==Qt::Checked)
- val2=true;
- bool prop = _container->storeContext();
- int prop2((int)val2);
- std::ostringstream oss; oss << prop2;
- //_properties[YACS::ENGINE::Container::AOC_ENTRY]=oss.str();
- _container->setStoreContext(val2);
- if(prop!=val2)
- onModified();
-}
public slots:
void onModifyType(const QString &text);
void onModifyAOC(int val);
- void onModifyStorePyCache(int val);
private:
QComboBox *cb_type;
};
</property>
</widget>
</item>
- <item>
- <widget class="QCheckBox" name="ch_pycache">
- <property name="text">
- <string>Store python cache</string>
- </property>
- </widget>
- </item>
</layout>
</item>
</layout>
squeezeMemoryRemote();
}
//
- if(!storeContext())
+ if(!isUsingPythonCache())
{
if(!CORBA::is_nil(_pynode))
{
std::string PythonNode::pythonEntryName()const
{
- if(storeContext())
+ if(isUsingPythonCache())
return "DEFAULT_NAME_FOR_UNIQUE_PYTHON_NODE_ENTRY";
else
return getName();
}
-bool PythonNode::storeContext()const
+bool PythonNode::isUsingPythonCache()const
{
bool found = false;
if(_container)
- found = _container->storeContext();
+ found = _container->isUsingPythonCache();
return found;
}
void imposeResource(const std::string& resource_name,
const std::string& container_name) override;
bool canAcceptImposedResource()override;
- bool storeContext()const;
+ bool isUsingPythonCache()const;
std::string getContainerLog();
PythonNode* cloneNode(const std::string& name);
virtual std::string typeName() { return "YACS__ENGINE__PythonNode"; }
--- /dev/null
+<?xml version='1.0' encoding='iso-8859-1' ?>
+<proc name="2foreach_with_cache">
+ <property name="executor" value="workloadmanager"/>
+ <type name="string" kind="string"/>
+ <struct name="Engines/dataref">
+ <member name="ref" type="string"/>
+ </struct>
+ <type name="bool" kind="bool"/>
+ <sequence name="boolvec" content="bool"/>
+ <type name="double" kind="double"/>
+ <sequence name="dblevec" content="double"/>
+ <objref name="file" id="file"/>
+ <type name="int" kind="int"/>
+ <sequence name="intvec" content="int"/>
+ <struct name="stringpair">
+ <member name="name" type="string"/>
+ <member name="value" type="string"/>
+ </struct>
+ <sequence name="propvec" content="stringpair"/>
+ <objref name="pyobj" id="python:obj:1.0"/>
+ <sequence name="seqboolvec" content="boolvec"/>
+ <sequence name="seqdblevec" content="dblevec"/>
+ <sequence name="seqint" content="int"/>
+ <sequence name="seqintvec" content="intvec"/>
+ <sequence name="seqpyobj" content="pyobj"/>
+ <sequence name="stringvec" content="string"/>
+ <sequence name="seqstringvec" content="stringvec"/>
+ <container name="DefaultContainer">
+ <property name="container_kind" value="Salome"/>
+ <property name="attached_on_cloning" value="0"/>
+ <property name="container_name" value="FactoryServer"/>
+ <property name="name" value="localhost"/>
+ </container>
+ <container name="c1">
+ <property name="container_kind" value="Salome"/>
+ <property name="attached_on_cloning" value="0"/>
+ <property name="nb_parallel_procs" value="1"/>
+ <property name="type" value="multi"/>
+ <property name="use_py_cache" value="1"/>
+ </container>
+ <container name="c2">
+ <property name="container_kind" value="Salome"/>
+ <property name="attached_on_cloning" value="0"/>
+ <property name="nb_parallel_procs" value="4"/>
+ <property name="type" value="multi"/>
+ <property name="use_py_cache" value="1"/>
+ </container>
+ <inline name="Begin">
+ <script><code><![CDATA[import time
+vals = [ i for i in range(40)]
+nbbranches = len(vals)
+t0=time.time()
+]]></code></script>
+ <outport name="t0" type="pyobj"/>
+ <outport name="vals" type="intvec"/>
+ <outport name="nbbranches" type="int"/>
+ </inline>
+ <inline name="End">
+ <script><code><![CDATA[import time
+tf=time.time()
+total_time=int(tf-t0)
+
+ok = True
+s_expected = sum(vals)
+
+tr1={}
+for cont_name, s in r1:
+ if cont_name in tr1.keys():
+ v = tr1[cont_name]
+ if s > v:
+ tr1[cont_name] = s
+ else:
+ tr1[cont_name] = s
+
+s1 = sum(tr1.values())
+if s_expected != s1:
+ ok = False
+ print("Error on the first foreach.")
+ print("Expected sum:", s_expected)
+ print("Obtained sum:", s1)
+
+tr2={}
+for cont_name, s in r2:
+ if cont_name in tr2.keys():
+ v = tr2[cont_name]
+ if s > v:
+ tr2[cont_name] = s
+ else:
+ tr2[cont_name] = s
+
+s2 = sum(tr2.values())
+if s_expected != s2:
+ ok = False
+ print("Error on the second foreach.")
+ print("Expected sum:", s_expected)
+ print("Obtained sum:", s2)
+
+#print(r1)
+#print(r2)
+]]></code></script>
+ <load container="DefaultContainer"/>
+ <inport name="t0" type="pyobj"/>
+ <inport name="r1" type="seqpyobj"/>
+ <inport name="r2" type="seqpyobj"/>
+ <inport name="vals" type="intvec"/>
+ <outport name="total_time" type="int"/>
+ <outport name="ok" type="bool"/>
+ </inline>
+ <foreach name="ForEach1" nbranch="1" loopWeight="-1" type="int">
+ <remote name="PyNode1" elementaryWeight="-1">
+ <script><code><![CDATA[import time
+if "s" not in globals():
+ # long initialization here to be done only once
+ time.sleep(1)
+ s=0
+s += v
+container_name = my_container.name
+result = (container_name, s)
+time.sleep(1)
+]]></code></script>
+ <load container="c1"/>
+ <inport name="v" type="int"/>
+ <outport name="result" type="pyobj"/>
+ </remote>
+ </foreach>
+ <foreach name="ForEach2" nbranch="1" loopWeight="-1" type="int">
+ <remote name="PyNode2" elementaryWeight="-1">
+ <script><code><![CDATA[import time
+if "s" not in globals():
+ # long initialization here to be done only once
+ time.sleep(1)
+ s=0
+s += v
+container_name = my_container.name
+result = (container_name, s)
+time.sleep(1)
+]]></code></script>
+ <load container="c2"/>
+ <inport name="v" type="int"/>
+ <outport name="result" type="pyobj"/>
+ </remote>
+ </foreach>
+ <control> <fromnode>Begin</fromnode> <tonode>ForEach1</tonode> </control>
+ <control> <fromnode>Begin</fromnode> <tonode>ForEach2</tonode> </control>
+ <control> <fromnode>Begin</fromnode> <tonode>End</tonode> </control>
+ <control> <fromnode>ForEach1</fromnode> <tonode>End</tonode> </control>
+ <control> <fromnode>ForEach2</fromnode> <tonode>End</tonode> </control>
+ <datalink control="false">
+ <fromnode>Begin</fromnode> <fromport>t0</fromport>
+ <tonode>End</tonode> <toport>t0</toport>
+ </datalink>
+ <datalink control="false">
+ <fromnode>Begin</fromnode> <fromport>vals</fromport>
+ <tonode>End</tonode> <toport>vals</toport>
+ </datalink>
+ <datalink control="false">
+ <fromnode>Begin</fromnode> <fromport>vals</fromport>
+ <tonode>ForEach1</tonode> <toport>SmplsCollection</toport>
+ </datalink>
+ <datalink control="false">
+ <fromnode>Begin</fromnode> <fromport>vals</fromport>
+ <tonode>ForEach2</tonode> <toport>SmplsCollection</toport>
+ </datalink>
+ <datalink control="false">
+ <fromnode>Begin</fromnode> <fromport>nbbranches</fromport>
+ <tonode>ForEach2</tonode> <toport>nbBranches</toport>
+ </datalink>
+ <datalink control="false">
+ <fromnode>Begin</fromnode> <fromport>nbbranches</fromport>
+ <tonode>ForEach1</tonode> <toport>nbBranches</toport>
+ </datalink>
+ <datalink control="false">
+ <fromnode>ForEach1</fromnode> <fromport>evalSamples</fromport>
+ <tonode>ForEach1.PyNode1</tonode> <toport>v</toport>
+ </datalink>
+ <datalink control="false">
+ <fromnode>ForEach2</fromnode> <fromport>evalSamples</fromport>
+ <tonode>ForEach2.PyNode2</tonode> <toport>v</toport>
+ </datalink>
+ <datalink control="false">
+ <fromnode>ForEach1.PyNode1</fromnode> <fromport>result</fromport>
+ <tonode>End</tonode> <toport>r1</toport>
+ </datalink>
+ <datalink control="false">
+ <fromnode>ForEach2.PyNode2</fromnode> <fromport>result</fromport>
+ <tonode>End</tonode> <toport>r2</toport>
+ </datalink>
+ <parameter>
+ <tonode>ForEach1</tonode><toport>nbBranches</toport>
+ <value><int>1</int></value>
+ </parameter>
+ <parameter>
+ <tonode>ForEach2</tonode><toport>nbBranches</toport>
+ <value><int>1</int></value>
+ </parameter>
+ <presentation name="Begin" x="9" y="86" width="158" height="117" expanded="1" expx="9" expy="86" expWidth="158" expHeight="117" shownState="0"/>
+ <presentation name="End" x="402.5" y="85" width="158" height="144" expanded="1" expx="402.5" expy="85" expWidth="158" expHeight="144" shownState="0"/>
+ <presentation name="ForEach1.PyNode1" x="9.5" y="88" width="158" height="63" expanded="1" expx="9.5" expy="88" expWidth="158" expHeight="63" shownState="0"/>
+ <presentation name="ForEach2.PyNode2" x="13" y="92" width="158" height="63" expanded="1" expx="13" expy="92" expWidth="158" expHeight="63" shownState="0"/>
+ <presentation name="ForEach1" x="201.5" y="146" width="171.5" height="155" expanded="1" expx="201.5" expy="146" expWidth="171.5" expHeight="155" shownState="0"/>
+ <presentation name="ForEach2" x="204" y="313.5" width="175" height="159" expanded="1" expx="204" expy="313.5" expWidth="175" expHeight="159" shownState="0"/>
+ <presentation name="__ROOT__" x="0" y="0" width="564.5" height="476.5" expanded="1" expx="0" expy="0" expWidth="564.5" expHeight="476.5" shownState="0"/>
+</proc>
export TESTCOMPONENT_ROOT_DIR=`pwd`/../runtime
#python3 -m unittest discover
-MODULES_TO_TEST=testEdit testExec testFEDyn testFEDyn2 testLoader testProgress \
-testRefcount testResume testSave testSaveLoadRun testValidationChecks
+MODULES_TO_TEST="testEdit testExec testLoader testProgress testRefcount \
+testResume testSave testSaveLoadRun testValidationChecks"
python3 -m unittest $MODULES_TO_TEST
ret=$?
# if no property is set, the old executor is used
proc.setProperty("executor", "workloadmanager")
# reuse the same python context for every execution
- cont.setStoreContext(True)
+ cont.usePythonCache(True)
# save & reload
schema_file = os.path.join(dir_test,"pynode_with_cache1.xml")
proc.saveSchema(schema_file)
proc.edAddChild(n2)
proc.edAddCFLink(n1,n2)
# reuse the same python context for every execution
- cont.setStoreContext(True)
+ cont.usePythonCache(True)
# save & reload
schema_file = os.path.join(dir_test,"pynode_with_cache2.xml")
proc.saveSchema(schema_file)
# lower time means some resources are overloaded
self.assertTrue(execution_time > 13)
# The containers need some time to be launched.
- # We need some room for that.
+ # We need some delay to add to the 15s.
+ self.assertTrue(execution_time < 20)
+
+ def test2(self):
+ """ Two parallel foreach-s with different containers and python nodes
+ using cache.
+ """
+ proc = self.l.load("samples/wlm_2foreach_with_cache.xml")
+ self.e.RunW(proc,0)
+ ok = proc.getChildByName("End").getOutputPort("ok")
+ self.assertTrue(ok)
+ total_time = proc.getChildByName("End").getOutputPort("total_time")
+ # theoretical time should be 16s
+ execution_time = total_time.getPyObj()
+ # lower time means some resources are overloaded
+ self.assertTrue(execution_time > 14)
+ # The containers need some time to be launched.
+ # We need some delay to add to the 15s.
self.assertTrue(execution_time < 20)
if __name__ == '__main__':
