From cc7fc6db381023cd476d0bdb3928b0d00e9cc52e Mon Sep 17 00:00:00 2001 From: Ovidiu MIRCESCU Date: Fri, 13 Nov 2020 15:27:47 +0100 Subject: [PATCH] Multijob ready for persalys. --- src/cpp/CMakeLists.txt | 3 + src/cpp/Launcher.hxx | 46 +++++++++ src/cpp/PyStudyJob.cxx | 153 ++++++++++++++++++++++++++++++ src/cpp/PyStudyJob.hxx | 49 ++++++++++ src/cpp/TPyStudyJob.hxx | 135 ++++++++++++++++++++++++++ src/cpp/Test/PyTestMain.cxx | 88 +++++++++++++++++ src/cpp/Test/StudyGeneralTest.cxx | 89 +++++++++++++++-- src/cpp/Test/StudyGeneralTest.hxx | 2 + src/pydefx/plugins/jobexecutor.py | 1 + 9 files changed, 560 insertions(+), 6 deletions(-) create mode 100644 src/cpp/PyStudyJob.cxx create mode 100644 src/cpp/PyStudyJob.hxx create mode 100644 src/cpp/TPyStudyJob.hxx create mode 100644 src/cpp/Test/PyTestMain.cxx diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 841f83f..417350a 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -30,6 +30,7 @@ SET(ydefx_SOURCES JobParametersProxy.cxx Exceptions.cxx MonoPyJob.cxx + PyStudyJob.cxx ) SET(ydefx_HEADERS @@ -44,6 +45,8 @@ SET(ydefx_HEADERS TMonoPyJob.hxx Job.hxx Launcher.hxx + PyStudyJob.hxx + TPyStudyJob.hxx ) SET(ydefx_LINK diff --git a/src/cpp/Launcher.hxx b/src/cpp/Launcher.hxx index 60c1404..4d102cd 100644 --- a/src/cpp/Launcher.hxx +++ b/src/cpp/Launcher.hxx @@ -20,6 +20,7 @@ #define YDEFX_LAUNCHER_H #include "TMonoPyJob.hxx" +#include "TPyStudyJob.hxx" namespace ydefx { @@ -39,6 +40,12 @@ public: Sample& sample, const JobParametersProxy& params); + template + Job* submitPyStudyJob(py2cpp::PyPtr& pyStudyObj, + const PyStudyFunction& fnScript, + Sample& sample, + const JobParametersProxy& params); + /*! * Connect to an already created job. * Return nullptr in case of failure. Check the error with lastError(). @@ -97,6 +104,45 @@ Job* Launcher::submitMonoPyJob(const PyStudyFunction& fnScript, return result; } +template +Job* Launcher::submitPyStudyJob(py2cpp::PyPtr& pyStudyObj, + const PyStudyFunction& fnScript, + Sample& sample, + const JobParametersProxy& params) +{ + Job* result = nullptr; + _lastError = ""; + try + { + result = new TPyStudyJob(pyStudyObj, fnScript, sample, params); + } + catch(std::exception& e) + { + if(result != nullptr) + delete result; + result = nullptr; + _lastError = e.what(); + return result; + } + + if(!result->lastError().empty()) + { + _lastError = result->lastError(); + delete result; + result = nullptr; + return result; + } + + if(!result->launch()) + { + _lastError = "Failed to submit job.\n"; + _lastError += result->lastError(); + delete result; + result = nullptr; + } + return result; +} + template Job* Launcher::connectJob(const std::string& jobDump, Sample& sample) diff --git a/src/cpp/PyStudyJob.cxx b/src/cpp/PyStudyJob.cxx new file mode 100644 index 0000000..a7171a4 --- /dev/null +++ b/src/cpp/PyStudyJob.cxx @@ -0,0 +1,153 @@ +// Copyright (C) 2019 EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +#include "PyStudyJob.hxx" +#include + +namespace ydefx +{ +PyStudyJob::PyStudyJob() +: _pyStudy() +, _lastError() +, _waitDelay(10) +{ + py2cpp::PyFunction objConstructor; + objConstructor.loadExp("pydefx", "PyStudy"); + _pyStudy = objConstructor(); +} + +PyStudyJob::PyStudyJob(const std::string& pymodule_name, const std::string& pyclass_name) +: _pyStudy() +, _lastError() +, _waitDelay(10) +{ + py2cpp::PyFunction objConstructor; + objConstructor.loadExp(pymodule_name, pyclass_name); + _pyStudy = objConstructor(); +} + +PyStudyJob::PyStudyJob(py2cpp::PyPtr& pyStudyObj) +: _pyStudy(pyStudyObj) +, _lastError() +, _waitDelay(10) +{ +} + +PyStudyJob::~PyStudyJob() +{ +} + +std::string PyStudyJob::state() +{ + std::string result; + _lastError = ""; + try + { + py2cpp::PyFunction pyFn; + pyFn.loadExp(_pyStudy, "getJobState"); + py2cpp::pyResult(result) = pyFn(); + } + catch(std::exception& e) + { + _lastError = "An error occured while retrieving job's state.\n"; + _lastError += e.what(); + } + return result; +} + +double PyStudyJob::progress() +{ + double result; + py2cpp::PyFunction pyFn; + _lastError = ""; + try + { + pyFn.loadExp(_pyStudy, "getProgress"); + py2cpp::pyResult(result) = pyFn(); + } + catch(std::exception& e) + { + _lastError = "An error occured while retrieving job's progress.\n"; + _lastError += e.what(); + } + return result; +} + +std::string PyStudyJob::dump() +{ + std::string result; + _lastError = ""; + try + { + py2cpp::PyFunction pyFn; + pyFn.loadExp(_pyStudy, "dump"); + py2cpp::pyResult(result) = pyFn(); + } + catch(std::exception& e) + { + _lastError = "An error occured while dumping the job.\n"; + _lastError += e.what(); + } + return result; +} + +bool PyStudyJob::launch() +{ + _lastError = ""; + try + { + py2cpp::PyFunction pyFn; + pyFn.loadExp(_pyStudy, "launch"); + pyFn(); + } + catch(std::exception& e) + { + _lastError = "An error occured while launching the job.\n"; + _lastError += e.what(); + } + return _lastError.empty(); +} + +const std::string& PyStudyJob::lastError() +{ + return _lastError; +} + +bool PyStudyJob::wait() +{ + _lastError = ""; + try + { + py2cpp::PyFunction pyFn; + pyFn.loadExp(_pyStudy, "wait"); + pyFn(_waitDelay); + } + catch(std::exception& e) + { + _lastError = "An error occured while waiting the end of the job.\n"; + _lastError += e.what(); + } + return _lastError.empty(); +} + +void PyStudyJob::configureWaitDelay(int seconds) +{ + _waitDelay = seconds; +} + +} diff --git a/src/cpp/PyStudyJob.hxx b/src/cpp/PyStudyJob.hxx new file mode 100644 index 0000000..9daf91d --- /dev/null +++ b/src/cpp/PyStudyJob.hxx @@ -0,0 +1,49 @@ +// Copyright (C) 2019 EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +#ifndef YDEFX_PYSTUDYJOB_HXX +#define YDEFX_PYSTUDYJOB_HXX +#include "Job.hxx" +#include + +namespace ydefx +{ +class PyStudyJob : public Job +{ +public: + PyStudyJob(const std::string& pymodule_name, const std::string& pyclass_name); + PyStudyJob(py2cpp::PyPtr& pyStudyObj); + PyStudyJob(); + virtual ~PyStudyJob(); + virtual std::string state(); + virtual double progress(); + virtual std::string dump(); + virtual bool launch(); // return false when it fails + virtual bool fetch()=0; // return false when it fails + virtual const std::string& lastError(); + virtual bool wait(); // Wait for the end of the job. Return false when it fails. + virtual void configureWaitDelay(int seconds); +protected: + py2cpp::PyPtr _pyStudy; + std::string _lastError; + int _waitDelay; +}; + +} + +#endif //YDEFX_PYSTUDYJOB_HXX diff --git a/src/cpp/TPyStudyJob.hxx b/src/cpp/TPyStudyJob.hxx new file mode 100644 index 0000000..57270c5 --- /dev/null +++ b/src/cpp/TPyStudyJob.hxx @@ -0,0 +1,135 @@ +// Copyright (C) 2019 EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +#ifndef YDEFX_TPYSTUDYJOB_HXX +#define YDEFX_TPYSTUDYJOB_HXX +#include "JobParametersProxy.hxx" +#include "PyStudyJob.hxx" +#include "SamplePyConversions.hxx" +#include "PyStudyFunction.hxx" + +namespace ydefx +{ +template +class TPyStudyJob : public PyStudyJob +{ +public: + //! Create a new job using the default pystudy class. + TPyStudyJob(const PyStudyFunction& fnScript, + Sample& sample, + const JobParametersProxy& params) + : PyStudyJob() + , _sample(sample) + { + createNewJob(fnScript, params); + /*if(_lastError.empty()) // no errors during parent construction + { + try + { + py2cpp::PyPtr pySample = createPySample(sample); + py2cpp::PyFunction pyFn; + pyFn.loadExp(_pyStudy, "createNewJob"); + pyFn(fnScript, pySample, params); + } + catch(std::exception& e) + { + _lastError = "An error occured while creating the job.\n"; + _lastError += e.what(); + } + }*/ + } + + TPyStudyJob(py2cpp::PyPtr& pyStudyObj, + const PyStudyFunction& fnScript, + Sample& sample, + const JobParametersProxy& params) + : PyStudyJob(pyStudyObj) + , _sample(sample) + { + createNewJob(fnScript, params); + } + + //! Connect to an existing job. + TPyStudyJob(const std::string& jobDump, Sample& sample) + : PyStudyJob() + , _sample(sample) + { + if(_lastError.empty()) // no errors during parent construction + { + try + { + py2cpp::PyFunction pyFn; + pyFn.loadExp(_pyStudy, "loadFromString"); + pyFn(jobDump); + } + catch(std::exception& e) + { + _lastError = "An error occured while creating the job.\n"; + _lastError += e.what(); + } + } + } + + virtual ~TPyStudyJob(){} + virtual bool fetch() + { + _lastError = ""; + try + { + py2cpp::PyFunction pyFn; + pyFn.loadExp(_pyStudy, "getResult"); + pyFn(); // python call: _pyStudy.getResult() + fetchResults(_pyStudy.getAttr("sample"), _sample); + } + catch(std::exception& e) + { + _lastError = "An error occured while fetching the results.\n"; + _lastError += e.what(); + } + return _lastError.empty(); + } + + const Sample& getSample()const{return _sample;} + +private: + void createNewJob(const PyStudyFunction& fnScript, const JobParametersProxy& params) + { + if(_lastError.empty()) // no errors during parent construction + { + try + { + py2cpp::PyPtr pySample = createPySample(_sample); + py2cpp::PyFunction pyFn; + pyFn.loadExp(_pyStudy, "createNewJob"); + pyFn(fnScript, pySample, params); + } + catch(std::exception& e) + { + _lastError = "An error occured while creating the job.\n"; + _lastError += e.what(); + } + } + } + +private: + Sample& _sample; +}; + +} + +#endif //YDEFX_TPYSTUDYJOB_HXX diff --git a/src/cpp/Test/PyTestMain.cxx b/src/cpp/Test/PyTestMain.cxx new file mode 100644 index 0000000..b5f960e --- /dev/null +++ b/src/cpp/Test/PyTestMain.cxx @@ -0,0 +1,88 @@ +// Copyright (C) 2019 EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +// ============================================================================ +/*! + * Main program source for Unit Tests with cppunit package does not depend + * on actual tests, so we use the same for all partial unit tests. + * This version of TestMain initializes the python library and it can be used + * if you have several tests which need Py_Initialize and salome_init. + */ +// ============================================================================ + +int main(int argc, char* argv[]) +{ + Py_Initialize(); + // --- Create the event manager and test controller + CPPUNIT_NS::TestResult controller; + + // --- Add a listener that collects test result + CPPUNIT_NS::TestResultCollector result; + controller.addListener( &result ); + + // --- Add a listener that print dots as test run. +#ifdef WIN32 + CPPUNIT_NS::TextTestProgressListener progress; +#else + CPPUNIT_NS::BriefTestProgressListener progress; +#endif + controller.addListener( &progress ); + + // --- Get the top level suite from the registry + + CPPUNIT_NS::Test *suite = + CPPUNIT_NS::TestFactoryRegistry::getRegistry().makeTest(); + + // --- Adds the test to the list of test to run + + CPPUNIT_NS::TestRunner runner; + runner.addTest( suite ); + runner.run( controller); + + // --- Print test in a compiler compatible format. + std::ofstream testFile; + testFile.open("test.log", std::ios::out | std::ios::app); + testFile << "------ Idefix test log:" << std::endl; + CPPUNIT_NS::CompilerOutputter outputter( &result, testFile ); + outputter.write(); + + // --- Run the tests. + + bool wasSucessful = result.wasSuccessful(); + testFile.close(); + Py_Finalize(); + + // --- Return error code 1 if the one of test failed. + + return wasSucessful ? 0 : 1; +} diff --git a/src/cpp/Test/StudyGeneralTest.cxx b/src/cpp/Test/StudyGeneralTest.cxx index cac120e..e633129 100644 --- a/src/cpp/Test/StudyGeneralTest.cxx +++ b/src/cpp/Test/StudyGeneralTest.cxx @@ -34,8 +34,6 @@ void SampleTest::cleanUp() void SampleTest::fullStudy() { - Py_Initialize(); - { std::list resources = ydefx::JobParametersProxy::AvailableResources(); CPPUNIT_ASSERT(resources.size() > 0); @@ -43,7 +41,7 @@ void SampleTest::fullStudy() jobParams.configureResource("localhost"); jobParams.work_directory(jobParams.work_directory() + "/GeneralTest"); jobParams.createResultDirectory("/tmp"); - std::string pyScript = + std::string pyScript = "def _exec(a, b):\n" " d = a / b\n" " t = ['object which needs pickel protocol']\n" @@ -109,9 +107,88 @@ void SampleTest::fullStudy() myJob = l.submitMonoPyJob(wrongStudy, sample, jobParams); CPPUNIT_ASSERT(myJob == nullptr); CPPUNIT_ASSERT(l.lastError().find("SyntaxError") != std::string::npos); - } - Py_Finalize(); +} + +void SampleTest::genericStudy() +{ + std::list resources = ydefx::JobParametersProxy::AvailableResources(); + CPPUNIT_ASSERT(resources.size() > 0); + + ydefx::JobParametersProxy jobParams; + jobParams.configureResource("localhost"); + jobParams.work_directory(jobParams.work_directory() + "/GenericTest"); + jobParams.createResultDirectory("/tmp"); + std::string pyScript = +"def _exec(a, b):\n" +" d = a / b\n" +" t = ['object which needs pickel protocol']\n" +" return d,t\n"; + + ydefx::PyStudyFunction studyFunction; + studyFunction.loadString(pyScript); + CPPUNIT_ASSERT(studyFunction.isValid()); + const std::list& inputs = studyFunction.inputNames(); + CPPUNIT_ASSERT(std::find(inputs.begin(), inputs.end(), "a")!=inputs.end()); + CPPUNIT_ASSERT(std::find(inputs.begin(), inputs.end(), "b")!=inputs.end()); + const std::list& outputs = studyFunction.outputNames(); + CPPUNIT_ASSERT(std::find(outputs.begin(), outputs.end(), "d") + != outputs.end()); + CPPUNIT_ASSERT(std::find(outputs.begin(), outputs.end(), "t") + != outputs.end()); + + ydefx::Sample sample; + std::vector a_vals = {1.1, 4.4, 9, 4}; + std::vector b_vals = {1.1, 2.2, 3, 1}; + sample.inputs().set("a", a_vals); + sample.inputs().set("b", b_vals); + sample.outputs().addName("d"); + sample.outputs().addName("t"); + + py2cpp::PyFunction objConstructor; + objConstructor.loadExp("pydefx", "PyStudy"); + py2cpp::PyPtr pyStudy = objConstructor(); + + ydefx::Launcher l; + ydefx::Job* myJob = l.submitPyStudyJob(pyStudy, studyFunction, sample, jobParams); + CPPUNIT_ASSERT(myJob); + CPPUNIT_ASSERT(l.lastError().empty()); + std::string jobDump = myJob->dump(); + CPPUNIT_ASSERT(myJob->lastError().empty()); + std::string jobState = myJob->state(); + CPPUNIT_ASSERT(myJob->lastError().empty()); + CPPUNIT_ASSERT(jobState == "QUEUED" || jobState == "RUNNING" + || jobState == "FINISHED"); + double progress = myJob->progress(); + CPPUNIT_ASSERT(progress >= 0.0 && progress <= 1.0 ); + CPPUNIT_ASSERT(myJob->lastError().empty()); + bool ok = myJob->wait(); + CPPUNIT_ASSERT(ok); + CPPUNIT_ASSERT(myJob->lastError().empty()); + jobState = myJob->state(); + CPPUNIT_ASSERT(jobState == "FINISHED"); + progress = myJob->progress(); + CPPUNIT_ASSERT(progress == 1.0); + ok = myJob->fetch(); + CPPUNIT_ASSERT(ok); + CPPUNIT_ASSERT(myJob->lastError().empty()); + std::vector expectedResult = {1,2,3,4}; + const std::vector& result = sample.outputs().get("d"); + CPPUNIT_ASSERT(expectedResult == result); + const std::vector& pyobjResult + = sample.outputs().get("t"); + for(const py2cpp::PyPtr& obj : pyobjResult) + CPPUNIT_ASSERT(obj.repr() == "['object which needs pickel protocol']"); + delete myJob; + + // test a case of error + std::string wrongScript = "wrong 'script"; + ydefx::PyStudyFunction wrongStudy; + wrongStudy.loadString(wrongScript); + CPPUNIT_ASSERT(!wrongStudy.isValid()); + myJob = l.submitPyStudyJob(pyStudy, wrongStudy, sample, jobParams); + CPPUNIT_ASSERT(myJob == nullptr); + CPPUNIT_ASSERT(l.lastError().find("SyntaxError") != std::string::npos); } CPPUNIT_TEST_SUITE_REGISTRATION( SampleTest ); -#include "TestMain.cxx" +#include "PyTestMain.cxx" diff --git a/src/cpp/Test/StudyGeneralTest.hxx b/src/cpp/Test/StudyGeneralTest.hxx index 473b645..739ce25 100644 --- a/src/cpp/Test/StudyGeneralTest.hxx +++ b/src/cpp/Test/StudyGeneralTest.hxx @@ -26,12 +26,14 @@ class SampleTest: public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(SampleTest); CPPUNIT_TEST(fullStudy); + CPPUNIT_TEST(genericStudy); CPPUNIT_TEST_SUITE_END(); public: void setUp(); void tearDown(); void cleanUp(); void fullStudy(); + void genericStudy(); }; #endif // YDEFX_SAMPLETEST_HXX diff --git a/src/pydefx/plugins/jobexecutor.py b/src/pydefx/plugins/jobexecutor.py index 4d56df0..0fdfab1 100644 --- a/src/pydefx/plugins/jobexecutor.py +++ b/src/pydefx/plugins/jobexecutor.py @@ -4,6 +4,7 @@ import pickle import time import traceback +pydefx.forceNoSalomeServers() class Context: def __init__(self): self.launcher = pydefx.salome_proxy.getLauncher() # getLauncher() -- 2.39.2