From ef1d58c84948c3de8a05eb9d47e99e7bd5bd4f1a Mon Sep 17 00:00:00 2001 From: Ovidiu Mircescu Date: Wed, 10 Jul 2019 14:25:02 +0200 Subject: [PATCH] Add insitu example as a test. --- src/CMakeLists.txt | 3 + src/CTestTestfileInstall.cmake | 1 + src/pydefx/samplecsviterator.py | 9 ++- src/pydefx/schemas/plugin.py | 10 ++- src/pydefx/studyexception.py | 41 +++++++++++++ src/pydefx/studyresult.py | 54 +++++++++++++++++ src/pyexample/CMakeLists.txt | 41 +++++++++++++ src/pyexample/CTestTestfileInstall.cmake | 25 ++++++++ .../__pycache__/insitumanager.cpython-36.pyc | Bin 0 -> 1366 bytes src/pyexample/insitu/insituiterator.py | 48 +++++++++++++++ src/pyexample/insitu/insitumanager.py | 46 ++++++++++++++ src/pyexample/runUnitTest.sh | 23 +++++++ src/pyexample/test_insitu.py | 57 ++++++++++++++++++ .../{tests_prescript.py => test_prescript.py} | 6 ++ 14 files changed, 353 insertions(+), 11 deletions(-) create mode 100644 src/pydefx/studyexception.py create mode 100644 src/pydefx/studyresult.py create mode 100644 src/pyexample/CMakeLists.txt create mode 100644 src/pyexample/CTestTestfileInstall.cmake create mode 100644 src/pyexample/insitu/__pycache__/insitumanager.cpython-36.pyc create mode 100644 src/pyexample/insitu/insituiterator.py create mode 100644 src/pyexample/insitu/insitumanager.py create mode 100755 src/pyexample/runUnitTest.sh create mode 100644 src/pyexample/test_insitu.py rename src/pyexample/{tests_prescript.py => test_prescript.py} (78%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 767b2b5..0080c28 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -29,3 +29,6 @@ ADD_SUBDIRECTORY(pydefx) IF(YDEFX_BUILD_GUI) ADD_SUBDIRECTORY(gui) ENDIF(YDEFX_BUILD_GUI) +IF(SALOME_BUILD_TESTS) + ADD_SUBDIRECTORY(pyexample) +ENDIF(SALOME_BUILD_TESTS) diff --git a/src/CTestTestfileInstall.cmake b/src/CTestTestfileInstall.cmake index 4809113..6fae9d7 100644 --- a/src/CTestTestfileInstall.cmake +++ b/src/CTestTestfileInstall.cmake @@ -24,4 +24,5 @@ SET(TIMEOUT 500) SUBDIRS( cpp + pyexample ) diff --git a/src/pydefx/samplecsviterator.py b/src/pydefx/samplecsviterator.py index e8063e5..4b23d62 100644 --- a/src/pydefx/samplecsviterator.py +++ b/src/pydefx/samplecsviterator.py @@ -41,10 +41,12 @@ class SampleIterator: if directory: datapath = os.path.join(directory, SampleIterator.DATAFILE) outputnamespath = os.path.join(directory, SampleIterator.OUTPUTNAMESFILE) + self.result_directory = os.path.join(directory, SampleIterator.RESULTDIR) self.directory = directory else: datapath = SampleIterator.DATAFILE outputnamespath = SampleIterator.OUTPUTNAMESFILE + self.result_directory = SampleIterator.RESULTDIR self.directory = None self.result_file = None self.datafile = open(datapath, newline='') @@ -67,14 +69,12 @@ class SampleIterator: result file. """ if self.directory: - resultdir = os.path.join(self.directory, SampleIterator.RESULTDIR) outputnamespath = os.path.join(self.directory, SampleIterator.OUTPUTNAMESFILE) else: - resultdir = SampleIterator.RESULTDIR outputnamespath = SampleIterator.OUTPUTNAMESFILE - os.makedirs(resultdir, exist_ok=True) - resultpath = os.path.join(resultdir, SampleIterator.RESULTFILE) + os.makedirs(self.result_directory, exist_ok=True) + resultpath = os.path.join(self.result_directory, SampleIterator.RESULTFILE) result_columns = [SampleIterator.IDCOLUMN] result_columns.extend(self.inputnames) result_columns.extend(self.outputnames) @@ -133,7 +133,6 @@ class SampleIterator: self.result_file.close() self.result_file = None - # Private functions def _loadOutputNames(filepath): outputnames = [] diff --git a/src/pydefx/schemas/plugin.py b/src/pydefx/schemas/plugin.py index d49a3e4..551a032 100644 --- a/src/pydefx/schemas/plugin.py +++ b/src/pydefx/schemas/plugin.py @@ -6,6 +6,7 @@ import importlib class myalgosync(SALOMERuntime.OptimizerAlgSync): def __init__(self): SALOMERuntime.OptimizerAlgSync.__init__(self, None) + self.started = False def setPool(self,pool): """Must be implemented to set the pool""" @@ -38,6 +39,7 @@ class myalgosync(SALOMERuntime.OptimizerAlgSync): """Start to fill the pool with samples to evaluate.""" itModuleName = self.config["sampleIterator"] itModule = importlib.import_module(itModuleName) + self.started = True self.manager = itModule.SampleIterator() self.manager.writeHeaders() values=None @@ -68,13 +70,9 @@ class myalgosync(SALOMERuntime.OptimizerAlgSync): def finish(self): """Optional method called when the algorithm has finished, successfully or not, to perform any necessary clean up.""" - # We need a try catch because finish is also called at the beginning of - # the algorthm, before any other call. - try: + if self.started : self.manager.terminate() - except: - pass - self.pool.destroyAll() + self.pool.destroyAll() def getAlgoResult(self): """return the result of the algorithm. diff --git a/src/pydefx/studyexception.py b/src/pydefx/studyexception.py new file mode 100644 index 0000000..6b069a9 --- /dev/null +++ b/src/pydefx/studyexception.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2019 EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +class StudyException(Exception): + """ + Root of exceptions raised by a study. + """ + def __init__(self, message): + super().__init__(message) + +class StudyUseException(StudyException): + """ + Exception used when there is a bad utilisation of the study - functions called + in a wrong order. + """ + def __init__(self, message): + super().__init__(message) + +class StudyRunException(StudyException): + """ + Exception used when there is a problem when running the study. + """ + def __init__(self, message): + super().__init__(message) diff --git a/src/pydefx/studyresult.py b/src/pydefx/studyresult.py new file mode 100644 index 0000000..b9e20c1 --- /dev/null +++ b/src/pydefx/studyresult.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2019 EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +class StudyResult: + """ + This class gathers global information about the execution of the study (global + errors, global result). + """ + def __init__(self): + self.result = None + self.exit_code = None + self.error_message = None + + def isExitCodeAvailable(self): + return not self.exit_code is None + + def getExitCode(self): + return self.exit_code + + def isResultAvailable(self): + return not self.exit_code is None + + def getResult(self): + return self.result + + def hasErrors(self): + return not self.error_message is None and len(self.error_message) > 0 + + def getErrors(self): + return self.error_message + + def __str__(self): + result = """Exit code : {} +Error message : {} +Result: +{}""".format(self.exit_code, self.error_message, self.result) + return result diff --git a/src/pyexample/CMakeLists.txt b/src/pyexample/CMakeLists.txt new file mode 100644 index 0000000..f00a9b0 --- /dev/null +++ b/src/pyexample/CMakeLists.txt @@ -0,0 +1,41 @@ +# Copyright (C) 2019 EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# +ADD_TEST(YdefxPyExampleTest runUnitTest.sh) +SET_TESTS_PROPERTIES(YdefxPyExampleTest PROPERTIES ENVIRONMENT + "PYTHONPATH=${CMAKE_SOURCE_DIR}/src:$ENV{PYTHONPATH}") +IF(SALOME_BUILD_TESTS) + # For salome test + SET(LOCAL_TEST_DIR ${SALOME_YDEFX_INSTALL_TEST}/pyexample) + SET(TESTFILES + test_insitu.py + test_prescript.py + ) + INSTALL(FILES ${TESTFILES} DESTINATION ${LOCAL_TEST_DIR}) + INSTALL(PROGRAMS runUnitTest.sh + DESTINATION ${LOCAL_TEST_DIR}) + SET(INSITU_TESTFILES + insitu/insituiterator.py + insitu/insitumanager.py + ) + INSTALL(FILES ${INSITU_TESTFILES} DESTINATION ${LOCAL_TEST_DIR}/insitu) + + INSTALL(FILES CTestTestfileInstall.cmake + DESTINATION ${LOCAL_TEST_DIR} + RENAME CTestTestfile.cmake) +ENDIF() diff --git a/src/pyexample/CTestTestfileInstall.cmake b/src/pyexample/CTestTestfileInstall.cmake new file mode 100644 index 0000000..6dc4cef --- /dev/null +++ b/src/pyexample/CTestTestfileInstall.cmake @@ -0,0 +1,25 @@ +# Copyright (C) 2019 EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +SET(TEST_NAME ${COMPONENT_NAME}_PyExampleTest) +ADD_TEST(${TEST_NAME} python ${SALOME_TEST_DRIVER} ${TIMEOUT} ./runUnitTest.sh) +SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES + LABELS "${COMPONENT_NAME}" + ) + diff --git a/src/pyexample/insitu/__pycache__/insitumanager.cpython-36.pyc b/src/pyexample/insitu/__pycache__/insitumanager.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..91e6490c02925fcba89ba245cf62c1f3c12b6e99 GIT binary patch literal 1366 zcmZWp&2HQ_5GEx`tKCXl+XbR_62GGqV2X>ypO-VT3b=sRFY z7d7?XlLZx2co$LXJKGlwY$gH`UXV0+PZ&uNKR#p7c$Hv=xJOkeuwPVinD$|);O{@$ck_ye1~w#L3b{nhe=F;8oPe zGpfGA6Sq*^MunaG9z|R7e_+q;ES^uso&M7*>z!~Ld5RE{%hv^-r)1jIz6I?<_ z?rL_)FDtMKT=#{;9I>pJriOSuysLN-u literal 0 HcmV?d00001 diff --git a/src/pyexample/insitu/insituiterator.py b/src/pyexample/insitu/insituiterator.py new file mode 100644 index 0000000..e73b93f --- /dev/null +++ b/src/pyexample/insitu/insituiterator.py @@ -0,0 +1,48 @@ +# Copyright (C) 2019 EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +# The following import is made in the execution environment and in the working +# directory of the job. +import samplecsviterator +import os + +class SampleIterator(samplecsviterator.SampleIterator): + """ + Example of an iterator which uses insitu computation. + """ + def __init__(self, directory=None): + super().__init__(directory) + self.insitu_result = 0 + + def addResult(self, currentId, currentInput, currentOutput, currentError): + """ + currentId : integer. Index of the curent point. + currentInput : dictionary of curent input values (name, value) + currentOutput: tuple with the output values + currentError : string. Empty if no error. + """ + super().addResult(currentId, currentInput, currentOutput, currentError) + value_of_interest = currentOutput + self.insitu_result = self.insitu_result + value_of_interest + + def terminate(self): + super().terminate() + result_file = os.path.join(self.result_directory, "insitu_result.txt") + with open(result_file, "w") as f: + f.write(repr(self.insitu_result)) diff --git a/src/pyexample/insitu/insitumanager.py b/src/pyexample/insitu/insitumanager.py new file mode 100644 index 0000000..bbf5d63 --- /dev/null +++ b/src/pyexample/insitu/insitumanager.py @@ -0,0 +1,46 @@ +# Copyright (C) 2019 EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +import pydefx.samplecsvmanager +import inspect +import os +import pathlib + +class InsituManager(pydefx.samplecsvmanager.SampleManager): + def prepareRun(self, sample, directory): + files_list = super().prepareRun(sample, directory) + # add the insituiterator file to the list + filename = inspect.getframeinfo(inspect.currentframe()).filename + install_directory = pathlib.Path(filename).resolve().parent + iteratorFile = os.path.join(install_directory, "insituiterator.py") + files_list.append(iteratorFile) + return files_list + + def loadResult(self, sample, directory): + super().loadResult(sample, directory) + # load the insitu result and return it + insitu_result_file = os.path.join(directory, + self.getResultFileName(), + "insitu_result.txt") + with open(insitu_result_file, "r") as f: + result_string = f.read() + return eval(result_string) + + def getModuleName(self): + return "insituiterator" diff --git a/src/pyexample/runUnitTest.sh b/src/pyexample/runUnitTest.sh new file mode 100755 index 0000000..fc4d6f1 --- /dev/null +++ b/src/pyexample/runUnitTest.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright (C) 2019 EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +python3 -m unittest test_insitu.py test_prescript.py +ret=$? +exit $ret diff --git a/src/pyexample/test_insitu.py b/src/pyexample/test_insitu.py new file mode 100644 index 0000000..5b56476 --- /dev/null +++ b/src/pyexample/test_insitu.py @@ -0,0 +1,57 @@ +import unittest +import insitu.insitumanager +import os + +class TestYdefx(unittest.TestCase): + def test_insitu(self): + """ + This test shows how to use insitu processing. + """ + import pydefx + + myParams = pydefx.Parameters() + myParams.configureResource("localhost") + mywd = os.path.join(myParams.salome_parameters.work_directory, + "prescript_test") + myParams.salome_parameters.work_directory = mywd + myParams.createResultDirectory("/tmp") + + pyScript = """ +def _exec(x): + with open("mydata.txt") as f: + mydata = f.read() + intdata = int(mydata) + y = x + intdata + return y""" + + myScript = pydefx.PyScript() + myScript.loadString(pyScript) + + mySample = myScript.CreateEmptySample() + mydata = {"x":list(range(10))} + mySample.setInputValues(mydata) + + # pre-processing script called before the first evaluation + myPrescript = """ +with open("mydata.txt", "w") as f: + f.write("1") +""" + mySchemaBuilder = pydefx.DefaultSchemaBuilder(myPrescript) + + mySampleManager = insitu.insitumanager.InsituManager() + + myStudy = pydefx.PyStudy(sampleManager=mySampleManager, + schemaBuilder=mySchemaBuilder) + myStudy.createNewJob(myScript, mySample, myParams) + + myStudy.launch() + myStudy.wait() + myStudy.getResult() + expected = """Exit code : 0 +Error message : None +Result: +55.0""" + self.assertEqual(str(myStudy.global_result),expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/pyexample/tests_prescript.py b/src/pyexample/test_prescript.py similarity index 78% rename from src/pyexample/tests_prescript.py rename to src/pyexample/test_prescript.py index 1f2cab1..6faf099 100644 --- a/src/pyexample/tests_prescript.py +++ b/src/pyexample/test_prescript.py @@ -1,13 +1,19 @@ import unittest +import os class TestYdefx(unittest.TestCase): def test_prescript(self): """ + This test shows how to use an initialization script which is called one time + before any evaluation of the study function. """ import pydefx myParams = pydefx.Parameters() myParams.configureResource("localhost") + mywd = os.path.join(myParams.salome_parameters.work_directory, + "prescript_test") + myParams.salome_parameters.work_directory = mywd myParams.createResultDirectory("/tmp") pyScript = """ -- 2.39.2