From 13e530810a427ab135c2910096887e597cac4a93 Mon Sep 17 00:00:00 2001 From: Ovidiu Mircescu Date: Wed, 19 Jun 2019 13:59:20 +0200 Subject: [PATCH] Execute a script before any evaluation. --- src/pydefx/CMakeLists.txt | 1 + src/pydefx/__init__.py | 1 + src/pydefx/defaultschemabuilder.py | 55 +++++++++++ src/pydefx/pystudy.py | 123 +++++++++++-------------- src/pydefx/samplecsvmanager.py | 5 +- src/pydefx/schemas/idefix_pyschema.xml | 9 ++ src/pyexample/tests_prescript.py | 44 +++++++++ 7 files changed, 166 insertions(+), 72 deletions(-) create mode 100644 src/pydefx/defaultschemabuilder.py create mode 100644 src/pyexample/tests_prescript.py diff --git a/src/pydefx/CMakeLists.txt b/src/pydefx/CMakeLists.txt index 5b85eb5..6d5d94c 100644 --- a/src/pydefx/CMakeLists.txt +++ b/src/pydefx/CMakeLists.txt @@ -25,6 +25,7 @@ SET(SCRIPTS sample.py samplecsviterator.py samplecsvmanager.py + defaultschemabuilder.py ) INSTALL(FILES ${SCRIPTS} DESTINATION ${SALOME_INSTALL_PYTHON}/pydefx) diff --git a/src/pydefx/__init__.py b/src/pydefx/__init__.py index b2430b1..29d9f05 100644 --- a/src/pydefx/__init__.py +++ b/src/pydefx/__init__.py @@ -20,6 +20,7 @@ from .parameters import Parameters from .pyscript import PyScript from .pystudy import PyStudy from .sample import Sample +from .defaultschemabuilder import DefaultSchemaBuilder import salome salome.salome_init() diff --git a/src/pydefx/defaultschemabuilder.py b/src/pydefx/defaultschemabuilder.py new file mode 100644 index 0000000..915af07 --- /dev/null +++ b/src/pydefx/defaultschemabuilder.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2019 EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# +import inspect +import pathlib +import os +import shutil + +class DefaultSchemaBuilder: + def __init__(self, prescript=None): + """ + This object builds the YACS schema for the parametric computation. + prescript: contains python code that is executed before any evaluation. + """ + self.prescript = prescript + + def buildSchema(self, local_work_dir): + """ + Create the YACS schema and copy it to local_work_dir. + local_work_dir : path where the schema will be created. + return: + path to the created file, + list of additional files needed for running. + """ + # use generic schema + filename = inspect.getframeinfo(inspect.currentframe()).filename + install_directory = pathlib.Path(filename).resolve().parent + yacs_schema_path = os.path.join(install_directory, "schemas", + "idefix_pyschema.xml") + plugin_path = os.path.join(install_directory, "schemas", "plugin.py") + yacs_schema_path = shutil.copy(yacs_schema_path, local_work_dir) + plugin_path = shutil.copy(plugin_path, local_work_dir) + files = [plugin_path] + if self.prescript: + prescript_path = os.path.join(local_work_dir, "idefix_prescript.py") + with open(prescript_path, "w") as f: + f.write(self.prescript) + files.append(prescript_path) + return yacs_schema_path, files diff --git a/src/pydefx/pystudy.py b/src/pydefx/pystudy.py index 39a0496..6852dec 100644 --- a/src/pydefx/pystudy.py +++ b/src/pydefx/pystudy.py @@ -26,58 +26,26 @@ import json from . import samplecsvmanager from . import parameters from . import configuration +from . import defaultschemabuilder def defaultSampleManager(): return samplecsvmanager.SampleManager() -def prepareDirectoryForLaunch(sample, result_directory, nb_branches, script, - sampleManager=None): - """ - sample : Sample type - result_directory : path to a result working directory. - nb_branches : int - script : script / pyscript type - return: - extra_in_files: list of files to add to salome_parameters.in_files - yacs_schema_path: path to the yacs schema (xml file). - """ - if sampleManager is None: - sampleManager = defaultSampleManager() - if not os.path.exists(result_directory): - os.makedirs(result_directory) - # export sample to result_directory - inputFiles = sampleManager.prepareRun(script, sample, result_directory) - - # export nbbranches - configpath = os.path.join(result_directory, "idefixconfig.json") - dicconfig = {} - dicconfig["nbbranches"] = nb_branches - dicconfig["studymodule"] = "idefixstudy" - dicconfig["sampleIterator"] = sampleManager.getModuleName() - with open(configpath, "w") as f: - json.dump(dicconfig, f, indent=2) - studypath = os.path.join(result_directory, "idefixstudy.py") - with open(studypath, "w") as f: - f.write(script.script) - # find generic schema - filename = inspect.getframeinfo(inspect.currentframe()).filename - install_directory = pathlib.Path(filename).resolve().parent - yacs_schema_path = os.path.join(install_directory, "schemas", - "idefix_pyschema.xml") - plugin_path = os.path.join(install_directory, "schemas", "plugin.py") - # create salome params - extra_in_files = [] - extra_in_files.extend([configpath, studypath, plugin_path]) - extra_in_files.extend(inputFiles) - return extra_in_files, yacs_schema_path - class PyStudy: JOB_DUMP_NAME = "jobDump.xml" - def __init__(self): + def __init__(self, sampleManager=None, schemaBuilder=None): self.job_id = -1 + if sampleManager is None: + self.sampleManager = defaultSampleManager() + else: + self.sampleManager = sampleManager + if schemaBuilder is None: + self.schemaBuilder = defaultschemabuilder.DefaultSchemaBuilder() + else: + self.schemaBuilder = schemaBuilder # Study creation functions - def createNewJob(self, script, sample, params, sampleManager=None): + def createNewJob(self, script, sample, params): """ Create a new job out of those parameters: script : script / pyscript type @@ -89,30 +57,21 @@ class PyStudy: self.sample = sample self.params = params self.params.salome_parameters.job_type = self.jobType() - if sampleManager is None: - self.sampleManager = defaultSampleManager() - else: - self.sampleManager = sampleManager tmp_workdir = self.params.salome_parameters.result_directory - extra_in_files, yacs_schema_path = prepareDirectoryForLaunch(self.sample, - tmp_workdir, - self.params.nb_branches, - script, - self.sampleManager) - self.params.salome_parameters.in_files.extend(extra_in_files) - self.params.salome_parameters.job_file = yacs_schema_path + schema_path, extra_files = self._prepareDirectoryForLaunch(tmp_workdir, + script) + + self.params.salome_parameters.in_files.extend(extra_files) + self.params.salome_parameters.job_file = schema_path launcher = salome.naming_service.Resolve('/SalomeLauncher') self.job_id = launcher.createJob(self.params.salome_parameters) return self.job_id - def loadFromDirectory(self, path, sampleManager=None): + def loadFromDirectory(self, path): """ Recover a study from a result directory where a previous study was launched. """ - if sampleManager is None: - sampleManager = defaultSampleManager() - self.sampleManager = sampleManager - self.sample = sampleManager.loadSample(path) + self.sample = self.sampleManager.loadSample(path) job_string = loadJobString(path) launcher = salome.naming_service.Resolve('/SalomeLauncher') self.job_id = launcher.restoreJob(job_string) @@ -122,14 +81,11 @@ class PyStudy: self.getResult() return self.job_id - def loadFromString(self, jobstring, sampleManager=None): + def loadFromString(self, jobstring): """ Recover a study from a string which contains the description of the job. This string can be obtained by launcher.dumpJob. """ - if sampleManager is None: - sampleManager = defaultSampleManager() - self.sampleManager = sampleManager launcher = salome.naming_service.Resolve('/SalomeLauncher') self.job_id = launcher.restoreJob(jobstring) self.params = None @@ -137,26 +93,26 @@ class PyStudy: if self.job_id >= 0: salome_params = launcher.getJobParameters(self.job_id) self.params = parameters.Parameters(salome_parameters=salome_params) - self.sample = sampleManager.loadSample(salome_params.result_directory) + #TODO: sampleManager should be loaded from result_directory + self.sample = self.sampleManager.loadSample( + salome_params.result_directory) self.getResult() else: raise Exception("Failed to restore the job.") - def loadFromId(self, jobid, sampleManager=None): + def loadFromId(self, jobid): """ Connect the study to an already created job. The result directory of the job must be already prepared for launch. """ if jobid < 0: return - if sampleManager is None: - sampleManager = defaultSampleManager() - self.sampleManager = sampleManager self.job_id = jobid launcher = salome.naming_service.Resolve('/SalomeLauncher') salome_params = launcher.getJobParameters(job_id) self.params = parameters.Parameters(salome_parameters=salome_params) - self.sample = sampleManager.loadSample(salome_params.result_directory) + #TODO: sampleManager should be loaded from result_directory + self.sample = self.sampleManager.loadSample(salome_params.result_directory) self.script = None return @@ -305,6 +261,35 @@ For further details, see {}/logs directory on {}.""".format( time.sleep(sleep_delay) jobState = launcher.getJobState(job_id) + def _prepareDirectoryForLaunch(self, result_directory, script): + """ + result_directory : path to a result working directory. + script : script / pyscript type + return: + yacs_schema_path: path to the yacs schema (xml file). + extra_in_files: list of files to add to salome_parameters.in_files + """ + if not os.path.exists(result_directory): + os.makedirs(result_directory) + # export sample to result_directory + inputFiles = self.sampleManager.prepareRun(self.sample, result_directory) + + # export nbbranches + configpath = os.path.join(result_directory, "idefixconfig.json") + dicconfig = {} + dicconfig["nbbranches"] = self.params.nb_branches + dicconfig["studymodule"] = "idefixstudy" + dicconfig["sampleIterator"] = self.sampleManager.getModuleName() + with open(configpath, "w") as f: + json.dump(dicconfig, f, indent=2) + studypath = os.path.join(result_directory, "idefixstudy.py") + with open(studypath, "w") as f: + f.write(script.script) + schema_path, extra_files = self.schemaBuilder.buildSchema(result_directory) + + extra_files.extend([configpath, studypath]) + extra_files.extend(inputFiles) + return schema_path, extra_files ### Deprecated!!!! def dumpJob(result_directory, jobString): diff --git a/src/pydefx/samplecsvmanager.py b/src/pydefx/samplecsvmanager.py index d626b40..819593f 100644 --- a/src/pydefx/samplecsvmanager.py +++ b/src/pydefx/samplecsvmanager.py @@ -28,10 +28,9 @@ class SampleManager: def __init__(self): pass - def prepareRun(self, script, sample, directory): + def prepareRun(self, sample, directory): """ Create a dump of the sample in the given directory. - script: PyScript object. sample: Sample object. directory: path to a local working directory where all the working files are copied. This directory should be already created. @@ -47,7 +46,7 @@ class SampleManager: outnamespath = os.path.join(directory, SampleIterator.OUTPUTNAMESFILE) with open(outnamespath, 'w') as outputfile: - for v in script.getOutputNames(): + for v in sample.getOutputNames(): outputfile.write(v+'\n') filename = inspect.getframeinfo(inspect.currentframe()).filename install_directory = pathlib.Path(filename).resolve().parent diff --git a/src/pydefx/schemas/idefix_pyschema.xml b/src/pydefx/schemas/idefix_pyschema.xml index 40b028c..19dd54c 100644 --- a/src/pydefx/schemas/idefix_pyschema.xml +++ b/src/pydefx/schemas/idefix_pyschema.xml @@ -62,6 +62,15 @@ with open("idefixconfig.json", "r") as f: config = json.load(f) nbbranches=config["nbbranches"] studymodule=config["studymodule"] + +prescript = None +try: + with open("idefix_prescript.py") as f: + prescript = f.read() +except FileNotFoundError: + pass +if not prescript is None: + exec(prescript) ]]> diff --git a/src/pyexample/tests_prescript.py b/src/pyexample/tests_prescript.py new file mode 100644 index 0000000..1f2cab1 --- /dev/null +++ b/src/pyexample/tests_prescript.py @@ -0,0 +1,44 @@ +import unittest + +class TestYdefx(unittest.TestCase): + def test_prescript(self): + """ + """ + import pydefx + + myParams = pydefx.Parameters() + myParams.configureResource("localhost") + myParams.createResultDirectory("/tmp") + + pyScript = """ +def _exec(name): + with open("mydata.txt") as f: + mydata = f.read() + message = mydata + name + return message""" + + myScript = pydefx.PyScript() + myScript.loadString(pyScript) + + mySample = myScript.CreateEmptySample() + mydata = {"name":["Jean", "Toto", "Titi", "Zizi"]} + mySample.setInputValues(mydata) + + myPrescript = """ +with open("mydata.txt", "w") as f: + f.write("Hello ") +""" + + mySchemaBuilder = pydefx.DefaultSchemaBuilder(myPrescript) + + myStudy = pydefx.PyStudy(schemaBuilder=mySchemaBuilder) + myStudy.createNewJob(myScript, mySample, myParams) + + myStudy.launch() + myStudy.wait() + myStudy.getResult() + expected = "name,message,messages\n'Jean','Hello Jean',\n'Toto','Hello Toto',\n'Titi','Hello Titi',\n'Zizi','Hello Zizi',\n" + self.assertEqual(str(myStudy.sample),expected) + +if __name__ == '__main__': + unittest.main() -- 2.39.2