From eebbef6d2f45d2a73159c6a02a25c0113e4808f8 Mon Sep 17 00:00:00 2001 From: Ovidiu Mircescu Date: Wed, 28 Oct 2020 11:42:11 +0100 Subject: [PATCH] Local study with examples. --- src/pydefx/CMakeLists.txt | 3 -- src/pydefx/configuration.py | 40 +++++++++++++++++++ src/pydefx/execconfiguration.py | 42 -------------------- src/pydefx/localstudy.py | 2 +- src/pyexample/multijob/jobstudy.py | 27 +++++++++++++ src/pyexample/multijob/launch_local_basic.py | 8 ---- src/pyexample/multijob/launch_local_jobs.py | 31 +++++++++++++++ src/pyexample/multijob/simple_command.sh | 1 + 8 files changed, 100 insertions(+), 54 deletions(-) delete mode 100644 src/pydefx/execconfiguration.py create mode 100644 src/pyexample/multijob/jobstudy.py create mode 100755 src/pyexample/multijob/launch_local_jobs.py create mode 100755 src/pyexample/multijob/simple_command.sh diff --git a/src/pydefx/CMakeLists.txt b/src/pydefx/CMakeLists.txt index abb0632..e5c500e 100644 --- a/src/pydefx/CMakeLists.txt +++ b/src/pydefx/CMakeLists.txt @@ -36,11 +36,8 @@ SET(SCRIPTS slurmstudy.py localbuilder.py localstudy.py - execconfiguration.py ) INSTALL(FILES ${SCRIPTS} DESTINATION ${SALOME_INSTALL_PYTHON}/pydefx) ADD_SUBDIRECTORY(schemas) ADD_SUBDIRECTORY(plugins) -#ADD_SUBDIRECTORY(multijob) -#ADD_SUBDIRECTORY(slurm) diff --git a/src/pydefx/configuration.py b/src/pydefx/configuration.py index 27d5ff0..4e07e82 100644 --- a/src/pydefx/configuration.py +++ b/src/pydefx/configuration.py @@ -17,8 +17,11 @@ # See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com # from . import salome_proxy +from . import parameters import tempfile import pathlib +import os +import json def defaultWorkingDir(resource): resManager = salome_proxy.getResourcesManager() @@ -57,3 +60,40 @@ def availableResources(): # GetFittingResources returns a tuple if in no salome session mode. # Force to list for uniformity between the two modes. return list(resManager.GetFittingResources(params.resource_required)) + +def exportConfig(dicconfig, directory = None): + """ Save the configuration to a directory. + dicconfig is a dictionary which contains the parameters to be saved. + If directory is None, the configuration is saved to the current directory. + """ + if directory is None: + directory = os.getcwd() + configpath = os.path.join(directory, "idefixconfig.json") + with open(configpath, "w") as f: + json.dump(dicconfig, f, indent=2) + +def loadConfig(directory = None): + """ Return the configuration dictionary from a directory. + If the directory is None, use the current directory. + """ + if directory is None: + directory = os.getcwd() + configpath = os.path.join(directory, "idefixconfig.json") + with open("idefixconfig.json", "r") as f: + config = json.load(f) + return config + +def loadJobConfig(directory = None): + """ Return the salome job parameters loaded from a directory which contains + a idefixconfig.json file. + If the directory is None, use the current directory. + """ + result = None + try: + config = loadConfig(directory) + params = parameters.Parameters() + params.loadDict(config["params"]) + result = params.salome_parameters + except: + result = None + return result diff --git a/src/pydefx/execconfiguration.py b/src/pydefx/execconfiguration.py deleted file mode 100644 index a6cc5bc..0000000 --- a/src/pydefx/execconfiguration.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (C) 2019 EDF R&D -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com -# -import json -from . import parameters - -def SalomeParameters(): - """ - This function can be called during the evaluation of a point in order to get - the parameters of the job. - """ - result = None - try: - with open("idefixconfig.json", "r") as f: - config = json.load(f) - params = parameters.Parameters() - params.loadDict(config["params"]) - result = params.salome_parameters - except: - result = None - return result - -def GetConfig(): - with open("idefixconfig.json", "r") as f: - config = json.load(f) - return config diff --git a/src/pydefx/localstudy.py b/src/pydefx/localstudy.py index 0d046ee..e733cb8 100644 --- a/src/pydefx/localstudy.py +++ b/src/pydefx/localstudy.py @@ -27,7 +27,7 @@ from . import salome_proxy class LocalStudy(pystudy.PyStudy): """ - This study uses one different job for each evaluation. + This study is always localy evaluated. """ def __init__(self, sampleManager=None, schemaBuilder=None): if schemaBuilder is None: diff --git a/src/pyexample/multijob/jobstudy.py b/src/pyexample/multijob/jobstudy.py new file mode 100644 index 0000000..6da5d38 --- /dev/null +++ b/src/pyexample/multijob/jobstudy.py @@ -0,0 +1,27 @@ +import pydefx.configuration +import pydefx.salome_proxy +import os +import time + +def _exec(n): + # get the job parameters + salome_parameters = pydefx.configuration.loadJobConfig() + + launcher = pydefx.salome_proxy.getLauncher() # CORBA or not CORBA + + # have a different working directory for each computation + resource = salome_parameters.resource_required.name + default_wd = pydefx.configuration.defaultWorkingDir(resource) + new_wd = os.path.join(default_wd, "myjob_"+str(n)) + salome_parameters.work_directory = new_wd + + # create and launch the job + job_id = launcher.createJob(salome_parameters) + launcher.launchJob(job_id) + + # wait for the end of the job + jobState = launcher.getJobState(job_id) + while jobState != "FINISHED" and jobState != "FAILED" : + time.sleep(5) + jobState = launcher.getJobState(job_id) + return jobState diff --git a/src/pyexample/multijob/launch_local_basic.py b/src/pyexample/multijob/launch_local_basic.py index 13687dc..282006b 100755 --- a/src/pyexample/multijob/launch_local_basic.py +++ b/src/pyexample/multijob/launch_local_basic.py @@ -3,16 +3,11 @@ import os myParams = pydefx.Parameters() myParams.configureResource("localhost") -#myParams.createResultDirectory("/tmp") myParams.nb_branches = 4 myParams.salome_parameters.resource_required.nb_proc = 1 -#myParams.salome_parameters.result_directory=os.path.join(os.getcwd(),"runbasic") -#myParams.salome_parameters.work_directory="/scratch/I35256/workingdir/test_multijob/" myParams.salome_parameters.work_directory=os.path.join(os.getcwd(),"runbasic") myParams.salome_parameters.local_directory = os.getcwd() -#myParams.salome_parameters.in_files=["template_jdd.txt", "mysolver.py"] -#pyScript = os.path.join(os.getcwd(), "mystudy.py") pyScript = """ def _exec(a,b): d = a / b @@ -20,11 +15,9 @@ def _exec(a,b): """ myScript = pydefx.PyScript() -#myScript.loadFile(pyScript) myScript.loadString(pyScript) mySample = myScript.CreateEmptySample() -#mydata = {"x":range(10)} mydata = {"a":[x // 10 for x in range(100)], "b":[x % 10 for x in range(100)]} mySample.setInputValues(mydata) @@ -37,4 +30,3 @@ myStudy.getJobState() myStudy.wait() print(myStudy.getResult()) print(myStudy.sample) -#print(myStudy.global_result) diff --git a/src/pyexample/multijob/launch_local_jobs.py b/src/pyexample/multijob/launch_local_jobs.py new file mode 100755 index 0000000..4017892 --- /dev/null +++ b/src/pyexample/multijob/launch_local_jobs.py @@ -0,0 +1,31 @@ +import pydefx +import os + +myParams = pydefx.Parameters() +myParams.nb_branches = 4 +myParams.salome_parameters.work_directory=os.path.join(os.getcwd(),"runbasic") +myParams.salome_parameters.local_directory = os.getcwd() +myParams.salome_parameters.resource_required.nb_proc = 1 +myParams.salome_parameters.job_name = "basic_job" +myParams.salome_parameters.job_type = "command" +myParams.salome_parameters.job_file = os.path.join(os.getcwd(), "simple_command.sh") +myParams.salome_parameters.resource_required.name = "eole" +myParams.salome_parameters.wckey = "P11N0:SALOME" + +myScript = pydefx.PyScript() + +pyScript = os.path.join(os.getcwd(), "jobstudy.py") +myScript.loadFile(pyScript) + +mySample = myScript.CreateEmptySample() +mydata = {"n":range(10)} +mySample.setInputValues(mydata) + +myStudy = pydefx.LocalStudy(schemaBuilder=pydefx.LocalBuilder("lightexecutor")) +myStudy.createNewJob(myScript, mySample, myParams) +myStudy.launch() + +myStudy.getJobState() +myStudy.wait() +print(myStudy.getResult()) +print(myStudy.sample) diff --git a/src/pyexample/multijob/simple_command.sh b/src/pyexample/multijob/simple_command.sh new file mode 100755 index 0000000..9e2740c --- /dev/null +++ b/src/pyexample/multijob/simple_command.sh @@ -0,0 +1 @@ +ls -- 2.39.2