From c14acfc207b81fc5fb33dd343bf06acb6b085fbb Mon Sep 17 00:00:00 2001 From: Konstantin Leontev Date: Fri, 11 Aug 2023 11:44:41 +0100 Subject: [PATCH] [bos #35138][EDF] (2023-T1) Specialization of resources in KERNEL. JobParameters resource_required type changed to ResourceParametersJob. --- src/pydefx/parameters.py | 26 +++----------------- src/pydefx/salome_proxy.py | 2 +- src/pydefx/slurmstudy.py | 2 -- src/pyexample/exemple_fr.py | 4 --- src/pyexample/multijob/launch.py | 1 - src/pyexample/multijob/launch_local_basic.py | 1 - src/pyexample/multijob/launch_local_jobs.py | 1 - src/pyexample/multijob/launch_multi.py | 1 - src/pyexample/multijob/launch_slurm_light.py | 1 - src/pyexample/multijob/launch_srun.py | 1 - src/pyexample/syrthes_launch.py | 1 - src/pytools/mpmcn.py | 6 ----- 12 files changed, 4 insertions(+), 43 deletions(-) diff --git a/src/pydefx/parameters.py b/src/pydefx/parameters.py index e2734bc..870bac9 100644 --- a/src/pydefx/parameters.py +++ b/src/pydefx/parameters.py @@ -20,6 +20,7 @@ import tempfile from . import salome_proxy from . import configuration +import copy class Parameters: def __init__(self, resource="localhost", @@ -33,12 +34,11 @@ class Parameters: job_params.work_directory = configuration.defaultWorkingDir(resource) if nb_branches is None: nb_branches = configuration.defaultNbBranches(resource) - job_params.resource_required.nb_proc = nb_branches self.nb_branches = nb_branches self.salome_parameters = job_params else: if nb_branches is None: - nb_branches = salome_parameters.resource_required.nb_proc + nb_branches = 1 self.nb_branches = nb_branches self.salome_parameters = salome_parameters @@ -47,7 +47,6 @@ class Parameters: self.salome_parameters.work_directory = configuration.defaultWorkingDir( resource) nb_branches = configuration.defaultNbBranches(resource) - self.salome_parameters.resource_required.nb_proc = nb_branches self.nb_branches = nb_branches self.salome_parameters.wckey = configuration.defaultWckey(resource) @@ -84,14 +83,7 @@ class Parameters: newobj.salome_parameters.wckey = self.salome_parameters.wckey newobj.salome_parameters.extra_params = self.salome_parameters.extra_params #newobj.salome_parameters.specific_parameters = self.salome_parameters.specific_parameters - newobj.salome_parameters.resource_required.name = self.salome_parameters.resource_required.name - newobj.salome_parameters.resource_required.hostname = self.salome_parameters.resource_required.hostname - newobj.salome_parameters.resource_required.OS = self.salome_parameters.resource_required.OS - newobj.salome_parameters.resource_required.nb_proc = self.salome_parameters.resource_required.nb_proc - newobj.salome_parameters.resource_required.mem_mb = self.salome_parameters.resource_required.mem_mb - newobj.salome_parameters.resource_required.cpu_clock = self.salome_parameters.resource_required.cpu_clock - newobj.salome_parameters.resource_required.nb_node = self.salome_parameters.resource_required.nb_node - newobj.salome_parameters.resource_required.nb_proc_per_node = self.salome_parameters.resource_required.nb_proc_per_node + newobj.salome_parameters.resource_required = copy.deepcopy(self.salome_parameters.resource_required) return newobj @@ -122,12 +114,6 @@ class Parameters: "resource_required" : { "name" : self.salome_parameters.resource_required.name, "hostname" : self.salome_parameters.resource_required.hostname, - "OS" : self.salome_parameters.resource_required.OS, - "nb_proc" : self.salome_parameters.resource_required.nb_proc, - "mem_mb" : self.salome_parameters.resource_required.mem_mb, - "cpu_clock" : self.salome_parameters.resource_required.cpu_clock, - "nb_node" : self.salome_parameters.resource_required.nb_node, - "nb_proc_per_node" : self.salome_parameters.resource_required.nb_proc_per_node } } } @@ -155,9 +141,3 @@ class Parameters: self.salome_parameters.extra_params = dico["salome_parameters"]["extra_params"] self.salome_parameters.resource_required.name = dico["salome_parameters"]["resource_required"]["name"] self.salome_parameters.resource_required.hostname = dico["salome_parameters"]["resource_required"]["hostname"] - self.salome_parameters.resource_required.OS = dico["salome_parameters"]["resource_required"]["OS"] - self.salome_parameters.resource_required.nb_proc = dico["salome_parameters"]["resource_required"]["nb_proc"] - self.salome_parameters.resource_required.mem_mb = dico["salome_parameters"]["resource_required"]["mem_mb"] - self.salome_parameters.resource_required.cpu_clock = dico["salome_parameters"]["resource_required"]["cpu_clock"] - self.salome_parameters.resource_required.nb_node = dico["salome_parameters"]["resource_required"]["nb_node"] - self.salome_parameters.resource_required.nb_proc_per_node = dico["salome_parameters"]["resource_required"]["nb_proc_per_node"] diff --git a/src/pydefx/salome_proxy.py b/src/pydefx/salome_proxy.py index dca0e41..1e55e5d 100644 --- a/src/pydefx/salome_proxy.py +++ b/src/pydefx/salome_proxy.py @@ -27,7 +27,7 @@ def createSalomeParameters(): _default() if _use_salome_servers: result = salome.JobParameters() - result.resource_required = salome.ResourceParametersContainer() + result.resource_required = salome.ResourceParametersJob() else: result = pylauncher.JobParameters_cpp() return result diff --git a/src/pydefx/slurmstudy.py b/src/pydefx/slurmstudy.py index 5554e2a..c296f2e 100644 --- a/src/pydefx/slurmstudy.py +++ b/src/pydefx/slurmstudy.py @@ -55,8 +55,6 @@ class SlurmStudy(pystudy.PyStudy): dicconfig["studymodule"] = "idefixstudy" dicconfig["sampleIterator"] = self.sampleManager.getModuleName() dicconfig["plugin"] = self.schemaBuilder.getPluginName() - nbproc = self.params.salome_parameters.resource_required.nb_proc - dicconfig["tasksPerEval"] = nbproc // self.params.nb_branches configpath = configuration.exportConfig(dicconfig, result_directory) studypath = os.path.join(result_directory, "idefixstudy.py") with open(studypath, "w") as f: diff --git a/src/pyexample/exemple_fr.py b/src/pyexample/exemple_fr.py index 0319129..6d7bbad 100644 --- a/src/pyexample/exemple_fr.py +++ b/src/pyexample/exemple_fr.py @@ -39,10 +39,6 @@ myParams.salome_parameters.in_files = [] print("ressource de calcul:", myParams.salome_parameters.resource_required.name) print("nombre d'évaluations parallèles:", myParams.nb_branches) -print("nombre de coeurs demandés:", - myParams.salome_parameters.resource_required.nb_proc) -print("nombre de noeuds demandés:", - myParams.salome_parameters.resource_required.nb_node) print("répertoire de travail:", myParams.salome_parameters.work_directory) print("répertoire local de gestion:", myParams.salome_parameters.result_directory) diff --git a/src/pyexample/multijob/launch.py b/src/pyexample/multijob/launch.py index f90755f..3414d5d 100755 --- a/src/pyexample/multijob/launch.py +++ b/src/pyexample/multijob/launch.py @@ -5,7 +5,6 @@ myParams = pydefx.Parameters() myParams.configureResource("eole") #myParams.createResultDirectory("/tmp") myParams.nb_branches = 4 -myParams.salome_parameters.resource_required.nb_proc = 4 myParams.salome_parameters.result_directory=os.path.join(os.getcwd(),"rundir") myParams.salome_parameters.work_directory="/scratch/I35256/workingdir/testjob/" myParams.salome_parameters.local_directory = os.getcwd() diff --git a/src/pyexample/multijob/launch_local_basic.py b/src/pyexample/multijob/launch_local_basic.py index 282006b..3fd7253 100755 --- a/src/pyexample/multijob/launch_local_basic.py +++ b/src/pyexample/multijob/launch_local_basic.py @@ -4,7 +4,6 @@ import os myParams = pydefx.Parameters() myParams.configureResource("localhost") myParams.nb_branches = 4 -myParams.salome_parameters.resource_required.nb_proc = 1 myParams.salome_parameters.work_directory=os.path.join(os.getcwd(),"runbasic") myParams.salome_parameters.local_directory = os.getcwd() diff --git a/src/pyexample/multijob/launch_local_jobs.py b/src/pyexample/multijob/launch_local_jobs.py index 4017892..582ac74 100755 --- a/src/pyexample/multijob/launch_local_jobs.py +++ b/src/pyexample/multijob/launch_local_jobs.py @@ -5,7 +5,6 @@ myParams = pydefx.Parameters() myParams.nb_branches = 4 myParams.salome_parameters.work_directory=os.path.join(os.getcwd(),"runbasic") myParams.salome_parameters.local_directory = os.getcwd() -myParams.salome_parameters.resource_required.nb_proc = 1 myParams.salome_parameters.job_name = "basic_job" myParams.salome_parameters.job_type = "command" myParams.salome_parameters.job_file = os.path.join(os.getcwd(), "simple_command.sh") diff --git a/src/pyexample/multijob/launch_multi.py b/src/pyexample/multijob/launch_multi.py index 95fdc64..ce9434e 100755 --- a/src/pyexample/multijob/launch_multi.py +++ b/src/pyexample/multijob/launch_multi.py @@ -5,7 +5,6 @@ myParams = pydefx.Parameters() myParams.configureResource("gaia") #myParams.createResultDirectory("/tmp") myParams.nb_branches = 4 -myParams.salome_parameters.resource_required.nb_proc = 1 myParams.salome_parameters.result_directory=os.path.join(os.getcwd(),"runmulti") myParams.salome_parameters.work_directory="/scratch/I35256/workingdir/test_multijob/" myParams.salome_parameters.local_directory = os.getcwd() diff --git a/src/pyexample/multijob/launch_slurm_light.py b/src/pyexample/multijob/launch_slurm_light.py index d8a42b6..bf3b770 100755 --- a/src/pyexample/multijob/launch_slurm_light.py +++ b/src/pyexample/multijob/launch_slurm_light.py @@ -11,7 +11,6 @@ import os myParams = pydefx.Parameters() myParams.configureResource("gaia") myParams.nb_branches = 4 -myParams.salome_parameters.resource_required.nb_proc = 4 myParams.salome_parameters.result_directory=os.path.join(os.getcwd(),"lightsrun") myParams.salome_parameters.work_directory="/scratch/I35256/workingdir/test_srunjob/" myParams.salome_parameters.local_directory = os.getcwd() diff --git a/src/pyexample/multijob/launch_srun.py b/src/pyexample/multijob/launch_srun.py index b47e2f3..4f83ad1 100755 --- a/src/pyexample/multijob/launch_srun.py +++ b/src/pyexample/multijob/launch_srun.py @@ -5,7 +5,6 @@ myParams = pydefx.Parameters() myParams.configureResource("eole") #myParams.createResultDirectory("/tmp") myParams.nb_branches = 4 -myParams.salome_parameters.resource_required.nb_proc = 4 myParams.salome_parameters.result_directory=os.path.join(os.getcwd(),"runsrun") myParams.salome_parameters.work_directory="/scratch/I35256/workingdir/test_srunjob/" myParams.salome_parameters.local_directory = os.getcwd() diff --git a/src/pyexample/syrthes_launch.py b/src/pyexample/syrthes_launch.py index 9158ead..e8e74ea 100755 --- a/src/pyexample/syrthes_launch.py +++ b/src/pyexample/syrthes_launch.py @@ -23,7 +23,6 @@ myParams.salome_parameters.work_directory = wd myParams.salome_parameters.local_directory = STUDY_DIR myParams.salome_parameters.in_files = ["brique_ech.syd", "Makefile", "Mesh", "run.sh", "syrthes.py", "user_cond.c"] -#myParams.salome_parameters.resource_required.nb_node = 2 myScript = pydefix.PyScript() myScript.loadFile(os.path.join(STUDY_DIR, "etude.py")) diff --git a/src/pytools/mpmcn.py b/src/pytools/mpmcn.py index ccfea22..d854e8f 100644 --- a/src/pytools/mpmcn.py +++ b/src/pytools/mpmcn.py @@ -130,12 +130,6 @@ def init(resourceName, resultDirectory = "/tmp"): Override default # of parallel evaluation : myParams.nb_branches - Override number of cores requested by the job when job resource is a cluster (for job scheduler query) : - myParams.salome_parameters.resource_required.nb_proc - - Override number of computationnal nodes on cluster to be allocated when job resource is a cluster (for job scheduler query) : - myParams.salome_parameters.resource_required.nb_node - Override working directory : myParams.salome_parameters.work_directory -- 2.39.2