From 151bf0929f949f39bfe9e1c219fe0fa3a2b9567e Mon Sep 17 00:00:00 2001 From: =?utf8?q?C=C3=A9dric=20Aguerre?= Date: Fri, 7 Apr 2017 13:46:50 +0200 Subject: [PATCH] Fix bug - jobs run simultaneously --- src/Launcher/BatchTest.cxx | 8 +-- src/Launcher/Launcher_Job.cxx | 10 ++- src/Launcher/Launcher_Job_Command.cxx | 12 ++-- src/Launcher/Launcher_Job_SALOME.cxx | 7 +- src/Launcher/Test/test_launcher.py | 94 +++++++++++++++------------ 5 files changed, 77 insertions(+), 54 deletions(-) diff --git a/src/Launcher/BatchTest.cxx b/src/Launcher/BatchTest.cxx index 788aca8b5..d81ef8a04 100644 --- a/src/Launcher/BatchTest.cxx +++ b/src/Launcher/BatchTest.cxx @@ -1,4 +1,4 @@ -// Copyright (C) 2007-2016 CEA/DEN, EDF R&D, OPEN CASCADE +// Copyright (C) 2007-2017 CEA/DEN, EDF R&D, OPEN CASCADE // // Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, // CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS @@ -22,6 +22,8 @@ #include "BatchTest.hxx" #include "Launcher.hxx" +#include "Basics_Utils.hxx" +#include "Basics_DirUtils.hxx" #ifdef WITH_LIBBATCH #include @@ -47,10 +49,8 @@ BatchTest::BatchTest(const Engines::ResourceDefinition& batch_descr) strftime(date, BUFSIZE, "%Y_%m_%d__%H_%M_%S", localtime(&curtime)); // Creating test temporary file - _test_filename = "/tmp/"; - _test_filename += std::string(date) + "_test_cluster_file_"; - _test_filename += _batch_descr.hostname.in(); _base_filename = std::string(date) + "_test_cluster_file_" + _batch_descr.hostname.in(); + _test_filename = Kernel_Utils::GetTmpDir() + _base_filename; #endif } diff --git a/src/Launcher/Launcher_Job.cxx b/src/Launcher/Launcher_Job.cxx index c52363bce..e046fd8cf 100644 --- a/src/Launcher/Launcher_Job.cxx +++ b/src/Launcher/Launcher_Job.cxx @@ -1,4 +1,4 @@ -// Copyright (C) 2009-2016 CEA/DEN, EDF R&D, OPEN CASCADE +// Copyright (C) 2009-2017 CEA/DEN, EDF R&D, OPEN CASCADE // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -27,7 +27,7 @@ #include #endif -using namespace std; +#include Launcher::Job::Job() { @@ -590,7 +590,11 @@ Launcher::Job::common_job_params() strftime(date, BUFSIZE, "%Y_%m_%d__%H_%M_%S", localtime(&curtime)); if(!_resource_definition.working_directory.empty()) { - std::string job_dir = std::string("/job_") + date; + std::string date_dir = std::string("/job_") + date; + std::ostringstream str_pid; + str_pid << ::getpid(); + std::string job_dir = date_dir + "-" + str_pid.str(); + _work_directory = _resource_definition.working_directory + job_dir; } else diff --git a/src/Launcher/Launcher_Job_Command.cxx b/src/Launcher/Launcher_Job_Command.cxx index 0fb681c35..bdb0f2ec4 100644 --- a/src/Launcher/Launcher_Job_Command.cxx +++ b/src/Launcher/Launcher_Job_Command.cxx @@ -1,4 +1,4 @@ -// Copyright (C) 2009-2016 CEA/DEN, EDF R&D, OPEN CASCADE +// Copyright (C) 2009-2017 CEA/DEN, EDF R&D, OPEN CASCADE // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -31,6 +31,8 @@ #define _chmod chmod #endif +#include + Launcher::Job_Command::Job_Command() {_job_type = "command";} Launcher::Job_Command::~Job_Command() {} @@ -46,7 +48,7 @@ Launcher::Job_Command::update_job() } #ifdef WITH_LIBBATCH -std::string +std::string Launcher::Job_Command::buildCommandScript(Batch::Parametre params, std::string launch_date) { // parameters @@ -54,10 +56,12 @@ Launcher::Job_Command::buildCommandScript(Batch::Parametre params, std::string l // File name std::string launch_date_port_file = launch_date; - std::string launch_script = Kernel_Utils::GetTmpDir() + "runCommand_" + _job_file_name + "_" + launch_date + ".sh"; + std::ostringstream str_pid; + str_pid << ::getpid(); + std::string launch_script = Kernel_Utils::GetTmpDir() + "runCommand_" + _job_file_name + "_" + launch_date + "-" + str_pid.str() + ".sh"; std::ofstream launch_script_stream; launch_script_stream.open(launch_script.c_str(), std::ofstream::out); - + // Script launch_script_stream << "#!/bin/sh -f" << std::endl; launch_script_stream << "cd " << work_directory << std::endl; diff --git a/src/Launcher/Launcher_Job_SALOME.cxx b/src/Launcher/Launcher_Job_SALOME.cxx index 75941e77b..ee2812c1b 100644 --- a/src/Launcher/Launcher_Job_SALOME.cxx +++ b/src/Launcher/Launcher_Job_SALOME.cxx @@ -31,6 +31,8 @@ #define _chmod chmod #endif +#include + Launcher::Job_SALOME::Job_SALOME() {} Launcher::Job_SALOME::~Job_SALOME() {} @@ -63,8 +65,9 @@ Launcher::Job_SALOME::buildSalomeScript(Batch::Parametre params) { // parameters std::string work_directory = params[Batch::WORKDIR].str(); - - std::string launch_script = Kernel_Utils::GetTmpDir() + "runSalome_" + _job_file_name + "_" + _launch_date + ".sh"; + std::ostringstream str_pid; + str_pid << ::getpid(); + std::string launch_script = Kernel_Utils::GetTmpDir() + "runSalome_" + _job_file_name + "_" + _launch_date + "-" + str_pid.str() + ".sh"; std::ofstream launch_script_stream; launch_script_stream.open(launch_script.c_str(), std::ofstream::out diff --git a/src/Launcher/Test/test_launcher.py b/src/Launcher/Test/test_launcher.py index 53fe67af5..546b0057f 100755 --- a/src/Launcher/Test/test_launcher.py +++ b/src/Launcher/Test/test_launcher.py @@ -5,6 +5,17 @@ import unittest import os import sys import time +import tempfile +import errno + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise # Test of SalomeLauncher. # This test should be run in the salome environment, using "salome shell" @@ -19,12 +30,13 @@ class TestCompo(unittest.TestCase): @classmethod def setUpClass(cls): # Prepare the test directory - import shutil - cls.test_dir = os.path.join(os.getcwd(), "test_dir") - cls.suffix = time.strftime("-%Y-%m-%d-%H-%M-%S") - shutil.rmtree(cls.test_dir, ignore_errors=True) - os.mkdir(cls.test_dir) - + temp = tempfile.NamedTemporaryFile() + cls.test_dir = os.path.join(temp.name, "test_dir") + name = os.path.basename(temp.name) + temp.close() + cls.suffix = time.strftime("-%Y-%m-%d-%H-%M-%S")+"-%s"%(os.getpid()) + mkdir_p(cls.test_dir) + # load catalogs # mc = salome.naming_service.Resolve('/Kernel/ModulCatalog') # ior = salome.orb.object_to_string(mc) @@ -54,11 +66,11 @@ class TestCompo(unittest.TestCase): ############################## def test_salome_py_job(self): case_test_dir = os.path.join(TestCompo.test_dir, "salome_py") - os.mkdir(case_test_dir) - + mkdir_p(case_test_dir) + old_dir = os.getcwd() os.chdir(case_test_dir) - + # job script script_file = "myScript.py" job_script_file = os.path.join(case_test_dir, script_file) @@ -82,8 +94,8 @@ f.close() f = open(job_script_file, "w") f.write(script_text) f.close() - - local_result_dir = os.path.join(case_test_dir, "result_py_job") + + local_result_dir = os.path.join(case_test_dir, "result_py_job-") job_params = salome.JobParameters() job_params.job_type = "python_salome" job_params.job_file = job_script_file @@ -91,9 +103,9 @@ f.close() job_params.out_files = ["result.txt", "subdir"] job_params.resource_required = salome.ResourceParameters() job_params.resource_required.nb_proc = 1 - + launcher = salome.naming_service.Resolve('/SalomeLauncher') - + for resource in self.ressources: print "Testing python_salome job on ", resource job_params.result_directory = local_result_dir + resource @@ -130,14 +142,14 @@ f.close() pass #for os.chdir(old_dir) - + ############################## # test of command job type ############################## def test_command(self): case_test_dir = os.path.join(TestCompo.test_dir, "command") - os.mkdir(case_test_dir) - + mkdir_p(case_test_dir) + # job script data_file = "in.txt" script_file = "myEnvScript.py" @@ -166,7 +178,7 @@ f.close() f.write(script_text) f.close() os.chmod(abs_script_file, 0o755) - + #environement script env_file = "myEnv.sh" env_text = """export ENV_TEST_VAR="expected" @@ -174,14 +186,14 @@ f.close() f = open(os.path.join(case_test_dir, env_file), "w") f.write(env_text) f.close() - + # write data file f = open(os.path.join(case_test_dir, data_file), "w") f.write("to be copied") f.close() - + # job params - local_result_dir = os.path.join(case_test_dir, "result_com_job") + local_result_dir = os.path.join(case_test_dir, "result_com_job-") job_params = salome.JobParameters() job_params.job_type = "command" job_params.job_file = script_file @@ -191,7 +203,7 @@ f.close() job_params.local_directory = case_test_dir job_params.resource_required = salome.ResourceParameters() job_params.resource_required.nb_proc = 1 - + # create and launch the job launcher = salome.naming_service.Resolve('/SalomeLauncher') resManager= salome.lcc.getResourcesManager() @@ -205,7 +217,7 @@ f.close() # use the working directory of the resource resParams = resManager.GetResourceDefinition(resource) wd = os.path.join(resParams.working_directory, - "CommandJob_" + self.suffix) + "CommandJob" + self.suffix) job_params.work_directory = wd job_id = launcher.createJob(job_params) @@ -247,10 +259,10 @@ f.close() yacs_path = os.getenv("YACS_ROOT_DIR", "") if not os.path.isdir(yacs_path): self.skipTest("Needs YACS module to run. Please define YACS_ROOT_DIR.") - + case_test_dir = os.path.join(TestCompo.test_dir, "yacs") - os.mkdir(case_test_dir) - + mkdir_p(case_test_dir) + #environement script env_file = "myEnv.sh" env_text = """export ENV_TEST_VAR="expected" @@ -258,7 +270,7 @@ f.close() f = open(os.path.join(case_test_dir, env_file), "w") f.write(env_text) f.close() - + # job script script_text = """ @@ -285,14 +297,14 @@ f.close() f = open(job_script_file, "w") f.write(script_text) f.close() - - local_result_dir = os.path.join(case_test_dir, "result_yacs_job") + + local_result_dir = os.path.join(case_test_dir, "result_yacs_job-") job_params = salome.JobParameters() job_params.job_type = "yacs_file" job_params.job_file = job_script_file job_params.env_file = os.path.join(case_test_dir,env_file) job_params.out_files = ["result.txt"] - + # define the interval between two YACS schema dumps (3 seconds) import Engines job_params.specific_parameters = [Engines.Parameter("EnableDumpYACS", "3")] @@ -301,7 +313,7 @@ f.close() launcher = salome.naming_service.Resolve('/SalomeLauncher') resManager= salome.lcc.getResourcesManager() - + for resource in self.ressources: print "Testing yacs job on ", resource job_params.result_directory = local_result_dir + resource @@ -311,7 +323,7 @@ f.close() # use the working directory of the resource resParams = resManager.GetResourceDefinition(resource) wd = os.path.join(resParams.working_directory, - "YacsJob_" + self.suffix) + "YacsJob" + self.suffix) job_params.work_directory = wd job_id = launcher.createJob(job_params) @@ -352,7 +364,7 @@ f.close() launcher.getJobResults(job_id, "") self.verifyFile(os.path.join(job_params.result_directory, "result.txt"), "expected") - + ############################## # test of yacs job type using "--init_port" driver option ############################## @@ -360,10 +372,10 @@ f.close() yacs_path = os.getenv("YACS_ROOT_DIR", "") if not os.path.isdir(yacs_path): self.skipTest("Needs YACS module to run. Please define YACS_ROOT_DIR.") - + case_test_dir = os.path.join(TestCompo.test_dir, "yacs_opt") - os.mkdir(case_test_dir) - + mkdir_p(case_test_dir) + # job script script_text = """ @@ -398,14 +410,14 @@ f.close() f = open(job_script_file, "w") f.write(script_text) f.close() - - local_result_dir = os.path.join(case_test_dir, "result_yacsopt_job") + + local_result_dir = os.path.join(case_test_dir, "result_yacsopt_job-") job_params = salome.JobParameters() job_params.job_type = "yacs_file" job_params.job_file = job_script_file #job_params.env_file = os.path.join(case_test_dir,env_file) job_params.out_files = ["result.txt"] - + # define the interval between two YACS schema dumps (3 seconds) import Engines job_params.specific_parameters = [Engines.Parameter("YACSDriverOptions", @@ -416,7 +428,7 @@ f.close() launcher = salome.naming_service.Resolve('/SalomeLauncher') resManager= salome.lcc.getResourcesManager() - + for resource in self.ressources: print "Testing yacs job with options on ", resource job_params.result_directory = local_result_dir + resource @@ -426,7 +438,7 @@ f.close() # use the working directory of the resource resParams = resManager.GetResourceDefinition(resource) wd = os.path.join(resParams.working_directory, - "YacsJobOpt_" + self.suffix) + "YacsJobOpt" + self.suffix) job_params.work_directory = wd job_id = launcher.createJob(job_params) @@ -452,4 +464,4 @@ if __name__ == '__main__': # creat study import salome salome.salome_init() - unittest.main() \ No newline at end of file + unittest.main() -- 2.39.2