From: barate Date: Thu, 26 Nov 2009 16:41:34 +0000 (+0000) Subject: Added environment variables for ePBS jobs. Added the corresponding tests. Merged... X-Git-Url: http://git.salome-platform.org/gitweb/?a=commitdiff_plain;h=afb71b7097250cf7526f0d3b1c8f2ff664bba8b9;p=tools%2Flibbatch.git Added environment variables for ePBS jobs. Added the corresponding tests. Merged Test_ePBS_SSH and Test_ePBS_RSH. --- diff --git a/src/Core/Test/SimpleParser.cxx b/src/Core/Test/SimpleParser.cxx index 83f6478..9942576 100644 --- a/src/Core/Test/SimpleParser.cxx +++ b/src/Core/Test/SimpleParser.cxx @@ -141,7 +141,6 @@ int SimpleParser::getValueAsInt(const string & key) const throw(ParserException) ostream & operator <<(ostream & os, const SimpleParser & parser) throw() { - os << "Configuration map:" << endl; if (parser._configmap.empty()) { os << "Empty map" << endl; } else { diff --git a/src/Core/Test/Test_SimpleParser.cxx b/src/Core/Test/Test_SimpleParser.cxx index bb446ef..6e60fbc 100644 --- a/src/Core/Test/Test_SimpleParser.cxx +++ b/src/Core/Test/Test_SimpleParser.cxx @@ -55,6 +55,7 @@ int main(int argc, char** argv) } // Print the configuration + cout << "Configuration map:" << endl; cout << parser << endl; return 0; diff --git a/src/PBS/Batch_BatchManager_ePBS.cxx b/src/PBS/Batch_BatchManager_ePBS.cxx index 724e9f8..9909b55 100644 --- a/src/PBS/Batch_BatchManager_ePBS.cxx +++ b/src/PBS/Batch_BatchManager_ePBS.cxx @@ -212,6 +212,7 @@ namespace Batch { { std::cerr << "BuildBatchScript" << std::endl; Parametre params = job.getParametre(); + Environnement env = job.getEnvironnement(); // Job Parameters string workDir = ""; @@ -266,6 +267,16 @@ namespace Batch { tempOutputFile << "#PBS -o " << workDir << "/logs/output.log." << rootNameToExecute << endl; tempOutputFile << "#PBS -e " << workDir << "/logs/error.log." << rootNameToExecute << endl; + // Define environment for the job + if (!env.empty()) { + tempOutputFile << "#PBS -v "; + Environnement::const_iterator iter; + for (iter = env.begin() ; iter != env.end() ; ++iter) { + tempOutputFile << iter->first << "=" << iter->second << ","; + } + tempOutputFile << endl; + } + // Abstraction of PBS_NODEFILE - TODO tempOutputFile << "export LIBBATCH_NODEFILE=$PBS_NODEFILE" << endl; diff --git a/src/PBS/Test/CMakeLists.txt b/src/PBS/Test/CMakeLists.txt index f584265..a96946a 100644 --- a/src/PBS/Test/CMakeLists.txt +++ b/src/PBS/Test/CMakeLists.txt @@ -32,16 +32,15 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..) include_directories(${CMAKE_CURRENT_BINARY_DIR}) # Build the test programs and add the tests +add_executable(Test_ePBS Test_ePBS.cxx) +target_link_libraries(Test_ePBS Batch SimpleParser) + IF (HAS_SSH) - add_executable(Test_ePBS_SSH Test_ePBS_SSH.cxx) - target_link_libraries(Test_ePBS_SSH Batch SimpleParser) - ADD_TEST(ePBS_SSH Test_ePBS_SSH) + ADD_TEST(ePBS_SSH Test_ePBS SSH) ENDIF (HAS_SSH) IF (HAS_RSH) - add_executable(Test_ePBS_RSH Test_ePBS_RSH.cxx) - target_link_libraries(Test_ePBS_RSH Batch SimpleParser) - ADD_TEST(ePBS_RSH Test_ePBS_RSH) + ADD_TEST(ePBS_RSH Test_ePBS RSH) ENDIF (HAS_RSH) IF (BUILD_PBS_INTERFACE AND PBS_FOUND) diff --git a/src/PBS/Test/Test_PBS.cxx b/src/PBS/Test/Test_PBS.cxx index e58278d..5ceb09f 100644 --- a/src/PBS/Test/Test_PBS.cxx +++ b/src/PBS/Test/Test_PBS.cxx @@ -47,6 +47,8 @@ using namespace std; using namespace Batch; +const int MAX_SLEEP_TIME = 600; + int main(int argc, char** argv) { cout << "*******************************************************************************************" << endl; @@ -92,6 +94,7 @@ int main(int argc, char** argv) job.setParametre(p); // ... and its environment Environnement e; + e["MYENVVAR"] = "MYVALUE"; job.setEnvironnement(e); cout << job << endl; @@ -107,13 +110,28 @@ int main(int argc, char** argv) cout << jobid.__repr__() << endl; // Wait for the end of the job - string state = "Undefined"; - for (int i=0 ; i -1); + bool timeoutReached = (testTimeout && time >= timeout); + JobInfo jinfo = jobid.queryJob(); + string state = jinfo.getParametre()["STATE"].str(); + cout << "State is \"" << state << "\""; + while (!timeoutReached && state != "U" && state != "C") { + cout << ", sleeping " << sleeptime << "s..." << endl; + sleep(sleeptime); + time += sleeptime; + timeoutReached = (testTimeout && time >= timeout); + sleeptime *= 2; + if (testTimeout && sleeptime > timeout - time) + sleeptime = timeout - time; + if (sleeptime > MAX_SLEEP_TIME) + sleeptime = MAX_SLEEP_TIME; + jinfo = jobid.queryJob(); state = jinfo.getParametre()["STATE"].str(); - cout << "State is \"" << state << "\"" << endl; + cout << "State is \"" << state << "\""; } + cout << endl; if (state == "U" || state == "C") { cout << "Job " << jobid.__repr__() << " is done" << endl; @@ -131,16 +149,21 @@ int main(int argc, char** argv) } // test the result file - string exp = "c = 12"; - string res; - ifstream f("result.txt"); - getline(f, res); - f.close(); - - cout << "result found : " << res << ", expected : " << exp << endl; - - if (res == exp) - return 0; - else + try { + SimpleParser resultParser; + resultParser.parse("result.txt"); + cout << "Result:" << endl << resultParser; + const string & envvar = resultParser.getValue("MYENVVAR"); + int result = resultParser.getValueAsInt("c"); + if (envvar == "MYVALUE" && result == 12) { + cout << "OK, Expected result found." << endl; + return 0; + } else { + cerr << "Error, result is not the expected one (MYENVVAR = MYVALUE, c = 12)." << endl; + return 1; + } + } catch (ParserException e) { + cerr << "Parser error on result file: " << e.what() << endl; return 1; + } } diff --git a/src/PBS/Test/Test_ePBS.cxx b/src/PBS/Test/Test_ePBS.cxx new file mode 100644 index 0000000..b603d75 --- /dev/null +++ b/src/PBS/Test/Test_ePBS.cxx @@ -0,0 +1,188 @@ +// Copyright (C) 2007-2008 CEA/DEN, EDF R&D, OPEN CASCADE +// +// Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +/* + * Test_ePBS.cxx : + * + * Author : Renaud BARATE - EDF R&D + * Date : April 2009 + * + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#ifdef WIN32 +#include +#define sleep(seconds) Sleep((seconds)*1000) +#define usleep(useconds) Sleep((useconds)/1000) +#endif + +using namespace std; +using namespace Batch; + +const int MAX_SLEEP_TIME = 600; + +void print_usage() +{ + cout << "usage: Test_ePBS PROTOCOL" << endl; + cout << " PROTOCOL \"SSH\" or \"RSH\"" << endl; +} + +int main(int argc, char** argv) +{ + // Parse argument + if (argc != 2) { + print_usage(); + return 1; + } + CommunicationProtocolType protocol; + if (strcmp(argv[1], "SSH") == 0) + protocol = SSH; + else if (strcmp(argv[1], "RSH") == 0) + protocol = RSH; + else { + print_usage(); + return 1; + } + + cout << "*******************************************************************************************" << endl; + cout << "This program tests the batch submission based on PBS emulation. Passwordless authentication" << endl; + cout << "must be used for this test to pass. For SSH, this can be configured with ssh-agent for" << endl; + cout << "instance. For RSH, this can be configured with the .rhosts file." << endl; + cout << "*******************************************************************************************" << endl; + + // eventually remove any previous result + remove("result.txt"); + + try { + // Parse the test configuration file + SimpleParser parser; + parser.parseTestConfigFile(); + const string & homedir = parser.getValue("TEST_EPBS_HOMEDIR"); + const string & host = parser.getValue("TEST_EPBS_HOST"); + const string & user = parser.getValue("TEST_EPBS_USER"); + const string & queue = parser.getValue("TEST_EPBS_QUEUE"); + int timeout = parser.getValueAsInt("TEST_EPBS_TIMEOUT"); + + // Define the job... + Job job; + // ... and its parameters ... + Parametre p; + p["EXECUTABLE"] = "./test-script.sh"; + p["NAME"] = string("Test_ePBS_") + argv[1]; + p["WORKDIR"] = homedir + "/tmp/Batch"; + p["INFILE"] = Couple("seta.sh", "tmp/Batch/seta.sh"); + p["INFILE"] += Couple("setb.sh", "tmp/Batch/setb.sh"); + p["OUTFILE"] = Couple("result.txt", "tmp/Batch/result.txt"); + p["TMPDIR"] = "tmp/Batch/"; + p["USER"] = user; + p["NBPROC"] = 1; + p["MAXWALLTIME"] = 1; + p["MAXRAMSIZE"] = 1000; + p["HOMEDIR"] = homedir; + p["QUEUE"] = queue; + job.setParametre(p); + // ... and its environment + Environnement e; + e["MYENVVAR"] = "MYVALUE"; + job.setEnvironnement(e); + cout << job << endl; + + // Get the catalog + BatchManagerCatalog& c = BatchManagerCatalog::getInstance(); + + // Create a BatchManager of type ePBS on localhost + FactBatchManager_eClient * fbm = (FactBatchManager_eClient *)(c("ePBS")); + BatchManager_eClient * bm = (*fbm)(host.c_str(), protocol, "lam"); + + // Submit the job to the BatchManager + JobId jobid = bm->submitJob(job); + cout << jobid.__repr__() << endl; + + // Wait for the end of the job + int time = 0; + int sleeptime = 1; + bool testTimeout = (timeout > -1); + bool timeoutReached = (testTimeout && time >= timeout); + JobInfo jinfo = jobid.queryJob(); + string state = jinfo.getParametre()["STATE"].str(); + cout << "State is \"" << state << "\""; + while (!timeoutReached && state != "U" && state != "C") { + cout << ", sleeping " << sleeptime << "s..." << endl; + sleep(sleeptime); + time += sleeptime; + timeoutReached = (testTimeout && time >= timeout); + sleeptime *= 2; + if (testTimeout && sleeptime > timeout - time) + sleeptime = timeout - time; + if (sleeptime > MAX_SLEEP_TIME) + sleeptime = MAX_SLEEP_TIME; + jinfo = jobid.queryJob(); + state = jinfo.getParametre()["STATE"].str(); + cout << "State is \"" << state << "\""; + } + cout << endl; + + if (state == "U" || state == "C") { + cout << "Job " << jobid.__repr__() << " is done" << endl; + bm->importOutputFiles(job, "."); + } else { + cerr << "Timeout while executing job" << endl; + return 1; + } + + } catch (GenericException e) { + cerr << "Error: " << e << endl; + return 1; + } catch (ParserException e) { + cerr << "Parser error: " << e.what() << endl; + return 1; + } + + // test the result file + try { + SimpleParser resultParser; + resultParser.parse("result.txt"); + cout << "Result:" << endl << resultParser; + const string & envvar = resultParser.getValue("MYENVVAR"); + int result = resultParser.getValueAsInt("c"); + if (envvar == "MYVALUE" && result == 12) { + cout << "OK, Expected result found." << endl; + return 0; + } else { + cerr << "Error, result is not the expected one (MYENVVAR = MYVALUE, c = 12)." << endl; + return 1; + } + } catch (ParserException e) { + cerr << "Parser error on result file: " << e.what() << endl; + return 1; + } +} diff --git a/src/PBS/Test/Test_ePBS_RSH.cxx b/src/PBS/Test/Test_ePBS_RSH.cxx deleted file mode 100644 index daafecb..0000000 --- a/src/PBS/Test/Test_ePBS_RSH.cxx +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (C) 2007-2008 CEA/DEN, EDF R&D, OPEN CASCADE -// -// Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, -// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com -// -/* - * Test_ePBS.cxx : - * - * Author : Renaud BARATE - EDF R&D - * Date : April 2009 - * - */ - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -#ifdef WIN32 -#include -#define sleep(seconds) Sleep((seconds)*1000) -#define usleep(useconds) Sleep((useconds)/1000) -#endif - -using namespace std; -using namespace Batch; - -int main(int argc, char** argv) -{ - cout << "*******************************************************************************************" << endl; - cout << "This program tests the batch submission based on PBS emulation with RSH. Passwordless RSH" << endl; - cout << "authentication must be used for this test to pass (this can be configured with the .rhosts" << endl; - cout << "file). You also need to create a directory \"tmp/Batch\" in your home directory on the PBS" << endl; - cout << "server before running this test." << endl; - cout << "*******************************************************************************************" << endl; - - // eventually remove any previous result - remove("result.txt"); - - try { - // Parse the test configuration file - SimpleParser parser; - parser.parseTestConfigFile(); - const string & homedir = parser.getValue("TEST_EPBS_HOMEDIR"); - const string & host = parser.getValue("TEST_EPBS_HOST"); - const string & user = parser.getValue("TEST_EPBS_USER"); - const string & queue = parser.getValue("TEST_EPBS_QUEUE"); - int timeout = parser.getValueAsInt("TEST_EPBS_TIMEOUT"); - - // Define the job... - Job job; - // ... and its parameters ... - Parametre p; - p["EXECUTABLE"] = "./test-script.sh"; - p["NAME"] = "Test_ePBS_RSH"; - p["WORKDIR"] = homedir + "/tmp/Batch"; - p["INFILE"] = Couple("seta.sh", "tmp/Batch/seta.sh"); - p["INFILE"] += Couple("setb.sh", "tmp/Batch/setb.sh"); - p["OUTFILE"] = Couple("result.txt", "tmp/Batch/result.txt"); - p["TMPDIR"] = "tmp/Batch/"; - p["USER"] = user; - p["NBPROC"] = 1; - p["MAXWALLTIME"] = 1; - p["MAXRAMSIZE"] = 4; - p["HOMEDIR"] = homedir; - p["QUEUE"] = queue; - job.setParametre(p); - // ... and its environment (SSH_AUTH_SOCK env var is important for ssh agent authentication) - Environnement e; - const char * sshAuthSock = getenv("SSH_AUTH_SOCK"); - if (sshAuthSock != NULL) e["SSH_AUTH_SOCK"] = sshAuthSock; - job.setEnvironnement(e); - cout << job << endl; - - // Get the catalog - BatchManagerCatalog& c = BatchManagerCatalog::getInstance(); - - // Create a BatchManager of type ePBS on localhost - FactBatchManager_eClient * fbm = (FactBatchManager_eClient *)(c("ePBS")); - BatchManager_eClient * bm = (*fbm)(host.c_str(), RSH, "lam"); - - // Submit the job to the BatchManager - JobId jobid = bm->submitJob(job); - cout << jobid.__repr__() << endl; - - // Wait for the end of the job - string state = "Undefined"; - for (int i=0 ; iimportOutputFiles(job, "."); - } else { - cerr << "Timeout while executing job" << endl; - return 1; - } - - } catch (GenericException e) { - cerr << "Error: " << e << endl; - return 1; - } catch (ParserException e) { - cerr << "Parser error: " << e.what() << endl; - return 1; - } - - // test the result file - string exp = "c = 12"; - string res; - ifstream f("result.txt"); - getline(f, res); - f.close(); - - cout << "result found : " << res << ", expected : " << exp << endl; - - if (res == exp) - return 0; - else - return 1; -} diff --git a/src/PBS/Test/Test_ePBS_SSH.cxx b/src/PBS/Test/Test_ePBS_SSH.cxx deleted file mode 100644 index 42e98c3..0000000 --- a/src/PBS/Test/Test_ePBS_SSH.cxx +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (C) 2007-2008 CEA/DEN, EDF R&D, OPEN CASCADE -// -// Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, -// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com -// -/* - * Test_ePBS.cxx : - * - * Author : Renaud BARATE - EDF R&D - * Date : April 2009 - * - */ - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -#ifdef WIN32 -#include -#define sleep(seconds) Sleep((seconds)*1000) -#define usleep(useconds) Sleep((useconds)/1000) -#endif - -using namespace std; -using namespace Batch; - -int main(int argc, char** argv) -{ - cout << "*******************************************************************************************" << endl; - cout << "This program tests the batch submission based on PBS emulation with SSH. Passwordless SSH" << endl; - cout << "authentication must be used for this test to pass (this can be configured with ssh-agent" << endl; - cout << "for instance). You also need to create a directory \"tmp/Batch\" in your home directory on" << endl; - cout << "the PBS server before running this test." << endl; - cout << "*******************************************************************************************" << endl; - - // eventually remove any previous result - remove("result.txt"); - - try { - // Parse the test configuration file - SimpleParser parser; - parser.parseTestConfigFile(); - const string & homedir = parser.getValue("TEST_EPBS_HOMEDIR"); - const string & host = parser.getValue("TEST_EPBS_HOST"); - const string & user = parser.getValue("TEST_EPBS_USER"); - const string & queue = parser.getValue("TEST_EPBS_QUEUE"); - int timeout = parser.getValueAsInt("TEST_EPBS_TIMEOUT"); - - // Define the job... - Job job; - // ... and its parameters ... - Parametre p; - p["EXECUTABLE"] = "./test-script.sh"; - p["NAME"] = "Test_ePBS_SSH"; - p["WORKDIR"] = homedir + "/tmp/Batch"; - p["INFILE"] = Couple("seta.sh", "tmp/Batch/seta.sh"); - p["INFILE"] += Couple("setb.sh", "tmp/Batch/setb.sh"); - p["OUTFILE"] = Couple("result.txt", "tmp/Batch/result.txt"); - p["TMPDIR"] = "tmp/Batch/"; - p["USER"] = user; - p["NBPROC"] = 1; - p["MAXWALLTIME"] = 1; - p["MAXRAMSIZE"] = 4; - p["HOMEDIR"] = homedir; - p["QUEUE"] = queue; - job.setParametre(p); - // ... and its environment (SSH_AUTH_SOCK env var is important for ssh agent authentication) - Environnement e; - const char * sshAuthSock = getenv("SSH_AUTH_SOCK"); - if (sshAuthSock != NULL) e["SSH_AUTH_SOCK"] = sshAuthSock; - job.setEnvironnement(e); - cout << job << endl; - - // Get the catalog - BatchManagerCatalog& c = BatchManagerCatalog::getInstance(); - - // Create a BatchManager of type ePBS on localhost - FactBatchManager_eClient * fbm = (FactBatchManager_eClient *)(c("ePBS")); - BatchManager_eClient * bm = (*fbm)(host.c_str(), SSH, "lam"); - - // Submit the job to the BatchManager - JobId jobid = bm->submitJob(job); - cout << jobid.__repr__() << endl; - - // Wait for the end of the job - string state = "Undefined"; - for (int i=0 ; iimportOutputFiles(job, "."); - } else { - cerr << "Timeout while executing job" << endl; - return 1; - } - - } catch (GenericException e) { - cerr << "Error: " << e << endl; - return 1; - } catch (ParserException e) { - cerr << "Parser error: " << e.what() << endl; - return 1; - } - - // test the result file - string exp = "c = 12"; - string res; - ifstream f("result.txt"); - getline(f, res); - f.close(); - - cout << "result found : " << res << ", expected : " << exp << endl; - - if (res == exp) - return 0; - else - return 1; -} diff --git a/src/PBS/Test/test-script.sh b/src/PBS/Test/test-script.sh index 64eb2fa..ae952c8 100755 --- a/src/PBS/Test/test-script.sh +++ b/src/PBS/Test/test-script.sh @@ -5,4 +5,5 @@ source setb.sh c=`expr $a "*" $b` -echo "c = $c" > result.txt +echo "MYENVVAR = $MYENVVAR" > result.txt +echo "c = $c" >> result.txt