From 2be7691d8b9f9cbddce1e7cd92eaab36de3deb2a Mon Sep 17 00:00:00 2001 From: barate Date: Wed, 28 Mar 2012 16:02:56 +0000 Subject: [PATCH] Merge from BR_DIET --- src/CMakeLists.txt | 1 + src/Core/Batch_BatchManager.cxx | 2 + src/Core/Batch_BatchManager_eClient.hxx | 2 +- src/Core/Batch_Utils.cxx | 7 + src/Core/Batch_Utils.hxx | 6 + src/Core/Test/batchtest.conf | 8 + src/Vishnu/Batch_BatchManager_eVishnu.cxx | 399 ++++++++++++++++++ src/Vishnu/Batch_BatchManager_eVishnu.hxx | 77 ++++ src/Vishnu/Batch_FactBatchManager_eVishnu.cxx | 62 +++ src/Vishnu/Batch_FactBatchManager_eVishnu.hxx | 57 +++ src/Vishnu/Batch_JobInfo_eVishnu.cxx | 98 +++++ src/Vishnu/Batch_JobInfo_eVishnu.hxx | 49 +++ src/Vishnu/CMakeLists.txt | 32 ++ src/Vishnu/Test/CMakeLists.txt | 44 ++ src/Vishnu/Test/Test_eVishnu.cxx | 137 ++++++ src/Vishnu/Test/seta.sh | 3 + src/Vishnu/Test/setb.sh | 3 + src/Vishnu/Test/test-script.sh | 9 + 18 files changed, 995 insertions(+), 1 deletion(-) create mode 100644 src/Vishnu/Batch_BatchManager_eVishnu.cxx create mode 100644 src/Vishnu/Batch_BatchManager_eVishnu.hxx create mode 100644 src/Vishnu/Batch_FactBatchManager_eVishnu.cxx create mode 100644 src/Vishnu/Batch_FactBatchManager_eVishnu.hxx create mode 100644 src/Vishnu/Batch_JobInfo_eVishnu.cxx create mode 100644 src/Vishnu/Batch_JobInfo_eVishnu.hxx create mode 100644 src/Vishnu/CMakeLists.txt create mode 100644 src/Vishnu/Test/CMakeLists.txt create mode 100644 src/Vishnu/Test/Test_eVishnu.cxx create mode 100644 src/Vishnu/Test/seta.sh create mode 100644 src/Vishnu/Test/setb.sh create mode 100755 src/Vishnu/Test/test-script.sh diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 11e07fb..2148bbf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -57,6 +57,7 @@ add_subdirectory (SGE) add_subdirectory (SSH) add_subdirectory (LoadLeveler) add_subdirectory (Slurm) +add_subdirectory (Vishnu) # Make a copy of the built value and clear the built value for the next run of cmake SET(SRC_FILES ${SRC_FILES_BUILD} CACHE INTERNAL "") diff --git a/src/Core/Batch_BatchManager.cxx b/src/Core/Batch_BatchManager.cxx index 6f34765..cbe725c 100644 --- a/src/Core/Batch_BatchManager.cxx +++ b/src/Core/Batch_BatchManager.cxx @@ -66,6 +66,7 @@ namespace Batch { // } BatchManager::BatchManager(const FactBatchManager * parent, const char * host) throw(InvalidArgumentException) : _hostname(host), jobid_map(), _parent(parent) { + /* #ifdef WIN32 WSADATA wsaData; WSAStartup(MAKEWORD(2, 2), &wsaData); // Initialize Winsock @@ -84,6 +85,7 @@ namespace Batch { msg += "\" unknown from the network"; throw InvalidArgumentException(msg.c_str()); } + */ } // Destructeur diff --git a/src/Core/Batch_BatchManager_eClient.hxx b/src/Core/Batch_BatchManager_eClient.hxx index b33ca17..d596f44 100644 --- a/src/Core/Batch_BatchManager_eClient.hxx +++ b/src/Core/Batch_BatchManager_eClient.hxx @@ -52,7 +52,7 @@ namespace Batch { const char * username="", CommunicationProtocolType protocolType = SSH, const char* mpiImpl="mpich1"); virtual ~BatchManager_eClient(); - void importOutputFiles( const Job & job, const std::string directory ); + virtual void importOutputFiles( const Job & job, const std::string directory ); bool importDumpStateFile( const Job & job, const std::string directory ); protected: diff --git a/src/Core/Batch_Utils.cxx b/src/Core/Batch_Utils.cxx index 3b706d2..83b43e3 100644 --- a/src/Core/Batch_Utils.cxx +++ b/src/Core/Batch_Utils.cxx @@ -61,4 +61,11 @@ int Utils::getCommandOutput(const string & command, string & output) return status; } +bool Utils::isAbsolutePath(const string & path) +{ + if (path.size() == 0) + return false; + return path[0] == '/'; +} + } diff --git a/src/Core/Batch_Utils.hxx b/src/Core/Batch_Utils.hxx index 24f17d1..ce59ccf 100644 --- a/src/Core/Batch_Utils.hxx +++ b/src/Core/Batch_Utils.hxx @@ -42,6 +42,12 @@ public: */ static int getCommandOutput(const std::string & command, std::string & output); + /** + * Test if the path in parameter is an absolute path (does not test the existence of + * a file or directory at this path). + */ + static bool isAbsolutePath(const std::string & path); + private: // No instanciation possible as this class provides only static methods diff --git a/src/Core/Test/batchtest.conf b/src/Core/Test/batchtest.conf index 570ec21..07034c4 100644 --- a/src/Core/Test/batchtest.conf +++ b/src/Core/Test/batchtest.conf @@ -43,3 +43,11 @@ TEST_ESLURM_HOST = "localhost" # Slurm server host TEST_ESLURM_USER = "username" # Login for the Slurm server TEST_ESLURM_HOMEDIR = "/home/username" # Home directory on Slurm server TEST_ESLURM_TIMEOUT = 120 # Execution timeout (in seconds) for Slurm Batch test + +TEST_EVISHNU_HOST = "localhost" # Vishnu server host +TEST_EVISHNU_USER = "username" # Login on the Vishnu server host +TEST_EVISHNU_VISHNU_USERID = "userid" # UserId for connecting to Vishnu +TEST_EVISHNU_VISHNU_PASSWORD = "password" # Password for connecting to Vishnu +TEST_EVISHNU_VISHNU_MACHINEID = "localhost" # ID of the machine where Vishnu will launch the job +TEST_EVISHNU_HOMEDIR = "/home/username" # Home directory on Vishnu server +TEST_EVISHNU_TIMEOUT = 120 # Execution timeout (in seconds) for Vishnu Batch test diff --git a/src/Vishnu/Batch_BatchManager_eVishnu.cxx b/src/Vishnu/Batch_BatchManager_eVishnu.cxx new file mode 100644 index 0000000..279dce3 --- /dev/null +++ b/src/Vishnu/Batch_BatchManager_eVishnu.cxx @@ -0,0 +1,399 @@ +// Copyright (C) 2007-2011 CEA/DEN, EDF R&D, OPEN CASCADE +// +// Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +/* + * Batch_BatchManager_eVishnu.cxx : + * + * Created on: 24 june 2011 + * Author : Renaud BARATE - EDF R&D + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include "Batch_FactBatchManager_eVishnu.hxx" +#include "Batch_BatchManager_eVishnu.hxx" +#include "Batch_JobInfo_eVishnu.hxx" + +using namespace std; + +namespace Batch { + + BatchManager_eVishnu::BatchManager_eVishnu(const FactBatchManager * parent, + const char * host, + const char * username, + CommunicationProtocolType protocolType, + const char * mpiImpl, + int nb_proc_per_node) + : BatchManager(parent, host), + // Force SH protocol for Vishnu + BatchManager_eClient(parent, host, username, SH, mpiImpl), + _nb_proc_per_node(nb_proc_per_node) + { + } + + BatchManager_eVishnu::~BatchManager_eVishnu() + { + } + + // Method to submit a job to the batch manager + const JobId BatchManager_eVishnu::submitJob(const Job & job) + { + int status; + Parametre params = job.getParametre(); + const string workDir = params[WORKDIR]; + + // export input files on cluster + exportInputFiles(job); + + // build command file to submit the job + string cmdFile = buildCommandFile(job); + + // define name of log file (local) + string logFile = generateTemporaryFileName("vishnu-submitlog"); + + // define command to submit batch + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect -p 2; "; + subCommand += "vishnu_submit_job " + _hostname + " " + cmdFile; + string command = _protocol.getExecCommand(subCommand, + _hostname, + _username); + command += " > "; + command += logFile; + cerr << command.c_str() << endl; + status = system(command.c_str()); + if (status) + { + ifstream error_message(logFile.c_str()); + string mess; + string temp; + while(getline(error_message, temp)) + mess += temp; + error_message.close(); + throw EmulationException("Error of connection on remote host, error was: " + mess); + } + + // read id of submitted job in log file + string jobref; + ifstream idfile(logFile.c_str()); + string line; + while (idfile && line.compare(0, 13, "Job Id : ") != 0) + getline(idfile, line); + idfile.close(); + if (line.compare(0, 13, "Job Id : ") == 0) + jobref = line.substr(13); + if (jobref.size() == 0) + throw EmulationException("Error in the submission of the job on the remote host"); + + JobId id(this, jobref); + return id; + } + + + void BatchManager_eVishnu::exportInputFiles(const Job& job) + { + int status; + Parametre params = job.getParametre(); + const Versatile & V = params[INFILE]; + Versatile::const_iterator Vit; + + // create remote directories + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect -p 2; "; + subCommand += "vishnu_create_dir -p " + _hostname + ":" + params[WORKDIR].str() + "/logs"; + string command = _protocol.getExecCommand(subCommand, + _hostname, + _username); + cerr << command.c_str() << endl; + status = system(command.c_str()); + if (status != 0) + throw EmulationException("Can't create remote directories"); + + // copy executable + string executeFile = params[EXECUTABLE]; + if (executeFile.size() != 0) { + + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect -p 2; "; + subCommand += "vishnu_copy_file " + executeFile + " " + _hostname + ":" + params[WORKDIR].str() + "/"; + string command = _protocol.getExecCommand(subCommand, + _hostname, + _username); + cerr << command.c_str() << endl; + status = system(command.c_str()); + if (status != 0) + throw EmulationException("Can't copy executable"); + } + + // copy filesToExportList + for (Vit=V.begin(); Vit!=V.end(); Vit++) { + CoupleType cpt = *static_cast< CoupleType * >(*Vit); + Couple inputFile = cpt; + + // Get absolute paths + char * buf = getcwd(NULL, 0); + string cwd = buf; + free(buf); + + string absremote = (Utils::isAbsolutePath(inputFile.getRemote()))? + inputFile.getRemote() : + params[WORKDIR].str() + "/" + inputFile.getRemote(); + string abslocal = (Utils::isAbsolutePath(inputFile.getLocal()))? + inputFile.getLocal() : + cwd + "/" + inputFile.getLocal(); + + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect -p 2; "; + subCommand += "vishnu_copy_file " + abslocal + " " + _hostname + ":" + absremote; + string command = _protocol.getExecCommand(subCommand, + _hostname, + _username); + cerr << command.c_str() << endl; + status = system(command.c_str()); + if (status != 0) + throw EmulationException("Can't copy file"); + } + } + + /** + * Create Vishnu command file and copy it on the server. + * Return the name of the remote file. + */ + string BatchManager_eVishnu::buildCommandFile(const Job & job) + { + Parametre params = job.getParametre(); + + // Job Parameters + string workDir = ""; + string fileToExecute = ""; + string queue = ""; + + // Mandatory parameters + if (params.find(WORKDIR) != params.end()) + workDir = params[WORKDIR].str(); + else + throw EmulationException("params[WORKDIR] is not defined. Please define it, cannot submit this job."); + if (params.find(EXECUTABLE) != params.end()) + fileToExecute = params[EXECUTABLE].str(); + else + throw EmulationException("params[EXECUTABLE] is not defined. Please define it, cannot submit this job."); + + string::size_type p1 = fileToExecute.find_last_of("/"); + string::size_type p2 = fileToExecute.find_last_of("."); + string rootNameToExecute = fileToExecute.substr(p1+1,p2-p1-1); + string fileNameToExecute = fileToExecute.substr(p1+1); + + // Create batch submit file + ofstream tempOutputFile; + string tmpFileName = createAndOpenTemporaryFile("vishnu-script", tempOutputFile); + + tempOutputFile << "#!/bin/sh" << endl; + tempOutputFile << "#% vishnu_output=" << workDir << "/logs/output.log." << rootNameToExecute << endl; + tempOutputFile << "#% vishnu_error=" << workDir << "/logs/error.log." << rootNameToExecute << endl; + + if (params.find(NAME) != params.end()) + tempOutputFile << "#% vishnu_job_name=\"" << params[NAME] << "\"" << endl; + + // Optional parameters + int nbproc = 1; + if (params.find(NBPROC) != params.end()) + nbproc = params[NBPROC]; + + //int nodes_requested = (nbproc + _nb_proc_per_node -1) / _nb_proc_per_node; + //tempOutputFile << "#SBATCH --nodes=" << nodes_requested << endl; + //tempOutputFile << "#SBATCH --ntasks-per-node=" << _nb_proc_per_node << endl; + + if (params.find(MAXWALLTIME) != params.end()) { + long totalMinutes = params[MAXWALLTIME]; + long h = totalMinutes / 60; + long m = totalMinutes - h * 60; + tempOutputFile << "#% vishnu_wallclocklimit=" << h << ":"; + if (m < 10) + tempOutputFile << "0"; + tempOutputFile << m << ":00" << endl; + } + //if (params.find(MAXRAMSIZE) != params.end()) + // tempOutputFile << "#SBATCH --mem=" << params[MAXRAMSIZE] << endl; + if (params.find(QUEUE) != params.end()) + tempOutputFile << "#% vishnu_queue=" << params[QUEUE] << endl; + + // Define environment for the job + Environnement env = job.getEnvironnement(); + for (Environnement::const_iterator iter = env.begin() ; iter != env.end() ; ++iter) { + tempOutputFile << "export " << iter->first << "=" << iter->second << endl; + } + + // generate nodes file + //tempOutputFile << "LIBBATCH_NODEFILE=`mktemp nodefile-XXXXXXXXXX`" << endl; + //tempOutputFile << "srun hostname > $LIBBATCH_NODEFILE" << endl; + //tempOutputFile << "export LIBBATCH_NODEFILE" << endl; + + // Launch the executable + tempOutputFile << "cd " << workDir << endl; + tempOutputFile << "./" + fileNameToExecute << endl; + + tempOutputFile.flush(); + tempOutputFile.close(); + + cerr << "Batch script file generated is: " << tmpFileName << endl; + return tmpFileName; + } + + void BatchManager_eVishnu::deleteJob(const JobId & jobid) + { + string vishnuJobId; + istringstream iss(jobid.getReference()); + getline(iss, vishnuJobId, ':'); + + // define command to delete job + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect -p 2; "; + subCommand += "vishnu_cancel_job " + _hostname + " " + vishnuJobId; + string command = _protocol.getExecCommand(subCommand, _hostname, _username); + cerr << command.c_str() << endl; + + int status = system(command.c_str()); + if (status) + throw EmulationException("Can't delete job " + jobid.getReference()); + + cerr << "job " << jobid.getReference() << " killed" << endl; + } + + void BatchManager_eVishnu::holdJob(const JobId & jobid) + { + throw NotYetImplementedException("BatchManager_eVishnu::holdJob"); + } + + void BatchManager_eVishnu::releaseJob(const JobId & jobid) + { + throw NotYetImplementedException("BatchManager_eVishnu::releaseJob"); + } + + void BatchManager_eVishnu::alterJob(const JobId & jobid, const Parametre & param, const Environnement & env) + { + throw NotYetImplementedException("BatchManager_eVishnu::alterJob"); + } + + void BatchManager_eVishnu::alterJob(const JobId & jobid, const Parametre & param) + { + throw NotYetImplementedException("BatchManager_eVishnu::alterJob"); + } + + void BatchManager_eVishnu::alterJob(const JobId & jobid, const Environnement & env) + { + throw NotYetImplementedException("BatchManager_eVishnu::alterJob"); + } + + JobInfo BatchManager_eVishnu::queryJob(const JobId & jobid) + { + // define name of log file (local) + string logFile = generateTemporaryFileName("vishnu-querylog-" + jobid.getReference()); + + string vishnuJobId; + istringstream iss(jobid.getReference()); + getline(iss, vishnuJobId, ':'); + + // define command to query batch + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect -p 2; "; + subCommand += "vishnu_get_job_info " + _hostname + " " + vishnuJobId; + string command = _protocol.getExecCommand(subCommand, _hostname, _username); + command += " > "; + command += logFile; + cerr << command.c_str() << endl; + int status = system(command.c_str()); + if (status != 0) + throw EmulationException("Can't query job " + jobid.getReference()); + + JobInfo_eVishnu jobinfo = JobInfo_eVishnu(jobid.getReference(), logFile); + return jobinfo; + } + + const JobId BatchManager_eVishnu::addJob(const Job & job, const string reference) + { + return JobId(this, reference); + } + + void BatchManager_eVishnu::importOutputFiles(const Job & job, const std::string directory) + { + Parametre params = job.getParametre(); + const Versatile & V = params[OUTFILE]; + Versatile::const_iterator Vit; + + // Create local result directory + char * buf = getcwd(NULL, 0); + string cwd = buf; + free(buf); + string absdir = (Utils::isAbsolutePath(directory))? directory : cwd + "/" + directory; + int status = CommunicationProtocol::getInstance(SH).makeDirectory(absdir, "", ""); + if (status) { + string mess("Directory creation failed. Status is :"); + ostringstream status_str; + status_str << status; + mess += status_str.str(); + cerr << mess << endl; + } + + for (Vit=V.begin(); Vit!=V.end(); Vit++) { + CoupleType cpt = *static_cast< CoupleType * >(*Vit); + Couple outputFile = cpt; + + // Get absolute paths + string absremote = (Utils::isAbsolutePath(outputFile.getRemote()))? + outputFile.getRemote() : + params[WORKDIR].str() + "/" + outputFile.getRemote(); + string abslocal = (Utils::isAbsolutePath(outputFile.getLocal()))? + outputFile.getLocal() : + absdir + "/" + outputFile.getLocal(); + + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect -p 2; "; + subCommand += "vishnu_copy_file " + _hostname + ":" + absremote + " " + abslocal; + string command = _protocol.getExecCommand(subCommand, + _hostname, + _username); + cerr << command.c_str() << endl; + status = system(command.c_str()); + if (status != 0) + throw EmulationException("Can't copy file"); + } + + // Copy logs + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect -p 2; "; + subCommand += "vishnu_copy_file -r " +_hostname + ":" + params[WORKDIR].str() + "/logs" + " " + absdir; + string command = _protocol.getExecCommand(subCommand, + _hostname, + _username); + cerr << command.c_str() << endl; + status = system(command.c_str()); + if (status != 0) + throw EmulationException("Can't copy logs"); + } + +} diff --git a/src/Vishnu/Batch_BatchManager_eVishnu.hxx b/src/Vishnu/Batch_BatchManager_eVishnu.hxx new file mode 100644 index 0000000..ec7106f --- /dev/null +++ b/src/Vishnu/Batch_BatchManager_eVishnu.hxx @@ -0,0 +1,77 @@ +// Copyright (C) 2007-2011 CEA/DEN, EDF R&D, OPEN CASCADE +// +// Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +/* + * Batch_BatchManager_eVishnu.hxx : + * + * Created on: 24 june 2011 + * Author : Renaud BARATE - EDF R&D + */ + +#ifndef _BATCHMANAGER_EVISHNU_H_ +#define _BATCHMANAGER_EVISHNU_H_ + +#include + +#include +#include +#include +#include +#include + +namespace Batch { + + class BATCH_EXPORT BatchManager_eVishnu : public BatchManager_eClient + { + public: + + BatchManager_eVishnu(const FactBatchManager * parent, + const char * host = "localhost", + const char * username = "", + CommunicationProtocolType protocolType = SSH, + const char * mpiImpl = "nompi", + int nb_proc_per_node = 1); + virtual ~BatchManager_eVishnu(); + + // Methods to control jobs + virtual const JobId submitJob(const Job & job); + virtual void deleteJob(const JobId & jobid); + virtual void holdJob(const JobId & jobid); + virtual void releaseJob(const JobId & jobid); + virtual void alterJob(const JobId & jobid, const Parametre & param, const Environnement & env); + virtual void alterJob(const JobId & jobid, const Parametre & param); + virtual void alterJob(const JobId & jobid, const Environnement & env); + virtual JobInfo queryJob(const JobId & jobid); + virtual const JobId addJob(const Job & job, const std::string reference); + virtual void importOutputFiles(const Job & job, const std::string directory); + + protected: + + std::string buildCommandFile(const Job & job); + void exportInputFiles(const Job & job); + + int _nb_proc_per_node; + + }; + +} + +#endif diff --git a/src/Vishnu/Batch_FactBatchManager_eVishnu.cxx b/src/Vishnu/Batch_FactBatchManager_eVishnu.cxx new file mode 100644 index 0000000..c0ede38 --- /dev/null +++ b/src/Vishnu/Batch_FactBatchManager_eVishnu.cxx @@ -0,0 +1,62 @@ +// Copyright (C) 2007-2011 CEA/DEN, EDF R&D, OPEN CASCADE +// +// Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +/* + * Batch_FactBatchManager_eVishnu.cxx : + * + * Created on: 24 june 2011 + * Author : Renaud BARATE - EDF R&D + */ + +#include + +#include "Batch_BatchManager_eVishnu.hxx" +#include "Batch_FactBatchManager_eVishnu.hxx" + +namespace Batch { + + static FactBatchManager_eVishnu sFBM_eVishnu; + + FactBatchManager_eVishnu::FactBatchManager_eVishnu() : FactBatchManager_eClient("eVISHNU") + { + } + + FactBatchManager_eVishnu::~FactBatchManager_eVishnu() + { + } + + BatchManager * FactBatchManager_eVishnu::operator() (const char * hostname) const + { + // MESSAGE("Building new BatchManager_eVishnu on host '" << hostname << "'"); + return new BatchManager_eVishnu(this, hostname); + } + + BatchManager_eClient * FactBatchManager_eVishnu::operator() (const char * hostname, + const char * username, + CommunicationProtocolType protocolType, + const char * mpiImpl, + int nb_proc_per_node) const + { + // MESSAGE("Building new BatchManager_eVishnu on host '" << hostname << "'"); + return new BatchManager_eVishnu(this, hostname, username, protocolType, mpiImpl, nb_proc_per_node); + } + +} diff --git a/src/Vishnu/Batch_FactBatchManager_eVishnu.hxx b/src/Vishnu/Batch_FactBatchManager_eVishnu.hxx new file mode 100644 index 0000000..07eee8e --- /dev/null +++ b/src/Vishnu/Batch_FactBatchManager_eVishnu.hxx @@ -0,0 +1,57 @@ +// Copyright (C) 2007-2011 CEA/DEN, EDF R&D, OPEN CASCADE +// +// Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +/* + * Batch_FactBatchManager_eVishnu.hxx : + * + * Created on: 24 june 2011 + * Author : Renaud BARATE - EDF R&D + */ + +#ifndef _FACTBATCHMANAGER_EVISHNU_H_ +#define _FACTBATCHMANAGER_EVISHNU_H_ + +#include +#include +#include +#include + +namespace Batch { + + class BATCH_EXPORT FactBatchManager_eVishnu : public FactBatchManager_eClient + { + public: + + FactBatchManager_eVishnu(); + virtual ~FactBatchManager_eVishnu(); + + virtual BatchManager * operator() (const char * hostname) const; + virtual BatchManager_eClient * operator() (const char * hostname, + const char * username, + CommunicationProtocolType protocolType, + const char * mpiImpl, + int nb_proc_per_node = 1) const; + + }; + +} + +#endif diff --git a/src/Vishnu/Batch_JobInfo_eVishnu.cxx b/src/Vishnu/Batch_JobInfo_eVishnu.cxx new file mode 100644 index 0000000..786103e --- /dev/null +++ b/src/Vishnu/Batch_JobInfo_eVishnu.cxx @@ -0,0 +1,98 @@ +// Copyright (C) 2007-2011 CEA/DEN, EDF R&D, OPEN CASCADE +// +// Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +/* + * Batch_JobInfo_eVishnu.cxx : + * + * Created on: 24 june 2011 + * Author : Renaud BARATE - EDF R&D + */ + +#include +#include +#include + +#include +#include + +#include "Batch_JobInfo_eVishnu.hxx" + +using namespace std; + +namespace Batch { + + JobInfo_eVishnu::JobInfo_eVishnu(const std::string & id, const std::string & logFile) + : JobInfo() + { + _param[ID] = id; + + // find the status in the log file + ifstream log(logFile.c_str()); + string status; + bool statusFound = false; + while (!statusFound && !log.eof()) { + string line; + getline(log, line); + size_t pos = line.find(':'); + if (pos != string::npos) { + string begline = line.substr(0, pos); + string keyword; + // Trim leading and trailing spaces of the string before ':' + size_t startpos = begline.find_first_not_of(" \t"); + size_t endpos = begline.find_last_not_of(" \t"); + if (startpos != string::npos && endpos != string::npos) + keyword = begline.substr(startpos, endpos-startpos+1); + + if (keyword == "Status") { + statusFound = true; + string endline = line.substr(pos + 1); + startpos = endline.find_first_not_of(" \t"); + endpos = endline.find_last_not_of(" \t"); + if (startpos != string::npos && endpos != string::npos) + status = endline.substr(startpos, endpos-startpos+1); + } + } + } + log.close(); + + if (status.size() == 0) { + // On some batch managers, the job is deleted as soon as it is finished, + // so we have to consider that an unknown job is a finished one, even if + // it is not always true. + _param[STATE] = FINISHED; + } else if (status == "QUEUED") { + _param[STATE] = QUEUED; + } else if (status == "RUNNING") { + _param[STATE] = RUNNING; + } else if (status == "TERMINATED") { + _param[STATE] = FINISHED; + } else if (status == "CANCELLED") { + _param[STATE] = FAILED; + } else { + throw RunTimeException("Unknown job state code: \"" + status + "\""); + } + } + + JobInfo_eVishnu::~JobInfo_eVishnu() + { + } + +} diff --git a/src/Vishnu/Batch_JobInfo_eVishnu.hxx b/src/Vishnu/Batch_JobInfo_eVishnu.hxx new file mode 100644 index 0000000..517e797 --- /dev/null +++ b/src/Vishnu/Batch_JobInfo_eVishnu.hxx @@ -0,0 +1,49 @@ +// Copyright (C) 2007-2011 CEA/DEN, EDF R&D, OPEN CASCADE +// +// Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +/* + * Batch_JobInfo_eVishnu.hxx : + * + * Created on: 24 june 2011 + * Author : Renaud BARATE - EDF R&D + */ + +#ifndef _JOBINFO_EVISHNU_H_ +#define _JOBINFO_EVISHNU_H_ + +#include + +#include + +namespace Batch { + + class JobInfo_eVishnu : public JobInfo + { + public: + + JobInfo_eVishnu(const std::string & id, const std::string & logFile); + virtual ~JobInfo_eVishnu(); + + }; + +} + +#endif diff --git a/src/Vishnu/CMakeLists.txt b/src/Vishnu/CMakeLists.txt new file mode 100644 index 0000000..02a7b8d --- /dev/null +++ b/src/Vishnu/CMakeLists.txt @@ -0,0 +1,32 @@ +# Copyright (C) 2007-2011 CEA/DEN, EDF R&D, OPEN CASCADE +# +# Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +# CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +SET(CLASS_LIST Vishnu/Batch_BatchManager_eVishnu + Vishnu/Batch_FactBatchManager_eVishnu + Vishnu/Batch_JobInfo_eVishnu + ) + +APPEND_CLASSES_TO_SRC_FILES(${CLASS_LIST}) + +IF (TEST_ENABLED) + add_subdirectory(Test) +ENDIF (TEST_ENABLED) diff --git a/src/Vishnu/Test/CMakeLists.txt b/src/Vishnu/Test/CMakeLists.txt new file mode 100644 index 0000000..b7606db --- /dev/null +++ b/src/Vishnu/Test/CMakeLists.txt @@ -0,0 +1,44 @@ +# Copyright (C) 2007-2011 CEA/DEN, EDF R&D, OPEN CASCADE +# +# Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +# CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +# Just copy the test scripts to the binary dir +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/seta.sh ${CMAKE_CURRENT_BINARY_DIR}/seta.sh COPYONLY) +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/setb.sh ${CMAKE_CURRENT_BINARY_DIR}/setb.sh COPYONLY) +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/test-script.sh ${CMAKE_CURRENT_BINARY_DIR}/test-script.sh COPYONLY) + +# set the include directories +include_directories(${CMAKE_SOURCE_DIR}/src/Core) +include_directories(${CMAKE_SOURCE_DIR}/src/Core/Test) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + +# Build the test programs and add the tests +add_executable(Test_eVishnu Test_eVishnu.cxx) +target_link_libraries(Test_eVishnu Batch SimpleParser) + +IF (HAS_SSH) + ADD_TEST(eVishnu_SSH Test_eVishnu SSH) +ENDIF (HAS_SSH) + +#IF (HAS_RSH) +# ADD_TEST(eVishnu_RSH Test_eVishnu RSH) +#ENDIF (HAS_RSH) diff --git a/src/Vishnu/Test/Test_eVishnu.cxx b/src/Vishnu/Test/Test_eVishnu.cxx new file mode 100644 index 0000000..4c191dc --- /dev/null +++ b/src/Vishnu/Test/Test_eVishnu.cxx @@ -0,0 +1,137 @@ +// Copyright (C) 2007-2011 CEA/DEN, EDF R&D, OPEN CASCADE +// +// Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +/* + * Test_eVishnu.cxx : + * + * Author : Renaud BARATE - EDF R&D + * Date : June 2011 + * + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace std; +using namespace Batch; + +int main(int argc, char** argv) +{ + cout << "*******************************************************************************************" << endl; + cout << "This program tests the batch submission based on Vishnu commands." << endl; + cout << "*******************************************************************************************" << endl; + + // eventually remove any previous result + remove("result.txt"); + + try { + // Parse the test configuration file + SimpleParser parser; + parser.parseTestConfigFile(); + const string & homedir = parser.getValue("TEST_EVISHNU_HOMEDIR"); + const string & host = parser.getValue("TEST_EVISHNU_HOST"); + const string & user = parser.getValue("TEST_EVISHNU_USER"); + int timeout = parser.getValueAsInt("TEST_EVISHNU_TIMEOUT"); + + // Define the job... + Job job; + // ... and its parameters ... + Parametre p; + p[EXECUTABLE] = "./test-script.sh"; + p[NAME] = "Test_eVISHNU"; + p[WORKDIR] = homedir + "/tmp/Batch"; + p[INFILE] = Couple("seta.sh", "seta.sh"); + p[INFILE] += Couple("setb.sh", "setb.sh"); + p[OUTFILE] = Couple("result.txt", "result.txt"); + p[NBPROC] = 1; + p[MAXWALLTIME] = 1; + p[MAXRAMSIZE] = 50; + job.setParametre(p); + // ... and its environment + Environnement e; + e["MYENVVAR"] = "MYVALUE"; + job.setEnvironnement(e); + cout << job << endl; + + // Get the catalog + BatchManagerCatalog& c = BatchManagerCatalog::getInstance(); + + // Create a BatchManager of type ePBS on localhost + FactBatchManager_eClient * fbm = (FactBatchManager_eClient *)(c("eVISHNU")); + BatchManager_eClient * bm = (*fbm)(host.c_str(), user.c_str(), SH); + + // Submit the job to the BatchManager + JobId jobid = bm->submitJob(job); + cout << jobid.__repr__() << endl; + + // Wait for the end of the job + string state = bm->waitForJobEnd(jobid, timeout); + + if (state == FINISHED) { + cout << "Job " << jobid.__repr__() << " is done" << endl; + bm->importOutputFiles(job, "resultdir/seconddirname"); + } else if (state == FAILED) { + cerr << "Job " << jobid.__repr__() << " finished in error" << endl; + bm->importOutputFiles(job, "resultdir/seconddirname"); + return 1; + } else { + cerr << "Timeout while executing job" << endl; + return 1; + } + + } catch (GenericException e) { + cerr << "Error: " << e << endl; + return 1; + } catch (ParserException e) { + cerr << "Parser error: " << e.what() << endl; + return 1; + } + + // test the result file + try { + SimpleParser resultParser; + resultParser.parse("resultdir/seconddirname/result.txt"); + cout << "Result:" << endl << resultParser; + const string & envvar = resultParser.getValue("MYENVVAR"); + int result = resultParser.getValueAsInt("c"); + if (envvar == "MYVALUE" && result == 12) { + cout << "OK, Expected result found." << endl; + return 0; + } else { + cerr << "Error, result is not the expected one (MYENVVAR = MYVALUE, c = 12)." << endl; + return 1; + } + } catch (ParserException e) { + cerr << "Parser error on result file: " << e.what() << endl; + return 1; + } +} diff --git a/src/Vishnu/Test/seta.sh b/src/Vishnu/Test/seta.sh new file mode 100644 index 0000000..42d1e38 --- /dev/null +++ b/src/Vishnu/Test/seta.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +a=4 diff --git a/src/Vishnu/Test/setb.sh b/src/Vishnu/Test/setb.sh new file mode 100644 index 0000000..8969060 --- /dev/null +++ b/src/Vishnu/Test/setb.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +b=3 diff --git a/src/Vishnu/Test/test-script.sh b/src/Vishnu/Test/test-script.sh new file mode 100755 index 0000000..1d56247 --- /dev/null +++ b/src/Vishnu/Test/test-script.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +. ./seta.sh +. ./setb.sh + +c=`expr $a "*" $b` + +echo "MYENVVAR = $MYENVVAR" > result.txt +echo "c = $c" >> result.txt -- 2.30.2