From 2ef26799d7ab5c02c708822d404f2bc50aeab8e2 Mon Sep 17 00:00:00 2001 From: barate Date: Fri, 21 May 2010 08:38:02 +0000 Subject: [PATCH] Changed memory unit to megabytes for batch mangers ePBS and eLSF. Added method to create the local directory to import the output files. --- Batch_config.h.in | 5 +++- CMakeLists.txt | 2 +- CMakeModules/FindLocal.cmake | 3 ++- src/Core/Batch_BatchManager_eClient.cxx | 29 ++++++++++++++-------- src/Core/Batch_CommunicationProtocol.cxx | 15 +++++++++++ src/Core/Batch_CommunicationProtocol.hxx | 6 +++++ src/Core/Batch_CommunicationProtocolSH.cxx | 12 ++++++++- src/Core/Batch_CommunicationProtocolSH.hxx | 4 ++- src/LSF/Batch_BatchManager_eLSF.cxx | 2 +- src/PBS/Batch_BatchManager_ePBS.cxx | 2 +- src/PBS/Test/Test_ePBS.cxx | 6 ++--- 11 files changed, 65 insertions(+), 21 deletions(-) diff --git a/Batch_config.h.in b/Batch_config.h.in index 4918f2f..daa346f 100644 --- a/Batch_config.h.in +++ b/Batch_config.h.in @@ -27,7 +27,7 @@ #cmakedefine MSVC #cmakedefine MINGW -/* SH tools (sh, cp, rm) found on the system */ +/* SH tools (sh, cp, rm, mkdir) found on the system */ #cmakedefine HAS_SH /* A path to a sh-like command */ @@ -39,6 +39,9 @@ /* A path to a cp-like command */ #cmakedefine CP_COMMAND "@CP_COMMAND@" +/* A path to a mkdir-like command */ +#cmakedefine MKDIR_COMMAND "@MKDIR_COMMAND@" + /* RSH tools (rsh, rcp) found on the system */ #cmakedefine HAS_RSH diff --git a/CMakeLists.txt b/CMakeLists.txt index 5872eb4..db519d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,7 +68,7 @@ SET(CPACK_GENERATOR TGZ) SET(CPACK_SOURCE_GENERATOR TGZ ZIP) SET(CPACK_PACKAGE_VERSION_MAJOR 1) SET(CPACK_PACKAGE_VERSION_MINOR 1) -SET(CPACK_PACKAGE_VERSION_PATCH 0rc2) +SET(CPACK_PACKAGE_VERSION_PATCH 0rc3) SET(CPACK_SOURCE_IGNORE_FILES /\\\\.;/CVS/;~) SET(CPACK_SOURCE_PACKAGE_FILE_NAME ${PROJECT_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}) diff --git a/CMakeModules/FindLocal.cmake b/CMakeModules/FindLocal.cmake index 7b30634..0e8c7df 100644 --- a/CMakeModules/FindLocal.cmake +++ b/CMakeModules/FindLocal.cmake @@ -55,11 +55,12 @@ MESSAGE(STATUS "Looking for commands needed for local submission...") FIND_LOCAL_COMMAND_WIN32_DEF(RM_COMMAND rm del) FIND_LOCAL_COMMAND(SH_COMMAND sh cmd.exe) FIND_LOCAL_COMMAND_WIN32_DEF(CP_COMMAND cp copy) +FIND_LOCAL_COMMAND_WIN32_DEF(MKDIR_COMMAND mkdir mkdir) FIND_LOCAL_COMMAND(RSH_COMMAND rsh rsh) FIND_LOCAL_COMMAND(RCP_COMMAND rcp rcp) FIND_LOCAL_COMMAND(SSH_COMMAND ssh plink) FIND_LOCAL_COMMAND(SCP_COMMAND scp pscp) -EVAL (HAS_SH SH_COMMAND AND CP_COMMAND AND RM_COMMAND) +EVAL (HAS_SH SH_COMMAND AND CP_COMMAND AND RM_COMMAND AND MKDIR_COMMAND) EVAL (HAS_RSH RSH_COMMAND AND RCP_COMMAND) EVAL (HAS_SSH SSH_COMMAND AND SCP_COMMAND) diff --git a/src/Core/Batch_BatchManager_eClient.cxx b/src/Core/Batch_BatchManager_eClient.cxx index ec5b8d8..1f92a24 100644 --- a/src/Core/Batch_BatchManager_eClient.cxx +++ b/src/Core/Batch_BatchManager_eClient.cxx @@ -83,10 +83,7 @@ namespace Batch { Versatile::iterator Vit; _username = string(params[USER]); - string subCommand = string("mkdir -p ") + string(params[TMPDIR]) + string("/logs"); - string command = _protocol.getExecCommand(subCommand, _hostname, _username); - cerr << command.c_str() << endl; - status = system(command.c_str()); + status = _protocol.makeDirectory(string(params[TMPDIR]) + "/logs", _hostname, _username); if(status) { std::ostringstream oss; oss << status; @@ -112,9 +109,9 @@ namespace Batch { #ifdef WIN32 // On Windows, we make the remote file executable afterward because // pscp does not preserve access permissions on files - subCommand = string("chmod u+x ") + string(params[TMPDIR]) + "/" + - string(params[EXECUTABLE]); - command = _protocol.getExecCommand(subCommand, _hostname, _username); + string subCommand = string("chmod u+x ") + string(params[TMPDIR]) + "/" + + string(params[EXECUTABLE]); + string command = _protocol.getExecCommand(subCommand, _hostname, _username); cerr << command.c_str() << endl; status = system(command.c_str()); if(status) { @@ -152,11 +149,21 @@ namespace Batch { Versatile::iterator Vit; _username = string(params[USER]); + // Create local result directory + int status = CommunicationProtocol::getInstance(SH).makeDirectory(directory, "", ""); + if (status) { + string mess("Directory creation failed. Status is :"); + ostringstream status_str; + status_str << status; + mess += status_str.str(); + cerr << mess << endl; + } + for(Vit=V.begin(); Vit!=V.end(); Vit++) { CoupleType cpt = *static_cast< CoupleType * >(*Vit); Couple outputFile = cpt; - int status = _protocol.copyFile(outputFile.getRemote(), _hostname, _username, - directory, "", ""); + status = _protocol.copyFile(outputFile.getRemote(), _hostname, _username, + directory, "", ""); if (status) { // Try to get what we can (logs files) // throw BatchException("Error of connection on remote host"); @@ -169,8 +176,8 @@ namespace Batch { } // Copy logs - int status = _protocol.copyFile(string(params[TMPDIR]) + string("/logs"), _hostname, _username, - directory, "", ""); + status = _protocol.copyFile(string(params[TMPDIR]) + string("/logs"), _hostname, _username, + directory, "", ""); if (status) { std::string mess("Copy logs directory failed ! status is :"); ostringstream status_str; diff --git a/src/Core/Batch_CommunicationProtocol.cxx b/src/Core/Batch_CommunicationProtocol.cxx index 8133aaa..c23c722 100644 --- a/src/Core/Batch_CommunicationProtocol.cxx +++ b/src/Core/Batch_CommunicationProtocol.cxx @@ -104,6 +104,11 @@ namespace Batch { return string("rm ") + path; } + string CommunicationProtocol::getMakeDirectorySubCommand(const string & path) const + { + return string("mkdir -p ") + path; + } + int CommunicationProtocol::removeFile(const std::string & path, const std::string & host, const std::string & user) const @@ -114,6 +119,16 @@ namespace Batch { return status; } + int CommunicationProtocol::makeDirectory(const std::string & path, + const std::string & host, + const std::string & user) const + { + string command = getExecCommand(getMakeDirectorySubCommand(path), host, user); + cout << command.c_str() << endl; + int status = system(command.c_str()); + return status; + } + string CommunicationProtocol::commandStringFromArgs(const vector & commandArgs) const { string commandStr; diff --git a/src/Core/Batch_CommunicationProtocol.hxx b/src/Core/Batch_CommunicationProtocol.hxx index f8b0916..c21ca8a 100644 --- a/src/Core/Batch_CommunicationProtocol.hxx +++ b/src/Core/Batch_CommunicationProtocol.hxx @@ -64,6 +64,10 @@ namespace Batch { const std::string & host, const std::string & user) const; + virtual int makeDirectory(const std::string & path, + const std::string & host, + const std::string & user) const; + protected: virtual std::vector getCopyCommandArgs(const std::string & sourcePath, @@ -75,6 +79,8 @@ namespace Batch { virtual std::string getRemoveSubCommand(const std::string & path) const; + virtual std::string getMakeDirectorySubCommand(const std::string & path) const; + std::string commandStringFromArgs(const std::vector & args) const; }; diff --git a/src/Core/Batch_CommunicationProtocolSH.cxx b/src/Core/Batch_CommunicationProtocolSH.cxx index 024fd0a..d7d5085 100644 --- a/src/Core/Batch_CommunicationProtocolSH.cxx +++ b/src/Core/Batch_CommunicationProtocolSH.cxx @@ -85,7 +85,17 @@ namespace Batch { string CommunicationProtocolSH::getRemoveSubCommand(const string & path) const { - return string(RM_COMMAND) + " " + path; + return string(RM_COMMAND) + " " + fixPath(path); + } + + string CommunicationProtocolSH::getMakeDirectorySubCommand(const string & path) const + { + string subCommand = MKDIR_COMMAND; +#ifndef WIN32 + subCommand += " -p"; +#endif + subCommand += " " + fixPath(path); + return subCommand; } } diff --git a/src/Core/Batch_CommunicationProtocolSH.hxx b/src/Core/Batch_CommunicationProtocolSH.hxx index a95d9ef..b696e70 100644 --- a/src/Core/Batch_CommunicationProtocolSH.hxx +++ b/src/Core/Batch_CommunicationProtocolSH.hxx @@ -51,9 +51,11 @@ namespace Batch { const std::string & destinationHost, const std::string & destinationUser) const; + protected: + std::string getRemoveSubCommand(const std::string & path) const; - protected: + std::string getMakeDirectorySubCommand(const std::string & path) const; std::string fixPath(const std::string & path) const; diff --git a/src/LSF/Batch_BatchManager_eLSF.cxx b/src/LSF/Batch_BatchManager_eLSF.cxx index 80d0346..f5fc22d 100644 --- a/src/LSF/Batch_BatchManager_eLSF.cxx +++ b/src/LSF/Batch_BatchManager_eLSF.cxx @@ -255,7 +255,7 @@ namespace Batch { if( edt > 0 ) tempOutputFile << "#BSUB -W " << getWallTime(edt) << endl ; if( mem > 0 ) - tempOutputFile << "#BSUB -M " << mem << endl ; + tempOutputFile << "#BSUB -M " << mem*1024 << endl ; tempOutputFile << "#BSUB -n " << nbproc << endl ; size_t pos = workDir.find("$HOME"); string baseDir; diff --git a/src/PBS/Batch_BatchManager_ePBS.cxx b/src/PBS/Batch_BatchManager_ePBS.cxx index e28d5f7..7512544 100644 --- a/src/PBS/Batch_BatchManager_ePBS.cxx +++ b/src/PBS/Batch_BatchManager_ePBS.cxx @@ -256,7 +256,7 @@ namespace Batch { if( edt > 0 ) tempOutputFile << "#PBS -l walltime=" << edt*60 << endl; if( mem > 0 ) - tempOutputFile << "#PBS -l mem=" << mem << "kb" << endl; + tempOutputFile << "#PBS -l mem=" << mem << "MB" << endl; tempOutputFile << "#PBS -o " << workDir << "/logs/output.log." << rootNameToExecute << endl; tempOutputFile << "#PBS -e " << workDir << "/logs/error.log." << rootNameToExecute << endl; diff --git a/src/PBS/Test/Test_ePBS.cxx b/src/PBS/Test/Test_ePBS.cxx index e6da684..c427f17 100644 --- a/src/PBS/Test/Test_ePBS.cxx +++ b/src/PBS/Test/Test_ePBS.cxx @@ -99,7 +99,7 @@ int main(int argc, char** argv) p[USER] = user; p[NBPROC] = 1; p[MAXWALLTIME] = 1; - p[MAXRAMSIZE] = 1000; + p[MAXRAMSIZE] = 1; p[HOMEDIR] = homedir; p[QUEUE] = queue; job.setParametre(p); @@ -125,7 +125,7 @@ int main(int argc, char** argv) if (state == FINISHED || state == FAILED) { cout << "Job " << jobid.__repr__() << " is done" << endl; - bm->importOutputFiles(job, "."); + bm->importOutputFiles(job, "resultdir/seconddirname"); } else { cerr << "Timeout while executing job" << endl; return 1; @@ -142,7 +142,7 @@ int main(int argc, char** argv) // test the result file try { SimpleParser resultParser; - resultParser.parse("result.txt"); + resultParser.parse("resultdir/seconddirname/result.txt"); cout << "Result:" << endl << resultParser; const string & envvar = resultParser.getValue("MYENVVAR"); int result = resultParser.getValueAsInt("c"); -- 2.39.2