From f8621378e3b8fb138e91cb0fe02f17f3c83f8956 Mon Sep 17 00:00:00 2001 From: barate Date: Fri, 18 Jan 2013 13:30:47 +0000 Subject: [PATCH] - Fixes for BatchManager_Local - Fixes for Windows --- src/CCC/BatchManager_CCC.cxx | 3 +- src/Core/BatchManager.cxx | 47 +++--- src/Core/CommunicationProtocol.cxx | 10 ++ src/Core/CommunicationProtocol.hxx | 6 + src/Core/CommunicationProtocolRSH.cxx | 5 + src/Core/CommunicationProtocolRSH.hxx | 8 + src/Core/CommunicationProtocolSH.cxx | 25 ++- src/Core/CommunicationProtocolSH.hxx | 6 +- src/Core/CommunicationProtocolSSH.cxx | 5 + src/Core/CommunicationProtocolSSH.hxx | 4 + src/Core/Defines.hxx | 8 +- src/Core/Test/batchtest.conf | 15 +- src/Core/Utils.cxx | 46 +++++- src/Core/Utils.hxx | 16 ++ src/Local/BatchManager_Local.cxx | 221 ++++++++++++++------------ src/Local/BatchManager_Local.hxx | 9 +- src/Local/Test/CMakeLists.txt | 8 +- src/Local/Test/Test_Local_RSH.cxx | 130 --------------- src/Local/Test/Test_Local_SSH.cxx | 133 ---------------- src/SGE/BatchManager_SGE.cxx | 3 +- 20 files changed, 266 insertions(+), 442 deletions(-) delete mode 100644 src/Local/Test/Test_Local_RSH.cxx delete mode 100644 src/Local/Test/Test_Local_SSH.cxx diff --git a/src/CCC/BatchManager_CCC.cxx b/src/CCC/BatchManager_CCC.cxx index 1703e5b..715a51f 100644 --- a/src/CCC/BatchManager_CCC.cxx +++ b/src/CCC/BatchManager_CCC.cxx @@ -36,7 +36,6 @@ #include #include #include -#include #include #include @@ -256,7 +255,7 @@ namespace Batch { tempOutputFile.flush(); tempOutputFile.close(); - BATCH_CHMOD(TmpFileName.c_str(), 0x1ED); + Utils::chmod(TmpFileName.c_str(), 0x1ED); cerr << "Batch script file generated is: " << TmpFileName.c_str() << endl; int status = _protocol.copyFile(TmpFileName, "", "", diff --git a/src/Core/BatchManager.cxx b/src/Core/BatchManager.cxx index 233ee46..c369f56 100644 --- a/src/Core/BatchManager.cxx +++ b/src/Core/BatchManager.cxx @@ -31,27 +31,16 @@ #include #include #include -#ifdef WIN32 -# include -#else -#include -# include -#endif #include "Constants.hxx" #include "Job.hxx" #include "JobId.hxx" #include "JobInfo.hxx" -#include "InvalidArgumentException.hxx" #include "FactBatchManager.hxx" #include "BatchManager.hxx" #include "Utils.hxx" #include "NotYetImplementedException.hxx" -#ifdef WIN32 -#define sleep(seconds) Sleep((seconds)*1000) -#endif - using namespace std; namespace Batch { @@ -162,7 +151,7 @@ namespace Batch { cout << "State is \"" << state << "\""; while (!timeoutReached && state != FINISHED && state != FAILED) { cout << ", sleeping " << sleeptime << "s..." << endl; - sleep(sleeptime); + Utils::sleep(sleeptime); time += sleeptime; timeoutReached = (testTimeout && time >= timeout); sleeptime *= 2; @@ -209,22 +198,24 @@ namespace Batch { } #ifdef WIN32 - // On Windows, we make the remote file executable afterward because - // pscp does not preserve access permissions on files - - string remoteExec = string(params[EXECUTABLE]); - remoteExec = remoteExec.substr(remoteExec.rfind("\\") + 1, remoteExec.length()); - remoteExec = string(params[WORKDIR]) + "/" + executable; - - string subCommand = string("chmod u+x ") + remoteExec; - string command = _protocol.getExecCommand(subCommand, _hostname, _username); - cerr << command.c_str() << endl; - status = system(command.c_str()); - if (status) { - std::ostringstream oss; - oss << "Cannot change permissions of file " << remoteExec << " on host " << _hostname; - oss << ". Return status is " << status; - throw RunTimeException(oss.str()); + if (_protocol.getType() != SH) { + // On Windows, we make the remote file executable afterward because + // pscp does not preserve access permissions on files + + string remoteExec = string(params[EXECUTABLE]); + remoteExec = remoteExec.substr(remoteExec.rfind("/") + 1, remoteExec.length()); + remoteExec = string(params[WORKDIR]) + "/" + remoteExec; + + string subCommand = string("chmod u+x ") + remoteExec; + string command = _protocol.getExecCommand(subCommand, _hostname, _username); + cerr << command.c_str() << endl; + status = system(command.c_str()); + if (status) { + std::ostringstream oss; + oss << "Cannot change permissions of file " << remoteExec << " on host " << _hostname; + oss << ". Return status is " << status; + throw RunTimeException(oss.str()); + } } #endif } diff --git a/src/Core/CommunicationProtocol.cxx b/src/Core/CommunicationProtocol.cxx index 416b1ee..b365bba 100644 --- a/src/Core/CommunicationProtocol.cxx +++ b/src/Core/CommunicationProtocol.cxx @@ -49,6 +49,11 @@ using namespace std; namespace Batch { + CommunicationProtocol::CommunicationProtocol(CommunicationProtocolType type) + : _type(type) + { + } + CommunicationProtocol::~CommunicationProtocol() { } @@ -164,4 +169,9 @@ namespace Batch { return commandStr; } + CommunicationProtocolType CommunicationProtocol::getType() const + { + return _type; + } + } diff --git a/src/Core/CommunicationProtocol.hxx b/src/Core/CommunicationProtocol.hxx index afd508e..43c31fa 100644 --- a/src/Core/CommunicationProtocol.hxx +++ b/src/Core/CommunicationProtocol.hxx @@ -68,8 +68,12 @@ namespace Batch { const std::string & host, const std::string & user) const; + CommunicationProtocolType getType() const; + protected: + CommunicationProtocol(CommunicationProtocolType type = SH); + virtual std::vector getCopyCommandArgs(const std::string & sourcePath, const std::string & sourceHost, const std::string & sourceUser, @@ -83,6 +87,8 @@ namespace Batch { std::string commandStringFromArgs(const std::vector & args) const; + CommunicationProtocolType _type; + }; } diff --git a/src/Core/CommunicationProtocolRSH.cxx b/src/Core/CommunicationProtocolRSH.cxx index bb7a308..59caf2d 100644 --- a/src/Core/CommunicationProtocolRSH.cxx +++ b/src/Core/CommunicationProtocolRSH.cxx @@ -41,6 +41,11 @@ using namespace std; namespace Batch { + CommunicationProtocolRSH::CommunicationProtocolRSH() + : CommunicationProtocol(RSH) + { + } + vector CommunicationProtocolRSH::getExecCommandArgs(const string & subCommand, const string & host, const string & user) const diff --git a/src/Core/CommunicationProtocolRSH.hxx b/src/Core/CommunicationProtocolRSH.hxx index 6a32d08..c62fc9a 100644 --- a/src/Core/CommunicationProtocolRSH.hxx +++ b/src/Core/CommunicationProtocolRSH.hxx @@ -39,6 +39,8 @@ namespace Batch { class BATCH_EXPORT CommunicationProtocolRSH : public CommunicationProtocol { + friend class CommunicationProtocol; + public: std::vector getExecCommandArgs(const std::string & subCommand, const std::string & host, @@ -62,12 +64,18 @@ namespace Batch { const std::string & destinationHost, const std::string & destinationUser) const; +#endif + protected: +#ifdef WIN32 + char getDriveLetter(const std::string & path, std::string * pathWithoutDrive = NULL) const; #endif + CommunicationProtocolRSH(); + }; } diff --git a/src/Core/CommunicationProtocolSH.cxx b/src/Core/CommunicationProtocolSH.cxx index 546e97c..89968d9 100644 --- a/src/Core/CommunicationProtocolSH.cxx +++ b/src/Core/CommunicationProtocolSH.cxx @@ -29,22 +29,15 @@ #include #include "CommunicationProtocolSH.hxx" +#include "Utils.hxx" using namespace std; namespace Batch { - // Simple method to fix path strings depending on the platform. On Windows, it will replace - // forward slashes '/' by backslashes '\'. On Unix, the path is just copied without change. - string CommunicationProtocolSH::fixPath(const string & path) const + CommunicationProtocolSH::CommunicationProtocolSH() + : CommunicationProtocol(SH) { - string fixedPath = path; - #ifdef WIN32 - for (unsigned int i=0 ; i CommunicationProtocolSH::getExecCommandArgs(const string & subCommand, @@ -53,7 +46,7 @@ namespace Batch { { vector cmd; - cmd.push_back(fixPath(SH_COMMAND)); + cmd.push_back(Utils::fixPath(SH_COMMAND)); #ifdef WIN32 cmd.push_back("/c"); @@ -61,7 +54,7 @@ namespace Batch { cmd.push_back("-c"); #endif - cmd.push_back(fixPath(subCommand)); + cmd.push_back(Utils::fixPath(subCommand)); return cmd; } @@ -78,14 +71,14 @@ namespace Batch { #ifndef WIN32 cmd.push_back("-r"); #endif - cmd.push_back(fixPath(sourcePath)); - cmd.push_back(fixPath(destinationPath)); + cmd.push_back(Utils::fixPath(sourcePath)); + cmd.push_back(Utils::fixPath(destinationPath)); return cmd; } string CommunicationProtocolSH::getRemoveSubCommand(const string & path) const { - return string(RM_COMMAND) + " " + fixPath(path); + return string(RM_COMMAND) + " " + Utils::fixPath(path); } string CommunicationProtocolSH::getMakeDirectorySubCommand(const string & path) const @@ -94,7 +87,7 @@ namespace Batch { #ifndef WIN32 subCommand += " -p"; #endif - subCommand += " " + fixPath(path); + subCommand += " " + Utils::fixPath(path); return subCommand; } diff --git a/src/Core/CommunicationProtocolSH.hxx b/src/Core/CommunicationProtocolSH.hxx index 778d5f9..b6b9a78 100644 --- a/src/Core/CommunicationProtocolSH.hxx +++ b/src/Core/CommunicationProtocolSH.hxx @@ -39,6 +39,8 @@ namespace Batch { class BATCH_EXPORT CommunicationProtocolSH : public CommunicationProtocol { + friend class CommunicationProtocol; + public: std::vector getExecCommandArgs(const std::string & subCommand, const std::string & host, @@ -53,12 +55,12 @@ namespace Batch { protected: + CommunicationProtocolSH(); + std::string getRemoveSubCommand(const std::string & path) const; std::string getMakeDirectorySubCommand(const std::string & path) const; - std::string fixPath(const std::string & path) const; - }; } diff --git a/src/Core/CommunicationProtocolSSH.cxx b/src/Core/CommunicationProtocolSSH.cxx index 3a40db1..b6883a4 100644 --- a/src/Core/CommunicationProtocolSSH.cxx +++ b/src/Core/CommunicationProtocolSSH.cxx @@ -34,6 +34,11 @@ using namespace std; namespace Batch { + CommunicationProtocolSSH::CommunicationProtocolSSH() + : CommunicationProtocol(SSH) + { + } + vector CommunicationProtocolSSH::getExecCommandArgs(const string & subCommand, const string & host, const string & user) const diff --git a/src/Core/CommunicationProtocolSSH.hxx b/src/Core/CommunicationProtocolSSH.hxx index 54145ce..3aadcce 100644 --- a/src/Core/CommunicationProtocolSSH.hxx +++ b/src/Core/CommunicationProtocolSSH.hxx @@ -39,6 +39,8 @@ namespace Batch { class BATCH_EXPORT CommunicationProtocolSSH : public CommunicationProtocol { + friend class CommunicationProtocol; + public: std::vector getExecCommandArgs(const std::string & subCommand, const std::string & host, @@ -53,6 +55,8 @@ namespace Batch { protected: + CommunicationProtocolSSH(); + }; } diff --git a/src/Core/Defines.hxx b/src/Core/Defines.hxx index 1a22c8d..0cecc1a 100644 --- a/src/Core/Defines.hxx +++ b/src/Core/Defines.hxx @@ -27,7 +27,7 @@ #define _BATCH_Defines_HXX_ #ifdef WIN32 -# if defined EXPORTS +# if defined batch_EXPORTS # define BATCH_EXPORT __declspec( dllexport ) # else # define BATCH_EXPORT __declspec( dllimport ) @@ -36,10 +36,4 @@ # define BATCH_EXPORT #endif -#ifdef WIN32 -#define BATCH_CHMOD(name, mode) _chmod(name, mode) -#else -#define BATCH_CHMOD(name, mode) chmod(name, mode) -#endif - #endif diff --git a/src/Core/Test/batchtest.conf b/src/Core/Test/batchtest.conf index 9deb2de..7ff4583 100644 --- a/src/Core/Test/batchtest.conf +++ b/src/Core/Test/batchtest.conf @@ -9,17 +9,10 @@ TEST_LOCAL_SH_WORK_DIR = "/tmp" # Work directory for local SH Batc TEST_LOCAL_SH_TIMEOUT = 2 # Execution timeout (in seconds) for local SH Batch test TEST_LOCAL_SH_FINALIZATION_TIME = 2 # Finalization time (in seconds) for local SH Batch test -TEST_LOCAL_RSH_EXECUTION_HOST = "localhost" # Execution host for RSH Batch test -TEST_LOCAL_RSH_USER = "username" # User name on the execution host for RSH Batch test -TEST_LOCAL_RSH_WORK_DIR = "/tmp" # Work directory for RSH Batch test -TEST_LOCAL_RSH_TIMEOUT = 10 # Execution timeout (in seconds) for local RSH Batch test -TEST_LOCAL_RSH_FINALIZATION_TIME = 5 # Finalization time (in seconds) for local RSH Batch test - -TEST_LOCAL_SSH_EXECUTION_HOST = "localhost" # Execution host for SSH Batch test -TEST_LOCAL_SSH_USER = "username" # User name on the execution host for SSH Batch test -TEST_LOCAL_SSH_WORK_DIR = "/tmp" # Work directory for SSH Batch test -TEST_LOCAL_SSH_TIMEOUT = 10 # Execution timeout (in seconds) for local SSH Batch test -TEST_LOCAL_SSH_FINALIZATION_TIME = 5 # Finalization time (in seconds) for local SSH Batch test +TEST_LOCAL_HOST = "localhost" # Execution host for LOCAL Batch test +TEST_LOCAL_USER = "username" # User name on the execution host for LOCAL Batch test +TEST_LOCAL_WORKDIR = "/tmp" # Work directory for LOCAL Batch test +TEST_LOCAL_TIMEOUT = 10 # Execution timeout (in seconds) for LOCAL Batch test TEST_EPBS_HOST = "localhost" # PBS server host TEST_EPBS_USER = "username" # Login for the PBS server diff --git a/src/Core/Utils.cxx b/src/Core/Utils.cxx index a75a90e..623bb60 100644 --- a/src/Core/Utils.cxx +++ b/src/Core/Utils.cxx @@ -28,10 +28,18 @@ #include #include -#include #include #include +#ifdef WIN32 +#include +#include +#include +#else +#include +#include +#endif + #include #include "Utils.hxx" #include "RunTimeException.hxx" @@ -70,6 +78,11 @@ bool Utils::isAbsolutePath(const string & path) { if (path.size() == 0) return false; +#ifdef WIN32 + // On Windows, absolute paths may begin with something like "C:" + if (path[1] == ':') + return true; +#endif return path[0] == '/'; } @@ -95,7 +108,7 @@ string Utils::createAndOpenTemporaryFile(const string & prefix, ofstream & outpu do { sprintf(randstr, "%06d", rand() % 1000000); fileName.replace(fileName.size()-6, 6, randstr); - } while (EXISTS(fileName.c_str())); + } while (_access_s(fileName.c_str(), 0) == 0); // Open the file as binary to avoid problems with Windows newlines outputStream.open(fileName.c_str(), ios_base::binary | ios_base::out); @@ -125,4 +138,33 @@ string Utils::createAndOpenTemporaryFile(const string & prefix, ofstream & outpu return fileName; } +int Utils::chmod(const char *path, int mode) +{ +#ifdef WIN32 + return _chmod(path, mode); +#else + return ::chmod(path, mode); +#endif +} + +void Utils::sleep(unsigned int seconds) +{ +#ifdef WIN32 + Sleep((seconds)*1000); +#else + ::sleep(seconds); +#endif +} + +string Utils::fixPath(const string & path) +{ + string fixedPath = path; +#ifdef WIN32 + for (unsigned int i=0 ; i BatchManager_Local::exec_command(const Parametre & param) const + string BatchManager_Local::ThreadAdapter::buildCommandFile(const Job_Local & job) { - ostringstream exec_sub_cmd; + Parametre param = job.getParametre(); + + // Mandatory parameters + string workDir = ""; + if (param.find(WORKDIR) != param.end()) + workDir = param[WORKDIR].str(); + else + throw RunTimeException("param[WORKDIR] is not defined. Please define it, cannot submit this job."); + string fileToExecute = ""; + if (param.find(EXECUTABLE) != param.end()) + fileToExecute = param[EXECUTABLE].str(); + else + throw RunTimeException("param[EXECUTABLE] is not defined. Please define it, cannot submit this job."); + + string::size_type p1 = fileToExecute.find_last_of("/"); + string::size_type p2 = fileToExecute.find_last_of("."); + string rootNameToExecute = fileToExecute.substr(p1+1,p2-p1-1); + string fileNameToExecute = fileToExecute.substr(p1+1); + string remotePath = workDir + "/" + rootNameToExecute + "_launch_job"; + + // Create batch submit file + ofstream tempOutputFile; + string tmpFileName = Utils::createAndOpenTemporaryFile("launch-job", tempOutputFile); #ifdef WIN32 - char drive[_MAX_DRIVE]; - _splitpath(string(param[WORKDIR]).c_str(), drive, NULL, NULL, NULL); - if (strlen(drive) > 0) exec_sub_cmd << drive << " && "; + if (_bm._protocol.getType() == SH) { + char drive[_MAX_DRIVE]; + _splitpath(workDir.c_str(), drive, NULL, NULL, NULL); + if (strlen(drive) > 0) tempOutputFile << drive << endl; + tempOutputFile << "cd " << Utils::fixPath(workDir) << endl; + // Launch the executable + tempOutputFile << fileNameToExecute; + if (param.find(ARGUMENTS) != param.end()) { + Versatile V = param[ARGUMENTS]; + for(Versatile::const_iterator it=V.begin(); it!=V.end(); it++) { + StringType argt = * static_cast(*it); + string arg = argt; + tempOutputFile << " " << arg; + } + } + remotePath += ".bat"; + } else { #endif - string fileToExecute = param[EXECUTABLE].str(); - string::size_type p1 = fileToExecute.find_last_of("/"); - string fileNameToExecute = fileToExecute.substr(p1+1); + tempOutputFile << "#!/bin/sh" << endl; + tempOutputFile << "cd " << workDir << endl; + + // Optional parameters + //if (param.find(MAXWALLTIME) != param.end()) + // tempOutputFile << "#SBATCH --time=" << param[MAXWALLTIME] << endl; + //if (param.find(MAXRAMSIZE) != param.end()) + // tempOutputFile << "#SBATCH --mem=" << param[MAXRAMSIZE] << endl; + + // Define environment for the job + Environnement env = job.getEnvironnement(); + for (Environnement::const_iterator iter = env.begin() ; iter != env.end() ; ++iter) { + tempOutputFile << "export " << iter->first << "=" << iter->second << endl; + } - exec_sub_cmd << "cd " << param[WORKDIR] << " && ./" << fileNameToExecute; + // generate nodes file + tempOutputFile << "LIBBATCH_NODEFILE=`mktemp nodefile-XXXXXXXXXX`" << endl; + tempOutputFile << "echo `hostname` > $LIBBATCH_NODEFILE" << endl; + tempOutputFile << "export LIBBATCH_NODEFILE" << endl; + // Launch the executable + tempOutputFile << "./" + fileNameToExecute; if (param.find(ARGUMENTS) != param.end()) { Versatile V = param[ARGUMENTS]; for(Versatile::const_iterator it=V.begin(); it!=V.end(); it++) { StringType argt = * static_cast(*it); string arg = argt; - exec_sub_cmd << " " << arg; + tempOutputFile << " " << arg; } } + // Standard input and output if (param.find(INFILE) != param.end()) { Versatile V = param[INFILE]; for(Versatile::const_iterator it=V.begin(); it!=V.end(); it++) { Couple cpl = * static_cast(*it); string remote = cpl.getRemote(); if (remote == "stdin") - exec_sub_cmd << " (*it); string remote = cpl.getRemote(); - if (remote == "stdout") exec_sub_cmd << " 1>stdout"; - if (remote == "stderr") exec_sub_cmd << " 2>stderr"; + if (remote == "stdout") stdoutFile = "stdout"; + if (remote == "stderr") stderrFile = "stderr"; } } + tempOutputFile << " 1>" << stdoutFile << " 2>" << stderrFile << endl; - return _protocol.getExecCommandArgs(exec_sub_cmd.str(), _hostname, _username); + // Remove the node file + tempOutputFile << "rm $LIBBATCH_NODEFILE" << endl; + +#ifdef WIN32 + } +#endif + + tempOutputFile.flush(); + tempOutputFile.close(); + + cerr << "Batch script file generated is: " << tmpFileName << endl; + + Utils::chmod(tmpFileName.c_str(), 0x1ED); + int status = _bm._protocol.copyFile(tmpFileName, "", "", + remotePath, _bm._hostname, _bm._username); + if (status) + throw RunTimeException("Cannot copy command file on host " + _bm._hostname); + +#ifdef WIN32 + if (_bm._protocol.getType() != SH) { + // On Windows, we make the remote file executable afterward because + // pscp does not preserve access permissions on files + string subCommand = string("chmod u+x ") + remotePath; + string command = _bm._protocol.getExecCommand(subCommand, _bm._hostname, _bm._username); + cerr << command.c_str() << endl; + status = system(command.c_str()); + if (status) { + std::ostringstream oss; + oss << "Cannot change permissions of file " << remotePath << " on host " << _bm._hostname; + oss << ". Return status is " << status; + throw RunTimeException(oss.str()); + } + } +#endif + + return remotePath; } @@ -576,17 +667,13 @@ namespace Batch { Parametre param = _job.getParametre(); Parametre::iterator it; - //char *const parmList[] = {"/usr/bin/ssh", "localhost", "-l", "aribes", "sleep 1 && echo end", NULL}; - //int result = execv("/usr/bin/ssh", parmList); - //UNDER_LOCK( cout << "*** debug_command = " << result << endl ); - //UNDER_LOCK( cout << "*** debug_command = " << strerror(errno) << endl ); try { - // EXECUTABLE is MANDATORY, if missing, we exit with failure notification - vector command; - if (param.find(EXECUTABLE) != param.end()) { - command = _bm.exec_command(param); - } else exit(1); + // build script file to launch the job and copy it on the server + string cmdFilePath = buildCommandFile(_job); + + // define command to submit the job + vector command = _bm._protocol.getExecCommandArgs(cmdFilePath, _bm._hostname, _bm._username); // Build the argument array argv from the command char ** argv = new char * [command.size() + 1]; @@ -594,43 +681,10 @@ namespace Batch { for (string::size_type i=0 ; i0) comstr += " # "; - comstr += command[i]; + comstr += command[i] + " "; } - argv[command.size()] = NULL; - UNDER_LOCK( cout << "*** debug_command = " << comstr << endl ); - UNDER_LOCK( cout << "*** debug_command = " << argv[0] << endl ); - - // Create the environment for the new process. Note (RB): Here we change the environment for - // the process launched in local. It would seem more logical to set the environment for the - // remote process. - Environnement env = _job.getEnvironnement(); - - char ** envp = NULL; - if(env.size() > 0) { - envp = new char * [env.size() + 1]; // 1 pour le NULL terminal - int i = 0; - for(Environnement::const_iterator it=env.begin(); it!=env.end(); it++, i++) { - const string & key = (*it).first; - const string & value = (*it).second; - ostringstream oss; - oss << key << "=" << value; - envp[i] = new char [oss.str().size() + 1]; - strncpy(envp[i], oss.str().c_str(), oss.str().size() + 1); - } - - // assert (i == env.size()) - envp[i] = NULL; - } - - //char *const parmList[] = {"/usr/bin/ssh", "localhost", "-l", "aribes", "sleep 1 && echo end", NULL}; - //int result = execv("/usr/bin/ssh", parmList); - //UNDER_LOCK( cout << "*** debug_command = " << result << endl ); - //UNDER_LOCK( cout << "*** debug_command = " << strerror(errno) << endl ); - - // On positionne les limites systeme imposees au fils // This part is deactivated because those limits should be set on the job process, not on @@ -661,26 +715,18 @@ namespace Batch { } */ - - //char *const parmList[] = {"/usr/bin/ssh", "localhost", "-l", "aribes", "sleep 1 && echo end", NULL}; - //int result = execv("/usr/bin/ssh", parmList); - //UNDER_LOCK( cout << "*** debug_command = " << result << endl ); - //UNDER_LOCK( cout << "*** debug_command = " << strerror(errno) << endl ); - // On cree une session pour le fils de facon a ce qu'il ne soit pas // detruit lorsque le shell se termine (le shell ouvre une session et // tue tous les process appartenant a la session en quittant) setsid(); - // On ferme les descripteurs de fichiers standards //close(STDIN_FILENO); //close(STDOUT_FILENO); //close(STDERR_FILENO); - // On execute la commande du fils - execve(argv[0], argv, envp); + execv(argv[0], argv); UNDER_LOCK( cout << "*** debug_command = " << strerror(errno) << endl ); // No need to deallocate since nothing happens after a successful exec @@ -706,46 +752,21 @@ namespace Batch { try { - // EXECUTABLE is MANDATORY, if missing, we throw an exception - vector exec_command; - if (param.find(EXECUTABLE) != param.end()) { - exec_command = _bm.exec_command(param); - } else { - throw RunTimeException("Parameter \"EXECUTABLE\" is mandatory for local batch submission"); - } + // build script file to launch the job and copy it on the server + string cmdFilePath = buildCommandFile(_job); + + // define command to submit the job + vector command = _bm._protocol.getExecCommandArgs(cmdFilePath, _bm._hostname, _bm._username); // Build the command string from the command argument vector string comstr; - for (unsigned int i=0 ; i0) comstr += " "; - comstr += exec_command[i]; + comstr += command[i]; } UNDER_LOCK( cout << "*** debug_command = " << comstr << endl ); - // Create the environment for the new process. Note (RB): Here we change the environment for - // the process launched in local. It would seem more logical to set the environment for the - // remote process. - // Note that if no environment is specified, we reuse the current environment. - Environnement env = _job.getEnvironnement(); - char * chNewEnv = NULL; - - if(env.size() > 0) { - chNewEnv = new char[4096]; - LPTSTR lpszCurrentVariable = chNewEnv; - for(Environnement::const_iterator it=env.begin() ; it!=env.end() ; it++) { - const string & key = (*it).first; - const string & value = (*it).second; - string envvar = key + "=" + value; - envvar.copy(lpszCurrentVariable, envvar.size()); - lpszCurrentVariable[envvar.size()] = '\0'; - lpszCurrentVariable += lstrlen(lpszCurrentVariable) + 1; - } - // Terminate the block with a NULL byte. - *lpszCurrentVariable = '\0'; - } - - STARTUPINFO si; ZeroMemory( &si, sizeof(si) ); si.cb = sizeof(si); @@ -756,7 +777,7 @@ namespace Batch { // launch the new process bool res = CreateProcess(NULL, buffer, NULL, NULL, FALSE, - CREATE_NO_WINDOW, chNewEnv, NULL, &si, &pi); + CREATE_NO_WINDOW, NULL, NULL, &si, &pi); if (buffer) free(buffer); if (!res) throw RunTimeException("Error while creating new process"); diff --git a/src/Local/BatchManager_Local.hxx b/src/Local/BatchManager_Local.hxx index 4946d6a..8058a3c 100644 --- a/src/Local/BatchManager_Local.hxx +++ b/src/Local/BatchManager_Local.hxx @@ -92,6 +92,12 @@ namespace Batch { Process launchWin32ChildProcess(); #endif + /** + * Create script to launch the remote command and copy it on the server. + * Return the path of the remote script. + */ + std::string buildCommandFile(const Job_Local & job); + }; enum Commande { @@ -143,9 +149,6 @@ namespace Batch { pthread_mutex_t _threads_mutex; std::map _threads; - // Methode qui renvoie la commande a executer - std::vector exec_command(const Parametre & param) const; - private: static void kill_child_on_exit(void * p_pid); static void delete_on_exit(void * arg); diff --git a/src/Local/Test/CMakeLists.txt b/src/Local/Test/CMakeLists.txt index 69b86d3..9a919e1 100644 --- a/src/Local/Test/CMakeLists.txt +++ b/src/Local/Test/CMakeLists.txt @@ -50,13 +50,9 @@ IF (HAS_SH) ENDIF (HAS_SH) IF (HAS_SSH) - add_executable(Test_Local_SSH Test_Local_SSH.cxx) - target_link_libraries(Test_Local_SSH batch SimpleParser) - ADD_TEST(Local_SSH Test_Local_SSH) + ADD_TEST(LOCAL_SSH ${CMAKE_BINARY_DIR}/src/Core/Test/Test_BatchManager LOCAL SSH) ENDIF (HAS_SSH) IF (HAS_RSH) - add_executable(Test_Local_RSH Test_Local_RSH.cxx) - target_link_libraries(Test_Local_RSH batch SimpleParser) - ADD_TEST(Local_RSH Test_Local_RSH) + ADD_TEST(LOCAL_RSH ${CMAKE_BINARY_DIR}/src/Core/Test/Test_BatchManager LOCAL RSH) ENDIF (HAS_RSH) diff --git a/src/Local/Test/Test_Local_RSH.cxx b/src/Local/Test/Test_Local_RSH.cxx deleted file mode 100644 index 25133a8..0000000 --- a/src/Local/Test/Test_Local_RSH.cxx +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (C) 2007-2012 CEA/DEN, EDF R&D, OPEN CASCADE -// -// Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, -// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com -// -/* - * Test_Local_RSH.cxx : - * - * Author : Renaud BARATE - EDF R&D - * Date : April 2009 - * - */ - -#include -#include - -#include -#include -#include -#include -#include - -#include - -using namespace std; -using namespace Batch; - -int main(int argc, char** argv) -{ - cout << "*******************************************************************************************" << endl; - cout << "This program tests the local batch submission based on RSH. Passwordless RSH authentication" << endl; - cout << "must be used for this test to pass (this can be configured with the .rhosts file)." << endl; - cout << "*******************************************************************************************" << endl; - - // eventually remove any previous result - remove("result.txt"); - - try { - // Parse the test configuration file - SimpleParser parser; - parser.parseTestConfigFile(); - const string & workdir = parser.getValue("TEST_LOCAL_RSH_WORK_DIR"); - const string & exechost = parser.getValue("TEST_LOCAL_RSH_EXECUTION_HOST"); - const string & user = parser.getValue("TEST_LOCAL_RSH_USER"); - int timeout = parser.getValueAsInt("TEST_LOCAL_RSH_TIMEOUT"); - - // Define the job... - Job job; - // ... and its parameters ... - Parametre p; - p[EXECUTABLE] = "test-script.sh"; - p[NAME] = "Test_Local_RSH"; - p[WORKDIR] = workdir; - p[INFILE] = Couple("seta.sh", workdir + "/copied-seta.sh"); - p[INFILE] += Couple("setb.sh", workdir + "/copied-setb.sh"); - p[OUTFILE] = Couple("result.txt", workdir + "/orig-result.txt"); - job.setParametre(p); - // ... and its environment - Environnement e; - job.setEnvironnement(e); - cout << job << endl; - - // Get the catalog - BatchManagerCatalog& c = BatchManagerCatalog::getInstance(); - - // Create a BatchManager of type Local_RSH on localhost - FactBatchManager * fbm = c("LOCAL"); - if (fbm == NULL) { - cerr << "Can't get RSH batch manager factory" << endl; - return 1; - } - BatchManager * bm = (*fbm)(exechost.c_str(), user.c_str(), RSH); - - // Submit the job to the BatchManager - JobId jobid = bm->submitJob(job); - cout << jobid.__repr__() << endl; - - // Wait for the end of the job - string state = bm->waitForJobEnd(jobid, timeout); - - if (state == FINISHED) { - cout << "Job " << jobid.__repr__() << " is done" << endl; - bm->importOutputFiles(job, "resultdir/seconddirname"); - } else if (state == FAILED) { - cerr << "Job " << jobid.__repr__() << " finished in error" << endl; - bm->importOutputFiles(job, "resultdir/seconddirname"); - return 1; - } else { - cerr << "Timeout while executing job" << endl; - return 1; - } - - } catch (GenericException e) { - cerr << "Error: " << e << endl; - return 1; - } catch (ParserException e) { - cerr << "Parser error: " << e.what() << endl; - return 1; - } - - // test the result file - string exp = "c = 12"; - string res; - ifstream f("resultdir/seconddirname/result.txt"); - getline(f, res); - f.close(); - - cout << "result found : " << res << ", expected : " << exp << endl; - - if (res == exp) - return 0; - else - return 1; -} diff --git a/src/Local/Test/Test_Local_SSH.cxx b/src/Local/Test/Test_Local_SSH.cxx deleted file mode 100644 index 82e857d..0000000 --- a/src/Local/Test/Test_Local_SSH.cxx +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (C) 2007-2012 CEA/DEN, EDF R&D, OPEN CASCADE -// -// Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, -// CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com -// -/* - * Test_Local_SSH.cxx : - * - * Author : Renaud BARATE - EDF R&D - * Date : April 2009 - * - */ - -#include -#include -#include - -#include -#include -#include -#include -#include - -#include - -using namespace std; -using namespace Batch; - -int main(int argc, char** argv) -{ - cout << "*******************************************************************************************" << endl; - cout << "This program tests the local batch submission based on SSH. Passwordless SSH authentication" << endl; - cout << "must be used for this test to pass (this can be configured with ssh-agent for instance)." << endl; - cout << "*******************************************************************************************" << endl; - - // eventually remove any previous result - remove("result.txt"); - - try { - // Parse the test configuration file - SimpleParser parser; - parser.parseTestConfigFile(); - const string & workdir = parser.getValue("TEST_LOCAL_SSH_WORK_DIR"); - const string & exechost = parser.getValue("TEST_LOCAL_SSH_EXECUTION_HOST"); - const string & user = parser.getValue("TEST_LOCAL_SSH_USER"); - int timeout = parser.getValueAsInt("TEST_LOCAL_SSH_TIMEOUT"); - - // Define the job... - Job job; - // ... and its parameters ... - Parametre p; - p[EXECUTABLE] = "test-script.sh"; - p[NAME] = "Test_Local_SSH"; - p[WORKDIR] = workdir; - p[INFILE] = Couple("seta.sh", workdir + "/copied-seta.sh"); - p[INFILE] += Couple("setb.sh", workdir + "/copied-setb.sh"); - p[OUTFILE] = Couple("result.txt", workdir + "/orig-result.txt"); - job.setParametre(p); - // ... and its environment (SSH_AUTH_SOCK env var is important for ssh agent authentication) - Environnement e; - const char * sshAuthSock = getenv("SSH_AUTH_SOCK"); - if (sshAuthSock != NULL) e["SSH_AUTH_SOCK"] = sshAuthSock; - job.setEnvironnement(e); - cout << job << endl; - - // Get the catalog - BatchManagerCatalog& c = BatchManagerCatalog::getInstance(); - - // Create a BatchManager of type Local_SSH on localhost - FactBatchManager * fbm = c("LOCAL"); - if (fbm == NULL) { - cerr << "Can't get SSH batch manager factory" << endl; - return 1; - } - BatchManager * bm = (*fbm)(exechost.c_str(), user.c_str(), SSH); - - // Submit the job to the BatchManager - JobId jobid = bm->submitJob(job); - cout << jobid.__repr__() << endl; - - // Wait for the end of the job - string state = bm->waitForJobEnd(jobid, timeout); - - if (state == FINISHED) { - cout << "Job " << jobid.__repr__() << " is done" << endl; - bm->importOutputFiles(job, "resultdir/seconddirname"); - } else if (state == FAILED) { - cerr << "Job " << jobid.__repr__() << " finished in error" << endl; - bm->importOutputFiles(job, "resultdir/seconddirname"); - return 1; - } else { - cerr << "Timeout while executing job" << endl; - return 1; - } - - } catch (GenericException e) { - cerr << "Error: " << e << endl; - return 1; - } catch (ParserException e) { - cerr << "Parser error: " << e.what() << endl; - return 1; - } - - // test the result file - string exp = "c = 12"; - string res; - ifstream f("resultdir/seconddirname/result.txt"); - getline(f, res); - f.close(); - - cout << "result found : " << res << ", expected : " << exp << endl; - - if (res == exp) - return 0; - else - return 1; -} diff --git a/src/SGE/BatchManager_SGE.cxx b/src/SGE/BatchManager_SGE.cxx index 6e6df08..72d8b3c 100644 --- a/src/SGE/BatchManager_SGE.cxx +++ b/src/SGE/BatchManager_SGE.cxx @@ -35,7 +35,6 @@ #include #include #include -#include #include #include @@ -219,7 +218,7 @@ namespace Batch { tempOutputFile.flush(); tempOutputFile.close(); - BATCH_CHMOD(TmpFileName.c_str(), 0x1ED); + Utils::chmod(TmpFileName.c_str(), 0x1ED); cerr << "Batch script file generated is: " << TmpFileName.c_str() << endl; int status = _protocol.copyFile(TmpFileName, "", "", -- 2.39.2