From 4231b8c4b75c2cfe3e40e7371207df6a593d2670 Mon Sep 17 00:00:00 2001 From: daix Date: Tue, 27 Mar 2012 09:17:57 +0000 Subject: [PATCH] Support for local VISHNU client added --- src/Vishnu/Batch_BatchManager_eVishnu.cxx | 228 +++--------------- src/Vishnu/Batch_BatchManager_eVishnu.hxx | 2 - src/Vishnu/Batch_FactBatchManager_eVishnu.cxx | 10 - src/Vishnu/Batch_FactBatchManager_eVishnu.hxx | 5 - 4 files changed, 40 insertions(+), 205 deletions(-) diff --git a/src/Vishnu/Batch_BatchManager_eVishnu.cxx b/src/Vishnu/Batch_BatchManager_eVishnu.cxx index dfd6615..b97722e 100644 --- a/src/Vishnu/Batch_BatchManager_eVishnu.cxx +++ b/src/Vishnu/Batch_BatchManager_eVishnu.cxx @@ -67,26 +67,21 @@ namespace Batch { // export input files on cluster - // This part will be removed in the final version - copyInputFilesFromLocalToVishnu(job); - // This part will be kept in the final version copyInputFilesFromVishnuToCluster(job); - // build command file to submit the job and copy it on the server + // build command file to submit the job string cmdFile = buildCommandFile(job); // define name of log file (local) string logFile = generateTemporaryFileName("vishnu-submitlog"); // define command to submit batch - string subCommand = string("cd ") + params[VISHNU_HOST_WORKDIR].str() + "; "; - subCommand += ". ~/.vishnu/vishnu.env ;"; - subCommand += "vishnu_connect -p 2 " + _username + - " -w " + params[VISHNU_PASSWORD].str() + "; "; + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect; "; subCommand += "vishnu_submit_job " + _hostname + " " + cmdFile; string command = _protocol.getExecCommand(subCommand, - params[VISHNU_HOST_NAME], - params[VISHNU_HOST_LOGIN]); + _hostname, + _username); command += " > "; command += logFile; cerr << command.c_str() << endl; @@ -114,75 +109,11 @@ namespace Batch { if (jobref.size() == 0) throw EmulationException("Error in the submission of the job on the remote host"); - // Store the Vishnu password, host name and login with the jobId in the job reference - // (for further queries) - jobref = params[VISHNU_PASSWORD].str() + ":" + params[VISHNU_HOST_NAME].str() + ":" + \ - params[VISHNU_HOST_LOGIN].str() + ":" + jobref; JobId id(this, jobref); return id; } - void BatchManager_eVishnu::copyInputFilesFromLocalToVishnu(const Job& job) - { - int status; - Parametre params = job.getParametre(); - const Versatile & V = params[INFILE]; - Versatile::const_iterator Vit; - - // Create working directory on Vishnu host - status = _protocol.makeDirectory(params[VISHNU_HOST_WORKDIR], - params[VISHNU_HOST_NAME], - params[VISHNU_HOST_LOGIN]); - if(status) { - std::ostringstream oss; - oss << status; - std::string ex_mess("Error of connection on remote host ! status = "); - ex_mess += oss.str(); - throw EmulationException(ex_mess.c_str()); - } - - // copy executable - string executeFile = params[EXECUTABLE]; - if (executeFile.size() != 0) { - status = _protocol.copyFile(executeFile, "", "", - params[VISHNU_HOST_WORKDIR], - params[VISHNU_HOST_NAME], - params[VISHNU_HOST_LOGIN]); - if(status) { - std::ostringstream oss; - oss << status; - std::string ex_mess("Error of connection on remote host ! status = "); - ex_mess += oss.str(); - throw EmulationException(ex_mess.c_str()); - } - - } - - // copy filesToExportList - for(Vit=V.begin(); Vit!=V.end(); Vit++) { - CoupleType cpt = *static_cast< CoupleType * >(*Vit); - Couple inputFile = cpt; - - // remote file -> transform to get path on Vishnu host - size_t found = inputFile.getRemote().find_last_of("/"); - string remote = params[VISHNU_HOST_WORKDIR].str() + "/" + inputFile.getRemote().substr(found+1); - - status = _protocol.copyFile(inputFile.getLocal(), "", "", - remote, - params[VISHNU_HOST_NAME], - params[VISHNU_HOST_LOGIN]); - if(status) { - std::ostringstream oss; - oss << status; - std::string ex_mess("Error of connection on remote host ! status = "); - ex_mess += oss.str(); - throw EmulationException(ex_mess.c_str()); - } - } - - } - void BatchManager_eVishnu::copyInputFilesFromVishnuToCluster(const Job& job) { int status; @@ -191,13 +122,13 @@ namespace Batch { Versatile::const_iterator Vit; // create remote directories - string subCommand = ". ~/.vishnu/vishnu.env ;"; - subCommand += "vishnu_connect -p 2 " + _username + " -w " + params[VISHNU_PASSWORD].str() + "; "; + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect; "; subCommand += "vishnu_create_dir " + _hostname + ":" + params[WORKDIR].str() + "; "; subCommand += "vishnu_create_dir " + _hostname + ":" + params[WORKDIR].str() + "/logs"; string command = _protocol.getExecCommand(subCommand, - params[VISHNU_HOST_NAME], - params[VISHNU_HOST_LOGIN]); + _hostname, + _username); cerr << command.c_str() << endl; status = system(command.c_str()); if (status != 0) @@ -207,16 +138,12 @@ namespace Batch { string executeFile = params[EXECUTABLE]; if (executeFile.size() != 0) { - // transform to get path on Vishnu host - size_t found = executeFile.find_last_of("/"); - string origin = params[VISHNU_HOST_WORKDIR].str() + "/" + executeFile.substr(found+1); - - string subCommand = ". ~/.vishnu/vishnu.env ;"; - subCommand += "vishnu_connect -p 2 " + _username + " -w " + params[VISHNU_PASSWORD].str() + "; "; - subCommand += "vishnu_copy_file " + origin + " " + _hostname + ":" + params[WORKDIR].str(); + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect; "; + subCommand += "vishnu_copy_file " + executeFile + " " + _hostname + ":" + params[WORKDIR].str() + "/"; string command = _protocol.getExecCommand(subCommand, - params[VISHNU_HOST_NAME], - params[VISHNU_HOST_LOGIN]); + _hostname, + _username); cerr << command.c_str() << endl; status = system(command.c_str()); if (status != 0) @@ -228,16 +155,12 @@ namespace Batch { CoupleType cpt = *static_cast< CoupleType * >(*Vit); Couple inputFile = cpt; - // transform to get path on Vishnu host - size_t found = inputFile.getRemote().find_last_of("/"); - string origin = params[VISHNU_HOST_WORKDIR].str() + "/" + inputFile.getRemote().substr(found+1); - - string subCommand = ". ~/.vishnu/vishnu.env ;"; - subCommand += "vishnu_connect -p 2 " + _username + " -w " + params[VISHNU_PASSWORD].str() + "; "; - subCommand += "vishnu_copy_file " + origin + " " + _hostname + ":" + inputFile.getRemote(); + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect; "; + subCommand += "vishnu_copy_file " + inputFile.getLocal() + " " + _hostname + ":" + params[WORKDIR].str() + "/" + inputFile.getRemote(); string command = _protocol.getExecCommand(subCommand, - params[VISHNU_HOST_NAME], - params[VISHNU_HOST_LOGIN]); + _hostname, + _username); cerr << command.c_str() << endl; status = system(command.c_str()); if (status != 0) @@ -329,9 +252,9 @@ namespace Batch { string remoteFileName = rootNameToExecute + "_vishnu"; int status = _protocol.copyFile(tmpFileName, "", "", - params[VISHNU_HOST_WORKDIR].str() + "/" + remoteFileName, - params[VISHNU_HOST_NAME], - params[VISHNU_HOST_LOGIN]); + remoteFileName, + _hostname, + _username); if (status) throw EmulationException("Cannot copy command file on host " + _hostname); @@ -340,19 +263,15 @@ namespace Batch { void BatchManager_eVishnu::deleteJob(const JobId & jobid) { - // split job reference in Vishnu password / host name / host login / jobId - string vishnuPassword, vishnuHostName, vishnuHostLogin, vishnuJobId; + string vishnuJobId; istringstream iss(jobid.getReference()); - getline(iss, vishnuPassword, ':'); - getline(iss, vishnuHostName, ':'); - getline(iss, vishnuHostLogin, ':'); getline(iss, vishnuJobId, ':'); // define command to delete job - string subCommand = ". ~/.vishnu/vishnu.env ;"; - subCommand += "vishnu_connect -p 2 " + _username + " -w " + vishnuPassword + "; "; + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect; "; subCommand += "vishnu_cancel_job " + _hostname + " " + vishnuJobId; - string command = _protocol.getExecCommand(subCommand, vishnuHostName, vishnuHostLogin); + string command = _protocol.getExecCommand(subCommand, _hostname, _username); cerr << command.c_str() << endl; int status = system(command.c_str()); @@ -392,19 +311,15 @@ namespace Batch { // define name of log file (local) string logFile = generateTemporaryFileName("vishnu-querylog-" + jobid.getReference()); - // split job reference in Vishnu password / host name / host login / jobId - string vishnuPassword, vishnuHostName, vishnuHostLogin, vishnuJobId; + string vishnuJobId; istringstream iss(jobid.getReference()); - getline(iss, vishnuPassword, ':'); - getline(iss, vishnuHostName, ':'); - getline(iss, vishnuHostLogin, ':'); getline(iss, vishnuJobId, ':'); // define command to query batch - string subCommand = ". ~/.vishnu/vishnu.env ;"; - subCommand += "vishnu_connect -p 2 " + _username + " -w " + vishnuPassword + "; "; + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect; "; subCommand += "vishnu_get_job_info " + _hostname + " " + vishnuJobId; - string command = _protocol.getExecCommand(subCommand, vishnuHostName, vishnuHostLogin); + string command = _protocol.getExecCommand(subCommand, _hostname, _username); command += " > "; command += logFile; cerr << command.c_str() << endl; @@ -423,11 +338,7 @@ namespace Batch { void BatchManager_eVishnu::importOutputFiles(const Job & job, const std::string directory) { - // This part will be kept in the final version copyOutputFilesFromClusterToVishnu(job); - - // This part will be removed in the final version - copyOutputFilesFromVishnuToLocal(job, directory); } void BatchManager_eVishnu::copyOutputFilesFromClusterToVishnu(const Job & job) @@ -441,16 +352,12 @@ namespace Batch { CoupleType cpt = *static_cast< CoupleType * >(*Vit); Couple outputFile = cpt; - // transform to get path on Vishnu host - size_t found = outputFile.getRemote().find_last_of("/"); - string dest = params[VISHNU_HOST_WORKDIR].str() + "/" + outputFile.getRemote().substr(found+1); - - string subCommand = ". ~/.vishnu/vishnu.env ;"; - subCommand += "vishnu_connect -p 2 " + _username + " -w " + params[VISHNU_PASSWORD].str() + "; "; - subCommand += "vishnu_copy_file " + _hostname + ":" + outputFile.getRemote() + " " + dest; + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect; "; + subCommand += "vishnu_copy_file " + _hostname + ":" + params[WORKDIR].str() + "/" + outputFile.getRemote() + " " + outputFile.getLocal(); string command = _protocol.getExecCommand(subCommand, - params[VISHNU_HOST_NAME], - params[VISHNU_HOST_LOGIN]); + _hostname, + _username); cerr << command.c_str() << endl; status = system(command.c_str()); if (status != 0) @@ -458,71 +365,16 @@ namespace Batch { } // Copy logs - string subCommand = ". ~/.vishnu/vishnu.env ;"; - subCommand += "vishnu_connect -p 2 " + _username + " -w " + params[VISHNU_PASSWORD].str() + "; "; - subCommand += "vishnu_copy_file -r " +_hostname + ":" + params[WORKDIR].str() + "/logs" + " " + \ - params[VISHNU_HOST_WORKDIR].str(); + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); + subCommand += "vishnu_connect; "; + subCommand += "vishnu_copy_file -r " +_hostname + ":" + params[WORKDIR].str() + "/logs" + " ."; string command = _protocol.getExecCommand(subCommand, - params[VISHNU_HOST_NAME], - params[VISHNU_HOST_LOGIN]); + _hostname, + _username); cerr << command.c_str() << endl; status = system(command.c_str()); if (status != 0) throw EmulationException("Can't copy logs"); } - void BatchManager_eVishnu::copyOutputFilesFromVishnuToLocal(const Job & job, - const std::string directory) - { - Parametre params = job.getParametre(); - const Versatile & V = params[OUTFILE]; - Versatile::const_iterator Vit; - - // Create local result directory - int status = CommunicationProtocol::getInstance(SH).makeDirectory(directory, "", ""); - if (status) { - string mess("Directory creation failed. Status is :"); - ostringstream status_str; - status_str << status; - mess += status_str.str(); - cerr << mess << endl; - } - - for(Vit=V.begin(); Vit!=V.end(); Vit++) { - CoupleType cpt = *static_cast< CoupleType * >(*Vit); - Couple outputFile = cpt; - - // remote file -> transform to get path on Vishnu host - size_t found = outputFile.getRemote().find_last_of("/"); - string remote = params[VISHNU_HOST_WORKDIR].str() + "/" + outputFile.getRemote().substr(found+1); - - status = _protocol.copyFile(remote, - params[VISHNU_HOST_NAME], - params[VISHNU_HOST_LOGIN], - directory, "", ""); - if (status) { - // Try to get what we can (logs files) - // throw BatchException("Error of connection on remote host"); - std::string mess("Copy command failed ! status is :"); - ostringstream status_str; - status_str << status; - mess += status_str.str(); - cerr << mess << endl; - } - } - - // Copy logs - status = _protocol.copyFile(params[VISHNU_HOST_WORKDIR].str() + "/logs", - params[VISHNU_HOST_NAME], - params[VISHNU_HOST_LOGIN], - directory, "", ""); - if (status) { - std::string mess("Copy logs directory failed ! status is :"); - ostringstream status_str; - status_str << status; - mess += status_str.str(); - cerr << mess << endl; - } - } - } diff --git a/src/Vishnu/Batch_BatchManager_eVishnu.hxx b/src/Vishnu/Batch_BatchManager_eVishnu.hxx index 195cb86..264344f 100644 --- a/src/Vishnu/Batch_BatchManager_eVishnu.hxx +++ b/src/Vishnu/Batch_BatchManager_eVishnu.hxx @@ -66,10 +66,8 @@ namespace Batch { protected: std::string buildCommandFile(const Job & job); - void copyInputFilesFromLocalToVishnu(const Job & job); void copyInputFilesFromVishnuToCluster(const Job & job); void copyOutputFilesFromClusterToVishnu(const Job & job); - void copyOutputFilesFromVishnuToLocal(const Job & job, const std::string directory); int _nb_proc_per_node; diff --git a/src/Vishnu/Batch_FactBatchManager_eVishnu.cxx b/src/Vishnu/Batch_FactBatchManager_eVishnu.cxx index b4dbb4e..c0ede38 100644 --- a/src/Vishnu/Batch_FactBatchManager_eVishnu.cxx +++ b/src/Vishnu/Batch_FactBatchManager_eVishnu.cxx @@ -33,20 +33,10 @@ namespace Batch { - def_Constant(VISHNU_PASSWORD); - def_Constant(VISHNU_HOST_NAME); - def_Constant(VISHNU_HOST_LOGIN); - def_Constant(VISHNU_HOST_WORKDIR); - static FactBatchManager_eVishnu sFBM_eVishnu; FactBatchManager_eVishnu::FactBatchManager_eVishnu() : FactBatchManager_eClient("eVISHNU") { - // Add specific parameters - ParameterTypeMap::getInstance().addParameter(VISHNU_PASSWORD, STRING, 1); - ParameterTypeMap::getInstance().addParameter(VISHNU_HOST_NAME, STRING, 1); - ParameterTypeMap::getInstance().addParameter(VISHNU_HOST_LOGIN, STRING, 1); - ParameterTypeMap::getInstance().addParameter(VISHNU_HOST_WORKDIR, STRING, 1); } FactBatchManager_eVishnu::~FactBatchManager_eVishnu() diff --git a/src/Vishnu/Batch_FactBatchManager_eVishnu.hxx b/src/Vishnu/Batch_FactBatchManager_eVishnu.hxx index 5e7fb41..07eee8e 100644 --- a/src/Vishnu/Batch_FactBatchManager_eVishnu.hxx +++ b/src/Vishnu/Batch_FactBatchManager_eVishnu.hxx @@ -36,11 +36,6 @@ namespace Batch { - decl_extern_Constant(VISHNU_PASSWORD); - decl_extern_Constant(VISHNU_HOST_NAME); - decl_extern_Constant(VISHNU_HOST_LOGIN); - decl_extern_Constant(VISHNU_HOST_WORKDIR); - class BATCH_EXPORT FactBatchManager_eVishnu : public FactBatchManager_eClient { public: -- 2.39.2