From a9e3787a4c142a91302c9a7e27698f4517e3fb1f Mon Sep 17 00:00:00 2001 From: barate Date: Wed, 28 Mar 2012 13:02:42 +0000 Subject: [PATCH] Fix some directory issues with Vishnu --- src/Core/Batch_Utils.cxx | 7 ++ src/Core/Batch_Utils.hxx | 6 ++ src/Vishnu/Batch_BatchManager_eVishnu.cxx | 82 ++++++++++++++--------- src/Vishnu/Batch_BatchManager_eVishnu.hxx | 3 +- src/Vishnu/Test/Test_eVishnu.cxx | 45 ++----------- 5 files changed, 70 insertions(+), 73 deletions(-) diff --git a/src/Core/Batch_Utils.cxx b/src/Core/Batch_Utils.cxx index 3b706d2..83b43e3 100644 --- a/src/Core/Batch_Utils.cxx +++ b/src/Core/Batch_Utils.cxx @@ -61,4 +61,11 @@ int Utils::getCommandOutput(const string & command, string & output) return status; } +bool Utils::isAbsolutePath(const string & path) +{ + if (path.size() == 0) + return false; + return path[0] == '/'; +} + } diff --git a/src/Core/Batch_Utils.hxx b/src/Core/Batch_Utils.hxx index 24f17d1..ce59ccf 100644 --- a/src/Core/Batch_Utils.hxx +++ b/src/Core/Batch_Utils.hxx @@ -42,6 +42,12 @@ public: */ static int getCommandOutput(const std::string & command, std::string & output); + /** + * Test if the path in parameter is an absolute path (does not test the existence of + * a file or directory at this path). + */ + static bool isAbsolutePath(const std::string & path); + private: // No instanciation possible as this class provides only static methods diff --git a/src/Vishnu/Batch_BatchManager_eVishnu.cxx b/src/Vishnu/Batch_BatchManager_eVishnu.cxx index b97722e..70f0a97 100644 --- a/src/Vishnu/Batch_BatchManager_eVishnu.cxx +++ b/src/Vishnu/Batch_BatchManager_eVishnu.cxx @@ -33,6 +33,7 @@ #include #include +#include #include "Batch_FactBatchManager_eVishnu.hxx" #include "Batch_BatchManager_eVishnu.hxx" @@ -66,8 +67,7 @@ namespace Batch { const string workDir = params[WORKDIR]; // export input files on cluster - - copyInputFilesFromVishnuToCluster(job); + exportInputFiles(job); // build command file to submit the job string cmdFile = buildCommandFile(job); @@ -77,7 +77,7 @@ namespace Batch { // define command to submit batch string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); - subCommand += "vishnu_connect; "; + subCommand += "vishnu_connect -p 2; "; subCommand += "vishnu_submit_job " + _hostname + " " + cmdFile; string command = _protocol.getExecCommand(subCommand, _hostname, @@ -114,7 +114,7 @@ namespace Batch { } - void BatchManager_eVishnu::copyInputFilesFromVishnuToCluster(const Job& job) + void BatchManager_eVishnu::exportInputFiles(const Job& job) { int status; Parametre params = job.getParametre(); @@ -123,9 +123,8 @@ namespace Batch { // create remote directories string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); - subCommand += "vishnu_connect; "; - subCommand += "vishnu_create_dir " + _hostname + ":" + params[WORKDIR].str() + "; "; - subCommand += "vishnu_create_dir " + _hostname + ":" + params[WORKDIR].str() + "/logs"; + subCommand += "vishnu_connect -p 2; "; + subCommand += "vishnu_create_dir -p " + _hostname + ":" + params[WORKDIR].str() + "/logs"; string command = _protocol.getExecCommand(subCommand, _hostname, _username); @@ -139,7 +138,7 @@ namespace Batch { if (executeFile.size() != 0) { string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); - subCommand += "vishnu_connect; "; + subCommand += "vishnu_connect -p 2; "; subCommand += "vishnu_copy_file " + executeFile + " " + _hostname + ":" + params[WORKDIR].str() + "/"; string command = _protocol.getExecCommand(subCommand, _hostname, @@ -155,9 +154,21 @@ namespace Batch { CoupleType cpt = *static_cast< CoupleType * >(*Vit); Couple inputFile = cpt; + // Get absolute paths + char * buf = getcwd(NULL, 0); + string cwd = buf; + free(buf); + + string absremote = (Utils::isAbsolutePath(inputFile.getRemote()))? + inputFile.getRemote() : + params[WORKDIR].str() + "/" + inputFile.getRemote(); + string abslocal = (Utils::isAbsolutePath(inputFile.getLocal()))? + inputFile.getLocal() : + cwd + "/" + inputFile.getLocal(); + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); - subCommand += "vishnu_connect; "; - subCommand += "vishnu_copy_file " + inputFile.getLocal() + " " + _hostname + ":" + params[WORKDIR].str() + "/" + inputFile.getRemote(); + subCommand += "vishnu_connect -p 2; "; + subCommand += "vishnu_copy_file " + abslocal + " " + _hostname + ":" + absremote; string command = _protocol.getExecCommand(subCommand, _hostname, _username); @@ -249,16 +260,7 @@ namespace Batch { tempOutputFile.close(); cerr << "Batch script file generated is: " << tmpFileName << endl; - - string remoteFileName = rootNameToExecute + "_vishnu"; - int status = _protocol.copyFile(tmpFileName, "", "", - remoteFileName, - _hostname, - _username); - if (status) - throw EmulationException("Cannot copy command file on host " + _hostname); - - return remoteFileName; + return tmpFileName; } void BatchManager_eVishnu::deleteJob(const JobId & jobid) @@ -269,7 +271,7 @@ namespace Batch { // define command to delete job string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); - subCommand += "vishnu_connect; "; + subCommand += "vishnu_connect -p 2; "; subCommand += "vishnu_cancel_job " + _hostname + " " + vishnuJobId; string command = _protocol.getExecCommand(subCommand, _hostname, _username); cerr << command.c_str() << endl; @@ -317,7 +319,7 @@ namespace Batch { // define command to query batch string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); - subCommand += "vishnu_connect; "; + subCommand += "vishnu_connect -p 2; "; subCommand += "vishnu_get_job_info " + _hostname + " " + vishnuJobId; string command = _protocol.getExecCommand(subCommand, _hostname, _username); command += " > "; @@ -338,23 +340,39 @@ namespace Batch { void BatchManager_eVishnu::importOutputFiles(const Job & job, const std::string directory) { - copyOutputFilesFromClusterToVishnu(job); - } - - void BatchManager_eVishnu::copyOutputFilesFromClusterToVishnu(const Job & job) - { - int status; Parametre params = job.getParametre(); const Versatile & V = params[OUTFILE]; Versatile::const_iterator Vit; + // Create local result directory + char * buf = getcwd(NULL, 0); + string cwd = buf; + free(buf); + string absdir = (Utils::isAbsolutePath(directory))? directory : cwd + "/" + directory; + int status = CommunicationProtocol::getInstance(SH).makeDirectory(absdir, "", ""); + if (status) { + string mess("Directory creation failed. Status is :"); + ostringstream status_str; + status_str << status; + mess += status_str.str(); + cerr << mess << endl; + } + for (Vit=V.begin(); Vit!=V.end(); Vit++) { CoupleType cpt = *static_cast< CoupleType * >(*Vit); Couple outputFile = cpt; + // Get absolute paths + string absremote = (Utils::isAbsolutePath(outputFile.getRemote()))? + outputFile.getRemote() : + params[WORKDIR].str() + "/" + outputFile.getRemote(); + string abslocal = (Utils::isAbsolutePath(outputFile.getLocal()))? + outputFile.getLocal() : + absdir + "/" + outputFile.getLocal(); + string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); - subCommand += "vishnu_connect; "; - subCommand += "vishnu_copy_file " + _hostname + ":" + params[WORKDIR].str() + "/" + outputFile.getRemote() + " " + outputFile.getLocal(); + subCommand += "vishnu_connect -p 2; "; + subCommand += "vishnu_copy_file " + _hostname + ":" + absremote + " " + abslocal; string command = _protocol.getExecCommand(subCommand, _hostname, _username); @@ -366,8 +384,8 @@ namespace Batch { // Copy logs string subCommand = string("export OMNIORB_CONFIG=$VISHNU_CONFIG_FILE; "); - subCommand += "vishnu_connect; "; - subCommand += "vishnu_copy_file -r " +_hostname + ":" + params[WORKDIR].str() + "/logs" + " ."; + subCommand += "vishnu_connect -p 2; "; + subCommand += "vishnu_copy_file -r " +_hostname + ":" + params[WORKDIR].str() + "/logs" + " " + absdir; string command = _protocol.getExecCommand(subCommand, _hostname, _username); diff --git a/src/Vishnu/Batch_BatchManager_eVishnu.hxx b/src/Vishnu/Batch_BatchManager_eVishnu.hxx index 264344f..ec7106f 100644 --- a/src/Vishnu/Batch_BatchManager_eVishnu.hxx +++ b/src/Vishnu/Batch_BatchManager_eVishnu.hxx @@ -66,8 +66,7 @@ namespace Batch { protected: std::string buildCommandFile(const Job & job); - void copyInputFilesFromVishnuToCluster(const Job & job); - void copyOutputFilesFromClusterToVishnu(const Job & job); + void exportInputFiles(const Job & job); int _nb_proc_per_node; diff --git a/src/Vishnu/Test/Test_eVishnu.cxx b/src/Vishnu/Test/Test_eVishnu.cxx index 8ca4cc8..4c191dc 100644 --- a/src/Vishnu/Test/Test_eVishnu.cxx +++ b/src/Vishnu/Test/Test_eVishnu.cxx @@ -44,33 +44,10 @@ using namespace std; using namespace Batch; -void print_usage() -{ - cout << "usage: Test_eVishnu PROTOCOL" << endl; - cout << " PROTOCOL \"SSH\" or \"RSH\"" << endl; -} - int main(int argc, char** argv) { - // Parse argument - if (argc != 2) { - print_usage(); - return 1; - } - CommunicationProtocolType protocol; - if (strcmp(argv[1], "SSH") == 0) - protocol = SSH; - else if (strcmp(argv[1], "RSH") == 0) - protocol = RSH; - else { - print_usage(); - return 1; - } - cout << "*******************************************************************************************" << endl; - cout << "This program tests the batch submission based on Vishnu emulation. Passwordless" << endl; - cout << "authentication must be used for this test to pass. For SSH, this can be configured with" << endl; - cout << "ssh-agent for instance. For RSH, this can be configured with the .rhosts file." << endl; + cout << "This program tests the batch submission based on Vishnu commands." << endl; cout << "*******************************************************************************************" << endl; // eventually remove any previous result @@ -83,10 +60,6 @@ int main(int argc, char** argv) const string & homedir = parser.getValue("TEST_EVISHNU_HOMEDIR"); const string & host = parser.getValue("TEST_EVISHNU_HOST"); const string & user = parser.getValue("TEST_EVISHNU_USER"); - const string & vishnu_password = parser.getValue("TEST_EVISHNU_VISHNU_PASSWORD"); - const string & vishnu_host_name = parser.getValue("TEST_EVISHNU_VISHNU_HOST_NAME"); - const string & vishnu_host_login = parser.getValue("TEST_EVISHNU_VISHNU_HOST_LOGIN"); - const string & vishnu_host_workdir = parser.getValue("TEST_EVISHNU_VISHNU_HOST_WORKDIR"); int timeout = parser.getValueAsInt("TEST_EVISHNU_TIMEOUT"); // Define the job... @@ -94,20 +67,14 @@ int main(int argc, char** argv) // ... and its parameters ... Parametre p; p[EXECUTABLE] = "./test-script.sh"; - p[NAME] = string("Test_eVISHNU_") + argv[1]; + p[NAME] = "Test_eVISHNU"; p[WORKDIR] = homedir + "/tmp/Batch"; - p[INFILE] = Couple("seta.sh", "tmp/Batch/seta.sh"); - p[INFILE] += Couple("setb.sh", "tmp/Batch/setb.sh"); - p[OUTFILE] = Couple("result.txt", "tmp/Batch/result.txt"); - p[TMPDIR] = "tmp/Batch/"; + p[INFILE] = Couple("seta.sh", "seta.sh"); + p[INFILE] += Couple("setb.sh", "setb.sh"); + p[OUTFILE] = Couple("result.txt", "result.txt"); p[NBPROC] = 1; p[MAXWALLTIME] = 1; p[MAXRAMSIZE] = 50; - p[HOMEDIR] = homedir; - p["VISHNU_PASSWORD"] = vishnu_password; - p["VISHNU_HOST_NAME"] = vishnu_host_name; - p["VISHNU_HOST_LOGIN"] = vishnu_host_login; - p["VISHNU_HOST_WORKDIR"] = vishnu_host_workdir; job.setParametre(p); // ... and its environment Environnement e; @@ -120,7 +87,7 @@ int main(int argc, char** argv) // Create a BatchManager of type ePBS on localhost FactBatchManager_eClient * fbm = (FactBatchManager_eClient *)(c("eVISHNU")); - BatchManager_eClient * bm = (*fbm)(host.c_str(), user.c_str(), protocol); + BatchManager_eClient * bm = (*fbm)(host.c_str(), user.c_str(), SH); // Submit the job to the BatchManager JobId jobid = bm->submitJob(job); -- 2.39.2