From b76554af6dd3104d010d0462690e05fb98e3c005 Mon Sep 17 00:00:00 2001 From: =?utf8?q?C=C3=A9dric=20Aguerre?= Date: Fri, 24 Oct 2014 14:51:29 +0200 Subject: [PATCH] run scripts on remote machines (substitute for runSalomeScript) --- bin/launchConfigureParser.py | 8 +- bin/runSession.py | 153 +++++++++++++++++++++++++++++++++-- bin/salomeContext.py | 8 +- bin/salomeContextUtils.py.in | 56 ++++++++++--- 4 files changed, 195 insertions(+), 30 deletions(-) diff --git a/bin/launchConfigureParser.py b/bin/launchConfigureParser.py index 77df1a60d..aa7c0f60e 100755 --- a/bin/launchConfigureParser.py +++ b/bin/launchConfigureParser.py @@ -1073,10 +1073,10 @@ def get_env(theAdditionalOptions=None, appname=salomeappname, cfgname=salomecfgn args[script_nam] = getScriptsAndArgs(cmd_args) new_args = [] if args[gui_nam] and args["session_gui"]: - for d in args[script_nam]: - for s, a in d.items(): - v = re.sub(r'^python.*\s+', r'', s) - new_args.append({v:a}) + from salomeContextUtils import ScriptAndArgs + for sa_obj in args[script_nam]: # args[script_nam] is a list of ScriptAndArgs objects + script = re.sub(r'^python.*\s+', r'', sa_obj.script) + new_args.append(ScriptAndArgs(script=script, args=sa_obj.args, out=sa_obj.out)) # args[script_nam] = new_args diff --git a/bin/runSession.py b/bin/runSession.py index f698e323d..5d5f71886 100644 --- a/bin/runSession.py +++ b/bin/runSession.py @@ -27,6 +27,8 @@ from optparse import OptionParser from NSparam import getNSparams import socket import subprocess +import re +from salomeContextUtils import getScriptsAndArgs, formatScriptsAndArgs # Use to display newlines (\n) in epilog class MyParser(OptionParser): @@ -34,6 +36,15 @@ class MyParser(OptionParser): return self.epilog # +class SessionParameters: + def __init__(self, mode, port, machine, user, directory): + self.mode = mode + self.port = port + self.machine = machine + self.user = user + self.directory = directory +# + def configureSession(args=None): if args is None: args = [] @@ -46,7 +57,12 @@ Python file arguments, if any, must be comma-separated (without blank characters If PORT and MACHINE are not given, try to connect to the last active session on the local machine. If PORT and MACHINE are given, try to connect to the remote session associated with PORT on MACHINE. If MACHINE is not given, try to connect to the session associated to PORT on the local machine. -If PORT is not given, try to connect to the remote session associated to port 2810 on MACHINE.\n\n""" +If PORT is not given, try to connect to the remote session associated to port 2810 on MACHINE. +\n +If MACHINE is remote, the following options MUST be provided: + * DIRECTORY: The full path to the salome command on remote machine. + * USER: The user on the computer to connect to. +""" parser = MyParser(usage=usage, epilog=epilog) parser.add_option("-p", "--port", metavar="", default=0, action="store", type="string", dest="port", @@ -56,6 +72,12 @@ If PORT is not given, try to connect to the remote session associated to port 28 action="store", type="string", dest="host", help="The machine to connect to." ) + parser.add_option('-d', '--directory', dest="directory", default=None, + help="[Remote mode] The full path to the salome command on remote machine." + ) + parser.add_option('-u', '--user', dest="user", default=None, + help="[Remote mode] The user on the computer to connect to." + ) try: (options, args) = parser.parse_args(args) except Exception, e: @@ -95,6 +117,16 @@ If PORT is not given, try to connect to the remote session associated to port 28 # os.environ['NSPORT'] = port os.environ['NSHOST'] = host + + # determine running mode, taht is either 'local' or 'remote' + from salomeContextUtils import getHostname + here = getHostname() + mode = "local" + if host != here and host != "localhost" and host != "no_host": + mode="remote" + pass + params = SessionParameters(mode, port, host, options.user, options.directory) + return params, args # # --- set the OMNIORB_CONFIG file and environment relative to this run of SALOME @@ -110,7 +142,7 @@ def _writeConfigFile(port, host): # command looks like a Bash command-line: # script1.py [args] ; script2.py [args] ; ... -def runSession(command): +def __runLocalSession(command): if command: sep = ";" if sys.platform == "win32": @@ -119,18 +151,16 @@ def runSession(command): outmsg = [] errmsg = [] for cmd in command: - save_cmd = cmd - cmd = cmd.strip().split(' ') - #proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - proc = subprocess.Popen(cmd) - (stdoutdata, stderrdata) = proc.communicate() + single_cmd = cmd.strip().split(' ') + proc = subprocess.Popen(single_cmd) + (stdoutdata, stderrdata) = proc.communicate() # Wait for process to terminate if stdoutdata: outmsg.append(stdoutdata) if stderrdata: errmsg.append(stderrdata) if proc.returncode != 0: - errmsg.append("Error raised when executing command: %s\n"%save_cmd) + errmsg.append("Error raised when executing command: %s\n"%cmd) if outmsg: sys.stdout.write("".join(outmsg)) if errmsg: @@ -144,3 +174,110 @@ def runSession(command): proc = subprocess.Popen(cmd, shell=False, close_fds=True) return proc.communicate() # + +def __copyFiles(user, machine, script, infiles, outfiles): + """Modify script, copy files to remote computer and return lists of copied files.""" + + namescript = os.path.basename(script) + import getpass + logname = getpass.getuser() + tmp_script = "/tmp/%s_%s_%s" % (logname, os.getpid(), namescript) + with open(script, 'r') as fscript: + script_text = fscript.read() + + list_infiles = [] + list_outfiles = [] + n = 0 + for infile in infiles: + # generate a temporary file name + namefile = os.path.basename(infile) + tmp_file = "/tmp/%s_%s_i%s_%s" % (logname, os.getpid(), n, namefile) + + # modify the salome script + script_text = re.sub(infile, tmp_file, script_text) + + # copy the infile to the remote server + cmd = "scp %s %s@%s:%s" % (infile, user, machine, tmp_file) + print "[ SCP ]", cmd + os.system(cmd) + + list_infiles.append(tmp_file) + n = n + 1 + # + n = 0 + for outfile in outfiles: + # generate a temporary file name + namefile = os.path.basename(outfile) + tmp_file = "/tmp/%s_%s_o%s_%s" % (logname, os.getpid(), n, namefile) + + # modify the salome script + script_text = re.sub(outfile, tmp_file, script_text) + + list_outfiles.append(tmp_file) + n = n + 1 + # + + with open(tmp_script,'w') as fscript: + fscript.write(script_text) + + # copy the salome script on the remote server + cmd = "scp %s %s@%s:%s" % (tmp_script, user, machine, tmp_script) + print "[ SCP ]", cmd + os.system(cmd) + + return list_infiles, list_outfiles, tmp_script +# + +# sa_obj is a ScriptAndArgs object (from salomeContextUtils) +def __runRemoteSession(sa_obj, params): + if not params.user: + print "ERROR: The user login on remote machine MUST be given." + return + if not params.directory: + print "ERROR: The remote directory MUST be given." + return + + # sa_obj.script may be 'python script.py' --> only process .py file + header = " ".join(sa_obj.script.split()[:-1]) + script = sa_obj.script.split()[-1] + + tmp_in, tmp_out, tmp_script = __copyFiles(params.user, params.machine, script, sa_obj.args or [], sa_obj.out or []) + + # execute command on the remote SALOME application + command = "ssh %s@%s %s/runSession " % (params.user, params.machine, params.directory) + if params.port: + command = command + "-p %s "%params.port + command = command + " ".join([header,tmp_script] + tmp_in) + print '[ SSH ] ' + command + os.system(command) + + # Get remote files and clean + temp_files = tmp_in + tmp_out + [tmp_script] + + # get the outfiles + for outfile in (sa_obj.out or []): + remote_outfile = tmp_out.pop(0) + command = "scp %s@%s:%s %s" %(params.user, params.machine, remote_outfile, outfile) + print "[ SCP ] " + command + os.system(command) + + # clean temporary files + command = "ssh %s@%s \\rm -f %s" % (params.user, params.machine, " ".join(temp_files)) + print '[ SSH ] ' + command + os.system(command) + os.remove(tmp_script) + +# + +def runSession(params, args): + scriptArgs = getScriptsAndArgs(args) + command = formatScriptsAndArgs(scriptArgs) + + if params.mode == "local": + command = formatScriptsAndArgs(scriptArgs) + return __runLocalSession(command) + + elif params.mode == "remote": + for sa_obj in scriptArgs: + __runRemoteSession(sa_obj, params) +# diff --git a/bin/salomeContext.py b/bin/salomeContext.py index d5cbd0169..bcf54f562 100644 --- a/bin/salomeContext.py +++ b/bin/salomeContext.py @@ -31,7 +31,6 @@ import subprocess import platform from salomeContextUtils import SalomeContextException -from salomeContextUtils import getScriptsAndArgs, formatScriptsAndArgs def usage(): #exeName = os.path.splitext(os.path.basename(__file__))[0] @@ -296,14 +295,13 @@ class SalomeContext: args = [] sys.argv = ['runSession'] + args import runSession - runSession.configureSession(args) + params, args = runSession.configureSession(args) + sys.argv = ['runSession'] + args import setenv setenv.main(True) - scriptArgs = getScriptsAndArgs(args) - command = formatScriptsAndArgs(scriptArgs) - return runSession.runSession(command) + return runSession.runSession(params, args) # def _runConsole(self, args=None): diff --git a/bin/salomeContextUtils.py.in b/bin/salomeContextUtils.py.in index 6e6a8fe0a..810d6ee79 100644 --- a/bin/salomeContextUtils.py.in +++ b/bin/salomeContextUtils.py.in @@ -24,6 +24,7 @@ import sys import glob import subprocess import re +import socket """ Define a specific exception class to manage exceptions related to SalomeContext @@ -101,7 +102,17 @@ def __getScriptPath(scriptName, searchPathList): return None # -# Return an array of dictionaries {script_name: [list_of_its_args]} +class ScriptAndArgs: + # script: the command to be run, e.g. python + # args: its input parameters + # out: its output parameters + def __init__(self, script = None, args = None, out = None): + self.script = script + self.args = args + self.out = out +# + +# Return an array of ScriptAndArgs objects def getScriptsAndArgs(args=None, searchPathList=None): if args is None: args = [] @@ -112,7 +123,9 @@ def getScriptsAndArgs(args=None, searchPathList=None): scriptArgs = [] currentKey = None argsPrefix = "args:" + outPrefix = "out:" callPython = False + afterArgs = False currentScript = None for i in range(len(args)): @@ -122,11 +135,21 @@ def getScriptsAndArgs(args=None, searchPathList=None): if not currentKey or callPython: raise SalomeContextException("args list must follow corresponding script file in command line.") elt = elt.replace(argsPrefix, '') - scriptArgs[len(scriptArgs)-1][currentKey] = elt.split(",") + scriptArgs[len(scriptArgs)-1].args = elt.split(",") + currentKey = None + callPython = False + afterArgs = True + elif elt.startswith(outPrefix): + if (not currentKey and not afterArgs) or callPython: + raise SalomeContextException("out list must follow both corresponding script file and its args in command line.") + elt = elt.replace(outPrefix, '') + scriptArgs[len(scriptArgs)-1].out = elt.split(",") currentKey = None callPython = False + afterArgs = False elif elt.startswith("python"): callPython = True + afterArgs = False else: if not os.path.isfile(elt) and not os.path.isfile(elt+".py"): eltInSearchPath = __getScriptPath(elt, searchPathList) @@ -146,12 +169,12 @@ def getScriptsAndArgs(args=None, searchPathList=None): pass if currentScript and callPython: currentKey = "@PYTHONBIN@ "+currentScript - scriptArgs.append({currentKey:[]}) + scriptArgs.append(ScriptAndArgs(script=currentKey)) callPython = False elif currentScript: if not os.access(currentScript, os.X_OK): currentKey = "@PYTHONBIN@ "+currentScript - scriptArgs.append({currentKey:[]}) + scriptArgs.append(ScriptAndArgs(script=currentKey)) else: ispython = False try: @@ -170,26 +193,29 @@ def getScriptsAndArgs(args=None, searchPathList=None): else: currentKey = currentScript pass - scriptArgs.append({currentKey:[]}) + scriptArgs.append(ScriptAndArgs(script=currentKey)) + # CLOSE elif currentScript + afterArgs = False # end for loop return scriptArgs # # Formatting scripts and args as a Bash-like command-line: # script1.py [args] ; script2.py [args] ; ... +# scriptArgs is a list of ScriptAndArgs objects; their output parameters are omitted def formatScriptsAndArgs(scriptArgs=None): if scriptArgs is None: - scriptArgs = [] + return "" commands = [] - for sc_dict in scriptArgs: - for script, sc_args in sc_dict.items(): # single entry - cmd = script - if sc_args: - cmd = cmd + " " + " ".join(sc_args) - commands.append(cmd) + for sa_obj in scriptArgs: + cmd = sa_obj.script + if sa_obj.args: + cmd = " ".join([cmd]+sa_obj.args) + commands.append(cmd) + sep = " ; " if sys.platform == "win32": - sep= " & " + sep = " & " command = sep.join(["%s"%x for x in commands]) return command # @@ -219,3 +245,7 @@ def setOmniOrbUserPath(): pass os.environ["OMNIORB_USER_PATH"] = defaultOmniorbUserPath # + +def getHostname(): + return socket.gethostname().split('.')[0] +# -- 2.39.2