From e55bd9af0cf26b3cf46bda13f099d2ebf5522856 Mon Sep 17 00:00:00 2001 From: Christian Van Wambeke Date: Thu, 7 Jun 2018 16:48:14 +0200 Subject: [PATCH] sat jobs with test_501_paramiko.py --- commands/jobs.py | 102 ++++++++++--------- commands/launcher.py | 3 +- src/configManager.py | 35 +++---- src/utilsSat.py | 3 + test/test_501_paramiko.py | 201 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 275 insertions(+), 69 deletions(-) create mode 100755 test/test_501_paramiko.py diff --git a/commands/jobs.py b/commands/jobs.py index f9e7bd4..fcd5cd9 100644 --- a/commands/jobs.py +++ b/commands/jobs.py @@ -359,23 +359,27 @@ class Machine(object): self._connection_successful = False self.ssh.load_system_host_keys() self.ssh.set_missing_host_key_policy(self.paramiko.AutoAddPolicy()) + aDict = { + "name": self.name, + "host": self.host, + "port": self.port, + "distrib": self.distribution, # Will be filled after copying SAT on the machine + "user": self.user, + "password": self.password, + "sat_path": self.sat_path, + } + DBG.write("ssh.connect", aDict) try: - self.ssh.connect(self.host, - port=self.port, - username=self.user, - password = self.password) - except self.paramiko.AuthenticationException: - rc = RCO.ReturnCode("KO", "Authentication failed on %s" % self.host) - except self.paramiko.BadHostKeyException: - rc = RCO.ReturnCode("KO", "The server's host key could not be verified on %s" % self.host) - except self.paramiko.SSHException: - rc = RCO.ReturnCode("KO", "SSH Exception connecting on %s" % self.host) - except: - rc = RCO.ReturnCode("KO", "Problem connecting or establishing an SSH session on %s" % self.host) - else: - self._connection_successful = True - rc = RCO.ReturnCode("OK", "connecting SSH session done on %s" % self.host) + res = self.ssh.connect(self.host, port=self.port, username=self.user, password = self.password) + except Exception as e: + msg = "connecting SSH on %s as %s" % (self.host, str(e)) + rc = RCO.ReturnCode("KO", msg, e) # e for futur more explicit... + return rc + DBG.write("ssh.connect OK", res) + self._connection_successful = True + rc = RCO.ReturnCode("OK", "connecting SSH done on %s" % self.host, aDict) return rc + def successfully_connected(self, logger): """ @@ -1038,61 +1042,69 @@ The job will not be launched. config = self.runner.getConfig() logger = self.logger logger.info("\nEstablishing connection with all the machines:") + tabul = 40 res = [] # all connections - for machine in self.lmachines[0:2]: # TODO for debug [0:2] + for machine in self.lmachines: # little algorithm in order to display traces header = ("Connection to %s" % machine.name) step = "SSH connection" logger.logStep_begin(header, step) # the call to the method that initiate the ssh connection rc = machine.connect() - res.append(rc) if not rc.isOk(): - logger.logStep_end(rc, 40) + logger.logStep_end(rc, tabul) continue # Copy salomeTools to the remote machine if machine.successfully_connected(logger): # as rc.isOk() step = _("Remove SAT") - logger.info('\r%s%s%s' % (begin_line, endline, 20 * " ")) - logger.info('\r%s%s%s' % (begin_line, endline, step)) + logger.logStep(step) (__, out_dist, __) = machine.exec_command( "rm -rf %s" % machine.sat_path, logger) out_dist.read() step = _("Copy SAT") - logger.info('\r%s%s%s' % (begin_line, endline, 20 * " ")) - logger.info('\r%s%s%s' % (begin_line, endline, step)) - - res_copy = machine.copy_sat(config.VARS.salometoolsway, self.job_file_path) + logger.logStep(step) - # set the local settings of sat on the remote machine using - # the init command - sat = os.path.join(machine.sat_path, "sat") - cmd = sat + " init --base default --workdir default --log_dir default" - (__, out_dist, __) = machine.exec_command(cmd, logger) - out_dist.read() + rc = machine.copy_sat(config.VARS.salometoolsway, self.job_file_path) + logger.logStep(str(rc)) - # get the remote machine distribution using a sat command - cmd = sat + " config --value VARS.dist --no_label" - (__, out_dist, __) = machine.exec_command(cmd, logger) - machine.distribution = out_dist.read().decode().replace("\n", "") + if rc.isOk(): + # set the local settings of sat on the remote machine using the init command + sat = os.path.join(machine.sat_path, "sat") + cmd = sat + " init --base default --workdir default --log_dir default" + (__, out_dist, __) = machine.exec_command(cmd, logger) + out_dist.read() + + # get the remote machine distribution using a sat command + cmd = sat + " config --value VARS.dist" + (__, out_dist, __) = machine.exec_command(cmd, logger) + # machine.distribution = out_dist.read().decode().replace("\n", "") + rc = self.extractIn(out_dist.read(), "dist: ") + if rc.isOk(): + machine.distribution = rc.getValue() + rc = RCO.ReturnCode("OK", "remote distribution %s" % machine.distribution) - # Print the status of the copy - if res_copy == 0: - logger.info('\r%s' % \ - ((len(begin_line)+len(endline)+20) * " ")) - logger.info('\r%s%s%s' % (begin_line, endline, "")) - else: - logger.info('\r%s' % \ - ((len(begin_line)+len(endline)+20) * " "), 3) - logger.info('\r%s%s%s %s' % \ - (begin_line, endline, "", - _("Copy of SAT failed: %s") % res_copy)) + # end of one machine + res.append(rc) + # Print the status of the copy + logger.logStep_end(str(rc), tabul) return res + def extractIn(self, inStr, searchStr): + """ + find in multiples lines value at left of searchStr + return at first line OK + """ + out = inStr.decode().split("\n") + for line in out: + if searchStr in line: + res = line.split(searchStr)[1] + return RCO.ReturnCode("OK", "'%s' found as '%s'" % (searchStr, res), res) + return RCO.ReturnCode("KO", "'%s' not found in %s" % (searchStr, out)) + def is_occupied(self, hostname): """ diff --git a/commands/launcher.py b/commands/launcher.py index 9c14c1a..03cb005 100644 --- a/commands/launcher.py +++ b/commands/launcher.py @@ -150,8 +150,7 @@ def generate_launch_file(config, .replace("BIN_KERNEL_INSTALL_DIR", bin_kernel_install_dir)\ .replace("KERNEL_INSTALL_DIR", kernel_root_dir) - before, after = withProfile.split( - "# here your local standalone environment\n") + before, after = withProfile.split("# here your local standalone environment\n") # create an environment file writer writer = ENVI.FileEnvWriter(config, logger, pathlauncher, src_root=None, env_info=None) diff --git a/src/configManager.py b/src/configManager.py index 2bca0a2..efc842b 100644 --- a/src/configManager.py +++ b/src/configManager.py @@ -87,52 +87,44 @@ class ConfigManager: The repository that contain external data for salomeTools. :return: (dict) The dictionary that stores all information. """ + JOIN = os.path.join # shortcut var = {} var['user'] = ARCH.get_user() var['salometoolsway'] = os.path.dirname( os.path.dirname(os.path.abspath(__file__))) - var['srcDir'] = os.path.join(var['salometoolsway'], 'src') - var['internal_dir'] = os.path.join(var['srcDir'], 'internal_config') + var['srcDir'] = JOIN(var['salometoolsway'], 'src') + var['internal_dir'] = JOIN(var['srcDir'], 'internal_config') var['sep']= os.path.sep # datadir has a default location - var['datadir'] = os.path.join(var['salometoolsway'], 'data') + var['datadir'] = JOIN(var['salometoolsway'], 'data') if datadir is not None: var['datadir'] = datadir - var['personalDir'] = os.path.join(os.path.expanduser('~'), - '.salomeTools') + var['personalDir'] = JOIN(os.path.expanduser('~'), '.salomeTools') UTS.ensure_path_exists(var['personalDir']) - var['personal_applications_dir'] = os.path.join(var['personalDir'], - "Applications") + var['personal_applications_dir'] = JOIN(var['personalDir'], "Applications") UTS.ensure_path_exists(var['personal_applications_dir']) - var['personal_products_dir'] = os.path.join(var['personalDir'], - "products") + var['personal_products_dir'] = JOIN(var['personalDir'], "products") UTS.ensure_path_exists(var['personal_products_dir']) - var['personal_archives_dir'] = os.path.join(var['personalDir'], - "Archives") + var['personal_archives_dir'] = JOIN(var['personalDir'], "Archives") UTS.ensure_path_exists(var['personal_archives_dir']) - var['personal_jobs_dir'] = os.path.join(var['personalDir'], - "Jobs") + var['personal_jobs_dir'] = JOIN(var['personalDir'], "Jobs") UTS.ensure_path_exists(var['personal_jobs_dir']) - var['personal_machines_dir'] = os.path.join(var['personalDir'], - "Machines") + var['personal_machines_dir'] = JOIN(var['personalDir'], "Machines") UTS.ensure_path_exists(var['personal_machines_dir']) # read linux distributions dictionary - distrib_cfg = PYCONF.Config(os.path.join(var['srcDir'], - 'internal_config', - 'distrib.pyconf')) + distrib_cfg = PYCONF.Config(JOIN(var['srcDir'], 'internal_config', 'distrib.pyconf')) # set platform parameters dist_name = ARCH.get_distribution(codes=distrib_cfg.DISTRIBUTIONS) - dist_version = ARCH.get_distrib_version(dist_name, - codes=distrib_cfg.VERSIONS) + dist_version = ARCH.get_distrib_version(dist_name, codes=distrib_cfg.VERSIONS) dist = dist_name + dist_version var['dist_name'] = dist_name @@ -207,8 +199,7 @@ class ConfigManager: # ===================================================================== # create VARS section - var = self._create_vars(application=application, command=command, - datadir=datadir) + var = self._create_vars(application=application, command=command, datadir=datadir) # add VARS to config cfg.VARS = PYCONF.Mapping(cfg) for variable in var: diff --git a/src/utilsSat.py b/src/utilsSat.py index ef47802..585a673 100644 --- a/src/utilsSat.py +++ b/src/utilsSat.py @@ -31,6 +31,7 @@ import os import shutil import errno import stat +import time import re import tempfile @@ -786,3 +787,5 @@ def generate_catalog(machines, config, logger): """ % msg) return catfile +def sleep(sec): + time.sleep(sec) \ No newline at end of file diff --git a/test/test_501_paramiko.py b/test/test_501_paramiko.py new file mode 100755 index 0000000..defec03 --- /dev/null +++ b/test/test_501_paramiko.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- + +# Copyright (C) 2010-2018 CEA/DEN +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +""" +to set id_rsa from/to reflexive on local machine: + + @is231761/home/wambeke/.ssh>ssh wambeke@is231761 + Password: + Last login: Thu Jun 7 13:34:07 2018 from is231761.intra.cea.fr + @is231761/home/wambeke>exit + déconnexion + + @is231761/home/wambeke/.ssh> ssh-keygen + Generating public/private rsa key pair. + Enter file in which to save the key (/home/wambeke/.ssh/id_rsa): + Enter passphrase (empty for no passphrase): + Enter same passphrase again: + Your identification has been saved in /home/wambeke/.ssh/id_rsa. + Your public key has been saved in /home/wambeke/.ssh/id_rsa.pub. + The key fingerprint is: + SHA256:V0IU/wkuCRw42rA5bHFgdJlzDx9EIJyWIBrkzkL3GNA wambeke@is231761 + The key's randomart image is: + +---[RSA 2048]----+ + |ooo.=+o*o=*. | + + | | + +----[SHA256]-----+ + + @is231761/home/wambeke/.ssh> ls + id_rsa id_rsa.pub known_hosts + @is231761/home/wambeke/.ssh> rm known_hosts + @is231761/home/wambeke/.ssh> ls + id_rsa id_rsa.pub + + @is231761/home/wambeke/.ssh> ssh wambeke@is231761 + The authenticity of host 'is231761 (127.0.0.1)' can't be established. + ECDSA key fingerprint is SHA256:QvrU7Abrbily0bzMjYbRPeKCxDkXT9rQ6pSpcm+yFN4. + ECDSA key fingerprint is MD5:6c:95:b7:c7:cd:de:c5:07:8b:3a:9b:14:d1:69:6b:c6. + Are you sure you want to continue connecting (yes/no)? yes + Warning: Permanently added 'is231761' (ECDSA) to the list of known hosts. + Password: + Last login: Thu Jun 7 13:35:07 2018 from is231761.intra.cea.fr + @is231761/home/wambeke>exit + déconnexion + Connection to is231761 closed. + + + @is231761/home/wambeke/.ssh> lst + total 124K + -rw-r--r-- 1 wambeke lgls 170 7 juin 13:36 known_hosts + drwx------ 2 wambeke lgls 4,0K 7 juin 13:36 . + -rw-r--r-- 1 wambeke lgls 398 7 juin 13:35 id_rsa.pub + -rw------- 1 wambeke lgls 1,7K 7 juin 13:35 id_rsa + drwxr-xr-x 182 wambeke lmpe 104K 6 juin 13:39 .. + + + + @is231761/home/wambeke/.ssh> ssh-copy-id -i ~/.ssh/id_rsa.pub is231761 + /usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/home/wambeke/.ssh/id_rsa.pub" + /usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed + /usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys + Password: + + Number of key(s) added: 1 + + Now try logging into the machine, with: "ssh 'is231761'" + and check to make sure that only the key(s) you wanted were added. + + @is231761/home/wambeke/.ssh> ssh wambeke@is231761 + Last login: Thu Jun 7 13:36:42 2018 from is231761.intra.cea.fr + @is231761/home/wambeke>exit + déconnexion + Connection to is231761 closed. + +""" + +import os +import sys +import unittest +import getpass + +import paramiko as PK + +verbose = False + +class TestCase(unittest.TestCase): + "Test a paramiko connection""" + + def setLoggerParamiko(self): + """to get logs of paramiko, useful if problems""" + import logging as LOGI + loggerPrmk = LOGI.getLogger("paramiko") + if len(loggerPrmk.handlers) != 0: + print("logging.__file__ %s" % LOGI.__file__) + print("logger paramiko have handler set yet, is a surprise") + return + if not verbose: + # stay as it, null + return + + #set a paramiko logger verbose + handler = LOGI.StreamHandler() + msg = "create paramiko logger, with handler on stdout" + + # handler = LOGI.MemoryHandler() + # etc... https://docs.python.org/2/library/logging.handlers.html + # msg = "create paramiko logger, with handler in memory" + + # original frm from paramiko + # frm = '%(levelname)-.3s [%(asctime)s.%(msecs)03d] thr=%(thread)-3d %(name)s: %(message)s' # noqa + frm = '%(levelname)-5s :: %(asctime)s :: %(name)s :: %(message)s' + handler.setFormatter(LOGI.Formatter(frm, '%y%m%d_%H%M%S')) + loggerPrmk.addHandler(handler) + + # logger is not notset but low, handlers needs setlevel greater + loggerPrmk.setLevel(LOGI.DEBUG) + handler.setLevel(LOGI.INFO) # LOGI.DEBUG) # may be other one + + loggerPrmk.info(msg) + + + '''example from internet + def fetch_netmask(self, hostname, port=22): + private_key = os.path.expanduser('~/.ssh/id_rsa') + connection = open_ssh_connection('wambeke', hostname, port=port, key=private_key) + + get_netmask = ("ip -oneline -family inet address show | grep {}").format(hostname) + stdin, stdout, stderr = connection.exec_command(get_netmask) + address = parse_address(hostname, stdout) + connection.close() + return address + + def open_ssh_connection(self, username, hostname, port=22, key=None): + client = PK.SSHClient() + client.set_missing_host_key_policy(PK.AutoAddPolicy()) + client.connect(hostname, port=port, timeout=5, username=username, key_filename=key) + return client + ''' + + def test_000(self): + self.setLoggerParamiko() + + + def test_010(self): + # http://docs.paramiko.org/en/2.4/api/agent.html + + # port=22 # useless + username = getpass.getuser() + hostname = os.uname()[1] + aFile = "/tmp/%s_test_paramiko.tmp" % username + cmd = ("pwd; ls -alt {0}; cat {0}").format(aFile) + + # connect + client = PK.SSHClient() + client.set_missing_host_key_policy(PK.AutoAddPolicy()) + # client.connect(hostname, username=username, password="xxxxx") + # client.connect(hostname, username=username, passphrase="yyyy", key_filename="/home/wambeke/.ssh/id_rsa_satjobs_passphrase") + # client.connect(hostname, username=username) + + # timeout in seconds + client.connect(hostname, username=username, timeout=1.) + + # obtain session + session = client.get_transport().open_session() + # Forward local agent + PK.agent.AgentRequestHandler(session) + # commands executed after this point will see the forwarded agent on the remote end. + + # one api + session.exec_command("date > %s" % aFile) + cmd = ("pwd; ls -alt {0}; cat {0} && echo OK").format(aFile) + # another api + stdin, stdout, stderr = client.exec_command(cmd) + output = stdout.read() + if verbose: + print('stdout:\n%s' % output) + self.assertTrue(aFile in output) + self.assertTrue("OK" in output) + client.close() + +if __name__ == '__main__': + # verbose = True # human eyes + unittest.main(exit=False) + pass -- 2.39.2