X-Git-Url: http://git.salome-platform.org/gitweb/?a=blobdiff_plain;f=commands%2Fjobs.py;h=c043d7048e094d8463f6f40b6f9f26b9a6c3c7dc;hb=606f9b6618e0e0659d4029c607eaa04d9b3501cc;hp=89fedbb5f210e4f22bbf6609e806ee6b08941639;hpb=5cf872ae63e295449857177bfe89c3f19929a22f;p=tools%2Fsat.git diff --git a/commands/jobs.py b/commands/jobs.py index 89fedbb..c043d70 100644 --- a/commands/jobs.py +++ b/commands/jobs.py @@ -17,16 +17,28 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import os +import sys +import tempfile +import traceback import datetime import time import csv import shutil import itertools import re -import paramiko + +# generate problem +try: + import paramiko +except: + paramiko = "import paramiko impossible" + pass import src + +import src.ElementTree as etree + STYLESHEET_GLOBAL = "jobs_global_report.xsl" STYLESHEET_BOARD = "jobs_board_report.xsl" @@ -35,9 +47,9 @@ CSV_DELIMITER = ";" parser = src.options.Options() -parser.add_option('n', 'name', 'string', 'jobs_cfg', +parser.add_option('n', 'name', 'list2', 'jobs_cfg', _('Mandatory: The name of the config file that contains' - ' the jobs configuration')) + ' the jobs configuration. Can be a list.')) parser.add_option('o', 'only_jobs', 'list2', 'only_jobs', _('Optional: the list of jobs to launch, by their name. ')) parser.add_option('l', 'list', 'boolean', 'list', @@ -288,8 +300,7 @@ class Job(object): " job --jobs_config " + os.path.join(self.machine.sat_path, self.name_remote_jobs_pyconf) + - " --name " + - self.name) + " --name " + self.name) if prefix: self.command = prefix + ' "' + self.command +'"' @@ -314,13 +325,16 @@ class Job(object): :return: (the output of the kill, the error of the kill) :rtype: (str, str) ''' - - pids = self.get_pids() + try: + pids = self.get_pids() + except: + return ("Unable to get the pid of the command.", "") + cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids]) (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill, self.logger) time.sleep(wait) - return (out_kill, err_kill) + return (out_kill.read().decode(), err_kill.read().decode()) def has_begun(self): '''Returns True if the job has already begun @@ -355,7 +369,10 @@ class Job(object): # Put end time self._Tf = time.time() # And get the remote command status and log files - self.get_log_files() + try: + self.get_log_files() + except Exception as e: + self.err += _("Unable to get remote log files: %s" % e) return self._has_finished @@ -502,14 +519,13 @@ class Job(object): self._has_finished = True self._has_timouted = True self._Tf = time.time() - self.get_pids() - (out_kill, _) = self.kill_remote_process() - self.out += "TIMEOUT \n" + out_kill.read().decode() + (out_kill, __) = self.kill_remote_process() + self.out += "TIMEOUT \n" + out_kill self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout) try: self.get_log_files() except Exception as e: - self.err += _("Unable to get remote log files: %s" % e) + self.err += _("Unable to get remote log files!\n%s\n" % str(e)) def total_duration(self): """Give the total duration of the job @@ -757,8 +773,11 @@ class Jobs(object): msg = _("WARNING: The job \"%(job_name)s\" requires the " "machine \"%(machine_name)s\" but this machine " "is not defined in the configuration file.\n" - "The job will not be launched") - self.logger.write(src.printcolors.printcWarning(msg)) + "The job will not be launched\n") + self.logger.write(src.printcolors.printcWarning( + msg % {"job_name" : job_def.name, + "machine_name" : name_machine})) + continue a_job = self.define_job(job_def, a_machine) @@ -809,6 +828,16 @@ class Jobs(object): self.logger.flush() res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway, self.job_file_path) + + # set the local settings of sat on the remote machine using + # the init command + (__, out_dist, __) = machine.exec_command( + os.path.join(machine.sat_path, + "sat init --base default --workdir" + " default --log_dir default"), + self.logger) + out_dist.read() + # get the remote machine distribution using a sat command (__, out_dist, __) = machine.exec_command( os.path.join(machine.sat_path, @@ -816,6 +845,7 @@ class Jobs(object): self.logger) machine.distribution = out_dist.read().decode().replace("\n", "") + # Print the status of the copy if res_copy == 0: self.logger.write('\r%s' % @@ -1167,7 +1197,8 @@ class Gui(object): continue for board in self.d_xml_board_files: if board_job == board: - if distrib is not None and distrib not in d_dist[board]: + if (distrib not in [None, ''] and + distrib not in d_dist[board]): d_dist[board].append(distrib) src.xmlManager.add_simple_node( self.d_xml_board_files[board].xmlroot.find( @@ -1176,7 +1207,7 @@ class Gui(object): attrib={"name" : distrib}) if board_job == board: - if (application is not None and + if (application not in [None, ''] and application not in d_application[board]): d_application[board].append(application) src.xmlManager.add_simple_node( @@ -1308,7 +1339,6 @@ class Gui(object): self.logger.write("%s\n" % src.printcolors.printcWarning( msg), 5) - # Construct the dictionnary self.history for job in l_jobs + l_jobs_not_today: l_links = [] @@ -1501,6 +1531,16 @@ class Gui(object): src.xmlManager.add_simple_node(xmlj, "remote_log_file_path", "nothing") + # Search for the test log if there is any + l_test_log_files = self.find_test_log(job.remote_log_files) + xml_test = src.xmlManager.add_simple_node(xmlj, + "test_log_file_path") + for test_log_path, res_test, nb_fails in l_test_log_files: + test_path_node = src.xmlManager.add_simple_node(xml_test, + "path", + test_log_path) + test_path_node.attrib["res"] = res_test + test_path_node.attrib["nb_fails"] = nb_fails xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after) # get the job father @@ -1542,6 +1582,33 @@ class Gui(object): datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}) + def find_test_log(self, l_remote_log_files): + '''Find if there is a test log (board) in the remote log files and + the path to it. There can be several test command, so the result is + a list. + + :param l_remote_log_files List: the list of all remote log files + :return: the list of (test log files path, res of the command) + :rtype: List + ''' + res = [] + for file_path in l_remote_log_files: + dirname = os.path.basename(os.path.dirname(file_path)) + file_name = os.path.basename(file_path) + regex = src.logger.log_all_command_file_expression + oExpr = re.compile(regex) + if dirname == "TEST" and oExpr.search(file_name): + # find the res of the command + prod_node = etree.parse(file_path).getroot().find("product") + res_test = prod_node.attrib["global_res"] + # find the number of fails + testbase_node = prod_node.find("tests").find("testbase") + nb_fails = int(testbase_node.attrib["failed"]) + # put the file path, the res of the test command and the number + # of fails in the output + res.append((file_path, res_test, nb_fails)) + + return res def last_update(self, finish_status = "finished"): '''update information about the jobs for the file xml_file @@ -1574,6 +1641,56 @@ class Gui(object): for xml_file in self.d_xml_board_files.values(): self.write_xml_file(xml_file, STYLESHEET_BOARD) +def get_config_file_path(job_config_name, l_cfg_dir): + found = False + file_jobs_cfg = None + if os.path.exists(job_config_name) and job_config_name.endswith(".pyconf"): + found = True + file_jobs_cfg = job_config_name + else: + for cfg_dir in l_cfg_dir: + file_jobs_cfg = os.path.join(cfg_dir, job_config_name) + if not file_jobs_cfg.endswith('.pyconf'): + file_jobs_cfg += '.pyconf' + + if not os.path.exists(file_jobs_cfg): + continue + else: + found = True + break + return found, file_jobs_cfg + +def develop_factorized_jobs(config_jobs): + '''update information about the jobs for the file xml_file + + :param config_jobs Config: the config corresponding to the jos description + ''' + developed_jobs_list = [] + for jb in config_jobs.jobs: + # case where the jobs are not developed + if type(jb.machine) == type(""): + developed_jobs_list.append(jb) + continue + # Case where the jobs must be developed + # Example: + # machine : ["CO7.2 physique", ["CO6.4 physique", $MONDAY, $TUESDAY ], "FD22"] + name_job = jb.name + for machine in jb.machine: + new_job = src.pyconf.deepCopyMapping(jb) + # case where there is a jobs on the machine corresponding to all + # days in when variable. + if type(machine) == type(""): + new_job.machine = machine + new_job.name = name_job + " / " + machine + else: + # case the days are re defined + new_job.machine = machine[0] + new_job.name = name_job + " / " + machine[0] + new_job.when = machine[1:] + developed_jobs_list.append(new_job) + + config_jobs.jobs = developed_jobs_list + ## # Describes the command @@ -1612,49 +1729,62 @@ def run(args, runner, logger): return 1 # Find the file in the directories, unless it is a full path - found = False - if os.path.exists(options.jobs_cfg): - found = True - file_jobs_cfg = options.jobs_cfg - else: - for cfg_dir in l_cfg_dir: - file_jobs_cfg = os.path.join(cfg_dir, options.jobs_cfg) - if not file_jobs_cfg.endswith('.pyconf'): - file_jobs_cfg += '.pyconf' - - if not os.path.exists(file_jobs_cfg): - continue - else: - found = True - break - - if not found: - msg = _("The file configuration %(name_file)s was not found." - "\nUse the --list option to get the possible files.") - src.printcolors.printcError(msg) - return 1 + # merge all in a config + merger = src.pyconf.ConfigMerger() + config_jobs = src.pyconf.Config() + l_conf_files_path = [] + for config_file in options.jobs_cfg: + found, file_jobs_cfg = get_config_file_path(config_file, l_cfg_dir) + if not found: + msg = _("The file configuration %s was not found." + "\nUse the --list option to get the " + "possible files." % config_file) + logger.write("%s\n" % src.printcolors.printcError(msg), 1) + return 1 + l_conf_files_path.append(file_jobs_cfg) + # Read the config that is in the file + one_config_jobs = src.read_config_from_a_file(file_jobs_cfg) + merger.merge(config_jobs, one_config_jobs) info = [ (_("Platform"), runner.cfg.VARS.dist), - (_("File containing the jobs configuration"), file_jobs_cfg) + (_("Files containing the jobs configuration"), l_conf_files_path) ] src.print_info(logger, info) - # Read the config that is in the file - config_jobs = src.read_config_from_a_file(file_jobs_cfg) if options.only_jobs: l_jb = src.pyconf.Sequence() for jb in config_jobs.jobs: if jb.name in options.only_jobs: l_jb.append(jb, - "Adding a job that was given in only_jobs option parameters") + "Job that was given in only_jobs option parameters\n") config_jobs.jobs = l_jb - + + # Parse the config jobs in order to develop all the factorized jobs + develop_factorized_jobs(config_jobs) + + # Make a unique file that contain all the jobs in order to use it + # on every machine + name_pyconf = "_".join([os.path.basename(path)[:-len('.pyconf')] + for path in l_conf_files_path]) + ".pyconf" + path_pyconf = src.get_tmp_filename(runner.cfg, name_pyconf) + #Save config + f = file( path_pyconf , 'w') + config_jobs.__save__(f) + + # log the paramiko problems + log_dir = src.get_log_path(runner.cfg) + paramiko_log_dir_path = os.path.join(log_dir, "JOBS") + src.ensure_path_exists(paramiko_log_dir_path) + paramiko.util.log_to_file(os.path.join(paramiko_log_dir_path, + logger.txtFileName)) + # Initialization today_jobs = Jobs(runner, logger, - file_jobs_cfg, + path_pyconf, config_jobs) + # SSH connection to all machines today_jobs.ssh_connection_all_machines() if options.test_connection: @@ -1667,18 +1797,21 @@ def run(args, runner, logger): logger.flush() # Copy the stylesheets in the log directory - log_dir = runner.cfg.USER.log_dir + log_dir = log_dir xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl') files_to_copy = [] files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL)) files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD)) + files_to_copy.append(os.path.join(xsl_dir, "command.xsl")) files_to_copy.append(os.path.join(xsl_dir, "running.gif")) for file_path in files_to_copy: - shutil.copy2(file_path, log_dir) + # OP We use copy instead of copy2 to update the creation date + # So we can clean the LOGS directories easily + shutil.copy(file_path, log_dir) # Instanciate the Gui in order to produce the xml files that contain all # the boards - gui = Gui(runner.cfg.USER.log_dir, + gui = Gui(log_dir, today_jobs.ljobs, today_jobs.ljobs_not_today, runner.cfg.VARS.datehour, @@ -1711,6 +1844,18 @@ def run(args, runner, logger): interruped = True logger.write("\n\n%s\n\n" % (src.printcolors.printcWarning(_("Forced interruption"))), 1) + except Exception as e: + msg = _("CRITICAL ERROR: The jobs loop has been interrupted\n") + logger.write("\n\n%s\n" % src.printcolors.printcError(msg) ) + logger.write("%s\n" % str(e)) + # get stack + __, __, exc_traceback = sys.exc_info() + fp = tempfile.TemporaryFile() + traceback.print_tb(exc_traceback, file=fp) + fp.seek(0) + stack = fp.read() + logger.write("\nTRACEBACK: %s\n" % stack.replace('"',"'"), 1) + finally: res = 0 if interruped: @@ -1738,4 +1883,7 @@ def run(args, runner, logger): today_jobs.gui.last_update() # Output the results today_jobs.write_all_results() + # Remove the temporary pyconf file + if os.path.exists(path_pyconf): + os.remove(path_pyconf) return res