X-Git-Url: http://git.salome-platform.org/gitweb/?a=blobdiff_plain;f=commands%2Fjobs.py;h=2b29f3aa11c5a5cb2702ce8fbbde21cfa3c6deb6;hb=e78f0659aceb8e693fd02b01531215180256543c;hp=39b55a70bbf91547ee40eeef164be0eab1b88af8;hpb=88fc4f3bac78f633f48e1d5d931658d46bead109;p=tools%2Fsat.git diff --git a/commands/jobs.py b/commands/jobs.py index 39b55a7..2b29f3a 100644 --- a/commands/jobs.py +++ b/commands/jobs.py @@ -21,24 +21,22 @@ import datetime import time import csv import shutil +import itertools +import re import paramiko import src STYLESHEET_GLOBAL = "jobs_global_report.xsl" STYLESHEET_BOARD = "jobs_board_report.xsl" -d_INT_DAY = {0 : "monday", - 1 : "tuesday", - 2 : "wednesday", - 3 : "thursday", - 4 : "friday", - 5 : "saturday", - 6 : "sunday"} + +DAYS_SEPARATOR = "," +CSV_DELIMITER = ";" parser = src.options.Options() -parser.add_option('j', 'jobs_config', 'string', 'jobs_cfg', - _('The name of the config file that contains' +parser.add_option('n', 'name', 'string', 'jobs_cfg', + _('Mandatory: The name of the config file that contains' ' the jobs configuration')) parser.add_option('o', 'only_jobs', 'list2', 'only_jobs', _('Optional: the list of jobs to launch, by their name. ')) @@ -52,10 +50,10 @@ parser.add_option('p', 'publish', 'boolean', 'publish', _("Optional: generate an xml file that can be read in a " "browser to display the jobs status."), False) -parser.add_option('i', 'input_boards', 'list2', 'input_boards', _("Optional: " - "the list of path to csv files that contain " - "the expected boards."),[]) -parser.add_option('n', 'completion', 'boolean', 'no_label', +parser.add_option('i', 'input_boards', 'string', 'input_boards', _("Optional: " + "the path to csv file that contain " + "the expected boards."),"") +parser.add_option('', 'completion', 'boolean', 'no_label', _("Optional (internal use): do not print labels, Works only " "with --list."), False) @@ -139,11 +137,9 @@ class Machine(object): self.put_dir(sat_local_path, self.sat_path, filters = ['.git']) # put the job configuration file in order to make it reachable # on the remote machine - job_file_name = os.path.basename(job_file) - self.sftp.put(job_file, os.path.join(self.sat_path, - "data", - "jobs", - job_file_name)) + self.sftp.put(job_file, os.path.join(".salomeTools", + "Jobs", + ".jobs_command_file.pyconf")) except Exception as e: res = str(e) self._connection_successful = False @@ -242,7 +238,7 @@ class Job(object): '''Class to manage one job ''' def __init__(self, name, machine, application, board, - commands, timeout, config, logger, job_file, after=None): + commands, timeout, config, logger, after=None): self.name = name self.machine = machine @@ -270,16 +266,15 @@ class Job(object): self._stdout = None # Store the command outputs field self._stderr = None # Store the command errors field - self.out = None # Contains something only if the job is finished - self.err = None # Contains something only if the job is finished + self.out = "" + self.err = "" self.commands = commands self.command = (os.path.join(self.machine.sat_path, "sat") + " -l " + os.path.join(self.machine.sat_path, "list_log_files.txt") + - " job --jobs_config " + - job_file + + " job --jobs_config .jobs_command_file" + " --name " + self.name) @@ -340,8 +335,8 @@ class Job(object): if self._stdout.channel.closed: self._has_finished = True # Store the result outputs - self.out = self._stdout.read().decode() - self.err = self._stderr.read().decode() + self.out += self._stdout.read().decode() + self.err += self._stderr.read().decode() # Put end time self._Tf = time.time() # And get the remote command status and log files @@ -351,7 +346,8 @@ class Job(object): def get_log_files(self): """Get the log files produced by the command launched - on the remote machine. + on the remote machine, and put it in the log directory of the user, + so they can be accessible from """ # Do not get the files if the command is not finished if not self.has_finished(): @@ -361,8 +357,9 @@ class Job(object): # First get the file that contains the list of log files to get tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt") + remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt") self.machine.sftp.get( - os.path.join(self.machine.sat_path, "list_log_files.txt"), + remote_path, tmp_file_path) # Read the file and get the result of the command and all the log files @@ -372,8 +369,13 @@ class Job(object): file_lines = [line.replace("\n", "") for line in file_lines] fstream_tmp.close() os.remove(tmp_file_path) - # The first line is the result of the command (0 success or 1 fail) - self.res_job = file_lines[0] + + try : + # The first line is the result of the command (0 success or 1 fail) + self.res_job = file_lines[0] + except Exception as e: + self.err += _("Unable to get status from remote file %s: %s" % + (remote_path, str(e))) for i, job_path_remote in enumerate(file_lines[1:]): try: @@ -382,7 +384,10 @@ class Job(object): # internal traces. # 2- The txt file containing the system command traces (like # traces produced by the "make" command) - if os.path.basename(os.path.dirname(job_path_remote)) != 'OUT': + # 3- In case of the test command, there is another file to get : + # the xml board that contain the test results + dirname = os.path.basename(os.path.dirname(job_path_remote)) + if dirname != 'OUT' and dirname != 'TEST': # Case 1- local_path = os.path.join(os.path.dirname( self.logger.logFilePath), @@ -392,19 +397,27 @@ class Job(object): "job", self.res_job, self.command) - else: + elif dirname == 'OUT': # Case 2- local_path = os.path.join(os.path.dirname( self.logger.logFilePath), 'OUT', os.path.basename(job_path_remote)) + elif dirname == 'TEST': + # Case 3- + local_path = os.path.join(os.path.dirname( + self.logger.logFilePath), + 'TEST', + os.path.basename(job_path_remote)) + # Get the file if not os.path.exists(local_path): self.machine.sftp.get(job_path_remote, local_path) self.remote_log_files.append(local_path) except Exception as e: self.err += _("Unable to get %s log file from remote: %s" % - (job_path_remote, str(e))) + (str(job_path_remote), + str(e))) def has_failed(self): '''Returns True if the job has failed. @@ -429,11 +442,13 @@ class Job(object): """In case of a failing job, one has to cancel every job that depend on it. This method put the job as failed and will not be executed. """ + if self.cancelled: + return self._has_begun = True self._has_finished = True self.cancelled = True - self.out = _("This job was not launched because its father has failed.") - self.err = _("This job was not launched because its father has failed.") + self.out += _("This job was not launched because its father has failed.") + self.err += _("This job was not launched because its father has failed.") def is_running(self): '''Returns True if the job commands are running @@ -474,8 +489,8 @@ class Job(object): self._Tf = time.time() self.get_pids() (out_kill, _) = self.kill_remote_process() - self.out = "TIMEOUT \n" + out_kill.read().decode() - self.err = "TIMEOUT : %s seconds elapsed\n" % str(self.timeout) + self.out += "TIMEOUT \n" + out_kill.read().decode() + self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout) try: self.get_log_files() except Exception as e: @@ -506,7 +521,7 @@ class Job(object): if not self.machine.successfully_connected(self.logger): self._has_finished = True self.out = "N\A" - self.err = ("Connection to machine (name : %s, host: %s, port:" + self.err += ("Connection to machine (name : %s, host: %s, port:" " %s, user: %s) has failed\nUse the log command " "to get more information." % (self.machine.name, @@ -523,8 +538,8 @@ class Job(object): if (self._stdin, self._stdout, self._stderr) == (None, None, None): self._has_finished = True self._Tf = time.time() - self.out = "N\A" - self.err = "The server failed to execute the command" + self.out += "N\A" + self.err += "The server failed to execute the command" # Put the beginning flag to true. self._has_begun = True @@ -553,15 +568,12 @@ class Job(object): self.machine.write_info(self.logger) self.logger.write(src.printcolors.printcInfo("out : \n")) - if self.out is None: + if self.out == "": self.logger.write("Unable to get output\n") else: self.logger.write(self.out + "\n") self.logger.write(src.printcolors.printcInfo("err : \n")) - if self.err is None: - self.logger.write("Unable to get error\n") - else: - self.logger.write(self.err + "\n") + self.logger.write(self.err + "\n") def get_status(self): """Get the status of the job (used by the Gui for xml display) @@ -591,13 +603,11 @@ class Jobs(object): def __init__(self, runner, logger, - job_file, job_file_path, config_jobs, lenght_columns = 20): # The jobs configuration self.cfg_jobs = config_jobs - self.job_file = job_file self.job_file_path = job_file_path # The machine that will be used today self.lmachines = [] @@ -633,7 +643,10 @@ class Jobs(object): ''' name = job_def.name cmmnds = job_def.commands - timeout = job_def.timeout + if not "timeout" in job_def: + timeout = 4*60*60 # default timeout = 4h + else: + timeout = job_def.timeout after = None if 'after' in job_def: after = job_def.after @@ -652,7 +665,6 @@ class Jobs(object): timeout, self.runner.cfg, self.logger, - self.job_file, after = after) def determine_jobs_and_machines(self): @@ -789,8 +801,8 @@ class Jobs(object): self.logger.write('\r%s%s%s %s' % (begin_line, endline, - src.printcolors.printc(src.OK_STATUS), - _("Copy of SAT failed")), 3) + src.printcolors.printc(src.KO_STATUS), + _("Copy of SAT failed: %s" % res_copy)), 3) else: self.logger.write('\r%s' % ((len(begin_line)+len(endline)+20) * " "), 3) @@ -1015,31 +1027,45 @@ class Gui(object): see the jobs states ''' - def __init__(self, xml_dir_path, l_jobs, l_jobs_not_today, l_file_boards = []): + def __init__(self, xml_dir_path, l_jobs, l_jobs_not_today, prefix, file_boards=""): '''Initialization :param xml_dir_path str: The path to the directory where to put the xml resulting files :param l_jobs List: the list of jobs that run today :param l_jobs_not_today List: the list of jobs that do not run today - :param l_file_boards List: the list of file path from which to read the + :param file_boards str: the file path from which to read the expected boards ''' + # The prefix to add to the xml files : date_hour + self.prefix = prefix + # The path of the csv files to read to fill the expected boards - self.l_file_boards = l_file_boards + self.file_boards = file_boards - today = d_INT_DAY[datetime.date.weekday(datetime.date.today())] - self.parse_csv_boards(today) + if file_boards != "": + today = datetime.date.weekday(datetime.date.today()) + self.parse_csv_boards(today) + else: + self.d_input_boards = {} # The path of the global xml file self.xml_dir_path = xml_dir_path # Initialize the xml files - xml_global_path = os.path.join(self.xml_dir_path, "global_report.xml") + self.global_name = "global_report" + xml_global_path = os.path.join(self.xml_dir_path, + self.global_name + ".xml") self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path, "JobsReport") + + # Find history for each job + self.history = {} + self.find_history(l_jobs, l_jobs_not_today) + # The xml files that corresponds to the boards. # {name_board : xml_object}} self.d_xml_board_files = {} + # Create the lines and columns self.initialize_boards(l_jobs, l_jobs_not_today) @@ -1047,6 +1073,9 @@ class Gui(object): self.update_xml_files(l_jobs) def add_xml_board(self, name): + '''Add a board to the board list + :param name str: the board name + ''' xml_board_path = os.path.join(self.xml_dir_path, name + ".xml") self.d_xml_board_files[name] = src.xmlManager.XmlLogFile( xml_board_path, @@ -1073,6 +1102,9 @@ class Gui(object): for board in list(self.d_input_boards.keys()): if board not in self.d_xml_board_files: self.add_xml_board(board) + root_node = self.d_xml_board_files[board].xmlroot + src.xmlManager.append_node_attrib(root_node, + {"input_file" : self.file_boards}) # Loop over all jobs in order to get the lines and columns for each # xml file @@ -1155,10 +1187,28 @@ class Gui(object): # that will not be launched today self.put_jobs_not_today(l_jobs_not_today, xml_jobs) + # add also the infos node xml_file.add_simple_node("infos", attrib={"name" : "last update", "JobsCommandStatus" : "running"}) - + + # and put the history node + history_node = xml_file.add_simple_node("history") + name_board = os.path.basename(xml_file.logFile)[:-len(".xml")] + # serach for board files + expression = "^[0-9]{8}_+[0-9]{6}_" + name_board + ".xml$" + oExpr = re.compile(expression) + # Get the list of xml borad files that are in the log directory + for file_name in os.listdir(self.xml_dir_path): + if oExpr.search(file_name): + date = os.path.basename(file_name).split("_")[0] + file_path = os.path.join(self.xml_dir_path, file_name) + src.xmlManager.add_simple_node(history_node, + "link", + text=file_path, + attrib={"date" : date}) + + # Find in each board the squares that needs to be filled regarding the # input csv files but that are not covered by a today job for board in self.d_input_boards.keys(): @@ -1177,7 +1227,46 @@ class Gui(object): "job", attrib={"distribution" : row, "application" : column }) - + + def find_history(self, l_jobs, l_jobs_not_today): + """find, for each job, in the existent xml boards the results for the + job. Store the results in the dictionnary self.history = {name_job : + list of (date, status, list links)} + + :param l_jobs List: the list of jobs to run today + :param l_jobs_not_today List: the list of jobs that do not run today + """ + # load the all the history + expression = "^[0-9]{8}_+[0-9]{6}_" + self.global_name + ".xml$" + oExpr = re.compile(expression) + # Get the list of global xml that are in the log directory + l_globalxml = [] + for file_name in os.listdir(self.xml_dir_path): + if oExpr.search(file_name): + file_path = os.path.join(self.xml_dir_path, file_name) + global_xml = src.xmlManager.ReadXmlFile(file_path) + l_globalxml.append(global_xml) + + # Construct the dictionnary self.history + for job in l_jobs + l_jobs_not_today: + l_links = [] + for global_xml in l_globalxml: + date = os.path.basename(global_xml.filePath).split("_")[0] + global_root_node = global_xml.xmlroot.find("jobs") + job_node = src.xmlManager.find_node_by_attrib( + global_root_node, + "job", + "name", + job.name) + if job_node: + if job_node.find("remote_log_file_path") is not None: + link = job_node.find("remote_log_file_path").text + res_job = job_node.find("res").text + if link != "nothing": + l_links.append((date, res_job, link)) + + self.history[job.name] = l_links + def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs): '''Get all the first information needed for each file and write the first version of the files @@ -1203,23 +1292,32 @@ class Gui(object): src.xmlManager.add_simple_node(xmlj, "user", job.machine.user) src.xmlManager.add_simple_node(xmlj, "sat_path", job.machine.sat_path) + xml_history = src.xmlManager.add_simple_node(xmlj, "history") + for date, res_job, link in self.history[job.name]: + src.xmlManager.add_simple_node(xml_history, + "link", + text=link, + attrib={"date" : date, + "res" : res_job}) def parse_csv_boards(self, today): - """ Parse the csv files that describes the boards to produce and fill + """ Parse the csv file that describes the boards to produce and fill the dict d_input_boards that contain the csv file contain - :param today str: the current day of the week + :param today int: the current day of the week """ - # loop over each csv file and read its content - l_boards = [] - for file_path in self.l_file_boards: - l_read = [] - with open(file_path, 'r') as f: - reader = csv.reader(f) - for row in reader: - l_read.append(row) - l_boards.append(l_read) - + # open the csv file and read its content + l_read = [] + with open(self.file_boards, 'r') as f: + reader = csv.reader(f,delimiter=CSV_DELIMITER) + for row in reader: + l_read.append(row) + # get the delimiter for the boards (empty line) + boards_delimiter = [''] * len(l_read[0]) + # Make the list of boards, by splitting with the delimiter + l_boards = [list(y) for x, y in itertools.groupby(l_read, + lambda z: z == boards_delimiter) if not x] + # loop over the csv lists of lines and get the rows, columns and jobs d_boards = {} for input_board in l_boards: @@ -1230,21 +1328,22 @@ class Gui(object): columns = input_board[0][1:] rows = [] - columns_out = [] jobs = [] for line in input_board[1:]: row = line[0] for i, square in enumerate(line[1:]): - if today in square: + if square=='': + continue + days = square.split(DAYS_SEPARATOR) + days = [int(day) for day in days] + if today in days: if row not in rows: rows.append(row) - if columns[i] not in columns_out: - columns_out.append(columns[i]) job = (row, columns[i]) jobs.append(job) - + d_boards[board_name] = {"rows" : rows, - "columns" : columns_out, + "columns" : columns, "jobs" : jobs} self.d_input_boards = d_boards @@ -1294,6 +1393,14 @@ class Gui(object): src.xmlManager.add_simple_node(xmlj, "host", job.machine.host) src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port)) src.xmlManager.add_simple_node(xmlj, "user", job.machine.user) + xml_history = src.xmlManager.add_simple_node(xmlj, "history") + for date, res_job, link in self.history[job.name]: + src.xmlManager.add_simple_node(xml_history, + "link", + text=link, + attrib={"date" : date, + "res" : res_job}) + src.xmlManager.add_simple_node(xmlj, "sat_path", job.machine.sat_path) src.xmlManager.add_simple_node(xmlj, "application", job.application) @@ -1373,33 +1480,40 @@ class Gui(object): attrib={"JobsCommandStatus" : finish_status}) # Write the file self.write_xml_files() - + + def write_xml_file(self, xml_file, stylesheet): + ''' Write one xml file and the same file with prefix + ''' + xml_file.write_tree(stylesheet) + file_path = xml_file.logFile + file_dir = os.path.dirname(file_path) + file_name = os.path.basename(file_path) + file_name_with_prefix = self.prefix + "_" + file_name + xml_file.write_tree(stylesheet, os.path.join(file_dir, + file_name_with_prefix)) + def write_xml_files(self): ''' Write the xml files ''' - self.xml_global_file.write_tree(STYLESHEET_GLOBAL) + self.write_xml_file(self.xml_global_file, STYLESHEET_GLOBAL) for xml_file in self.d_xml_board_files.values(): - xml_file.write_tree(STYLESHEET_BOARD) - + self.write_xml_file(xml_file, STYLESHEET_BOARD) + + ## # Describes the command def description(): return _("The jobs command launches maintenances that are described" - " in the dedicated jobs configuration file.") + " in the dedicated jobs configuration file.\n\nexample:\nsat " + "jobs --name my_jobs --publish") ## # Runs the command. def run(args, runner, logger): (options, args) = parser.parse_args(args) - - jobs_cfg_files_dir = runner.cfg.SITE.jobs.config_path - - l_cfg_dir = [os.path.join(runner.cfg.VARS.datadir, "jobs"), - jobs_cfg_files_dir] - - # Make sure the path to the jobs config files directory exists - src.ensure_path_exists(jobs_cfg_files_dir) + + l_cfg_dir = runner.cfg.PATHS.JOBPATH # list option : display all the available config files if options.list: @@ -1445,7 +1559,7 @@ def run(args, runner, logger): (_("File containing the jobs configuration"), file_jobs_cfg) ] src.print_info(logger, info) - + # Read the config that is in the file config_jobs = src.read_config_from_a_file(file_jobs_cfg) if options.only_jobs: @@ -1459,7 +1573,6 @@ def run(args, runner, logger): # Initialization today_jobs = Jobs(runner, logger, - options.jobs_cfg, file_jobs_cfg, config_jobs) # SSH connection to all machines @@ -1470,7 +1583,7 @@ def run(args, runner, logger): gui = None if options.publish: # Copy the stylesheets in the log directory - log_dir = runner.cfg.SITE.log.log_dir + log_dir = runner.cfg.USER.log_dir xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl') files_to_copy = [] files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL)) @@ -1481,10 +1594,23 @@ def run(args, runner, logger): # Instanciate the Gui in order to produce the xml files that contain all # the boards - gui = Gui(runner.cfg.SITE.log.log_dir, + gui = Gui(runner.cfg.USER.log_dir, today_jobs.ljobs, today_jobs.ljobs_not_today, - l_file_boards = options.input_boards) + runner.cfg.VARS.datehour, + file_boards = options.input_boards) + + # Display the list of the xml files + logger.write(src.printcolors.printcInfo(("Here is the list of published" + " files :\n")), 4) + logger.write("%s\n" % gui.xml_global_file.logFile, 4) + for board in gui.d_xml_board_files.keys(): + file_path = gui.d_xml_board_files[board].logFile + file_name = os.path.basename(file_path) + logger.write("%s\n" % file_path, 4) + logger.add_link(file_name, "board", 0, board) + + logger.write("\n", 4) today_jobs.gui = gui @@ -1505,7 +1631,11 @@ def run(args, runner, logger): # find the potential not finished jobs and kill them for jb in today_jobs.ljobs: if not jb.has_finished(): - jb.kill_remote_process() + try: + jb.kill_remote_process() + except Exception as e: + msg = _("Failed to kill job %s: %s\n" % (jb.name, e)) + logger.write(src.printcolors.printcWarning(msg)) if interruped: if today_jobs.gui: today_jobs.gui.last_update(_("Forced interruption"))