X-Git-Url: http://git.salome-platform.org/gitweb/?a=blobdiff_plain;f=commands%2Fjobs.py;h=06602f2d941364c37cd0c457a26ed23b023b8936;hb=86042d1d4bc886479259b14865ad220f1a8c8a1a;hp=601a049f98f67b7535d82f2ffa9a1eb3cda1a19c;hpb=9e8d61df89449a009daa16dc7912e1edeb497cc3;p=tools%2Fsat.git diff --git a/commands/jobs.py b/commands/jobs.py index 601a049..06602f2 100644 --- a/commands/jobs.py +++ b/commands/jobs.py @@ -137,9 +137,9 @@ class Machine(object): self.put_dir(sat_local_path, self.sat_path, filters = ['.git']) # put the job configuration file in order to make it reachable # on the remote machine - self.sftp.put(job_file, os.path.join(".salomeTools", - "Jobs", - ".jobs_command_file.pyconf")) + remote_job_file_name = ".%s" % os.path.basename(job_file) + self.sftp.put(job_file, os.path.join(self.sat_path, + remote_job_file_name)) except Exception as e: res = str(e) self._connection_successful = False @@ -237,8 +237,18 @@ class Machine(object): class Job(object): '''Class to manage one job ''' - def __init__(self, name, machine, application, board, - commands, timeout, config, logger, after=None): + def __init__(self, + name, + machine, + application, + board, + commands, + timeout, + config, + job_file_path, + logger, + after=None, + prefix=None): self.name = name self.machine = machine @@ -268,15 +278,20 @@ class Job(object): self.out = "" self.err = "" - + + self.name_remote_jobs_pyconf = ".%s" % os.path.basename(job_file_path) self.commands = commands self.command = (os.path.join(self.machine.sat_path, "sat") + " -l " + os.path.join(self.machine.sat_path, "list_log_files.txt") + - " job --jobs_config .jobs_command_file" + + " job --jobs_config " + + os.path.join(self.machine.sat_path, + self.name_remote_jobs_pyconf) + " --name " + self.name) + if prefix: + self.command = prefix + ' "' + self.command +'"' def get_pids(self): """ Get the pid(s) corresponding to the command that have been launched @@ -656,6 +671,9 @@ class Jobs(object): board = None if 'board' in job_def: board = job_def.board + prefix = None + if "prefix" in job_def: + prefix = job_def.prefix return Job(name, machine, @@ -664,8 +682,10 @@ class Jobs(object): cmmnds, timeout, self.runner.cfg, + self.job_file_path, self.logger, - after = after) + after = after, + prefix = prefix) def determine_jobs_and_machines(self): '''Function that reads the pyconf jobs definition and instantiates all @@ -774,6 +794,15 @@ class Jobs(object): # Copy salomeTools to the remote machine if machine.successfully_connected(self.logger): + step = _("Remove SAT") + self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3) + self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3) + (__, out_dist, __) = machine.exec_command( + "rm -rf %s" % machine.sat_path, + self.logger) + out_dist.read() + + self.logger.flush() step = _("Copy SAT") self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3) self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3) @@ -1027,7 +1056,13 @@ class Gui(object): see the jobs states ''' - def __init__(self, xml_dir_path, l_jobs, l_jobs_not_today, prefix, file_boards=""): + def __init__(self, + xml_dir_path, + l_jobs, + l_jobs_not_today, + prefix, + logger, + file_boards=""): '''Initialization :param xml_dir_path str: The path to the directory where to put @@ -1037,6 +1072,9 @@ class Gui(object): :param file_boards str: the file path from which to read the expected boards ''' + # The logging instance + self.logger = logger + # The prefix to add to the xml files : date_hour self.prefix = prefix @@ -1068,7 +1106,7 @@ class Gui(object): # Create the lines and columns self.initialize_boards(l_jobs, l_jobs_not_today) - + # Write the xml file self.update_xml_files(l_jobs) @@ -1213,6 +1251,7 @@ class Gui(object): # input csv files but that are not covered by a today job for board in self.d_input_boards.keys(): xml_root_board = self.d_xml_board_files[board].xmlroot + # Find the missing jobs for today xml_missing = src.xmlManager.add_simple_node(xml_root_board, "missing_jobs") for row, column in self.d_input_boards[board]["jobs"]: @@ -1227,6 +1266,22 @@ class Gui(object): "job", attrib={"distribution" : row, "application" : column }) + # Find the missing jobs not today + xml_missing_not_today = src.xmlManager.add_simple_node( + xml_root_board, + "missing_jobs_not_today") + for row, column in self.d_input_boards[board]["jobs_not_today"]: + found = False + for job in l_jobs_not_today: + if (job.application == column and + job.machine.distribution == row): + found = True + break + if not found: + src.xmlManager.add_simple_node(xml_missing_not_today, + "job", + attrib={"distribution" : row, + "application" : column }) def find_history(self, l_jobs, l_jobs_not_today): """find, for each job, in the existent xml boards the results for the @@ -1244,8 +1299,15 @@ class Gui(object): for file_name in os.listdir(self.xml_dir_path): if oExpr.search(file_name): file_path = os.path.join(self.xml_dir_path, file_name) - global_xml = src.xmlManager.ReadXmlFile(file_path) - l_globalxml.append(global_xml) + try: + global_xml = src.xmlManager.ReadXmlFile(file_path) + l_globalxml.append(global_xml) + except Exception as e: + msg = _("\nWARNING: the file %s can not be read, it will be " + "ignored\n%s" % (file_path, e)) + self.logger.write("%s\n" % src.printcolors.printcWarning( + msg), 5) + # Construct the dictionnary self.history for job in l_jobs + l_jobs_not_today: @@ -1264,7 +1326,7 @@ class Gui(object): res_job = job_node.find("res").text if link != "nothing": l_links.append((date, res_job, link)) - + l_links = sorted(l_links, reverse=True) self.history[job.name] = l_links def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs): @@ -1293,12 +1355,22 @@ class Gui(object): src.xmlManager.add_simple_node(xmlj, "sat_path", job.machine.sat_path) xml_history = src.xmlManager.add_simple_node(xmlj, "history") - for date, res_job, link in self.history[job.name]: - src.xmlManager.add_simple_node(xml_history, - "link", - text=link, - attrib={"date" : date, - "res" : res_job}) + for i, (date, res_job, link) in enumerate(self.history[job.name]): + if i==0: + # tag the first one (the last one) + src.xmlManager.add_simple_node(xml_history, + "link", + text=link, + attrib={"date" : date, + "res" : res_job, + "last" : "yes"}) + else: + src.xmlManager.add_simple_node(xml_history, + "link", + text=link, + attrib={"date" : date, + "res" : res_job, + "last" : "no"}) def parse_csv_boards(self, today): """ Parse the csv file that describes the boards to produce and fill @@ -1329,6 +1401,7 @@ class Gui(object): rows = [] jobs = [] + jobs_not_today = [] for line in input_board[1:]: row = line[0] rows.append(row) @@ -1337,13 +1410,16 @@ class Gui(object): continue days = square.split(DAYS_SEPARATOR) days = [int(day) for day in days] + job = (row, columns[i]) if today in days: - job = (row, columns[i]) jobs.append(job) + else: + jobs_not_today.append(job) d_boards[board_name] = {"rows" : rows, "columns" : columns, - "jobs" : jobs} + "jobs" : jobs, + "jobs_not_today" : jobs_not_today} self.d_input_boards = d_boards @@ -1425,6 +1501,12 @@ class Gui(object): src.xmlManager.add_simple_node(xmlj, "remote_log_file_path", "nothing") + # Search for the test log if there is any + l_test_log_files = self.find_test_log(job.remote_log_files) + xml_test = src.xmlManager.add_simple_node(xmlj, + "test_log_file_path") + for test_log_path in l_test_log_files: + src.xmlManager.add_simple_node(xml_test, "path", test_log_path) xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after) # get the job father @@ -1466,6 +1548,21 @@ class Gui(object): datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}) + def find_test_log(self, l_remote_log_files): + '''Find if there is a test log (board) in the remote log files and + the path to it. There can be several test command, so the result is + a list. + + :param l_remote_log_files List: the list of all remote log files + :return: the list of test log files path + :rtype: List + ''' + res = [] + for file_path in l_remote_log_files: + dirname = os.path.basename(os.path.dirname(file_path)) + if dirname == "TEST": + res.append(file_path) + return res def last_update(self, finish_status = "finished"): '''update information about the jobs for the file xml_file @@ -1520,7 +1617,8 @@ def run(args, runner, logger): if not options.no_label: logger.write("------ %s\n" % src.printcolors.printcHeader(cfg_dir)) - + if not os.path.exists(cfg_dir): + continue for f in sorted(os.listdir(cfg_dir)): if not f.endswith('.pyconf'): continue @@ -1534,18 +1632,22 @@ def run(args, runner, logger): src.printcolors.printcError(message) return 1 - # Find the file in the directories + # Find the file in the directories, unless it is a full path found = False - for cfg_dir in l_cfg_dir: - file_jobs_cfg = os.path.join(cfg_dir, options.jobs_cfg) - if not file_jobs_cfg.endswith('.pyconf'): - file_jobs_cfg += '.pyconf' - - if not os.path.exists(file_jobs_cfg): - continue - else: - found = True - break + if os.path.exists(options.jobs_cfg): + found = True + file_jobs_cfg = options.jobs_cfg + else: + for cfg_dir in l_cfg_dir: + file_jobs_cfg = os.path.join(cfg_dir, options.jobs_cfg) + if not file_jobs_cfg.endswith('.pyconf'): + file_jobs_cfg += '.pyconf' + + if not os.path.exists(file_jobs_cfg): + continue + else: + found = True + break if not found: msg = _("The file configuration %(name_file)s was not found." @@ -1581,6 +1683,10 @@ def run(args, runner, logger): gui = None if options.publish: + logger.write(src.printcolors.printcInfo( + _("Initialize the xml boards : ")), 5) + logger.flush() + # Copy the stylesheets in the log directory log_dir = runner.cfg.USER.log_dir xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl') @@ -1597,8 +1703,13 @@ def run(args, runner, logger): today_jobs.ljobs, today_jobs.ljobs_not_today, runner.cfg.VARS.datehour, + logger, file_boards = options.input_boards) + logger.write(src.printcolors.printcSuccess("OK"), 5) + logger.write("\n\n", 5) + logger.flush() + # Display the list of the xml files logger.write(src.printcolors.printcInfo(("Here is the list of published" " files :\n")), 4) @@ -1608,9 +1719,9 @@ def run(args, runner, logger): file_name = os.path.basename(file_path) logger.write("%s\n" % file_path, 4) logger.add_link(file_name, "board", 0, board) - + logger.write("\n", 4) - + today_jobs.gui = gui interruped = False @@ -1622,7 +1733,9 @@ def run(args, runner, logger): logger.write("\n\n%s\n\n" % (src.printcolors.printcWarning(_("Forced interruption"))), 1) finally: + res = 0 if interruped: + res = 1 msg = _("Killing the running jobs and trying" " to get the corresponding logs\n") logger.write(src.printcolors.printcWarning(msg)) @@ -1630,11 +1743,14 @@ def run(args, runner, logger): # find the potential not finished jobs and kill them for jb in today_jobs.ljobs: if not jb.has_finished(): + res = 1 try: jb.kill_remote_process() except Exception as e: msg = _("Failed to kill job %s: %s\n" % (jb.name, e)) logger.write(src.printcolors.printcWarning(msg)) + if jb.res_job != "0": + res = 1 if interruped: if today_jobs.gui: today_jobs.gui.last_update(_("Forced interruption")) @@ -1643,3 +1759,4 @@ def run(args, runner, logger): today_jobs.gui.last_update() # Output the results today_jobs.write_all_results() + return res