3 # Copyright (C) 2010-2013 CEA/DEN
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Lesser General Public
7 # License as published by the Free Software Foundation; either
8 # version 2.1 of the License.
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # Lesser General Public License for more details.
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with this library; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
28 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
29 STYLESHEET_BOARD = "jobs_board_report.xsl"
30 d_INT_DAY = {0 : "monday",
38 parser = src.options.Options()
40 parser.add_option('n', 'name', 'string', 'jobs_cfg',
41 _('The name of the config file that contains'
42 ' the jobs configuration'))
43 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
44 _('Optional: the list of jobs to launch, by their name. '))
45 parser.add_option('l', 'list', 'boolean', 'list',
46 _('Optional: list all available config files.'))
47 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
48 _("Optional: try to connect to the machines. "
49 "Not executing the jobs."),
51 parser.add_option('p', 'publish', 'boolean', 'publish',
52 _("Optional: generate an xml file that can be read in a "
53 "browser to display the jobs status."),
55 parser.add_option('i', 'input_boards', 'list2', 'input_boards', _("Optional: "
56 "the list of path to csv files that contain "
57 "the expected boards."),[])
58 parser.add_option('n', 'completion', 'boolean', 'no_label',
59 _("Optional (internal use): do not print labels, Works only "
63 class Machine(object):
64 '''Class to manage a ssh connection on a machine
72 sat_path="salomeTools"):
76 self.distribution = None # Will be filled after copying SAT on the machine
78 self.password = passwd
79 self.sat_path = sat_path
80 self.ssh = paramiko.SSHClient()
81 self._connection_successful = None
83 def connect(self, logger):
84 '''Initiate the ssh connection to the remote machine
86 :param logger src.logger.Logger: The logger instance
91 self._connection_successful = False
92 self.ssh.load_system_host_keys()
93 self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
95 self.ssh.connect(self.host,
98 password = self.password)
99 except paramiko.AuthenticationException:
100 message = src.KO_STATUS + _("Authentication failed")
101 except paramiko.BadHostKeyException:
102 message = (src.KO_STATUS +
103 _("The server's host key could not be verified"))
104 except paramiko.SSHException:
105 message = ( _("SSHException error connecting or "
106 "establishing an SSH session"))
108 message = ( _("Error connecting or establishing an SSH session"))
110 self._connection_successful = True
114 def successfully_connected(self, logger):
115 '''Verify if the connection to the remote machine has succeed
117 :param logger src.logger.Logger: The logger instance
118 :return: True if the connection has succeed, False if not
121 if self._connection_successful == None:
122 message = _("Warning : trying to ask if the connection to "
123 "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
124 " no connection request" %
125 (self.name, self.host, self.port, self.user))
126 logger.write( src.printcolors.printcWarning(message))
127 return self._connection_successful
129 def copy_sat(self, sat_local_path, job_file):
130 '''Copy salomeTools to the remote machine in self.sat_path
134 # open a sftp connection
135 self.sftp = self.ssh.open_sftp()
136 # Create the sat directory on remote machine if it is not existing
137 self.mkdir(self.sat_path, ignore_existing=True)
139 self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
140 # put the job configuration file in order to make it reachable
141 # on the remote machine
142 self.sftp.put(job_file, os.path.join(".salomeTools",
144 ".jobs_command_file.pyconf"))
145 except Exception as e:
147 self._connection_successful = False
151 def put_dir(self, source, target, filters = []):
152 ''' Uploads the contents of the source directory to the target path. The
153 target directory needs to exists. All sub-directories in source are
154 created under target.
156 for item in os.listdir(source):
159 source_path = os.path.join(source, item)
160 destination_path = os.path.join(target, item)
161 if os.path.islink(source_path):
162 linkto = os.readlink(source_path)
164 self.sftp.symlink(linkto, destination_path)
165 self.sftp.chmod(destination_path,
166 os.stat(source_path).st_mode)
170 if os.path.isfile(source_path):
171 self.sftp.put(source_path, destination_path)
172 self.sftp.chmod(destination_path,
173 os.stat(source_path).st_mode)
175 self.mkdir(destination_path, ignore_existing=True)
176 self.put_dir(source_path, destination_path)
178 def mkdir(self, path, mode=511, ignore_existing=False):
179 ''' Augments mkdir by adding an option to not fail
183 self.sftp.mkdir(path, mode)
190 def exec_command(self, command, logger):
191 '''Execute the command on the remote machine
193 :param command str: The command to be run
194 :param logger src.logger.Logger: The logger instance
195 :return: the stdin, stdout, and stderr of the executing command,
197 :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
198 paramiko.channel.ChannelFile)
201 # Does not wait the end of the command
202 (stdin, stdout, stderr) = self.ssh.exec_command(command)
203 except paramiko.SSHException:
204 message = src.KO_STATUS + _(
205 ": the server failed to execute the command\n")
206 logger.write( src.printcolors.printcError(message))
207 return (None, None, None)
209 logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
210 return (None, None, None)
212 return (stdin, stdout, stderr)
215 '''Close the ssh connection
221 def write_info(self, logger):
222 '''Prints the informations relative to the machine in the logger
223 (terminal traces and log file)
225 :param logger src.logger.Logger: The logger instance
229 logger.write("host : " + self.host + "\n")
230 logger.write("port : " + str(self.port) + "\n")
231 logger.write("user : " + str(self.user) + "\n")
232 if self.successfully_connected(logger):
233 status = src.OK_STATUS
235 status = src.KO_STATUS
236 logger.write("Connection : " + status + "\n\n")
240 '''Class to manage one job
242 def __init__(self, name, machine, application, board,
243 commands, timeout, config, logger, after=None):
246 self.machine = machine
248 self.timeout = timeout
249 self.application = application
253 # The list of log files to download from the remote machine
254 self.remote_log_files = []
256 # The remote command status
257 # -1 means that it has not been launched,
258 # 0 means success and 1 means fail
260 self.cancelled = False
264 self._has_begun = False
265 self._has_finished = False
266 self._has_timouted = False
267 self._stdin = None # Store the command inputs field
268 self._stdout = None # Store the command outputs field
269 self._stderr = None # Store the command errors field
274 self.commands = commands
275 self.command = (os.path.join(self.machine.sat_path, "sat") +
277 os.path.join(self.machine.sat_path,
278 "list_log_files.txt") +
279 " job --jobs_config .jobs_command_file" +
284 """ Get the pid(s) corresponding to the command that have been launched
285 On the remote machine
287 :return: The list of integers corresponding to the found pids
291 cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
292 (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
293 pids_cmd = out_pid.readlines()
294 pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
298 def kill_remote_process(self, wait=1):
299 '''Kills the process on the remote machine.
301 :return: (the output of the kill, the error of the kill)
305 pids = self.get_pids()
306 cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
307 (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
310 return (out_kill, err_kill)
313 '''Returns True if the job has already begun
315 :return: True if the job has already begun
318 return self._has_begun
320 def has_finished(self):
321 '''Returns True if the job has already finished
322 (i.e. all the commands have been executed)
323 If it is finished, the outputs are stored in the fields out and err.
325 :return: True if the job has already finished
329 # If the method has already been called and returned True
330 if self._has_finished:
333 # If the job has not begun yet
334 if not self.has_begun():
337 if self._stdout.channel.closed:
338 self._has_finished = True
339 # Store the result outputs
340 self.out += self._stdout.read().decode()
341 self.err += self._stderr.read().decode()
343 self._Tf = time.time()
344 # And get the remote command status and log files
347 return self._has_finished
349 def get_log_files(self):
350 """Get the log files produced by the command launched
351 on the remote machine, and put it in the log directory of the user,
352 so they can be accessible from
354 # Do not get the files if the command is not finished
355 if not self.has_finished():
356 msg = _("Trying to get log files whereas the job is not finished.")
357 self.logger.write(src.printcolors.printcWarning(msg))
360 # First get the file that contains the list of log files to get
361 tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
362 remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt")
363 self.machine.sftp.get(
367 # Read the file and get the result of the command and all the log files
369 fstream_tmp = open(tmp_file_path, "r")
370 file_lines = fstream_tmp.readlines()
371 file_lines = [line.replace("\n", "") for line in file_lines]
373 os.remove(tmp_file_path)
376 # The first line is the result of the command (0 success or 1 fail)
377 self.res_job = file_lines[0]
378 except Exception as e:
379 self.err += _("Unable to get status from remote file %s: %s" %
380 (remote_path, str(e)))
382 for i, job_path_remote in enumerate(file_lines[1:]):
384 # For each command, there is two files to get :
385 # 1- The xml file describing the command and giving the
387 # 2- The txt file containing the system command traces (like
388 # traces produced by the "make" command)
389 # 3- In case of the test command, there is another file to get :
390 # the xml board that contain the test results
391 dirname = os.path.basename(os.path.dirname(job_path_remote))
392 if dirname != 'OUT' and dirname != 'TEST':
394 local_path = os.path.join(os.path.dirname(
395 self.logger.logFilePath),
396 os.path.basename(job_path_remote))
397 if i==0: # The first is the job command
398 self.logger.add_link(os.path.basename(job_path_remote),
402 elif dirname == 'OUT':
404 local_path = os.path.join(os.path.dirname(
405 self.logger.logFilePath),
407 os.path.basename(job_path_remote))
408 elif dirname == 'TEST':
410 local_path = os.path.join(os.path.dirname(
411 self.logger.logFilePath),
413 os.path.basename(job_path_remote))
416 if not os.path.exists(local_path):
417 self.machine.sftp.get(job_path_remote, local_path)
418 self.remote_log_files.append(local_path)
419 except Exception as e:
420 self.err += _("Unable to get %s log file from remote: %s" %
421 (job_path_remote, str(e)))
423 def has_failed(self):
424 '''Returns True if the job has failed.
425 A job is considered as failed if the machine could not be reached,
426 if the remote command failed,
427 or if the job finished with a time out.
429 :return: True if the job has failed
432 if not self.has_finished():
434 if not self.machine.successfully_connected(self.logger):
436 if self.is_timeout():
438 if self.res_job == "1":
443 """In case of a failing job, one has to cancel every job that depend
444 on it. This method put the job as failed and will not be executed.
446 self._has_begun = True
447 self._has_finished = True
448 self.cancelled = True
449 self.out += _("This job was not launched because its father has failed.")
450 self.err += _("This job was not launched because its father has failed.")
452 def is_running(self):
453 '''Returns True if the job commands are running
455 :return: True if the job is running
458 return self.has_begun() and not self.has_finished()
460 def is_timeout(self):
461 '''Returns True if the job commands has finished with timeout
463 :return: True if the job has finished with timeout
466 return self._has_timouted
468 def time_elapsed(self):
469 """Get the time elapsed since the job launching
471 :return: The number of seconds
474 if not self.has_begun():
477 return T_now - self._T0
479 def check_time(self):
480 """Verify that the job has not exceeded its timeout.
481 If it has, kill the remote command and consider the job as finished.
483 if not self.has_begun():
485 if self.time_elapsed() > self.timeout:
486 self._has_finished = True
487 self._has_timouted = True
488 self._Tf = time.time()
490 (out_kill, _) = self.kill_remote_process()
491 self.out += "TIMEOUT \n" + out_kill.read().decode()
492 self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
495 except Exception as e:
496 self.err += _("Unable to get remote log files: %s" % e)
498 def total_duration(self):
499 """Give the total duration of the job
501 :return: the total duration of the job in seconds
504 return self._Tf - self._T0
507 """Launch the job by executing the remote command.
510 # Prevent multiple run
512 msg = _("Warning: A job can only be launched one time")
513 msg2 = _("Trying to launch the job \"%s\" whereas it has "
514 "already been launched." % self.name)
515 self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
519 # Do not execute the command if the machine could not be reached
520 if not self.machine.successfully_connected(self.logger):
521 self._has_finished = True
523 self.err += ("Connection to machine (name : %s, host: %s, port:"
524 " %s, user: %s) has failed\nUse the log command "
525 "to get more information."
526 % (self.machine.name,
531 # Usual case : Launch the command on remote machine
532 self._T0 = time.time()
533 self._stdin, self._stdout, self._stderr = self.machine.exec_command(
536 # If the results are not initialized, finish the job
537 if (self._stdin, self._stdout, self._stderr) == (None, None, None):
538 self._has_finished = True
539 self._Tf = time.time()
541 self.err += "The server failed to execute the command"
543 # Put the beginning flag to true.
544 self._has_begun = True
546 def write_results(self):
547 """Display on the terminal all the job's information
549 self.logger.write("name : " + self.name + "\n")
551 self.logger.write("after : %s\n" % self.after)
552 self.logger.write("Time elapsed : %4imin %2is \n" %
553 (self.total_duration()//60 , self.total_duration()%60))
555 self.logger.write("Begin time : %s\n" %
556 time.strftime('%Y-%m-%d %H:%M:%S',
557 time.localtime(self._T0)) )
559 self.logger.write("End time : %s\n\n" %
560 time.strftime('%Y-%m-%d %H:%M:%S',
561 time.localtime(self._Tf)) )
563 machine_head = "Informations about connection :\n"
564 underline = (len(machine_head) - 2) * "-"
565 self.logger.write(src.printcolors.printcInfo(
566 machine_head+underline+"\n"))
567 self.machine.write_info(self.logger)
569 self.logger.write(src.printcolors.printcInfo("out : \n"))
571 self.logger.write("Unable to get output\n")
573 self.logger.write(self.out + "\n")
574 self.logger.write(src.printcolors.printcInfo("err : \n"))
575 self.logger.write(self.err + "\n")
577 def get_status(self):
578 """Get the status of the job (used by the Gui for xml display)
580 :return: The current status of the job
583 if not self.machine.successfully_connected(self.logger):
584 return "SSH connection KO"
585 if not self.has_begun():
586 return "Not launched"
589 if self.is_running():
590 return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
591 time.localtime(self._T0))
592 if self.has_finished():
593 if self.is_timeout():
594 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
595 time.localtime(self._Tf))
596 return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
597 time.localtime(self._Tf))
600 '''Class to manage the jobs to be run
607 lenght_columns = 20):
608 # The jobs configuration
609 self.cfg_jobs = config_jobs
610 self.job_file_path = job_file_path
611 # The machine that will be used today
613 # The list of machine (hosts, port) that will be used today
614 # (a same host can have several machine instances since there
615 # can be several ssh parameters)
617 # The jobs to be launched today
619 # The jobs that will not be launched today
620 self.ljobs_not_today = []
623 self.len_columns = lenght_columns
625 # the list of jobs that have not been run yet
626 self._l_jobs_not_started = []
627 # the list of jobs that have already ran
628 self._l_jobs_finished = []
629 # the list of jobs that are running
630 self._l_jobs_running = []
632 self.determine_jobs_and_machines()
634 def define_job(self, job_def, machine):
635 '''Takes a pyconf job definition and a machine (from class machine)
636 and returns the job instance corresponding to the definition.
638 :param job_def src.config.Mapping: a job definition
639 :param machine machine: the machine on which the job will run
640 :return: The corresponding job in a job class instance
644 cmmnds = job_def.commands
645 timeout = job_def.timeout
647 if 'after' in job_def:
648 after = job_def.after
650 if 'application' in job_def:
651 application = job_def.application
653 if 'board' in job_def:
654 board = job_def.board
666 def determine_jobs_and_machines(self):
667 '''Function that reads the pyconf jobs definition and instantiates all
668 the machines and jobs to be done today.
673 today = datetime.date.weekday(datetime.date.today())
676 for job_def in self.cfg_jobs.jobs :
678 if not "machine" in job_def:
679 msg = _('WARNING: The job "%s" do not have the key '
680 '"machine", this job is ignored.\n\n' % job_def.name)
681 self.logger.write(src.printcolors.printcWarning(msg))
683 name_machine = job_def.machine
686 for mach in self.lmachines:
687 if mach.name == name_machine:
691 if a_machine == None:
692 for machine_def in self.cfg_jobs.machines:
693 if machine_def.name == name_machine:
694 if 'host' not in machine_def:
695 host = self.runner.cfg.VARS.hostname
697 host = machine_def.host
699 if 'user' not in machine_def:
700 user = self.runner.cfg.VARS.user
702 user = machine_def.user
704 if 'port' not in machine_def:
707 port = machine_def.port
709 if 'password' not in machine_def:
712 passwd = machine_def.password
714 if 'sat_path' not in machine_def:
715 sat_path = "salomeTools"
717 sat_path = machine_def.sat_path
728 self.lmachines.append(a_machine)
729 if (host, port) not in host_list:
730 host_list.append((host, port))
732 if a_machine == None:
733 msg = _("WARNING: The job \"%(job_name)s\" requires the "
734 "machine \"%(machine_name)s\" but this machine "
735 "is not defined in the configuration file.\n"
736 "The job will not be launched")
737 self.logger.write(src.printcolors.printcWarning(msg))
739 a_job = self.define_job(job_def, a_machine)
741 if today in job_def.when:
742 self.ljobs.append(a_job)
743 else: # today in job_def.when
744 self.ljobs_not_today.append(a_job)
746 self.lhosts = host_list
748 def ssh_connection_all_machines(self, pad=50):
749 '''Function that do the ssh connection to every machine
755 self.logger.write(src.printcolors.printcInfo((
756 "Establishing connection with all the machines :\n")))
757 for machine in self.lmachines:
758 # little algorithm in order to display traces
759 begin_line = (_("Connection to %s: " % machine.name))
760 if pad - len(begin_line) < 0:
763 endline = (pad - len(begin_line)) * "." + " "
765 step = "SSH connection"
766 self.logger.write( begin_line + endline + step)
768 # the call to the method that initiate the ssh connection
769 msg = machine.connect(self.logger)
771 # Copy salomeTools to the remote machine
772 if machine.successfully_connected(self.logger):
774 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
775 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
777 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
779 # get the remote machine distribution using a sat command
780 (__, out_dist, __) = machine.exec_command(
781 os.path.join(machine.sat_path,
782 "sat config --value VARS.dist --no_label"),
784 machine.distribution = out_dist.read().decode().replace("\n",
786 # Print the status of the copy
788 self.logger.write('\r%s' %
789 ((len(begin_line)+len(endline)+20) * " "), 3)
790 self.logger.write('\r%s%s%s' %
793 src.printcolors.printc(src.OK_STATUS)), 3)
795 self.logger.write('\r%s' %
796 ((len(begin_line)+len(endline)+20) * " "), 3)
797 self.logger.write('\r%s%s%s %s' %
800 src.printcolors.printc(src.OK_STATUS),
801 _("Copy of SAT failed")), 3)
803 self.logger.write('\r%s' %
804 ((len(begin_line)+len(endline)+20) * " "), 3)
805 self.logger.write('\r%s%s%s %s' %
808 src.printcolors.printc(src.KO_STATUS),
810 self.logger.write("\n", 3)
812 self.logger.write("\n")
815 def is_occupied(self, hostname):
816 '''Function that returns True if a job is running on
817 the machine defined by its host and its port.
819 :param hostname (str, int): the pair (host, port)
820 :return: the job that is running on the host,
821 or false if there is no job running on the host.
826 for jb in self.ljobs:
827 if jb.machine.host == host and jb.machine.port == port:
832 def update_jobs_states_list(self):
833 '''Function that updates the lists that store the currently
834 running jobs and the jobs that have already finished.
839 jobs_finished_list = []
840 jobs_running_list = []
841 for jb in self.ljobs:
843 jobs_running_list.append(jb)
845 if jb.has_finished():
846 jobs_finished_list.append(jb)
848 nb_job_finished_before = len(self._l_jobs_finished)
849 self._l_jobs_finished = jobs_finished_list
850 self._l_jobs_running = jobs_running_list
852 nb_job_finished_now = len(self._l_jobs_finished)
854 return nb_job_finished_now > nb_job_finished_before
856 def cancel_dependencies_of_failing_jobs(self):
857 '''Function that cancels all the jobs that depend on a failing one.
863 for job in self.ljobs:
864 if job.after is None:
866 father_job = self.find_job_that_has_name(job.after)
867 if father_job is not None and father_job.has_failed():
870 def find_job_that_has_name(self, name):
871 '''Returns the job by its name.
873 :param name str: a job name
874 :return: the job that has the name.
877 for jb in self.ljobs:
880 # the following is executed only if the job was not found
883 def str_of_length(self, text, length):
884 '''Takes a string text of any length and returns
885 the most close string of length "length".
887 :param text str: any string
888 :param length int: a length for the returned string
889 :return: the most close string of length "length"
892 if len(text) > length:
893 text_out = text[:length-3] + '...'
895 diff = length - len(text)
896 before = " " * (diff//2)
897 after = " " * (diff//2 + diff%2)
898 text_out = before + text + after
902 def display_status(self, len_col):
903 '''Takes a lenght and construct the display of the current status
904 of the jobs in an array that has a column for each host.
905 It displays the job that is currently running on the host
908 :param len_col int: the size of the column
914 for host_port in self.lhosts:
915 jb = self.is_occupied(host_port)
916 if not jb: # nothing running on the host
917 empty = self.str_of_length("empty", len_col)
918 display_line += "|" + empty
920 display_line += "|" + src.printcolors.printcInfo(
921 self.str_of_length(jb.name, len_col))
923 self.logger.write("\r" + display_line + "|")
928 '''The main method. Runs all the jobs on every host.
929 For each host, at a given time, only one job can be running.
930 The jobs that have the field after (that contain the job that has
931 to be run before it) are run after the previous job.
932 This method stops when all the jobs are finished.
939 self.logger.write(src.printcolors.printcInfo(
940 _('Executing the jobs :\n')))
942 for host_port in self.lhosts:
945 if port == 22: # default value
946 text_line += "|" + self.str_of_length(host, self.len_columns)
948 text_line += "|" + self.str_of_length(
949 "("+host+", "+str(port)+")", self.len_columns)
951 tiret_line = " " + "-"*(len(text_line)-1) + "\n"
952 self.logger.write(tiret_line)
953 self.logger.write(text_line + "|\n")
954 self.logger.write(tiret_line)
957 # The infinite loop that runs the jobs
958 l_jobs_not_started = src.deepcopy_list(self.ljobs)
959 while len(self._l_jobs_finished) != len(self.ljobs):
960 new_job_start = False
961 for host_port in self.lhosts:
963 if self.is_occupied(host_port):
966 for jb in l_jobs_not_started:
967 if (jb.machine.host, jb.machine.port) != host_port:
971 l_jobs_not_started.remove(jb)
975 jb_before = self.find_job_that_has_name(jb.after)
976 if jb_before is None:
978 msg = _("This job was not launched because its "
979 "father is not in the jobs list.")
983 if jb_before.has_finished():
985 l_jobs_not_started.remove(jb)
988 self.cancel_dependencies_of_failing_jobs()
989 new_job_finished = self.update_jobs_states_list()
991 if new_job_start or new_job_finished:
993 self.gui.update_xml_files(self.ljobs)
994 # Display the current status
995 self.display_status(self.len_columns)
997 # Make sure that the proc is not entirely busy
1000 self.logger.write("\n")
1001 self.logger.write(tiret_line)
1002 self.logger.write("\n\n")
1005 self.gui.update_xml_files(self.ljobs)
1006 self.gui.last_update()
1008 def write_all_results(self):
1009 '''Display all the jobs outputs.
1015 for jb in self.ljobs:
1016 self.logger.write(src.printcolors.printcLabel(
1017 "#------- Results for job %s -------#\n" % jb.name))
1019 self.logger.write("\n\n")
1022 '''Class to manage the the xml data that can be displayed in a browser to
1026 def __init__(self, xml_dir_path, l_jobs, l_jobs_not_today, l_file_boards = []):
1029 :param xml_dir_path str: The path to the directory where to put
1030 the xml resulting files
1031 :param l_jobs List: the list of jobs that run today
1032 :param l_jobs_not_today List: the list of jobs that do not run today
1033 :param l_file_boards List: the list of file path from which to read the
1036 # The path of the csv files to read to fill the expected boards
1037 self.l_file_boards = l_file_boards
1039 today = d_INT_DAY[datetime.date.weekday(datetime.date.today())]
1040 self.parse_csv_boards(today)
1042 # The path of the global xml file
1043 self.xml_dir_path = xml_dir_path
1044 # Initialize the xml files
1045 xml_global_path = os.path.join(self.xml_dir_path, "global_report.xml")
1046 self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1048 # The xml files that corresponds to the boards.
1049 # {name_board : xml_object}}
1050 self.d_xml_board_files = {}
1051 # Create the lines and columns
1052 self.initialize_boards(l_jobs, l_jobs_not_today)
1054 # Write the xml file
1055 self.update_xml_files(l_jobs)
1057 def add_xml_board(self, name):
1058 xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1059 self.d_xml_board_files[name] = src.xmlManager.XmlLogFile(
1062 self.d_xml_board_files[name].add_simple_node("distributions")
1063 self.d_xml_board_files[name].add_simple_node("applications")
1064 self.d_xml_board_files[name].add_simple_node("board", text=name)
1066 def initialize_boards(self, l_jobs, l_jobs_not_today):
1067 '''Get all the first information needed for each file and write the
1068 first version of the files
1069 :param l_jobs List: the list of jobs that run today
1070 :param l_jobs_not_today List: the list of jobs that do not run today
1072 # Get the boards to fill and put it in a dictionary
1073 # {board_name : xml instance corresponding to the board}
1074 for job in l_jobs + l_jobs_not_today:
1076 if (board is not None and
1077 board not in self.d_xml_board_files.keys()):
1078 self.add_xml_board(board)
1080 # Verify that the boards given as input are done
1081 for board in list(self.d_input_boards.keys()):
1082 if board not in self.d_xml_board_files:
1083 self.add_xml_board(board)
1085 # Loop over all jobs in order to get the lines and columns for each
1089 for board in self.d_xml_board_files:
1091 d_application[board] = []
1095 for job in l_jobs + l_jobs_not_today:
1097 if (job.machine.host, job.machine.port) not in l_hosts_ports:
1098 l_hosts_ports.append((job.machine.host, job.machine.port))
1100 distrib = job.machine.distribution
1101 application = job.application
1103 board_job = job.board
1106 for board in self.d_xml_board_files:
1107 if board_job == board:
1108 if distrib is not None and distrib not in d_dist[board]:
1109 d_dist[board].append(distrib)
1110 src.xmlManager.add_simple_node(
1111 self.d_xml_board_files[board].xmlroot.find(
1114 attrib={"name" : distrib})
1116 if board_job == board:
1117 if (application is not None and
1118 application not in d_application[board]):
1119 d_application[board].append(application)
1120 src.xmlManager.add_simple_node(
1121 self.d_xml_board_files[board].xmlroot.find(
1125 "name" : application})
1127 # Verify that there are no missing application or distribution in the
1128 # xml board files (regarding the input boards)
1129 for board in self.d_xml_board_files:
1130 l_dist = d_dist[board]
1131 if board not in self.d_input_boards.keys():
1133 for dist in self.d_input_boards[board]["rows"]:
1134 if dist not in l_dist:
1135 src.xmlManager.add_simple_node(
1136 self.d_xml_board_files[board].xmlroot.find(
1139 attrib={"name" : dist})
1140 l_appli = d_application[board]
1141 for appli in self.d_input_boards[board]["columns"]:
1142 if appli not in l_appli:
1143 src.xmlManager.add_simple_node(
1144 self.d_xml_board_files[board].xmlroot.find(
1147 attrib={"name" : appli})
1149 # Initialize the hosts_ports node for the global file
1150 self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1152 for host, port in l_hosts_ports:
1153 host_port = "%s:%i" % (host, port)
1154 src.xmlManager.add_simple_node(self.xmlhosts_ports,
1156 attrib={"name" : host_port})
1158 # Initialize the jobs node in all files
1159 for xml_file in [self.xml_global_file] + list(
1160 self.d_xml_board_files.values()):
1161 xml_jobs = xml_file.add_simple_node("jobs")
1162 # Get the jobs present in the config file but
1163 # that will not be launched today
1164 self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1166 xml_file.add_simple_node("infos",
1167 attrib={"name" : "last update",
1168 "JobsCommandStatus" : "running"})
1170 # Find in each board the squares that needs to be filled regarding the
1171 # input csv files but that are not covered by a today job
1172 for board in self.d_input_boards.keys():
1173 xml_root_board = self.d_xml_board_files[board].xmlroot
1174 xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1176 for row, column in self.d_input_boards[board]["jobs"]:
1179 if (job.application == column and
1180 job.machine.distribution == row):
1184 src.xmlManager.add_simple_node(xml_missing,
1186 attrib={"distribution" : row,
1187 "application" : column })
1189 def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1190 '''Get all the first information needed for each file and write the
1191 first version of the files
1193 :param xml_node_jobs etree.Element: the node corresponding to a job
1194 :param l_jobs_not_today List: the list of jobs that do not run today
1196 for job in l_jobs_not_today:
1197 xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1199 attrib={"name" : job.name})
1200 src.xmlManager.add_simple_node(xmlj, "application", job.application)
1201 src.xmlManager.add_simple_node(xmlj,
1203 job.machine.distribution)
1204 src.xmlManager.add_simple_node(xmlj, "board", job.board)
1205 src.xmlManager.add_simple_node(xmlj,
1206 "commands", " ; ".join(job.commands))
1207 src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1208 src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1209 src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1210 src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1211 src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1212 src.xmlManager.add_simple_node(xmlj, "sat_path",
1213 job.machine.sat_path)
1215 def parse_csv_boards(self, today):
1216 """ Parse the csv files that describes the boards to produce and fill
1217 the dict d_input_boards that contain the csv file contain
1219 :param today str: the current day of the week
1221 # loop over each csv file and read its content
1223 for file_path in self.l_file_boards:
1225 with open(file_path, 'r') as f:
1226 reader = csv.reader(f)
1229 l_boards.append(l_read)
1231 # loop over the csv lists of lines and get the rows, columns and jobs
1233 for input_board in l_boards:
1235 board_name = input_board[0][0]
1238 columns = input_board[0][1:]
1243 for line in input_board[1:]:
1245 for i, square in enumerate(line[1:]):
1249 if columns[i] not in columns_out:
1250 columns_out.append(columns[i])
1251 job = (row, columns[i])
1254 d_boards[board_name] = {"rows" : rows,
1255 "columns" : columns_out,
1258 self.d_input_boards = d_boards
1260 def update_xml_files(self, l_jobs):
1261 '''Write all the xml files with updated information about the jobs
1263 :param l_jobs List: the list of jobs that run today
1265 for xml_file in [self.xml_global_file] + list(
1266 self.d_xml_board_files.values()):
1267 self.update_xml_file(l_jobs, xml_file)
1270 self.write_xml_files()
1272 def update_xml_file(self, l_jobs, xml_file):
1273 '''update information about the jobs for the file xml_file
1275 :param l_jobs List: the list of jobs that run today
1276 :param xml_file xmlManager.XmlLogFile: the xml instance to update
1279 xml_node_jobs = xml_file.xmlroot.find('jobs')
1280 # Update the job names and status node
1282 # Find the node corresponding to the job and delete it
1283 # in order to recreate it
1284 for xmljob in xml_node_jobs.findall('job'):
1285 if xmljob.attrib['name'] == job.name:
1286 xml_node_jobs.remove(xmljob)
1290 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1291 time.localtime(job._T0))
1294 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1295 time.localtime(job._Tf))
1297 # recreate the job node
1298 xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1300 attrib={"name" : job.name})
1301 src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1302 src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1303 src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1304 src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1305 src.xmlManager.add_simple_node(xmlj, "sat_path",
1306 job.machine.sat_path)
1307 src.xmlManager.add_simple_node(xmlj, "application", job.application)
1308 src.xmlManager.add_simple_node(xmlj, "distribution",
1309 job.machine.distribution)
1310 src.xmlManager.add_simple_node(xmlj, "board", job.board)
1311 src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1312 src.xmlManager.add_simple_node(xmlj, "commands",
1313 " ; ".join(job.commands))
1314 src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1315 src.xmlManager.add_simple_node(xmlj, "begin", T0)
1316 src.xmlManager.add_simple_node(xmlj, "end", Tf)
1317 src.xmlManager.add_simple_node(xmlj, "out",
1318 src.printcolors.cleancolor(job.out))
1319 src.xmlManager.add_simple_node(xmlj, "err",
1320 src.printcolors.cleancolor(job.err))
1321 src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1322 if len(job.remote_log_files) > 0:
1323 src.xmlManager.add_simple_node(xmlj,
1324 "remote_log_file_path",
1325 job.remote_log_files[0])
1327 src.xmlManager.add_simple_node(xmlj,
1328 "remote_log_file_path",
1331 xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1332 # get the job father
1333 if job.after is not None:
1336 if jb.name == job.after:
1339 if (job_father is not None and
1340 len(job_father.remote_log_files) > 0):
1341 link = job_father.remote_log_files[0]
1344 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1346 # Verify that the job is to be done today regarding the input csv
1348 if job.board and job.board in self.d_input_boards.keys():
1350 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1351 if (job.machine.distribution == dist
1352 and job.application == appli):
1354 src.xmlManager.add_simple_node(xmlj,
1359 src.xmlManager.add_simple_node(xmlj,
1365 xml_node_infos = xml_file.xmlroot.find('infos')
1366 src.xmlManager.append_node_attrib(xml_node_infos,
1368 datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1372 def last_update(self, finish_status = "finished"):
1373 '''update information about the jobs for the file xml_file
1375 :param l_jobs List: the list of jobs that run today
1376 :param xml_file xmlManager.XmlLogFile: the xml instance to update
1378 for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1379 xml_node_infos = xml_file.xmlroot.find('infos')
1380 src.xmlManager.append_node_attrib(xml_node_infos,
1381 attrib={"JobsCommandStatus" : finish_status})
1383 self.write_xml_files()
1385 def write_xml_files(self):
1386 ''' Write the xml files
1388 self.xml_global_file.write_tree(STYLESHEET_GLOBAL)
1389 for xml_file in self.d_xml_board_files.values():
1390 xml_file.write_tree(STYLESHEET_BOARD)
1393 # Describes the command
1395 return _("The jobs command launches maintenances that are described"
1396 " in the dedicated jobs configuration file.")
1400 def run(args, runner, logger):
1402 (options, args) = parser.parse_args(args)
1404 l_cfg_dir = runner.cfg.PATHS.JOBPATH
1406 # list option : display all the available config files
1408 for cfg_dir in l_cfg_dir:
1409 if not options.no_label:
1410 logger.write("------ %s\n" %
1411 src.printcolors.printcHeader(cfg_dir))
1413 for f in sorted(os.listdir(cfg_dir)):
1414 if not f.endswith('.pyconf'):
1417 logger.write("%s\n" % cfilename)
1420 # Make sure the jobs_config option has been called
1421 if not options.jobs_cfg:
1422 message = _("The option --jobs_config is required\n")
1423 src.printcolors.printcError(message)
1426 # Find the file in the directories
1428 for cfg_dir in l_cfg_dir:
1429 file_jobs_cfg = os.path.join(cfg_dir, options.jobs_cfg)
1430 if not file_jobs_cfg.endswith('.pyconf'):
1431 file_jobs_cfg += '.pyconf'
1433 if not os.path.exists(file_jobs_cfg):
1440 msg = _("The file configuration %(name_file)s was not found."
1441 "\nUse the --list option to get the possible files.")
1442 src.printcolors.printcError(msg)
1446 (_("Platform"), runner.cfg.VARS.dist),
1447 (_("File containing the jobs configuration"), file_jobs_cfg)
1449 src.print_info(logger, info)
1451 # Read the config that is in the file
1452 config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1453 if options.only_jobs:
1454 l_jb = src.pyconf.Sequence()
1455 for jb in config_jobs.jobs:
1456 if jb.name in options.only_jobs:
1458 "Adding a job that was given in only_jobs option parameters")
1459 config_jobs.jobs = l_jb
1462 today_jobs = Jobs(runner,
1466 # SSH connection to all machines
1467 today_jobs.ssh_connection_all_machines()
1468 if options.test_connection:
1473 # Copy the stylesheets in the log directory
1474 log_dir = runner.cfg.SITE.log.log_dir
1475 xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1477 files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1478 files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1479 files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1480 for file_path in files_to_copy:
1481 shutil.copy2(file_path, log_dir)
1483 # Instanciate the Gui in order to produce the xml files that contain all
1485 gui = Gui(runner.cfg.SITE.log.log_dir,
1487 today_jobs.ljobs_not_today,
1488 l_file_boards = options.input_boards)
1490 # Display the list of the xml files
1491 logger.write(src.printcolors.printcInfo(("Here is the list of published"
1493 logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1494 for board in gui.d_xml_board_files.keys():
1495 logger.write("%s\n" % gui.d_xml_board_files[board].logFile, 4)
1497 logger.write("\n", 4)
1499 today_jobs.gui = gui
1503 # Run all the jobs contained in config_jobs
1504 today_jobs.run_jobs()
1505 except KeyboardInterrupt:
1507 logger.write("\n\n%s\n\n" %
1508 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1511 msg = _("Killing the running jobs and trying"
1512 " to get the corresponding logs\n")
1513 logger.write(src.printcolors.printcWarning(msg))
1515 # find the potential not finished jobs and kill them
1516 for jb in today_jobs.ljobs:
1517 if not jb.has_finished():
1519 jb.kill_remote_process()
1520 except Exception as e:
1521 msg = _("Failed to kill job %s: %s\n" % (jb.name, e))
1522 logger.write(src.printcolors.printcWarning(msg))
1525 today_jobs.gui.last_update(_("Forced interruption"))
1528 today_jobs.gui.last_update()
1529 # Output the results
1530 today_jobs.write_all_results()