commands/jobs.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import datetime
  21 import time
  22 import csv
  23 import shutil
  24 import paramiko
  25
  26 import src
  27
  28 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
  29 STYLESHEET_BOARD = "jobs_board_report.xsl"
  30 d_INT_DAY = {0 : "monday",
  31              1 : "tuesday",
  32              2 : "wednesday",
  33              3 : "thursday",
  34              4 : "friday",
  35              5 : "saturday",
  36              6 : "sunday"}
  37
  38 parser = src.options.Options()
  39
  40 parser.add_option('n', 'name', 'string', 'jobs_cfg',
  41                   _('The name of the config file that contains'
  42                   ' the jobs configuration'))
  43 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
  44                   _('Optional: the list of jobs to launch, by their name. '))
  45 parser.add_option('l', 'list', 'boolean', 'list',
  46                   _('Optional: list all available config files.'))
  47 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
  48                   _("Optional: try to connect to the machines. "
  49                     "Not executing the jobs."),
  50                   False)
  51 parser.add_option('p', 'publish', 'boolean', 'publish',
  52                   _("Optional: generate an xml file that can be read in a "
  53                     "browser to display the jobs status."),
  54                   False)
  55 parser.add_option('i', 'input_boards', 'list2', 'input_boards', _("Optional: "
  56                                 "the list of path to csv files that contain "
  57                                 "the expected boards."),[])
  58 parser.add_option('n', 'completion', 'boolean', 'no_label',
  59                   _("Optional (internal use): do not print labels, Works only "
  60                     "with --list."),
  61                   False)
  62
  63 class Machine(object):
  64     '''Class to manage a ssh connection on a machine
  65     '''
  66     def __init__(self,
  67                  name,
  68                  host,
  69                  user,
  70                  port=22,
  71                  passwd=None,
  72                  sat_path="salomeTools"):
  73         self.name = name
  74         self.host = host
  75         self.port = port
  76         self.distribution = None # Will be filled after copying SAT on the machine
  77         self.user = user
  78         self.password = passwd
  79         self.sat_path = sat_path
  80         self.ssh = paramiko.SSHClient()
  81         self._connection_successful = None
  82
  83     def connect(self, logger):
  84         '''Initiate the ssh connection to the remote machine
  85
  86         :param logger src.logger.Logger: The logger instance
  87         :return: Nothing
  88         :rtype: N\A
  89         '''
  90
  91         self._connection_successful = False
  92         self.ssh.load_system_host_keys()
  93         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  94         try:
  95             self.ssh.connect(self.host,
  96                              port=self.port,
  97                              username=self.user,
  98                              password = self.password)
  99         except paramiko.AuthenticationException:
 100             message = src.KO_STATUS + _("Authentication failed")
 101         except paramiko.BadHostKeyException:
 102             message = (src.KO_STATUS +
 103                        _("The server's host key could not be verified"))
 104         except paramiko.SSHException:
 105             message = ( _("SSHException error connecting or "
 106                           "establishing an SSH session"))
 107         except:
 108             message = ( _("Error connecting or establishing an SSH session"))
 109         else:
 110             self._connection_successful = True
 111             message = ""
 112         return message
 113
 114     def successfully_connected(self, logger):
 115         '''Verify if the connection to the remote machine has succeed
 116
 117         :param logger src.logger.Logger: The logger instance
 118         :return: True if the connection has succeed, False if not
 119         :rtype: bool
 120         '''
 121         if self._connection_successful == None:
 122             message = _("Warning : trying to ask if the connection to "
 123             "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
 124             " no connection request" %
 125                         (self.name, self.host, self.port, self.user))
 126             logger.write( src.printcolors.printcWarning(message))
 127         return self._connection_successful
 128
 129     def copy_sat(self, sat_local_path, job_file):
 130         '''Copy salomeTools to the remote machine in self.sat_path
 131         '''
 132         res = 0
 133         try:
 134             # open a sftp connection
 135             self.sftp = self.ssh.open_sftp()
 136             # Create the sat directory on remote machine if it is not existing
 137             self.mkdir(self.sat_path, ignore_existing=True)
 138             # Put sat
 139             self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
 140             # put the job configuration file in order to make it reachable
 141             # on the remote machine
 142             self.sftp.put(job_file, os.path.join(".salomeTools",
 143                                                  "Jobs",
 144                                                  ".jobs_command_file.pyconf"))
 145         except Exception as e:
 146             res = str(e)
 147             self._connection_successful = False
 148
 149         return res
 150
 151     def put_dir(self, source, target, filters = []):
 152         ''' Uploads the contents of the source directory to the target path. The
 153             target directory needs to exists. All sub-directories in source are
 154             created under target.
 155         '''
 156         for item in os.listdir(source):
 157             if item in filters:
 158                 continue
 159             source_path = os.path.join(source, item)
 160             destination_path = os.path.join(target, item)
 161             if os.path.islink(source_path):
 162                 linkto = os.readlink(source_path)
 163                 try:
 164                     self.sftp.symlink(linkto, destination_path)
 165                     self.sftp.chmod(destination_path,
 166                                     os.stat(source_path).st_mode)
 167                 except IOError:
 168                     pass
 169             else:
 170                 if os.path.isfile(source_path):
 171                     self.sftp.put(source_path, destination_path)
 172                     self.sftp.chmod(destination_path,
 173                                     os.stat(source_path).st_mode)
 174                 else:
 175                     self.mkdir(destination_path, ignore_existing=True)
 176                     self.put_dir(source_path, destination_path)
 177
 178     def mkdir(self, path, mode=511, ignore_existing=False):
 179         ''' Augments mkdir by adding an option to not fail
 180             if the folder exists
 181         '''
 182         try:
 183             self.sftp.mkdir(path, mode)
 184         except IOError:
 185             if ignore_existing:
 186                 pass
 187             else:
 188                 raise
 189
 190     def exec_command(self, command, logger):
 191         '''Execute the command on the remote machine
 192
 193         :param command str: The command to be run
 194         :param logger src.logger.Logger: The logger instance
 195         :return: the stdin, stdout, and stderr of the executing command,
 196                  as a 3-tuple
 197         :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
 198                 paramiko.channel.ChannelFile)
 199         '''
 200         try:
 201             # Does not wait the end of the command
 202             (stdin, stdout, stderr) = self.ssh.exec_command(command)
 203         except paramiko.SSHException:
 204             message = src.KO_STATUS + _(
 205                             ": the server failed to execute the command\n")
 206             logger.write( src.printcolors.printcError(message))
 207             return (None, None, None)
 208         except:
 209             logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
 210             return (None, None, None)
 211         else:
 212             return (stdin, stdout, stderr)
 213
 214     def close(self):
 215         '''Close the ssh connection
 216
 217         :rtype: N\A
 218         '''
 219         self.ssh.close()
 220
 221     def write_info(self, logger):
 222         '''Prints the informations relative to the machine in the logger
 223            (terminal traces and log file)
 224
 225         :param logger src.logger.Logger: The logger instance
 226         :return: Nothing
 227         :rtype: N\A
 228         '''
 229         logger.write("host : " + self.host + "\n")
 230         logger.write("port : " + str(self.port) + "\n")
 231         logger.write("user : " + str(self.user) + "\n")
 232         if self.successfully_connected(logger):
 233             status = src.OK_STATUS
 234         else:
 235             status = src.KO_STATUS
 236         logger.write("Connection : " + status + "\n\n")
 237
 238
 239 class Job(object):
 240     '''Class to manage one job
 241     '''
 242     def __init__(self, name, machine, application, board,
 243                  commands, timeout, config, logger, after=None):
 244
 245         self.name = name
 246         self.machine = machine
 247         self.after = after
 248         self.timeout = timeout
 249         self.application = application
 250         self.board = board
 251         self.config = config
 252         self.logger = logger
 253         # The list of log files to download from the remote machine
 254         self.remote_log_files = []
 255
 256         # The remote command status
 257         # -1 means that it has not been launched,
 258         # 0 means success and 1 means fail
 259         self.res_job = "-1"
 260         self.cancelled = False
 261
 262         self._T0 = -1
 263         self._Tf = -1
 264         self._has_begun = False
 265         self._has_finished = False
 266         self._has_timouted = False
 267         self._stdin = None # Store the command inputs field
 268         self._stdout = None # Store the command outputs field
 269         self._stderr = None # Store the command errors field
 270
 271         self.out = None # Contains something only if the job is finished
 272         self.err = None # Contains something only if the job is finished
 273
 274         self.commands = commands
 275         self.command = (os.path.join(self.machine.sat_path, "sat") +
 276                         " -l " +
 277                         os.path.join(self.machine.sat_path,
 278                                      "list_log_files.txt") +
 279                         " job --jobs_config .jobs_command_file" +
 280                         " --name " +
 281                         self.name)
 282
 283     def get_pids(self):
 284         """ Get the pid(s) corresponding to the command that have been launched
 285             On the remote machine
 286
 287         :return: The list of integers corresponding to the found pids
 288         :rtype: List
 289         """
 290         pids = []
 291         cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
 292         (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
 293         pids_cmd = out_pid.readlines()
 294         pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
 295         pids+=pids_cmd
 296         return pids
 297
 298     def kill_remote_process(self, wait=1):
 299         '''Kills the process on the remote machine.
 300
 301         :return: (the output of the kill, the error of the kill)
 302         :rtype: (str, str)
 303         '''
 304
 305         pids = self.get_pids()
 306         cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
 307         (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
 308                                                             self.logger)
 309         time.sleep(wait)
 310         return (out_kill, err_kill)
 311
 312     def has_begun(self):
 313         '''Returns True if the job has already begun
 314
 315         :return: True if the job has already begun
 316         :rtype: bool
 317         '''
 318         return self._has_begun
 319
 320     def has_finished(self):
 321         '''Returns True if the job has already finished
 322            (i.e. all the commands have been executed)
 323            If it is finished, the outputs are stored in the fields out and err.
 324
 325         :return: True if the job has already finished
 326         :rtype: bool
 327         '''
 328
 329         # If the method has already been called and returned True
 330         if self._has_finished:
 331             return True
 332
 333         # If the job has not begun yet
 334         if not self.has_begun():
 335             return False
 336
 337         if self._stdout.channel.closed:
 338             self._has_finished = True
 339             # Store the result outputs
 340             self.out = self._stdout.read().decode()
 341             self.err = self._stderr.read().decode()
 342             # Put end time
 343             self._Tf = time.time()
 344             # And get the remote command status and log files
 345             self.get_log_files()
 346
 347         return self._has_finished
 348
 349     def get_log_files(self):
 350         """Get the log files produced by the command launched
 351            on the remote machine.
 352         """
 353         # Do not get the files if the command is not finished
 354         if not self.has_finished():
 355             msg = _("Trying to get log files whereas the job is not finished.")
 356             self.logger.write(src.printcolors.printcWarning(msg))
 357             return
 358
 359         # First get the file that contains the list of log files to get
 360         tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
 361         self.machine.sftp.get(
 362                     os.path.join(self.machine.sat_path, "list_log_files.txt"),
 363                     tmp_file_path)
 364
 365         # Read the file and get the result of the command and all the log files
 366         # to get
 367         fstream_tmp = open(tmp_file_path, "r")
 368         file_lines = fstream_tmp.readlines()
 369         file_lines = [line.replace("\n", "") for line in file_lines]
 370         fstream_tmp.close()
 371         os.remove(tmp_file_path)
 372         # The first line is the result of the command (0 success or 1 fail)
 373         self.res_job = file_lines[0]
 374
 375         for i, job_path_remote in enumerate(file_lines[1:]):
 376             try:
 377                 # For each command, there is two files to get :
 378                 # 1- The xml file describing the command and giving the
 379                 # internal traces.
 380                 # 2- The txt file containing the system command traces (like
 381                 # traces produced by the "make" command)
 382                 if os.path.basename(os.path.dirname(job_path_remote)) != 'OUT':
 383                     # Case 1-
 384                     local_path = os.path.join(os.path.dirname(
 385                                                         self.logger.logFilePath),
 386                                               os.path.basename(job_path_remote))
 387                     if i==0: # The first is the job command
 388                         self.logger.add_link(os.path.basename(job_path_remote),
 389                                              "job",
 390                                              self.res_job,
 391                                              self.command)
 392                 else:
 393                     # Case 2-
 394                     local_path = os.path.join(os.path.dirname(
 395                                                         self.logger.logFilePath),
 396                                               'OUT',
 397                                               os.path.basename(job_path_remote))
 398                 # Get the file
 399                 if not os.path.exists(local_path):
 400                     self.machine.sftp.get(job_path_remote, local_path)
 401                 self.remote_log_files.append(local_path)
 402             except Exception as e:
 403                 self.err += _("Unable to get %s log file from remote: %s" %
 404                                                     (job_path_remote, str(e)))
 405
 406     def has_failed(self):
 407         '''Returns True if the job has failed.
 408            A job is considered as failed if the machine could not be reached,
 409            if the remote command failed,
 410            or if the job finished with a time out.
 411
 412         :return: True if the job has failed
 413         :rtype: bool
 414         '''
 415         if not self.has_finished():
 416             return False
 417         if not self.machine.successfully_connected(self.logger):
 418             return True
 419         if self.is_timeout():
 420             return True
 421         if self.res_job == "1":
 422             return True
 423         return False
 424
 425     def cancel(self):
 426         """In case of a failing job, one has to cancel every job that depend
 427            on it. This method put the job as failed and will not be executed.
 428         """
 429         self._has_begun = True
 430         self._has_finished = True
 431         self.cancelled = True
 432         self.out = _("This job was not launched because its father has failed.")
 433         self.err = _("This job was not launched because its father has failed.")
 434
 435     def is_running(self):
 436         '''Returns True if the job commands are running
 437
 438         :return: True if the job is running
 439         :rtype: bool
 440         '''
 441         return self.has_begun() and not self.has_finished()
 442
 443     def is_timeout(self):
 444         '''Returns True if the job commands has finished with timeout
 445
 446         :return: True if the job has finished with timeout
 447         :rtype: bool
 448         '''
 449         return self._has_timouted
 450
 451     def time_elapsed(self):
 452         """Get the time elapsed since the job launching
 453
 454         :return: The number of seconds
 455         :rtype: int
 456         """
 457         if not self.has_begun():
 458             return -1
 459         T_now = time.time()
 460         return T_now - self._T0
 461
 462     def check_time(self):
 463         """Verify that the job has not exceeded its timeout.
 464            If it has, kill the remote command and consider the job as finished.
 465         """
 466         if not self.has_begun():
 467             return
 468         if self.time_elapsed() > self.timeout:
 469             self._has_finished = True
 470             self._has_timouted = True
 471             self._Tf = time.time()
 472             self.get_pids()
 473             (out_kill, _) = self.kill_remote_process()
 474             self.out = "TIMEOUT \n" + out_kill.read().decode()
 475             self.err = "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
 476             try:
 477                 self.get_log_files()
 478             except Exception as e:
 479                 self.err += _("Unable to get remote log files: %s" % e)
 480
 481     def total_duration(self):
 482         """Give the total duration of the job
 483
 484         :return: the total duration of the job in seconds
 485         :rtype: int
 486         """
 487         return self._Tf - self._T0
 488
 489     def run(self):
 490         """Launch the job by executing the remote command.
 491         """
 492
 493         # Prevent multiple run
 494         if self.has_begun():
 495             msg = _("Warning: A job can only be launched one time")
 496             msg2 = _("Trying to launch the job \"%s\" whereas it has "
 497                      "already been launched." % self.name)
 498             self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
 499                                                                         msg2)))
 500             return
 501
 502         # Do not execute the command if the machine could not be reached
 503         if not self.machine.successfully_connected(self.logger):
 504             self._has_finished = True
 505             self.out = "N\A"
 506             self.err = ("Connection to machine (name : %s, host: %s, port:"
 507                         " %s, user: %s) has failed\nUse the log command "
 508                         "to get more information."
 509                         % (self.machine.name,
 510                            self.machine.host,
 511                            self.machine.port,
 512                            self.machine.user))
 513         else:
 514             # Usual case : Launch the command on remote machine
 515             self._T0 = time.time()
 516             self._stdin, self._stdout, self._stderr = self.machine.exec_command(
 517                                                                   self.command,
 518                                                                   self.logger)
 519             # If the results are not initialized, finish the job
 520             if (self._stdin, self._stdout, self._stderr) == (None, None, None):
 521                 self._has_finished = True
 522                 self._Tf = time.time()
 523                 self.out = "N\A"
 524                 self.err = "The server failed to execute the command"
 525
 526         # Put the beginning flag to true.
 527         self._has_begun = True
 528
 529     def write_results(self):
 530         """Display on the terminal all the job's information
 531         """
 532         self.logger.write("name : " + self.name + "\n")
 533         if self.after:
 534             self.logger.write("after : %s\n" % self.after)
 535         self.logger.write("Time elapsed : %4imin %2is \n" %
 536                      (self.total_duration()//60 , self.total_duration()%60))
 537         if self._T0 != -1:
 538             self.logger.write("Begin time : %s\n" %
 539                          time.strftime('%Y-%m-%d %H:%M:%S',
 540                                        time.localtime(self._T0)) )
 541         if self._Tf != -1:
 542             self.logger.write("End time   : %s\n\n" %
 543                          time.strftime('%Y-%m-%d %H:%M:%S',
 544                                        time.localtime(self._Tf)) )
 545
 546         machine_head = "Informations about connection :\n"
 547         underline = (len(machine_head) - 2) * "-"
 548         self.logger.write(src.printcolors.printcInfo(
 549                                                 machine_head+underline+"\n"))
 550         self.machine.write_info(self.logger)
 551
 552         self.logger.write(src.printcolors.printcInfo("out : \n"))
 553         if self.out is None:
 554             self.logger.write("Unable to get output\n")
 555         else:
 556             self.logger.write(self.out + "\n")
 557         self.logger.write(src.printcolors.printcInfo("err : \n"))
 558         if self.err is None:
 559             self.logger.write("Unable to get error\n")
 560         else:
 561             self.logger.write(self.err + "\n")
 562
 563     def get_status(self):
 564         """Get the status of the job (used by the Gui for xml display)
 565
 566         :return: The current status of the job
 567         :rtype: String
 568         """
 569         if not self.machine.successfully_connected(self.logger):
 570             return "SSH connection KO"
 571         if not self.has_begun():
 572             return "Not launched"
 573         if self.cancelled:
 574             return "Cancelled"
 575         if self.is_running():
 576             return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
 577                                                     time.localtime(self._T0))
 578         if self.has_finished():
 579             if self.is_timeout():
 580                 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
 581                                                     time.localtime(self._Tf))
 582             return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
 583                                                      time.localtime(self._Tf))
 584
 585 class Jobs(object):
 586     '''Class to manage the jobs to be run
 587     '''
 588     def __init__(self,
 589                  runner,
 590                  logger,
 591                  job_file_path,
 592                  config_jobs,
 593                  lenght_columns = 20):
 594         # The jobs configuration
 595         self.cfg_jobs = config_jobs
 596         self.job_file_path = job_file_path
 597         # The machine that will be used today
 598         self.lmachines = []
 599         # The list of machine (hosts, port) that will be used today
 600         # (a same host can have several machine instances since there
 601         # can be several ssh parameters)
 602         self.lhosts = []
 603         # The jobs to be launched today
 604         self.ljobs = []
 605         # The jobs that will not be launched today
 606         self.ljobs_not_today = []
 607         self.runner = runner
 608         self.logger = logger
 609         self.len_columns = lenght_columns
 610
 611         # the list of jobs that have not been run yet
 612         self._l_jobs_not_started = []
 613         # the list of jobs that have already ran
 614         self._l_jobs_finished = []
 615         # the list of jobs that are running
 616         self._l_jobs_running = []
 617
 618         self.determine_jobs_and_machines()
 619
 620     def define_job(self, job_def, machine):
 621         '''Takes a pyconf job definition and a machine (from class machine)
 622            and returns the job instance corresponding to the definition.
 623
 624         :param job_def src.config.Mapping: a job definition
 625         :param machine machine: the machine on which the job will run
 626         :return: The corresponding job in a job class instance
 627         :rtype: job
 628         '''
 629         name = job_def.name
 630         cmmnds = job_def.commands
 631         timeout = job_def.timeout
 632         after = None
 633         if 'after' in job_def:
 634             after = job_def.after
 635         application = None
 636         if 'application' in job_def:
 637             application = job_def.application
 638         board = None
 639         if 'board' in job_def:
 640             board = job_def.board
 641
 642         return Job(name,
 643                    machine,
 644                    application,
 645                    board,
 646                    cmmnds,
 647                    timeout,
 648                    self.runner.cfg,
 649                    self.logger,
 650                    after = after)
 651
 652     def determine_jobs_and_machines(self):
 653         '''Function that reads the pyconf jobs definition and instantiates all
 654            the machines and jobs to be done today.
 655
 656         :return: Nothing
 657         :rtype: N\A
 658         '''
 659         today = datetime.date.weekday(datetime.date.today())
 660         host_list = []
 661
 662         for job_def in self.cfg_jobs.jobs :
 663
 664             if not "machine" in job_def:
 665                 msg = _('WARNING: The job "%s" do not have the key '
 666                        '"machine", this job is ignored.\n\n' % job_def.name)
 667                 self.logger.write(src.printcolors.printcWarning(msg))
 668                 continue
 669             name_machine = job_def.machine
 670
 671             a_machine = None
 672             for mach in self.lmachines:
 673                 if mach.name == name_machine:
 674                     a_machine = mach
 675                     break
 676
 677             if a_machine == None:
 678                 for machine_def in self.cfg_jobs.machines:
 679                     if machine_def.name == name_machine:
 680                         if 'host' not in machine_def:
 681                             host = self.runner.cfg.VARS.hostname
 682                         else:
 683                             host = machine_def.host
 684
 685                         if 'user' not in machine_def:
 686                             user = self.runner.cfg.VARS.user
 687                         else:
 688                             user = machine_def.user
 689
 690                         if 'port' not in machine_def:
 691                             port = 22
 692                         else:
 693                             port = machine_def.port
 694
 695                         if 'password' not in machine_def:
 696                             passwd = None
 697                         else:
 698                             passwd = machine_def.password
 699
 700                         if 'sat_path' not in machine_def:
 701                             sat_path = "salomeTools"
 702                         else:
 703                             sat_path = machine_def.sat_path
 704
 705                         a_machine = Machine(
 706                                             machine_def.name,
 707                                             host,
 708                                             user,
 709                                             port=port,
 710                                             passwd=passwd,
 711                                             sat_path=sat_path
 712                                             )
 713
 714                         self.lmachines.append(a_machine)
 715                         if (host, port) not in host_list:
 716                             host_list.append((host, port))
 717
 718                 if a_machine == None:
 719                     msg = _("WARNING: The job \"%(job_name)s\" requires the "
 720                             "machine \"%(machine_name)s\" but this machine "
 721                             "is not defined in the configuration file.\n"
 722                             "The job will not be launched")
 723                     self.logger.write(src.printcolors.printcWarning(msg))
 724
 725             a_job = self.define_job(job_def, a_machine)
 726
 727             if today in job_def.when:
 728                 self.ljobs.append(a_job)
 729             else: # today in job_def.when
 730                 self.ljobs_not_today.append(a_job)
 731
 732         self.lhosts = host_list
 733
 734     def ssh_connection_all_machines(self, pad=50):
 735         '''Function that do the ssh connection to every machine
 736            to be used today.
 737
 738         :return: Nothing
 739         :rtype: N\A
 740         '''
 741         self.logger.write(src.printcolors.printcInfo((
 742                         "Establishing connection with all the machines :\n")))
 743         for machine in self.lmachines:
 744             # little algorithm in order to display traces
 745             begin_line = (_("Connection to %s: " % machine.name))
 746             if pad - len(begin_line) < 0:
 747                 endline = " "
 748             else:
 749                 endline = (pad - len(begin_line)) * "." + " "
 750
 751             step = "SSH connection"
 752             self.logger.write( begin_line + endline + step)
 753             self.logger.flush()
 754             # the call to the method that initiate the ssh connection
 755             msg = machine.connect(self.logger)
 756
 757             # Copy salomeTools to the remote machine
 758             if machine.successfully_connected(self.logger):
 759                 step = _("Copy SAT")
 760                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 761                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 762                 self.logger.flush()
 763                 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
 764                                             self.job_file_path)
 765                 # get the remote machine distribution using a sat command
 766                 (__, out_dist, __) = machine.exec_command(
 767                                 os.path.join(machine.sat_path,
 768                                     "sat config --value VARS.dist --no_label"),
 769                                 self.logger)
 770                 machine.distribution = out_dist.read().decode().replace("\n",
 771                                                                         "")
 772                 # Print the status of the copy
 773                 if res_copy == 0:
 774                     self.logger.write('\r%s' %
 775                                 ((len(begin_line)+len(endline)+20) * " "), 3)
 776                     self.logger.write('\r%s%s%s' %
 777                         (begin_line,
 778                          endline,
 779                          src.printcolors.printc(src.OK_STATUS)), 3)
 780                 else:
 781                     self.logger.write('\r%s' %
 782                             ((len(begin_line)+len(endline)+20) * " "), 3)
 783                     self.logger.write('\r%s%s%s %s' %
 784                         (begin_line,
 785                          endline,
 786                          src.printcolors.printc(src.OK_STATUS),
 787                          _("Copy of SAT failed")), 3)
 788             else:
 789                 self.logger.write('\r%s' %
 790                                   ((len(begin_line)+len(endline)+20) * " "), 3)
 791                 self.logger.write('\r%s%s%s %s' %
 792                     (begin_line,
 793                      endline,
 794                      src.printcolors.printc(src.KO_STATUS),
 795                      msg), 3)
 796             self.logger.write("\n", 3)
 797
 798         self.logger.write("\n")
 799
 800
 801     def is_occupied(self, hostname):
 802         '''Function that returns True if a job is running on
 803            the machine defined by its host and its port.
 804
 805         :param hostname (str, int): the pair (host, port)
 806         :return: the job that is running on the host,
 807                 or false if there is no job running on the host.
 808         :rtype: job / bool
 809         '''
 810         host = hostname[0]
 811         port = hostname[1]
 812         for jb in self.ljobs:
 813             if jb.machine.host == host and jb.machine.port == port:
 814                 if jb.is_running():
 815                     return jb
 816         return False
 817
 818     def update_jobs_states_list(self):
 819         '''Function that updates the lists that store the currently
 820            running jobs and the jobs that have already finished.
 821
 822         :return: Nothing.
 823         :rtype: N\A
 824         '''
 825         jobs_finished_list = []
 826         jobs_running_list = []
 827         for jb in self.ljobs:
 828             if jb.is_running():
 829                 jobs_running_list.append(jb)
 830                 jb.check_time()
 831             if jb.has_finished():
 832                 jobs_finished_list.append(jb)
 833
 834         nb_job_finished_before = len(self._l_jobs_finished)
 835         self._l_jobs_finished = jobs_finished_list
 836         self._l_jobs_running = jobs_running_list
 837
 838         nb_job_finished_now = len(self._l_jobs_finished)
 839
 840         return nb_job_finished_now > nb_job_finished_before
 841
 842     def cancel_dependencies_of_failing_jobs(self):
 843         '''Function that cancels all the jobs that depend on a failing one.
 844
 845         :return: Nothing.
 846         :rtype: N\A
 847         '''
 848
 849         for job in self.ljobs:
 850             if job.after is None:
 851                 continue
 852             father_job = self.find_job_that_has_name(job.after)
 853             if father_job is not None and father_job.has_failed():
 854                 job.cancel()
 855
 856     def find_job_that_has_name(self, name):
 857         '''Returns the job by its name.
 858
 859         :param name str: a job name
 860         :return: the job that has the name.
 861         :rtype: job
 862         '''
 863         for jb in self.ljobs:
 864             if jb.name == name:
 865                 return jb
 866         # the following is executed only if the job was not found
 867         return None
 868
 869     def str_of_length(self, text, length):
 870         '''Takes a string text of any length and returns
 871            the most close string of length "length".
 872
 873         :param text str: any string
 874         :param length int: a length for the returned string
 875         :return: the most close string of length "length"
 876         :rtype: str
 877         '''
 878         if len(text) > length:
 879             text_out = text[:length-3] + '...'
 880         else:
 881             diff = length - len(text)
 882             before = " " * (diff//2)
 883             after = " " * (diff//2 + diff%2)
 884             text_out = before + text + after
 885
 886         return text_out
 887
 888     def display_status(self, len_col):
 889         '''Takes a lenght and construct the display of the current status
 890            of the jobs in an array that has a column for each host.
 891            It displays the job that is currently running on the host
 892            of the column.
 893
 894         :param len_col int: the size of the column
 895         :return: Nothing
 896         :rtype: N\A
 897         '''
 898
 899         display_line = ""
 900         for host_port in self.lhosts:
 901             jb = self.is_occupied(host_port)
 902             if not jb: # nothing running on the host
 903                 empty = self.str_of_length("empty", len_col)
 904                 display_line += "|" + empty
 905             else:
 906                 display_line += "|" + src.printcolors.printcInfo(
 907                                         self.str_of_length(jb.name, len_col))
 908
 909         self.logger.write("\r" + display_line + "|")
 910         self.logger.flush()
 911
 912
 913     def run_jobs(self):
 914         '''The main method. Runs all the jobs on every host.
 915            For each host, at a given time, only one job can be running.
 916            The jobs that have the field after (that contain the job that has
 917            to be run before it) are run after the previous job.
 918            This method stops when all the jobs are finished.
 919
 920         :return: Nothing
 921         :rtype: N\A
 922         '''
 923
 924         # Print header
 925         self.logger.write(src.printcolors.printcInfo(
 926                                                 _('Executing the jobs :\n')))
 927         text_line = ""
 928         for host_port in self.lhosts:
 929             host = host_port[0]
 930             port = host_port[1]
 931             if port == 22: # default value
 932                 text_line += "|" + self.str_of_length(host, self.len_columns)
 933             else:
 934                 text_line += "|" + self.str_of_length(
 935                                 "("+host+", "+str(port)+")", self.len_columns)
 936
 937         tiret_line = " " + "-"*(len(text_line)-1) + "\n"
 938         self.logger.write(tiret_line)
 939         self.logger.write(text_line + "|\n")
 940         self.logger.write(tiret_line)
 941         self.logger.flush()
 942
 943         # The infinite loop that runs the jobs
 944         l_jobs_not_started = src.deepcopy_list(self.ljobs)
 945         while len(self._l_jobs_finished) != len(self.ljobs):
 946             new_job_start = False
 947             for host_port in self.lhosts:
 948
 949                 if self.is_occupied(host_port):
 950                     continue
 951
 952                 for jb in l_jobs_not_started:
 953                     if (jb.machine.host, jb.machine.port) != host_port:
 954                         continue
 955                     if jb.after == None:
 956                         jb.run()
 957                         l_jobs_not_started.remove(jb)
 958                         new_job_start = True
 959                         break
 960                     else:
 961                         jb_before = self.find_job_that_has_name(jb.after)
 962                         if jb_before is None:
 963                             jb.cancel()
 964                             msg = _("This job was not launched because its "
 965                                     "father is not in the jobs list.")
 966                             jb.out = msg
 967                             jb.err = msg
 968                             break
 969                         if jb_before.has_finished():
 970                             jb.run()
 971                             l_jobs_not_started.remove(jb)
 972                             new_job_start = True
 973                             break
 974             self.cancel_dependencies_of_failing_jobs()
 975             new_job_finished = self.update_jobs_states_list()
 976
 977             if new_job_start or new_job_finished:
 978                 if self.gui:
 979                     self.gui.update_xml_files(self.ljobs)
 980                 # Display the current status
 981                 self.display_status(self.len_columns)
 982
 983             # Make sure that the proc is not entirely busy
 984             time.sleep(0.001)
 985
 986         self.logger.write("\n")
 987         self.logger.write(tiret_line)
 988         self.logger.write("\n\n")
 989
 990         if self.gui:
 991             self.gui.update_xml_files(self.ljobs)
 992             self.gui.last_update()
 993
 994     def write_all_results(self):
 995         '''Display all the jobs outputs.
 996
 997         :return: Nothing
 998         :rtype: N\A
 999         '''
1000
1001         for jb in self.ljobs:
1002             self.logger.write(src.printcolors.printcLabel(
1003                         "#------- Results for job %s -------#\n" % jb.name))
1004             jb.write_results()
1005             self.logger.write("\n\n")
1006
1007 class Gui(object):
1008     '''Class to manage the the xml data that can be displayed in a browser to
1009        see the jobs states
1010     '''
1011
1012     def __init__(self, xml_dir_path, l_jobs, l_jobs_not_today, l_file_boards = []):
1013         '''Initialization
1014
1015         :param xml_dir_path str: The path to the directory where to put
1016                                  the xml resulting files
1017         :param l_jobs List: the list of jobs that run today
1018         :param l_jobs_not_today List: the list of jobs that do not run today
1019         :param l_file_boards List: the list of file path from which to read the
1020                                    expected boards
1021         '''
1022         # The path of the csv files to read to fill the expected boards
1023         self.l_file_boards = l_file_boards
1024
1025         today = d_INT_DAY[datetime.date.weekday(datetime.date.today())]
1026         self.parse_csv_boards(today)
1027
1028         # The path of the global xml file
1029         self.xml_dir_path = xml_dir_path
1030         # Initialize the xml files
1031         xml_global_path = os.path.join(self.xml_dir_path, "global_report.xml")
1032         self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1033                                                          "JobsReport")
1034         # The xml files that corresponds to the boards.
1035         # {name_board : xml_object}}
1036         self.d_xml_board_files = {}
1037         # Create the lines and columns
1038         self.initialize_boards(l_jobs, l_jobs_not_today)
1039
1040         # Write the xml file
1041         self.update_xml_files(l_jobs)
1042
1043     def add_xml_board(self, name):
1044         xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1045         self.d_xml_board_files[name] =  src.xmlManager.XmlLogFile(
1046                                                     xml_board_path,
1047                                                     "JobsReport")
1048         self.d_xml_board_files[name].add_simple_node("distributions")
1049         self.d_xml_board_files[name].add_simple_node("applications")
1050         self.d_xml_board_files[name].add_simple_node("board", text=name)
1051
1052     def initialize_boards(self, l_jobs, l_jobs_not_today):
1053         '''Get all the first information needed for each file and write the
1054            first version of the files
1055         :param l_jobs List: the list of jobs that run today
1056         :param l_jobs_not_today List: the list of jobs that do not run today
1057         '''
1058         # Get the boards to fill and put it in a dictionary
1059         # {board_name : xml instance corresponding to the board}
1060         for job in l_jobs + l_jobs_not_today:
1061             board = job.board
1062             if (board is not None and
1063                                 board not in self.d_xml_board_files.keys()):
1064                 self.add_xml_board(board)
1065
1066         # Verify that the boards given as input are done
1067         for board in list(self.d_input_boards.keys()):
1068             if board not in self.d_xml_board_files:
1069                 self.add_xml_board(board)
1070
1071         # Loop over all jobs in order to get the lines and columns for each
1072         # xml file
1073         d_dist = {}
1074         d_application = {}
1075         for board in self.d_xml_board_files:
1076             d_dist[board] = []
1077             d_application[board] = []
1078
1079         l_hosts_ports = []
1080
1081         for job in l_jobs + l_jobs_not_today:
1082
1083             if (job.machine.host, job.machine.port) not in l_hosts_ports:
1084                 l_hosts_ports.append((job.machine.host, job.machine.port))
1085
1086             distrib = job.machine.distribution
1087             application = job.application
1088
1089             board_job = job.board
1090             if board is None:
1091                 continue
1092             for board in self.d_xml_board_files:
1093                 if board_job == board:
1094                     if distrib is not None and distrib not in d_dist[board]:
1095                         d_dist[board].append(distrib)
1096                         src.xmlManager.add_simple_node(
1097                             self.d_xml_board_files[board].xmlroot.find(
1098                                                             'distributions'),
1099                                                    "dist",
1100                                                    attrib={"name" : distrib})
1101
1102                 if board_job == board:
1103                     if (application is not None and
1104                                     application not in d_application[board]):
1105                         d_application[board].append(application)
1106                         src.xmlManager.add_simple_node(
1107                             self.d_xml_board_files[board].xmlroot.find(
1108                                                                 'applications'),
1109                                                    "application",
1110                                                    attrib={
1111                                                         "name" : application})
1112
1113         # Verify that there are no missing application or distribution in the
1114         # xml board files (regarding the input boards)
1115         for board in self.d_xml_board_files:
1116             l_dist = d_dist[board]
1117             if board not in self.d_input_boards.keys():
1118                 continue
1119             for dist in self.d_input_boards[board]["rows"]:
1120                 if dist not in l_dist:
1121                     src.xmlManager.add_simple_node(
1122                             self.d_xml_board_files[board].xmlroot.find(
1123                                                             'distributions'),
1124                                                    "dist",
1125                                                    attrib={"name" : dist})
1126             l_appli = d_application[board]
1127             for appli in self.d_input_boards[board]["columns"]:
1128                 if appli not in l_appli:
1129                     src.xmlManager.add_simple_node(
1130                             self.d_xml_board_files[board].xmlroot.find(
1131                                                                 'applications'),
1132                                                    "application",
1133                                                    attrib={"name" : appli})
1134
1135         # Initialize the hosts_ports node for the global file
1136         self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1137                                                                 "hosts_ports")
1138         for host, port in l_hosts_ports:
1139             host_port = "%s:%i" % (host, port)
1140             src.xmlManager.add_simple_node(self.xmlhosts_ports,
1141                                            "host_port",
1142                                            attrib={"name" : host_port})
1143
1144         # Initialize the jobs node in all files
1145         for xml_file in [self.xml_global_file] + list(
1146                                             self.d_xml_board_files.values()):
1147             xml_jobs = xml_file.add_simple_node("jobs")
1148             # Get the jobs present in the config file but
1149             # that will not be launched today
1150             self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1151
1152             xml_file.add_simple_node("infos",
1153                                      attrib={"name" : "last update",
1154                                              "JobsCommandStatus" : "running"})
1155
1156         # Find in each board the squares that needs to be filled regarding the
1157         # input csv files but that are not covered by a today job
1158         for board in self.d_input_boards.keys():
1159             xml_root_board = self.d_xml_board_files[board].xmlroot
1160             xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1161                                                  "missing_jobs")
1162             for row, column in self.d_input_boards[board]["jobs"]:
1163                 found = False
1164                 for job in l_jobs:
1165                     if (job.application == column and
1166                         job.machine.distribution == row):
1167                         found = True
1168                         break
1169                 if not found:
1170                     src.xmlManager.add_simple_node(xml_missing,
1171                                             "job",
1172                                             attrib={"distribution" : row,
1173                                                     "application" : column })
1174
1175     def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1176         '''Get all the first information needed for each file and write the
1177            first version of the files
1178
1179         :param xml_node_jobs etree.Element: the node corresponding to a job
1180         :param l_jobs_not_today List: the list of jobs that do not run today
1181         '''
1182         for job in l_jobs_not_today:
1183             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1184                                                  "job",
1185                                                  attrib={"name" : job.name})
1186             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1187             src.xmlManager.add_simple_node(xmlj,
1188                                            "distribution",
1189                                            job.machine.distribution)
1190             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1191             src.xmlManager.add_simple_node(xmlj,
1192                                        "commands", " ; ".join(job.commands))
1193             src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1194             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1195             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1196             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1197             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1198             src.xmlManager.add_simple_node(xmlj, "sat_path",
1199                                                         job.machine.sat_path)
1200
1201     def parse_csv_boards(self, today):
1202         """ Parse the csv files that describes the boards to produce and fill
1203             the dict d_input_boards that contain the csv file contain
1204
1205         :param today str: the current day of the week
1206         """
1207         # loop over each csv file and read its content
1208         l_boards = []
1209         for file_path in self.l_file_boards:
1210             l_read = []
1211             with open(file_path, 'r') as f:
1212                 reader = csv.reader(f)
1213                 for row in reader:
1214                     l_read.append(row)
1215             l_boards.append(l_read)
1216
1217         # loop over the csv lists of lines and get the rows, columns and jobs
1218         d_boards = {}
1219         for input_board in l_boards:
1220             # get board name
1221             board_name = input_board[0][0]
1222
1223             # Get columns list
1224             columns = input_board[0][1:]
1225
1226             rows = []
1227             columns_out = []
1228             jobs = []
1229             for line in input_board[1:]:
1230                 row = line[0]
1231                 for i, square in enumerate(line[1:]):
1232                     if today in square:
1233                         if row not in rows:
1234                             rows.append(row)
1235                         if columns[i] not in columns_out:
1236                             columns_out.append(columns[i])
1237                         job = (row, columns[i])
1238                         jobs.append(job)
1239
1240             d_boards[board_name] = {"rows" : rows,
1241                                     "columns" : columns_out,
1242                                     "jobs" : jobs}
1243
1244         self.d_input_boards = d_boards
1245
1246     def update_xml_files(self, l_jobs):
1247         '''Write all the xml files with updated information about the jobs
1248
1249         :param l_jobs List: the list of jobs that run today
1250         '''
1251         for xml_file in [self.xml_global_file] + list(
1252                                             self.d_xml_board_files.values()):
1253             self.update_xml_file(l_jobs, xml_file)
1254
1255         # Write the file
1256         self.write_xml_files()
1257
1258     def update_xml_file(self, l_jobs, xml_file):
1259         '''update information about the jobs for the file xml_file
1260
1261         :param l_jobs List: the list of jobs that run today
1262         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1263         '''
1264
1265         xml_node_jobs = xml_file.xmlroot.find('jobs')
1266         # Update the job names and status node
1267         for job in l_jobs:
1268             # Find the node corresponding to the job and delete it
1269             # in order to recreate it
1270             for xmljob in xml_node_jobs.findall('job'):
1271                 if xmljob.attrib['name'] == job.name:
1272                     xml_node_jobs.remove(xmljob)
1273
1274             T0 = str(job._T0)
1275             if T0 != "-1":
1276                 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1277                                        time.localtime(job._T0))
1278             Tf = str(job._Tf)
1279             if Tf != "-1":
1280                 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1281                                        time.localtime(job._Tf))
1282
1283             # recreate the job node
1284             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1285                                                   "job",
1286                                                   attrib={"name" : job.name})
1287             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1288             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1289             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1290             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1291             src.xmlManager.add_simple_node(xmlj, "sat_path",
1292                                            job.machine.sat_path)
1293             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1294             src.xmlManager.add_simple_node(xmlj, "distribution",
1295                                            job.machine.distribution)
1296             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1297             src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1298             src.xmlManager.add_simple_node(xmlj, "commands",
1299                                            " ; ".join(job.commands))
1300             src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1301             src.xmlManager.add_simple_node(xmlj, "begin", T0)
1302             src.xmlManager.add_simple_node(xmlj, "end", Tf)
1303             src.xmlManager.add_simple_node(xmlj, "out",
1304                                            src.printcolors.cleancolor(job.out))
1305             src.xmlManager.add_simple_node(xmlj, "err",
1306                                            src.printcolors.cleancolor(job.err))
1307             src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1308             if len(job.remote_log_files) > 0:
1309                 src.xmlManager.add_simple_node(xmlj,
1310                                                "remote_log_file_path",
1311                                                job.remote_log_files[0])
1312             else:
1313                 src.xmlManager.add_simple_node(xmlj,
1314                                                "remote_log_file_path",
1315                                                "nothing")
1316
1317             xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1318             # get the job father
1319             if job.after is not None:
1320                 job_father = None
1321                 for jb in l_jobs:
1322                     if jb.name == job.after:
1323                         job_father = jb
1324
1325                 if (job_father is not None and
1326                         len(job_father.remote_log_files) > 0):
1327                     link = job_father.remote_log_files[0]
1328                 else:
1329                     link = "nothing"
1330                 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1331
1332             # Verify that the job is to be done today regarding the input csv
1333             # files
1334             if job.board and job.board in self.d_input_boards.keys():
1335                 found = False
1336                 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1337                     if (job.machine.distribution == dist
1338                         and job.application == appli):
1339                         found = True
1340                         src.xmlManager.add_simple_node(xmlj,
1341                                                "extra_job",
1342                                                "no")
1343                         break
1344                 if not found:
1345                     src.xmlManager.add_simple_node(xmlj,
1346                                                "extra_job",
1347                                                "yes")
1348
1349
1350         # Update the date
1351         xml_node_infos = xml_file.xmlroot.find('infos')
1352         src.xmlManager.append_node_attrib(xml_node_infos,
1353                     attrib={"value" :
1354                     datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1355
1356
1357
1358     def last_update(self, finish_status = "finished"):
1359         '''update information about the jobs for the file xml_file
1360
1361         :param l_jobs List: the list of jobs that run today
1362         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1363         '''
1364         for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1365             xml_node_infos = xml_file.xmlroot.find('infos')
1366             src.xmlManager.append_node_attrib(xml_node_infos,
1367                         attrib={"JobsCommandStatus" : finish_status})
1368         # Write the file
1369         self.write_xml_files()
1370
1371     def write_xml_files(self):
1372         ''' Write the xml files
1373         '''
1374         self.xml_global_file.write_tree(STYLESHEET_GLOBAL)
1375         for xml_file in self.d_xml_board_files.values():
1376             xml_file.write_tree(STYLESHEET_BOARD)
1377
1378 ##
1379 # Describes the command
1380 def description():
1381     return _("The jobs command launches maintenances that are described"
1382              " in the dedicated jobs configuration file.")
1383
1384 ##
1385 # Runs the command.
1386 def run(args, runner, logger):
1387
1388     (options, args) = parser.parse_args(args)
1389
1390     l_cfg_dir = runner.cfg.PATHS.JOBPATH
1391
1392     # list option : display all the available config files
1393     if options.list:
1394         for cfg_dir in l_cfg_dir:
1395             if not options.no_label:
1396                 logger.write("------ %s\n" %
1397                                  src.printcolors.printcHeader(cfg_dir))
1398
1399             for f in sorted(os.listdir(cfg_dir)):
1400                 if not f.endswith('.pyconf'):
1401                     continue
1402                 cfilename = f[:-7]
1403                 logger.write("%s\n" % cfilename)
1404         return 0
1405
1406     # Make sure the jobs_config option has been called
1407     if not options.jobs_cfg:
1408         message = _("The option --jobs_config is required\n")
1409         src.printcolors.printcError(message)
1410         return 1
1411
1412     # Find the file in the directories
1413     found = False
1414     for cfg_dir in l_cfg_dir:
1415         file_jobs_cfg = os.path.join(cfg_dir, options.jobs_cfg)
1416         if not file_jobs_cfg.endswith('.pyconf'):
1417             file_jobs_cfg += '.pyconf'
1418
1419         if not os.path.exists(file_jobs_cfg):
1420             continue
1421         else:
1422             found = True
1423             break
1424
1425     if not found:
1426         msg = _("The file configuration %(name_file)s was not found."
1427                 "\nUse the --list option to get the possible files.")
1428         src.printcolors.printcError(msg)
1429         return 1
1430
1431     info = [
1432         (_("Platform"), runner.cfg.VARS.dist),
1433         (_("File containing the jobs configuration"), file_jobs_cfg)
1434     ]
1435     src.print_info(logger, info)
1436
1437     # Read the config that is in the file
1438     config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1439     if options.only_jobs:
1440         l_jb = src.pyconf.Sequence()
1441         for jb in config_jobs.jobs:
1442             if jb.name in options.only_jobs:
1443                 l_jb.append(jb,
1444                 "Adding a job that was given in only_jobs option parameters")
1445         config_jobs.jobs = l_jb
1446
1447     # Initialization
1448     today_jobs = Jobs(runner,
1449                       logger,
1450                       file_jobs_cfg,
1451                       config_jobs)
1452     # SSH connection to all machines
1453     today_jobs.ssh_connection_all_machines()
1454     if options.test_connection:
1455         return 0
1456
1457     gui = None
1458     if options.publish:
1459         # Copy the stylesheets in the log directory
1460         log_dir = runner.cfg.SITE.log.log_dir
1461         xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1462         files_to_copy = []
1463         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1464         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1465         files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1466         for file_path in files_to_copy:
1467             shutil.copy2(file_path, log_dir)
1468
1469         # Instanciate the Gui in order to produce the xml files that contain all
1470         # the boards
1471         gui = Gui(runner.cfg.SITE.log.log_dir,
1472                   today_jobs.ljobs,
1473                   today_jobs.ljobs_not_today,
1474                   l_file_boards = options.input_boards)
1475
1476         # Display the list of the xml files
1477         logger.write(src.printcolors.printcInfo(("Here is the list of published"
1478                                                  " files :\n")), 4)
1479         logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1480         for board in gui.d_xml_board_files.keys():
1481             logger.write("%s\n" % gui.d_xml_board_files[board].logFile, 4)
1482
1483         logger.write("\n", 4)
1484
1485     today_jobs.gui = gui
1486
1487     interruped = False
1488     try:
1489         # Run all the jobs contained in config_jobs
1490         today_jobs.run_jobs()
1491     except KeyboardInterrupt:
1492         interruped = True
1493         logger.write("\n\n%s\n\n" %
1494                 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1495     finally:
1496         if interruped:
1497             msg = _("Killing the running jobs and trying"
1498                     " to get the corresponding logs\n")
1499             logger.write(src.printcolors.printcWarning(msg))
1500
1501         # find the potential not finished jobs and kill them
1502         for jb in today_jobs.ljobs:
1503             if not jb.has_finished():
1504                 jb.kill_remote_process()
1505         if interruped:
1506             if today_jobs.gui:
1507                 today_jobs.gui.last_update(_("Forced interruption"))
1508         else:
1509             if today_jobs.gui:
1510                 today_jobs.gui.last_update()
1511         # Output the results
1512         today_jobs.write_all_results()