commands/jobs.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import datetime
  21 import time
  22 import csv
  23 import shutil
  24 import itertools
  25 import paramiko
  26
  27 import src
  28
  29 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
  30 STYLESHEET_BOARD = "jobs_board_report.xsl"
  31
  32 DAYS_SEPARATOR = ","
  33 CSV_DELIMITER = ";"
  34
  35 parser = src.options.Options()
  36
  37 parser.add_option('n', 'name', 'string', 'jobs_cfg',
  38                   _('The name of the config file that contains'
  39                   ' the jobs configuration'))
  40 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
  41                   _('Optional: the list of jobs to launch, by their name. '))
  42 parser.add_option('l', 'list', 'boolean', 'list',
  43                   _('Optional: list all available config files.'))
  44 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
  45                   _("Optional: try to connect to the machines. "
  46                     "Not executing the jobs."),
  47                   False)
  48 parser.add_option('p', 'publish', 'boolean', 'publish',
  49                   _("Optional: generate an xml file that can be read in a "
  50                     "browser to display the jobs status."),
  51                   False)
  52 parser.add_option('i', 'input_boards', 'string', 'input_boards', _("Optional: "
  53                                 "the path to csv file that contain "
  54                                 "the expected boards."),"")
  55 parser.add_option('n', 'completion', 'boolean', 'no_label',
  56                   _("Optional (internal use): do not print labels, Works only "
  57                     "with --list."),
  58                   False)
  59
  60 class Machine(object):
  61     '''Class to manage a ssh connection on a machine
  62     '''
  63     def __init__(self,
  64                  name,
  65                  host,
  66                  user,
  67                  port=22,
  68                  passwd=None,
  69                  sat_path="salomeTools"):
  70         self.name = name
  71         self.host = host
  72         self.port = port
  73         self.distribution = None # Will be filled after copying SAT on the machine
  74         self.user = user
  75         self.password = passwd
  76         self.sat_path = sat_path
  77         self.ssh = paramiko.SSHClient()
  78         self._connection_successful = None
  79
  80     def connect(self, logger):
  81         '''Initiate the ssh connection to the remote machine
  82
  83         :param logger src.logger.Logger: The logger instance
  84         :return: Nothing
  85         :rtype: N\A
  86         '''
  87
  88         self._connection_successful = False
  89         self.ssh.load_system_host_keys()
  90         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  91         try:
  92             self.ssh.connect(self.host,
  93                              port=self.port,
  94                              username=self.user,
  95                              password = self.password)
  96         except paramiko.AuthenticationException:
  97             message = src.KO_STATUS + _("Authentication failed")
  98         except paramiko.BadHostKeyException:
  99             message = (src.KO_STATUS +
 100                        _("The server's host key could not be verified"))
 101         except paramiko.SSHException:
 102             message = ( _("SSHException error connecting or "
 103                           "establishing an SSH session"))
 104         except:
 105             message = ( _("Error connecting or establishing an SSH session"))
 106         else:
 107             self._connection_successful = True
 108             message = ""
 109         return message
 110
 111     def successfully_connected(self, logger):
 112         '''Verify if the connection to the remote machine has succeed
 113
 114         :param logger src.logger.Logger: The logger instance
 115         :return: True if the connection has succeed, False if not
 116         :rtype: bool
 117         '''
 118         if self._connection_successful == None:
 119             message = _("Warning : trying to ask if the connection to "
 120             "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
 121             " no connection request" %
 122                         (self.name, self.host, self.port, self.user))
 123             logger.write( src.printcolors.printcWarning(message))
 124         return self._connection_successful
 125
 126     def copy_sat(self, sat_local_path, job_file):
 127         '''Copy salomeTools to the remote machine in self.sat_path
 128         '''
 129         res = 0
 130         try:
 131             # open a sftp connection
 132             self.sftp = self.ssh.open_sftp()
 133             # Create the sat directory on remote machine if it is not existing
 134             self.mkdir(self.sat_path, ignore_existing=True)
 135             # Put sat
 136             self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
 137             # put the job configuration file in order to make it reachable
 138             # on the remote machine
 139             self.sftp.put(job_file, os.path.join(".salomeTools",
 140                                                  "Jobs",
 141                                                  ".jobs_command_file.pyconf"))
 142         except Exception as e:
 143             res = str(e)
 144             self._connection_successful = False
 145
 146         return res
 147
 148     def put_dir(self, source, target, filters = []):
 149         ''' Uploads the contents of the source directory to the target path. The
 150             target directory needs to exists. All sub-directories in source are
 151             created under target.
 152         '''
 153         for item in os.listdir(source):
 154             if item in filters:
 155                 continue
 156             source_path = os.path.join(source, item)
 157             destination_path = os.path.join(target, item)
 158             if os.path.islink(source_path):
 159                 linkto = os.readlink(source_path)
 160                 try:
 161                     self.sftp.symlink(linkto, destination_path)
 162                     self.sftp.chmod(destination_path,
 163                                     os.stat(source_path).st_mode)
 164                 except IOError:
 165                     pass
 166             else:
 167                 if os.path.isfile(source_path):
 168                     self.sftp.put(source_path, destination_path)
 169                     self.sftp.chmod(destination_path,
 170                                     os.stat(source_path).st_mode)
 171                 else:
 172                     self.mkdir(destination_path, ignore_existing=True)
 173                     self.put_dir(source_path, destination_path)
 174
 175     def mkdir(self, path, mode=511, ignore_existing=False):
 176         ''' Augments mkdir by adding an option to not fail
 177             if the folder exists
 178         '''
 179         try:
 180             self.sftp.mkdir(path, mode)
 181         except IOError:
 182             if ignore_existing:
 183                 pass
 184             else:
 185                 raise
 186
 187     def exec_command(self, command, logger):
 188         '''Execute the command on the remote machine
 189
 190         :param command str: The command to be run
 191         :param logger src.logger.Logger: The logger instance
 192         :return: the stdin, stdout, and stderr of the executing command,
 193                  as a 3-tuple
 194         :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
 195                 paramiko.channel.ChannelFile)
 196         '''
 197         try:
 198             # Does not wait the end of the command
 199             (stdin, stdout, stderr) = self.ssh.exec_command(command)
 200         except paramiko.SSHException:
 201             message = src.KO_STATUS + _(
 202                             ": the server failed to execute the command\n")
 203             logger.write( src.printcolors.printcError(message))
 204             return (None, None, None)
 205         except:
 206             logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
 207             return (None, None, None)
 208         else:
 209             return (stdin, stdout, stderr)
 210
 211     def close(self):
 212         '''Close the ssh connection
 213
 214         :rtype: N\A
 215         '''
 216         self.ssh.close()
 217
 218     def write_info(self, logger):
 219         '''Prints the informations relative to the machine in the logger
 220            (terminal traces and log file)
 221
 222         :param logger src.logger.Logger: The logger instance
 223         :return: Nothing
 224         :rtype: N\A
 225         '''
 226         logger.write("host : " + self.host + "\n")
 227         logger.write("port : " + str(self.port) + "\n")
 228         logger.write("user : " + str(self.user) + "\n")
 229         if self.successfully_connected(logger):
 230             status = src.OK_STATUS
 231         else:
 232             status = src.KO_STATUS
 233         logger.write("Connection : " + status + "\n\n")
 234
 235
 236 class Job(object):
 237     '''Class to manage one job
 238     '''
 239     def __init__(self, name, machine, application, board,
 240                  commands, timeout, config, logger, after=None):
 241
 242         self.name = name
 243         self.machine = machine
 244         self.after = after
 245         self.timeout = timeout
 246         self.application = application
 247         self.board = board
 248         self.config = config
 249         self.logger = logger
 250         # The list of log files to download from the remote machine
 251         self.remote_log_files = []
 252
 253         # The remote command status
 254         # -1 means that it has not been launched,
 255         # 0 means success and 1 means fail
 256         self.res_job = "-1"
 257         self.cancelled = False
 258
 259         self._T0 = -1
 260         self._Tf = -1
 261         self._has_begun = False
 262         self._has_finished = False
 263         self._has_timouted = False
 264         self._stdin = None # Store the command inputs field
 265         self._stdout = None # Store the command outputs field
 266         self._stderr = None # Store the command errors field
 267
 268         self.out = ""
 269         self.err = ""
 270
 271         self.commands = commands
 272         self.command = (os.path.join(self.machine.sat_path, "sat") +
 273                         " -l " +
 274                         os.path.join(self.machine.sat_path,
 275                                      "list_log_files.txt") +
 276                         " job --jobs_config .jobs_command_file" +
 277                         " --name " +
 278                         self.name)
 279
 280     def get_pids(self):
 281         """ Get the pid(s) corresponding to the command that have been launched
 282             On the remote machine
 283
 284         :return: The list of integers corresponding to the found pids
 285         :rtype: List
 286         """
 287         pids = []
 288         cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
 289         (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
 290         pids_cmd = out_pid.readlines()
 291         pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
 292         pids+=pids_cmd
 293         return pids
 294
 295     def kill_remote_process(self, wait=1):
 296         '''Kills the process on the remote machine.
 297
 298         :return: (the output of the kill, the error of the kill)
 299         :rtype: (str, str)
 300         '''
 301
 302         pids = self.get_pids()
 303         cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
 304         (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
 305                                                             self.logger)
 306         time.sleep(wait)
 307         return (out_kill, err_kill)
 308
 309     def has_begun(self):
 310         '''Returns True if the job has already begun
 311
 312         :return: True if the job has already begun
 313         :rtype: bool
 314         '''
 315         return self._has_begun
 316
 317     def has_finished(self):
 318         '''Returns True if the job has already finished
 319            (i.e. all the commands have been executed)
 320            If it is finished, the outputs are stored in the fields out and err.
 321
 322         :return: True if the job has already finished
 323         :rtype: bool
 324         '''
 325
 326         # If the method has already been called and returned True
 327         if self._has_finished:
 328             return True
 329
 330         # If the job has not begun yet
 331         if not self.has_begun():
 332             return False
 333
 334         if self._stdout.channel.closed:
 335             self._has_finished = True
 336             # Store the result outputs
 337             self.out += self._stdout.read().decode()
 338             self.err += self._stderr.read().decode()
 339             # Put end time
 340             self._Tf = time.time()
 341             # And get the remote command status and log files
 342             self.get_log_files()
 343
 344         return self._has_finished
 345
 346     def get_log_files(self):
 347         """Get the log files produced by the command launched
 348            on the remote machine, and put it in the log directory of the user,
 349            so they can be accessible from
 350         """
 351         # Do not get the files if the command is not finished
 352         if not self.has_finished():
 353             msg = _("Trying to get log files whereas the job is not finished.")
 354             self.logger.write(src.printcolors.printcWarning(msg))
 355             return
 356
 357         # First get the file that contains the list of log files to get
 358         tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
 359         remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt")
 360         self.machine.sftp.get(
 361                     remote_path,
 362                     tmp_file_path)
 363
 364         # Read the file and get the result of the command and all the log files
 365         # to get
 366         fstream_tmp = open(tmp_file_path, "r")
 367         file_lines = fstream_tmp.readlines()
 368         file_lines = [line.replace("\n", "") for line in file_lines]
 369         fstream_tmp.close()
 370         os.remove(tmp_file_path)
 371
 372         try :
 373             # The first line is the result of the command (0 success or 1 fail)
 374             self.res_job = file_lines[0]
 375         except Exception as e:
 376             self.err += _("Unable to get status from remote file %s: %s" %
 377                                                     (remote_path, str(e)))
 378
 379         for i, job_path_remote in enumerate(file_lines[1:]):
 380             try:
 381                 # For each command, there is two files to get :
 382                 # 1- The xml file describing the command and giving the
 383                 # internal traces.
 384                 # 2- The txt file containing the system command traces (like
 385                 # traces produced by the "make" command)
 386                 # 3- In case of the test command, there is another file to get :
 387                 # the xml board that contain the test results
 388                 dirname = os.path.basename(os.path.dirname(job_path_remote))
 389                 if dirname != 'OUT' and dirname != 'TEST':
 390                     # Case 1-
 391                     local_path = os.path.join(os.path.dirname(
 392                                                         self.logger.logFilePath),
 393                                               os.path.basename(job_path_remote))
 394                     if i==0: # The first is the job command
 395                         self.logger.add_link(os.path.basename(job_path_remote),
 396                                              "job",
 397                                              self.res_job,
 398                                              self.command)
 399                 elif dirname == 'OUT':
 400                     # Case 2-
 401                     local_path = os.path.join(os.path.dirname(
 402                                                         self.logger.logFilePath),
 403                                               'OUT',
 404                                               os.path.basename(job_path_remote))
 405                 elif dirname == 'TEST':
 406                     # Case 3-
 407                     local_path = os.path.join(os.path.dirname(
 408                                                         self.logger.logFilePath),
 409                                               'TEST',
 410                                               os.path.basename(job_path_remote))
 411
 412                 # Get the file
 413                 if not os.path.exists(local_path):
 414                     self.machine.sftp.get(job_path_remote, local_path)
 415                 self.remote_log_files.append(local_path)
 416             except Exception as e:
 417                 self.err += _("Unable to get %s log file from remote: %s" %
 418                                                     (str(job_path_remote),
 419                                                      str(e)))
 420
 421     def has_failed(self):
 422         '''Returns True if the job has failed.
 423            A job is considered as failed if the machine could not be reached,
 424            if the remote command failed,
 425            or if the job finished with a time out.
 426
 427         :return: True if the job has failed
 428         :rtype: bool
 429         '''
 430         if not self.has_finished():
 431             return False
 432         if not self.machine.successfully_connected(self.logger):
 433             return True
 434         if self.is_timeout():
 435             return True
 436         if self.res_job == "1":
 437             return True
 438         return False
 439
 440     def cancel(self):
 441         """In case of a failing job, one has to cancel every job that depend
 442            on it. This method put the job as failed and will not be executed.
 443         """
 444         self._has_begun = True
 445         self._has_finished = True
 446         self.cancelled = True
 447         self.out += _("This job was not launched because its father has failed.")
 448         self.err += _("This job was not launched because its father has failed.")
 449
 450     def is_running(self):
 451         '''Returns True if the job commands are running
 452
 453         :return: True if the job is running
 454         :rtype: bool
 455         '''
 456         return self.has_begun() and not self.has_finished()
 457
 458     def is_timeout(self):
 459         '''Returns True if the job commands has finished with timeout
 460
 461         :return: True if the job has finished with timeout
 462         :rtype: bool
 463         '''
 464         return self._has_timouted
 465
 466     def time_elapsed(self):
 467         """Get the time elapsed since the job launching
 468
 469         :return: The number of seconds
 470         :rtype: int
 471         """
 472         if not self.has_begun():
 473             return -1
 474         T_now = time.time()
 475         return T_now - self._T0
 476
 477     def check_time(self):
 478         """Verify that the job has not exceeded its timeout.
 479            If it has, kill the remote command and consider the job as finished.
 480         """
 481         if not self.has_begun():
 482             return
 483         if self.time_elapsed() > self.timeout:
 484             self._has_finished = True
 485             self._has_timouted = True
 486             self._Tf = time.time()
 487             self.get_pids()
 488             (out_kill, _) = self.kill_remote_process()
 489             self.out += "TIMEOUT \n" + out_kill.read().decode()
 490             self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
 491             try:
 492                 self.get_log_files()
 493             except Exception as e:
 494                 self.err += _("Unable to get remote log files: %s" % e)
 495
 496     def total_duration(self):
 497         """Give the total duration of the job
 498
 499         :return: the total duration of the job in seconds
 500         :rtype: int
 501         """
 502         return self._Tf - self._T0
 503
 504     def run(self):
 505         """Launch the job by executing the remote command.
 506         """
 507
 508         # Prevent multiple run
 509         if self.has_begun():
 510             msg = _("Warning: A job can only be launched one time")
 511             msg2 = _("Trying to launch the job \"%s\" whereas it has "
 512                      "already been launched." % self.name)
 513             self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
 514                                                                         msg2)))
 515             return
 516
 517         # Do not execute the command if the machine could not be reached
 518         if not self.machine.successfully_connected(self.logger):
 519             self._has_finished = True
 520             self.out = "N\A"
 521             self.err += ("Connection to machine (name : %s, host: %s, port:"
 522                         " %s, user: %s) has failed\nUse the log command "
 523                         "to get more information."
 524                         % (self.machine.name,
 525                            self.machine.host,
 526                            self.machine.port,
 527                            self.machine.user))
 528         else:
 529             # Usual case : Launch the command on remote machine
 530             self._T0 = time.time()
 531             self._stdin, self._stdout, self._stderr = self.machine.exec_command(
 532                                                                   self.command,
 533                                                                   self.logger)
 534             # If the results are not initialized, finish the job
 535             if (self._stdin, self._stdout, self._stderr) == (None, None, None):
 536                 self._has_finished = True
 537                 self._Tf = time.time()
 538                 self.out += "N\A"
 539                 self.err += "The server failed to execute the command"
 540
 541         # Put the beginning flag to true.
 542         self._has_begun = True
 543
 544     def write_results(self):
 545         """Display on the terminal all the job's information
 546         """
 547         self.logger.write("name : " + self.name + "\n")
 548         if self.after:
 549             self.logger.write("after : %s\n" % self.after)
 550         self.logger.write("Time elapsed : %4imin %2is \n" %
 551                      (self.total_duration()//60 , self.total_duration()%60))
 552         if self._T0 != -1:
 553             self.logger.write("Begin time : %s\n" %
 554                          time.strftime('%Y-%m-%d %H:%M:%S',
 555                                        time.localtime(self._T0)) )
 556         if self._Tf != -1:
 557             self.logger.write("End time   : %s\n\n" %
 558                          time.strftime('%Y-%m-%d %H:%M:%S',
 559                                        time.localtime(self._Tf)) )
 560
 561         machine_head = "Informations about connection :\n"
 562         underline = (len(machine_head) - 2) * "-"
 563         self.logger.write(src.printcolors.printcInfo(
 564                                                 machine_head+underline+"\n"))
 565         self.machine.write_info(self.logger)
 566
 567         self.logger.write(src.printcolors.printcInfo("out : \n"))
 568         if self.out == "":
 569             self.logger.write("Unable to get output\n")
 570         else:
 571             self.logger.write(self.out + "\n")
 572         self.logger.write(src.printcolors.printcInfo("err : \n"))
 573         self.logger.write(self.err + "\n")
 574
 575     def get_status(self):
 576         """Get the status of the job (used by the Gui for xml display)
 577
 578         :return: The current status of the job
 579         :rtype: String
 580         """
 581         if not self.machine.successfully_connected(self.logger):
 582             return "SSH connection KO"
 583         if not self.has_begun():
 584             return "Not launched"
 585         if self.cancelled:
 586             return "Cancelled"
 587         if self.is_running():
 588             return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
 589                                                     time.localtime(self._T0))
 590         if self.has_finished():
 591             if self.is_timeout():
 592                 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
 593                                                     time.localtime(self._Tf))
 594             return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
 595                                                      time.localtime(self._Tf))
 596
 597 class Jobs(object):
 598     '''Class to manage the jobs to be run
 599     '''
 600     def __init__(self,
 601                  runner,
 602                  logger,
 603                  job_file_path,
 604                  config_jobs,
 605                  lenght_columns = 20):
 606         # The jobs configuration
 607         self.cfg_jobs = config_jobs
 608         self.job_file_path = job_file_path
 609         # The machine that will be used today
 610         self.lmachines = []
 611         # The list of machine (hosts, port) that will be used today
 612         # (a same host can have several machine instances since there
 613         # can be several ssh parameters)
 614         self.lhosts = []
 615         # The jobs to be launched today
 616         self.ljobs = []
 617         # The jobs that will not be launched today
 618         self.ljobs_not_today = []
 619         self.runner = runner
 620         self.logger = logger
 621         self.len_columns = lenght_columns
 622
 623         # the list of jobs that have not been run yet
 624         self._l_jobs_not_started = []
 625         # the list of jobs that have already ran
 626         self._l_jobs_finished = []
 627         # the list of jobs that are running
 628         self._l_jobs_running = []
 629
 630         self.determine_jobs_and_machines()
 631
 632     def define_job(self, job_def, machine):
 633         '''Takes a pyconf job definition and a machine (from class machine)
 634            and returns the job instance corresponding to the definition.
 635
 636         :param job_def src.config.Mapping: a job definition
 637         :param machine machine: the machine on which the job will run
 638         :return: The corresponding job in a job class instance
 639         :rtype: job
 640         '''
 641         name = job_def.name
 642         cmmnds = job_def.commands
 643         if not "timeout" in job_def:
 644             timeout = 4*60*60 # default timeout = 4h
 645         else:
 646             timeout = job_def.timeout
 647         after = None
 648         if 'after' in job_def:
 649             after = job_def.after
 650         application = None
 651         if 'application' in job_def:
 652             application = job_def.application
 653         board = None
 654         if 'board' in job_def:
 655             board = job_def.board
 656
 657         return Job(name,
 658                    machine,
 659                    application,
 660                    board,
 661                    cmmnds,
 662                    timeout,
 663                    self.runner.cfg,
 664                    self.logger,
 665                    after = after)
 666
 667     def determine_jobs_and_machines(self):
 668         '''Function that reads the pyconf jobs definition and instantiates all
 669            the machines and jobs to be done today.
 670
 671         :return: Nothing
 672         :rtype: N\A
 673         '''
 674         today = datetime.date.weekday(datetime.date.today())
 675         host_list = []
 676
 677         for job_def in self.cfg_jobs.jobs :
 678
 679             if not "machine" in job_def:
 680                 msg = _('WARNING: The job "%s" do not have the key '
 681                        '"machine", this job is ignored.\n\n' % job_def.name)
 682                 self.logger.write(src.printcolors.printcWarning(msg))
 683                 continue
 684             name_machine = job_def.machine
 685
 686             a_machine = None
 687             for mach in self.lmachines:
 688                 if mach.name == name_machine:
 689                     a_machine = mach
 690                     break
 691
 692             if a_machine == None:
 693                 for machine_def in self.cfg_jobs.machines:
 694                     if machine_def.name == name_machine:
 695                         if 'host' not in machine_def:
 696                             host = self.runner.cfg.VARS.hostname
 697                         else:
 698                             host = machine_def.host
 699
 700                         if 'user' not in machine_def:
 701                             user = self.runner.cfg.VARS.user
 702                         else:
 703                             user = machine_def.user
 704
 705                         if 'port' not in machine_def:
 706                             port = 22
 707                         else:
 708                             port = machine_def.port
 709
 710                         if 'password' not in machine_def:
 711                             passwd = None
 712                         else:
 713                             passwd = machine_def.password
 714
 715                         if 'sat_path' not in machine_def:
 716                             sat_path = "salomeTools"
 717                         else:
 718                             sat_path = machine_def.sat_path
 719
 720                         a_machine = Machine(
 721                                             machine_def.name,
 722                                             host,
 723                                             user,
 724                                             port=port,
 725                                             passwd=passwd,
 726                                             sat_path=sat_path
 727                                             )
 728
 729                         self.lmachines.append(a_machine)
 730                         if (host, port) not in host_list:
 731                             host_list.append((host, port))
 732
 733                 if a_machine == None:
 734                     msg = _("WARNING: The job \"%(job_name)s\" requires the "
 735                             "machine \"%(machine_name)s\" but this machine "
 736                             "is not defined in the configuration file.\n"
 737                             "The job will not be launched")
 738                     self.logger.write(src.printcolors.printcWarning(msg))
 739
 740             a_job = self.define_job(job_def, a_machine)
 741
 742             if today in job_def.when:
 743                 self.ljobs.append(a_job)
 744             else: # today in job_def.when
 745                 self.ljobs_not_today.append(a_job)
 746
 747         self.lhosts = host_list
 748
 749     def ssh_connection_all_machines(self, pad=50):
 750         '''Function that do the ssh connection to every machine
 751            to be used today.
 752
 753         :return: Nothing
 754         :rtype: N\A
 755         '''
 756         self.logger.write(src.printcolors.printcInfo((
 757                         "Establishing connection with all the machines :\n")))
 758         for machine in self.lmachines:
 759             # little algorithm in order to display traces
 760             begin_line = (_("Connection to %s: " % machine.name))
 761             if pad - len(begin_line) < 0:
 762                 endline = " "
 763             else:
 764                 endline = (pad - len(begin_line)) * "." + " "
 765
 766             step = "SSH connection"
 767             self.logger.write( begin_line + endline + step)
 768             self.logger.flush()
 769             # the call to the method that initiate the ssh connection
 770             msg = machine.connect(self.logger)
 771
 772             # Copy salomeTools to the remote machine
 773             if machine.successfully_connected(self.logger):
 774                 step = _("Copy SAT")
 775                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 776                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 777                 self.logger.flush()
 778                 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
 779                                             self.job_file_path)
 780                 # get the remote machine distribution using a sat command
 781                 (__, out_dist, __) = machine.exec_command(
 782                                 os.path.join(machine.sat_path,
 783                                     "sat config --value VARS.dist --no_label"),
 784                                 self.logger)
 785                 machine.distribution = out_dist.read().decode().replace("\n",
 786                                                                         "")
 787                 # Print the status of the copy
 788                 if res_copy == 0:
 789                     self.logger.write('\r%s' %
 790                                 ((len(begin_line)+len(endline)+20) * " "), 3)
 791                     self.logger.write('\r%s%s%s' %
 792                         (begin_line,
 793                          endline,
 794                          src.printcolors.printc(src.OK_STATUS)), 3)
 795                 else:
 796                     self.logger.write('\r%s' %
 797                             ((len(begin_line)+len(endline)+20) * " "), 3)
 798                     self.logger.write('\r%s%s%s %s' %
 799                         (begin_line,
 800                          endline,
 801                          src.printcolors.printc(src.OK_STATUS),
 802                          _("Copy of SAT failed")), 3)
 803             else:
 804                 self.logger.write('\r%s' %
 805                                   ((len(begin_line)+len(endline)+20) * " "), 3)
 806                 self.logger.write('\r%s%s%s %s' %
 807                     (begin_line,
 808                      endline,
 809                      src.printcolors.printc(src.KO_STATUS),
 810                      msg), 3)
 811             self.logger.write("\n", 3)
 812
 813         self.logger.write("\n")
 814
 815
 816     def is_occupied(self, hostname):
 817         '''Function that returns True if a job is running on
 818            the machine defined by its host and its port.
 819
 820         :param hostname (str, int): the pair (host, port)
 821         :return: the job that is running on the host,
 822                 or false if there is no job running on the host.
 823         :rtype: job / bool
 824         '''
 825         host = hostname[0]
 826         port = hostname[1]
 827         for jb in self.ljobs:
 828             if jb.machine.host == host and jb.machine.port == port:
 829                 if jb.is_running():
 830                     return jb
 831         return False
 832
 833     def update_jobs_states_list(self):
 834         '''Function that updates the lists that store the currently
 835            running jobs and the jobs that have already finished.
 836
 837         :return: Nothing.
 838         :rtype: N\A
 839         '''
 840         jobs_finished_list = []
 841         jobs_running_list = []
 842         for jb in self.ljobs:
 843             if jb.is_running():
 844                 jobs_running_list.append(jb)
 845                 jb.check_time()
 846             if jb.has_finished():
 847                 jobs_finished_list.append(jb)
 848
 849         nb_job_finished_before = len(self._l_jobs_finished)
 850         self._l_jobs_finished = jobs_finished_list
 851         self._l_jobs_running = jobs_running_list
 852
 853         nb_job_finished_now = len(self._l_jobs_finished)
 854
 855         return nb_job_finished_now > nb_job_finished_before
 856
 857     def cancel_dependencies_of_failing_jobs(self):
 858         '''Function that cancels all the jobs that depend on a failing one.
 859
 860         :return: Nothing.
 861         :rtype: N\A
 862         '''
 863
 864         for job in self.ljobs:
 865             if job.after is None:
 866                 continue
 867             father_job = self.find_job_that_has_name(job.after)
 868             if father_job is not None and father_job.has_failed():
 869                 job.cancel()
 870
 871     def find_job_that_has_name(self, name):
 872         '''Returns the job by its name.
 873
 874         :param name str: a job name
 875         :return: the job that has the name.
 876         :rtype: job
 877         '''
 878         for jb in self.ljobs:
 879             if jb.name == name:
 880                 return jb
 881         # the following is executed only if the job was not found
 882         return None
 883
 884     def str_of_length(self, text, length):
 885         '''Takes a string text of any length and returns
 886            the most close string of length "length".
 887
 888         :param text str: any string
 889         :param length int: a length for the returned string
 890         :return: the most close string of length "length"
 891         :rtype: str
 892         '''
 893         if len(text) > length:
 894             text_out = text[:length-3] + '...'
 895         else:
 896             diff = length - len(text)
 897             before = " " * (diff//2)
 898             after = " " * (diff//2 + diff%2)
 899             text_out = before + text + after
 900
 901         return text_out
 902
 903     def display_status(self, len_col):
 904         '''Takes a lenght and construct the display of the current status
 905            of the jobs in an array that has a column for each host.
 906            It displays the job that is currently running on the host
 907            of the column.
 908
 909         :param len_col int: the size of the column
 910         :return: Nothing
 911         :rtype: N\A
 912         '''
 913
 914         display_line = ""
 915         for host_port in self.lhosts:
 916             jb = self.is_occupied(host_port)
 917             if not jb: # nothing running on the host
 918                 empty = self.str_of_length("empty", len_col)
 919                 display_line += "|" + empty
 920             else:
 921                 display_line += "|" + src.printcolors.printcInfo(
 922                                         self.str_of_length(jb.name, len_col))
 923
 924         self.logger.write("\r" + display_line + "|")
 925         self.logger.flush()
 926
 927
 928     def run_jobs(self):
 929         '''The main method. Runs all the jobs on every host.
 930            For each host, at a given time, only one job can be running.
 931            The jobs that have the field after (that contain the job that has
 932            to be run before it) are run after the previous job.
 933            This method stops when all the jobs are finished.
 934
 935         :return: Nothing
 936         :rtype: N\A
 937         '''
 938
 939         # Print header
 940         self.logger.write(src.printcolors.printcInfo(
 941                                                 _('Executing the jobs :\n')))
 942         text_line = ""
 943         for host_port in self.lhosts:
 944             host = host_port[0]
 945             port = host_port[1]
 946             if port == 22: # default value
 947                 text_line += "|" + self.str_of_length(host, self.len_columns)
 948             else:
 949                 text_line += "|" + self.str_of_length(
 950                                 "("+host+", "+str(port)+")", self.len_columns)
 951
 952         tiret_line = " " + "-"*(len(text_line)-1) + "\n"
 953         self.logger.write(tiret_line)
 954         self.logger.write(text_line + "|\n")
 955         self.logger.write(tiret_line)
 956         self.logger.flush()
 957
 958         # The infinite loop that runs the jobs
 959         l_jobs_not_started = src.deepcopy_list(self.ljobs)
 960         while len(self._l_jobs_finished) != len(self.ljobs):
 961             new_job_start = False
 962             for host_port in self.lhosts:
 963
 964                 if self.is_occupied(host_port):
 965                     continue
 966
 967                 for jb in l_jobs_not_started:
 968                     if (jb.machine.host, jb.machine.port) != host_port:
 969                         continue
 970                     if jb.after == None:
 971                         jb.run()
 972                         l_jobs_not_started.remove(jb)
 973                         new_job_start = True
 974                         break
 975                     else:
 976                         jb_before = self.find_job_that_has_name(jb.after)
 977                         if jb_before is None:
 978                             jb.cancel()
 979                             msg = _("This job was not launched because its "
 980                                     "father is not in the jobs list.")
 981                             jb.out = msg
 982                             jb.err = msg
 983                             break
 984                         if jb_before.has_finished():
 985                             jb.run()
 986                             l_jobs_not_started.remove(jb)
 987                             new_job_start = True
 988                             break
 989             self.cancel_dependencies_of_failing_jobs()
 990             new_job_finished = self.update_jobs_states_list()
 991
 992             if new_job_start or new_job_finished:
 993                 if self.gui:
 994                     self.gui.update_xml_files(self.ljobs)
 995                 # Display the current status
 996                 self.display_status(self.len_columns)
 997
 998             # Make sure that the proc is not entirely busy
 999             time.sleep(0.001)
1000
1001         self.logger.write("\n")
1002         self.logger.write(tiret_line)
1003         self.logger.write("\n\n")
1004
1005         if self.gui:
1006             self.gui.update_xml_files(self.ljobs)
1007             self.gui.last_update()
1008
1009     def write_all_results(self):
1010         '''Display all the jobs outputs.
1011
1012         :return: Nothing
1013         :rtype: N\A
1014         '''
1015
1016         for jb in self.ljobs:
1017             self.logger.write(src.printcolors.printcLabel(
1018                         "#------- Results for job %s -------#\n" % jb.name))
1019             jb.write_results()
1020             self.logger.write("\n\n")
1021
1022 class Gui(object):
1023     '''Class to manage the the xml data that can be displayed in a browser to
1024        see the jobs states
1025     '''
1026
1027     def __init__(self, xml_dir_path, l_jobs, l_jobs_not_today, file_boards=""):
1028         '''Initialization
1029
1030         :param xml_dir_path str: The path to the directory where to put
1031                                  the xml resulting files
1032         :param l_jobs List: the list of jobs that run today
1033         :param l_jobs_not_today List: the list of jobs that do not run today
1034         :param file_boards str: the file path from which to read the
1035                                    expected boards
1036         '''
1037         # The path of the csv files to read to fill the expected boards
1038         self.file_boards = file_boards
1039
1040         if file_boards != "":
1041             today = datetime.date.weekday(datetime.date.today())
1042             self.parse_csv_boards(today)
1043         else:
1044             self.d_input_boards = {}
1045
1046         # The path of the global xml file
1047         self.xml_dir_path = xml_dir_path
1048         # Initialize the xml files
1049         xml_global_path = os.path.join(self.xml_dir_path, "global_report.xml")
1050         self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1051                                                          "JobsReport")
1052         # The xml files that corresponds to the boards.
1053         # {name_board : xml_object}}
1054         self.d_xml_board_files = {}
1055         # Create the lines and columns
1056         self.initialize_boards(l_jobs, l_jobs_not_today)
1057
1058         # Write the xml file
1059         self.update_xml_files(l_jobs)
1060
1061     def add_xml_board(self, name):
1062         xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1063         self.d_xml_board_files[name] =  src.xmlManager.XmlLogFile(
1064                                                     xml_board_path,
1065                                                     "JobsReport")
1066         self.d_xml_board_files[name].add_simple_node("distributions")
1067         self.d_xml_board_files[name].add_simple_node("applications")
1068         self.d_xml_board_files[name].add_simple_node("board", text=name)
1069
1070     def initialize_boards(self, l_jobs, l_jobs_not_today):
1071         '''Get all the first information needed for each file and write the
1072            first version of the files
1073         :param l_jobs List: the list of jobs that run today
1074         :param l_jobs_not_today List: the list of jobs that do not run today
1075         '''
1076         # Get the boards to fill and put it in a dictionary
1077         # {board_name : xml instance corresponding to the board}
1078         for job in l_jobs + l_jobs_not_today:
1079             board = job.board
1080             if (board is not None and
1081                                 board not in self.d_xml_board_files.keys()):
1082                 self.add_xml_board(board)
1083
1084         # Verify that the boards given as input are done
1085         for board in list(self.d_input_boards.keys()):
1086             if board not in self.d_xml_board_files:
1087                 self.add_xml_board(board)
1088             root_node = self.d_xml_board_files[board].xmlroot
1089             src.xmlManager.append_node_attrib(root_node,
1090                                               {"input_file" : self.file_boards})
1091
1092         # Loop over all jobs in order to get the lines and columns for each
1093         # xml file
1094         d_dist = {}
1095         d_application = {}
1096         for board in self.d_xml_board_files:
1097             d_dist[board] = []
1098             d_application[board] = []
1099
1100         l_hosts_ports = []
1101
1102         for job in l_jobs + l_jobs_not_today:
1103
1104             if (job.machine.host, job.machine.port) not in l_hosts_ports:
1105                 l_hosts_ports.append((job.machine.host, job.machine.port))
1106
1107             distrib = job.machine.distribution
1108             application = job.application
1109
1110             board_job = job.board
1111             if board is None:
1112                 continue
1113             for board in self.d_xml_board_files:
1114                 if board_job == board:
1115                     if distrib is not None and distrib not in d_dist[board]:
1116                         d_dist[board].append(distrib)
1117                         src.xmlManager.add_simple_node(
1118                             self.d_xml_board_files[board].xmlroot.find(
1119                                                             'distributions'),
1120                                                    "dist",
1121                                                    attrib={"name" : distrib})
1122
1123                 if board_job == board:
1124                     if (application is not None and
1125                                     application not in d_application[board]):
1126                         d_application[board].append(application)
1127                         src.xmlManager.add_simple_node(
1128                             self.d_xml_board_files[board].xmlroot.find(
1129                                                                 'applications'),
1130                                                    "application",
1131                                                    attrib={
1132                                                         "name" : application})
1133
1134         # Verify that there are no missing application or distribution in the
1135         # xml board files (regarding the input boards)
1136         for board in self.d_xml_board_files:
1137             l_dist = d_dist[board]
1138             if board not in self.d_input_boards.keys():
1139                 continue
1140             for dist in self.d_input_boards[board]["rows"]:
1141                 if dist not in l_dist:
1142                     src.xmlManager.add_simple_node(
1143                             self.d_xml_board_files[board].xmlroot.find(
1144                                                             'distributions'),
1145                                                    "dist",
1146                                                    attrib={"name" : dist})
1147             l_appli = d_application[board]
1148             for appli in self.d_input_boards[board]["columns"]:
1149                 if appli not in l_appli:
1150                     src.xmlManager.add_simple_node(
1151                             self.d_xml_board_files[board].xmlroot.find(
1152                                                                 'applications'),
1153                                                    "application",
1154                                                    attrib={"name" : appli})
1155
1156         # Initialize the hosts_ports node for the global file
1157         self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1158                                                                 "hosts_ports")
1159         for host, port in l_hosts_ports:
1160             host_port = "%s:%i" % (host, port)
1161             src.xmlManager.add_simple_node(self.xmlhosts_ports,
1162                                            "host_port",
1163                                            attrib={"name" : host_port})
1164
1165         # Initialize the jobs node in all files
1166         for xml_file in [self.xml_global_file] + list(
1167                                             self.d_xml_board_files.values()):
1168             xml_jobs = xml_file.add_simple_node("jobs")
1169             # Get the jobs present in the config file but
1170             # that will not be launched today
1171             self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1172
1173             xml_file.add_simple_node("infos",
1174                                      attrib={"name" : "last update",
1175                                              "JobsCommandStatus" : "running"})
1176
1177         # Find in each board the squares that needs to be filled regarding the
1178         # input csv files but that are not covered by a today job
1179         for board in self.d_input_boards.keys():
1180             xml_root_board = self.d_xml_board_files[board].xmlroot
1181             xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1182                                                  "missing_jobs")
1183             for row, column in self.d_input_boards[board]["jobs"]:
1184                 found = False
1185                 for job in l_jobs:
1186                     if (job.application == column and
1187                         job.machine.distribution == row):
1188                         found = True
1189                         break
1190                 if not found:
1191                     src.xmlManager.add_simple_node(xml_missing,
1192                                             "job",
1193                                             attrib={"distribution" : row,
1194                                                     "application" : column })
1195
1196     def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1197         '''Get all the first information needed for each file and write the
1198            first version of the files
1199
1200         :param xml_node_jobs etree.Element: the node corresponding to a job
1201         :param l_jobs_not_today List: the list of jobs that do not run today
1202         '''
1203         for job in l_jobs_not_today:
1204             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1205                                                  "job",
1206                                                  attrib={"name" : job.name})
1207             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1208             src.xmlManager.add_simple_node(xmlj,
1209                                            "distribution",
1210                                            job.machine.distribution)
1211             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1212             src.xmlManager.add_simple_node(xmlj,
1213                                        "commands", " ; ".join(job.commands))
1214             src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1215             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1216             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1217             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1218             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1219             src.xmlManager.add_simple_node(xmlj, "sat_path",
1220                                                         job.machine.sat_path)
1221
1222     def parse_csv_boards(self, today):
1223         """ Parse the csv file that describes the boards to produce and fill
1224             the dict d_input_boards that contain the csv file contain
1225
1226         :param today int: the current day of the week
1227         """
1228         # open the csv file and read its content
1229         l_read = []
1230         with open(self.file_boards, 'r') as f:
1231             reader = csv.reader(f,delimiter=CSV_DELIMITER)
1232             for row in reader:
1233                 l_read.append(row)
1234         # get the delimiter for the boards (empty line)
1235         boards_delimiter = [''] * len(l_read[0])
1236         # Make the list of boards, by splitting with the delimiter
1237         l_boards = [list(y) for x, y in itertools.groupby(l_read,
1238                                     lambda z: z == boards_delimiter) if not x]
1239
1240         # loop over the csv lists of lines and get the rows, columns and jobs
1241         d_boards = {}
1242         for input_board in l_boards:
1243             # get board name
1244             board_name = input_board[0][0]
1245
1246             # Get columns list
1247             columns = input_board[0][1:]
1248
1249             rows = []
1250             jobs = []
1251             for line in input_board[1:]:
1252                 row = line[0]
1253                 for i, square in enumerate(line[1:]):
1254                     if square=='':
1255                         continue
1256                     days = square.split(DAYS_SEPARATOR)
1257                     days = [int(day) for day in days]
1258                     if today in days:
1259                         if row not in rows:
1260                             rows.append(row)
1261                         job = (row, columns[i])
1262                         jobs.append(job)
1263
1264             d_boards[board_name] = {"rows" : rows,
1265                                     "columns" : columns,
1266                                     "jobs" : jobs}
1267
1268         self.d_input_boards = d_boards
1269
1270     def update_xml_files(self, l_jobs):
1271         '''Write all the xml files with updated information about the jobs
1272
1273         :param l_jobs List: the list of jobs that run today
1274         '''
1275         for xml_file in [self.xml_global_file] + list(
1276                                             self.d_xml_board_files.values()):
1277             self.update_xml_file(l_jobs, xml_file)
1278
1279         # Write the file
1280         self.write_xml_files()
1281
1282     def update_xml_file(self, l_jobs, xml_file):
1283         '''update information about the jobs for the file xml_file
1284
1285         :param l_jobs List: the list of jobs that run today
1286         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1287         '''
1288
1289         xml_node_jobs = xml_file.xmlroot.find('jobs')
1290         # Update the job names and status node
1291         for job in l_jobs:
1292             # Find the node corresponding to the job and delete it
1293             # in order to recreate it
1294             for xmljob in xml_node_jobs.findall('job'):
1295                 if xmljob.attrib['name'] == job.name:
1296                     xml_node_jobs.remove(xmljob)
1297
1298             T0 = str(job._T0)
1299             if T0 != "-1":
1300                 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1301                                        time.localtime(job._T0))
1302             Tf = str(job._Tf)
1303             if Tf != "-1":
1304                 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1305                                        time.localtime(job._Tf))
1306
1307             # recreate the job node
1308             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1309                                                   "job",
1310                                                   attrib={"name" : job.name})
1311             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1312             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1313             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1314             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1315             src.xmlManager.add_simple_node(xmlj, "sat_path",
1316                                            job.machine.sat_path)
1317             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1318             src.xmlManager.add_simple_node(xmlj, "distribution",
1319                                            job.machine.distribution)
1320             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1321             src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1322             src.xmlManager.add_simple_node(xmlj, "commands",
1323                                            " ; ".join(job.commands))
1324             src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1325             src.xmlManager.add_simple_node(xmlj, "begin", T0)
1326             src.xmlManager.add_simple_node(xmlj, "end", Tf)
1327             src.xmlManager.add_simple_node(xmlj, "out",
1328                                            src.printcolors.cleancolor(job.out))
1329             src.xmlManager.add_simple_node(xmlj, "err",
1330                                            src.printcolors.cleancolor(job.err))
1331             src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1332             if len(job.remote_log_files) > 0:
1333                 src.xmlManager.add_simple_node(xmlj,
1334                                                "remote_log_file_path",
1335                                                job.remote_log_files[0])
1336             else:
1337                 src.xmlManager.add_simple_node(xmlj,
1338                                                "remote_log_file_path",
1339                                                "nothing")
1340
1341             xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1342             # get the job father
1343             if job.after is not None:
1344                 job_father = None
1345                 for jb in l_jobs:
1346                     if jb.name == job.after:
1347                         job_father = jb
1348
1349                 if (job_father is not None and
1350                         len(job_father.remote_log_files) > 0):
1351                     link = job_father.remote_log_files[0]
1352                 else:
1353                     link = "nothing"
1354                 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1355
1356             # Verify that the job is to be done today regarding the input csv
1357             # files
1358             if job.board and job.board in self.d_input_boards.keys():
1359                 found = False
1360                 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1361                     if (job.machine.distribution == dist
1362                         and job.application == appli):
1363                         found = True
1364                         src.xmlManager.add_simple_node(xmlj,
1365                                                "extra_job",
1366                                                "no")
1367                         break
1368                 if not found:
1369                     src.xmlManager.add_simple_node(xmlj,
1370                                                "extra_job",
1371                                                "yes")
1372
1373
1374         # Update the date
1375         xml_node_infos = xml_file.xmlroot.find('infos')
1376         src.xmlManager.append_node_attrib(xml_node_infos,
1377                     attrib={"value" :
1378                     datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1379
1380
1381
1382     def last_update(self, finish_status = "finished"):
1383         '''update information about the jobs for the file xml_file
1384
1385         :param l_jobs List: the list of jobs that run today
1386         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1387         '''
1388         for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1389             xml_node_infos = xml_file.xmlroot.find('infos')
1390             src.xmlManager.append_node_attrib(xml_node_infos,
1391                         attrib={"JobsCommandStatus" : finish_status})
1392         # Write the file
1393         self.write_xml_files()
1394
1395     def write_xml_files(self):
1396         ''' Write the xml files
1397         '''
1398         self.xml_global_file.write_tree(STYLESHEET_GLOBAL)
1399         for xml_file in self.d_xml_board_files.values():
1400             xml_file.write_tree(STYLESHEET_BOARD)
1401
1402 ##
1403 # Describes the command
1404 def description():
1405     return _("The jobs command launches maintenances that are described"
1406              " in the dedicated jobs configuration file.")
1407
1408 ##
1409 # Runs the command.
1410 def run(args, runner, logger):
1411
1412     (options, args) = parser.parse_args(args)
1413
1414     l_cfg_dir = runner.cfg.PATHS.JOBPATH
1415
1416     # list option : display all the available config files
1417     if options.list:
1418         for cfg_dir in l_cfg_dir:
1419             if not options.no_label:
1420                 logger.write("------ %s\n" %
1421                                  src.printcolors.printcHeader(cfg_dir))
1422
1423             for f in sorted(os.listdir(cfg_dir)):
1424                 if not f.endswith('.pyconf'):
1425                     continue
1426                 cfilename = f[:-7]
1427                 logger.write("%s\n" % cfilename)
1428         return 0
1429
1430     # Make sure the jobs_config option has been called
1431     if not options.jobs_cfg:
1432         message = _("The option --jobs_config is required\n")
1433         src.printcolors.printcError(message)
1434         return 1
1435
1436     # Find the file in the directories
1437     found = False
1438     for cfg_dir in l_cfg_dir:
1439         file_jobs_cfg = os.path.join(cfg_dir, options.jobs_cfg)
1440         if not file_jobs_cfg.endswith('.pyconf'):
1441             file_jobs_cfg += '.pyconf'
1442
1443         if not os.path.exists(file_jobs_cfg):
1444             continue
1445         else:
1446             found = True
1447             break
1448
1449     if not found:
1450         msg = _("The file configuration %(name_file)s was not found."
1451                 "\nUse the --list option to get the possible files.")
1452         src.printcolors.printcError(msg)
1453         return 1
1454
1455     info = [
1456         (_("Platform"), runner.cfg.VARS.dist),
1457         (_("File containing the jobs configuration"), file_jobs_cfg)
1458     ]
1459     src.print_info(logger, info)
1460
1461     # Read the config that is in the file
1462     config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1463     if options.only_jobs:
1464         l_jb = src.pyconf.Sequence()
1465         for jb in config_jobs.jobs:
1466             if jb.name in options.only_jobs:
1467                 l_jb.append(jb,
1468                 "Adding a job that was given in only_jobs option parameters")
1469         config_jobs.jobs = l_jb
1470
1471     # Initialization
1472     today_jobs = Jobs(runner,
1473                       logger,
1474                       file_jobs_cfg,
1475                       config_jobs)
1476     # SSH connection to all machines
1477     today_jobs.ssh_connection_all_machines()
1478     if options.test_connection:
1479         return 0
1480
1481     gui = None
1482     if options.publish:
1483         # Copy the stylesheets in the log directory
1484         log_dir = runner.cfg.USER.log_dir
1485         xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1486         files_to_copy = []
1487         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1488         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1489         files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1490         for file_path in files_to_copy:
1491             shutil.copy2(file_path, log_dir)
1492
1493         # Instanciate the Gui in order to produce the xml files that contain all
1494         # the boards
1495         gui = Gui(runner.cfg.USER.log_dir,
1496                   today_jobs.ljobs,
1497                   today_jobs.ljobs_not_today,
1498                   file_boards = options.input_boards)
1499
1500         # Display the list of the xml files
1501         logger.write(src.printcolors.printcInfo(("Here is the list of published"
1502                                                  " files :\n")), 4)
1503         logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1504         for board in gui.d_xml_board_files.keys():
1505             file_path = gui.d_xml_board_files[board].logFile
1506             file_name = os.path.basename(file_path)
1507             logger.write("%s\n" % file_path, 4)
1508             logger.add_link(file_name, "board", 0, board)
1509
1510         logger.write("\n", 4)
1511
1512     today_jobs.gui = gui
1513
1514     interruped = False
1515     try:
1516         # Run all the jobs contained in config_jobs
1517         today_jobs.run_jobs()
1518     except KeyboardInterrupt:
1519         interruped = True
1520         logger.write("\n\n%s\n\n" %
1521                 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1522     finally:
1523         if interruped:
1524             msg = _("Killing the running jobs and trying"
1525                     " to get the corresponding logs\n")
1526             logger.write(src.printcolors.printcWarning(msg))
1527
1528         # find the potential not finished jobs and kill them
1529         for jb in today_jobs.ljobs:
1530             if not jb.has_finished():
1531                 try:
1532                     jb.kill_remote_process()
1533                 except Exception as e:
1534                     msg = _("Failed to kill job %s: %s\n" % (jb.name, e))
1535                     logger.write(src.printcolors.printcWarning(msg))
1536         if interruped:
1537             if today_jobs.gui:
1538                 today_jobs.gui.last_update(_("Forced interruption"))
1539         else:
1540             if today_jobs.gui:
1541                 today_jobs.gui.last_update()
1542         # Output the results
1543         today_jobs.write_all_results()