commands/jobs.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import datetime
  21 import time
  22 import csv
  23 import shutil
  24 import itertools
  25 import paramiko
  26
  27 import src
  28
  29 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
  30 STYLESHEET_BOARD = "jobs_board_report.xsl"
  31
  32 DAYS_SEPARATOR = ","
  33 CSV_DELIMITER = ";"
  34
  35 parser = src.options.Options()
  36
  37 parser.add_option('n', 'name', 'string', 'jobs_cfg',
  38                   _('The name of the config file that contains'
  39                   ' the jobs configuration'))
  40 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
  41                   _('Optional: the list of jobs to launch, by their name. '))
  42 parser.add_option('l', 'list', 'boolean', 'list',
  43                   _('Optional: list all available config files.'))
  44 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
  45                   _("Optional: try to connect to the machines. "
  46                     "Not executing the jobs."),
  47                   False)
  48 parser.add_option('p', 'publish', 'boolean', 'publish',
  49                   _("Optional: generate an xml file that can be read in a "
  50                     "browser to display the jobs status."),
  51                   False)
  52 parser.add_option('i', 'input_boards', 'string', 'input_boards', _("Optional: "
  53                                 "the path to csv file that contain "
  54                                 "the expected boards."),"")
  55 parser.add_option('n', 'completion', 'boolean', 'no_label',
  56                   _("Optional (internal use): do not print labels, Works only "
  57                     "with --list."),
  58                   False)
  59
  60 class Machine(object):
  61     '''Class to manage a ssh connection on a machine
  62     '''
  63     def __init__(self,
  64                  name,
  65                  host,
  66                  user,
  67                  port=22,
  68                  passwd=None,
  69                  sat_path="salomeTools"):
  70         self.name = name
  71         self.host = host
  72         self.port = port
  73         self.distribution = None # Will be filled after copying SAT on the machine
  74         self.user = user
  75         self.password = passwd
  76         self.sat_path = sat_path
  77         self.ssh = paramiko.SSHClient()
  78         self._connection_successful = None
  79
  80     def connect(self, logger):
  81         '''Initiate the ssh connection to the remote machine
  82
  83         :param logger src.logger.Logger: The logger instance
  84         :return: Nothing
  85         :rtype: N\A
  86         '''
  87
  88         self._connection_successful = False
  89         self.ssh.load_system_host_keys()
  90         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  91         try:
  92             self.ssh.connect(self.host,
  93                              port=self.port,
  94                              username=self.user,
  95                              password = self.password)
  96         except paramiko.AuthenticationException:
  97             message = src.KO_STATUS + _("Authentication failed")
  98         except paramiko.BadHostKeyException:
  99             message = (src.KO_STATUS +
 100                        _("The server's host key could not be verified"))
 101         except paramiko.SSHException:
 102             message = ( _("SSHException error connecting or "
 103                           "establishing an SSH session"))
 104         except:
 105             message = ( _("Error connecting or establishing an SSH session"))
 106         else:
 107             self._connection_successful = True
 108             message = ""
 109         return message
 110
 111     def successfully_connected(self, logger):
 112         '''Verify if the connection to the remote machine has succeed
 113
 114         :param logger src.logger.Logger: The logger instance
 115         :return: True if the connection has succeed, False if not
 116         :rtype: bool
 117         '''
 118         if self._connection_successful == None:
 119             message = _("Warning : trying to ask if the connection to "
 120             "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
 121             " no connection request" %
 122                         (self.name, self.host, self.port, self.user))
 123             logger.write( src.printcolors.printcWarning(message))
 124         return self._connection_successful
 125
 126     def copy_sat(self, sat_local_path, job_file):
 127         '''Copy salomeTools to the remote machine in self.sat_path
 128         '''
 129         res = 0
 130         try:
 131             # open a sftp connection
 132             self.sftp = self.ssh.open_sftp()
 133             # Create the sat directory on remote machine if it is not existing
 134             self.mkdir(self.sat_path, ignore_existing=True)
 135             # Put sat
 136             self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
 137             # put the job configuration file in order to make it reachable
 138             # on the remote machine
 139             self.sftp.put(job_file, os.path.join(".salomeTools",
 140                                                  "Jobs",
 141                                                  ".jobs_command_file.pyconf"))
 142         except Exception as e:
 143             res = str(e)
 144             self._connection_successful = False
 145
 146         return res
 147
 148     def put_dir(self, source, target, filters = []):
 149         ''' Uploads the contents of the source directory to the target path. The
 150             target directory needs to exists. All sub-directories in source are
 151             created under target.
 152         '''
 153         for item in os.listdir(source):
 154             if item in filters:
 155                 continue
 156             source_path = os.path.join(source, item)
 157             destination_path = os.path.join(target, item)
 158             if os.path.islink(source_path):
 159                 linkto = os.readlink(source_path)
 160                 try:
 161                     self.sftp.symlink(linkto, destination_path)
 162                     self.sftp.chmod(destination_path,
 163                                     os.stat(source_path).st_mode)
 164                 except IOError:
 165                     pass
 166             else:
 167                 if os.path.isfile(source_path):
 168                     self.sftp.put(source_path, destination_path)
 169                     self.sftp.chmod(destination_path,
 170                                     os.stat(source_path).st_mode)
 171                 else:
 172                     self.mkdir(destination_path, ignore_existing=True)
 173                     self.put_dir(source_path, destination_path)
 174
 175     def mkdir(self, path, mode=511, ignore_existing=False):
 176         ''' Augments mkdir by adding an option to not fail
 177             if the folder exists
 178         '''
 179         try:
 180             self.sftp.mkdir(path, mode)
 181         except IOError:
 182             if ignore_existing:
 183                 pass
 184             else:
 185                 raise
 186
 187     def exec_command(self, command, logger):
 188         '''Execute the command on the remote machine
 189
 190         :param command str: The command to be run
 191         :param logger src.logger.Logger: The logger instance
 192         :return: the stdin, stdout, and stderr of the executing command,
 193                  as a 3-tuple
 194         :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
 195                 paramiko.channel.ChannelFile)
 196         '''
 197         try:
 198             # Does not wait the end of the command
 199             (stdin, stdout, stderr) = self.ssh.exec_command(command)
 200         except paramiko.SSHException:
 201             message = src.KO_STATUS + _(
 202                             ": the server failed to execute the command\n")
 203             logger.write( src.printcolors.printcError(message))
 204             return (None, None, None)
 205         except:
 206             logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
 207             return (None, None, None)
 208         else:
 209             return (stdin, stdout, stderr)
 210
 211     def close(self):
 212         '''Close the ssh connection
 213
 214         :rtype: N\A
 215         '''
 216         self.ssh.close()
 217
 218     def write_info(self, logger):
 219         '''Prints the informations relative to the machine in the logger
 220            (terminal traces and log file)
 221
 222         :param logger src.logger.Logger: The logger instance
 223         :return: Nothing
 224         :rtype: N\A
 225         '''
 226         logger.write("host : " + self.host + "\n")
 227         logger.write("port : " + str(self.port) + "\n")
 228         logger.write("user : " + str(self.user) + "\n")
 229         if self.successfully_connected(logger):
 230             status = src.OK_STATUS
 231         else:
 232             status = src.KO_STATUS
 233         logger.write("Connection : " + status + "\n\n")
 234
 235
 236 class Job(object):
 237     '''Class to manage one job
 238     '''
 239     def __init__(self, name, machine, application, board,
 240                  commands, timeout, config, logger, after=None):
 241
 242         self.name = name
 243         self.machine = machine
 244         self.after = after
 245         self.timeout = timeout
 246         self.application = application
 247         self.board = board
 248         self.config = config
 249         self.logger = logger
 250         # The list of log files to download from the remote machine
 251         self.remote_log_files = []
 252
 253         # The remote command status
 254         # -1 means that it has not been launched,
 255         # 0 means success and 1 means fail
 256         self.res_job = "-1"
 257         self.cancelled = False
 258
 259         self._T0 = -1
 260         self._Tf = -1
 261         self._has_begun = False
 262         self._has_finished = False
 263         self._has_timouted = False
 264         self._stdin = None # Store the command inputs field
 265         self._stdout = None # Store the command outputs field
 266         self._stderr = None # Store the command errors field
 267
 268         self.out = ""
 269         self.err = ""
 270
 271         self.commands = commands
 272         self.command = (os.path.join(self.machine.sat_path, "sat") +
 273                         " -l " +
 274                         os.path.join(self.machine.sat_path,
 275                                      "list_log_files.txt") +
 276                         " job --jobs_config .jobs_command_file" +
 277                         " --name " +
 278                         self.name)
 279
 280     def get_pids(self):
 281         """ Get the pid(s) corresponding to the command that have been launched
 282             On the remote machine
 283
 284         :return: The list of integers corresponding to the found pids
 285         :rtype: List
 286         """
 287         pids = []
 288         cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
 289         (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
 290         pids_cmd = out_pid.readlines()
 291         pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
 292         pids+=pids_cmd
 293         return pids
 294
 295     def kill_remote_process(self, wait=1):
 296         '''Kills the process on the remote machine.
 297
 298         :return: (the output of the kill, the error of the kill)
 299         :rtype: (str, str)
 300         '''
 301
 302         pids = self.get_pids()
 303         cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
 304         (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
 305                                                             self.logger)
 306         time.sleep(wait)
 307         return (out_kill, err_kill)
 308
 309     def has_begun(self):
 310         '''Returns True if the job has already begun
 311
 312         :return: True if the job has already begun
 313         :rtype: bool
 314         '''
 315         return self._has_begun
 316
 317     def has_finished(self):
 318         '''Returns True if the job has already finished
 319            (i.e. all the commands have been executed)
 320            If it is finished, the outputs are stored in the fields out and err.
 321
 322         :return: True if the job has already finished
 323         :rtype: bool
 324         '''
 325
 326         # If the method has already been called and returned True
 327         if self._has_finished:
 328             return True
 329
 330         # If the job has not begun yet
 331         if not self.has_begun():
 332             return False
 333
 334         if self._stdout.channel.closed:
 335             self._has_finished = True
 336             # Store the result outputs
 337             self.out += self._stdout.read().decode()
 338             self.err += self._stderr.read().decode()
 339             # Put end time
 340             self._Tf = time.time()
 341             # And get the remote command status and log files
 342             self.get_log_files()
 343
 344         return self._has_finished
 345
 346     def get_log_files(self):
 347         """Get the log files produced by the command launched
 348            on the remote machine, and put it in the log directory of the user,
 349            so they can be accessible from
 350         """
 351         # Do not get the files if the command is not finished
 352         if not self.has_finished():
 353             msg = _("Trying to get log files whereas the job is not finished.")
 354             self.logger.write(src.printcolors.printcWarning(msg))
 355             return
 356
 357         # First get the file that contains the list of log files to get
 358         tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
 359         remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt")
 360         self.machine.sftp.get(
 361                     remote_path,
 362                     tmp_file_path)
 363
 364         # Read the file and get the result of the command and all the log files
 365         # to get
 366         fstream_tmp = open(tmp_file_path, "r")
 367         file_lines = fstream_tmp.readlines()
 368         file_lines = [line.replace("\n", "") for line in file_lines]
 369         fstream_tmp.close()
 370         os.remove(tmp_file_path)
 371
 372         try :
 373             # The first line is the result of the command (0 success or 1 fail)
 374             self.res_job = file_lines[0]
 375         except Exception as e:
 376             self.err += _("Unable to get status from remote file %s: %s" %
 377                                                     (remote_path, str(e)))
 378
 379         for i, job_path_remote in enumerate(file_lines[1:]):
 380             try:
 381                 # For each command, there is two files to get :
 382                 # 1- The xml file describing the command and giving the
 383                 # internal traces.
 384                 # 2- The txt file containing the system command traces (like
 385                 # traces produced by the "make" command)
 386                 # 3- In case of the test command, there is another file to get :
 387                 # the xml board that contain the test results
 388                 dirname = os.path.basename(os.path.dirname(job_path_remote))
 389                 if dirname != 'OUT' and dirname != 'TEST':
 390                     # Case 1-
 391                     local_path = os.path.join(os.path.dirname(
 392                                                         self.logger.logFilePath),
 393                                               os.path.basename(job_path_remote))
 394                     if i==0: # The first is the job command
 395                         self.logger.add_link(os.path.basename(job_path_remote),
 396                                              "job",
 397                                              self.res_job,
 398                                              self.command)
 399                 elif dirname == 'OUT':
 400                     # Case 2-
 401                     local_path = os.path.join(os.path.dirname(
 402                                                         self.logger.logFilePath),
 403                                               'OUT',
 404                                               os.path.basename(job_path_remote))
 405                 elif dirname == 'TEST':
 406                     # Case 3-
 407                     local_path = os.path.join(os.path.dirname(
 408                                                         self.logger.logFilePath),
 409                                               'TEST',
 410                                               os.path.basename(job_path_remote))
 411
 412                 # Get the file
 413                 if not os.path.exists(local_path):
 414                     self.machine.sftp.get(job_path_remote, local_path)
 415                 self.remote_log_files.append(local_path)
 416             except Exception as e:
 417                 self.err += _("Unable to get %s log file from remote: %s" %
 418                                                     (str(job_path_remote),
 419                                                      str(e)))
 420
 421     def has_failed(self):
 422         '''Returns True if the job has failed.
 423            A job is considered as failed if the machine could not be reached,
 424            if the remote command failed,
 425            or if the job finished with a time out.
 426
 427         :return: True if the job has failed
 428         :rtype: bool
 429         '''
 430         if not self.has_finished():
 431             return False
 432         if not self.machine.successfully_connected(self.logger):
 433             return True
 434         if self.is_timeout():
 435             return True
 436         if self.res_job == "1":
 437             return True
 438         return False
 439
 440     def cancel(self):
 441         """In case of a failing job, one has to cancel every job that depend
 442            on it. This method put the job as failed and will not be executed.
 443         """
 444         self._has_begun = True
 445         self._has_finished = True
 446         self.cancelled = True
 447         self.out += _("This job was not launched because its father has failed.")
 448         self.err += _("This job was not launched because its father has failed.")
 449
 450     def is_running(self):
 451         '''Returns True if the job commands are running
 452
 453         :return: True if the job is running
 454         :rtype: bool
 455         '''
 456         return self.has_begun() and not self.has_finished()
 457
 458     def is_timeout(self):
 459         '''Returns True if the job commands has finished with timeout
 460
 461         :return: True if the job has finished with timeout
 462         :rtype: bool
 463         '''
 464         return self._has_timouted
 465
 466     def time_elapsed(self):
 467         """Get the time elapsed since the job launching
 468
 469         :return: The number of seconds
 470         :rtype: int
 471         """
 472         if not self.has_begun():
 473             return -1
 474         T_now = time.time()
 475         return T_now - self._T0
 476
 477     def check_time(self):
 478         """Verify that the job has not exceeded its timeout.
 479            If it has, kill the remote command and consider the job as finished.
 480         """
 481         if not self.has_begun():
 482             return
 483         if self.time_elapsed() > self.timeout:
 484             self._has_finished = True
 485             self._has_timouted = True
 486             self._Tf = time.time()
 487             self.get_pids()
 488             (out_kill, _) = self.kill_remote_process()
 489             self.out += "TIMEOUT \n" + out_kill.read().decode()
 490             self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
 491             try:
 492                 self.get_log_files()
 493             except Exception as e:
 494                 self.err += _("Unable to get remote log files: %s" % e)
 495
 496     def total_duration(self):
 497         """Give the total duration of the job
 498
 499         :return: the total duration of the job in seconds
 500         :rtype: int
 501         """
 502         return self._Tf - self._T0
 503
 504     def run(self):
 505         """Launch the job by executing the remote command.
 506         """
 507
 508         # Prevent multiple run
 509         if self.has_begun():
 510             msg = _("Warning: A job can only be launched one time")
 511             msg2 = _("Trying to launch the job \"%s\" whereas it has "
 512                      "already been launched." % self.name)
 513             self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
 514                                                                         msg2)))
 515             return
 516
 517         # Do not execute the command if the machine could not be reached
 518         if not self.machine.successfully_connected(self.logger):
 519             self._has_finished = True
 520             self.out = "N\A"
 521             self.err += ("Connection to machine (name : %s, host: %s, port:"
 522                         " %s, user: %s) has failed\nUse the log command "
 523                         "to get more information."
 524                         % (self.machine.name,
 525                            self.machine.host,
 526                            self.machine.port,
 527                            self.machine.user))
 528         else:
 529             # Usual case : Launch the command on remote machine
 530             self._T0 = time.time()
 531             self._stdin, self._stdout, self._stderr = self.machine.exec_command(
 532                                                                   self.command,
 533                                                                   self.logger)
 534             # If the results are not initialized, finish the job
 535             if (self._stdin, self._stdout, self._stderr) == (None, None, None):
 536                 self._has_finished = True
 537                 self._Tf = time.time()
 538                 self.out += "N\A"
 539                 self.err += "The server failed to execute the command"
 540
 541         # Put the beginning flag to true.
 542         self._has_begun = True
 543
 544     def write_results(self):
 545         """Display on the terminal all the job's information
 546         """
 547         self.logger.write("name : " + self.name + "\n")
 548         if self.after:
 549             self.logger.write("after : %s\n" % self.after)
 550         self.logger.write("Time elapsed : %4imin %2is \n" %
 551                      (self.total_duration()//60 , self.total_duration()%60))
 552         if self._T0 != -1:
 553             self.logger.write("Begin time : %s\n" %
 554                          time.strftime('%Y-%m-%d %H:%M:%S',
 555                                        time.localtime(self._T0)) )
 556         if self._Tf != -1:
 557             self.logger.write("End time   : %s\n\n" %
 558                          time.strftime('%Y-%m-%d %H:%M:%S',
 559                                        time.localtime(self._Tf)) )
 560
 561         machine_head = "Informations about connection :\n"
 562         underline = (len(machine_head) - 2) * "-"
 563         self.logger.write(src.printcolors.printcInfo(
 564                                                 machine_head+underline+"\n"))
 565         self.machine.write_info(self.logger)
 566
 567         self.logger.write(src.printcolors.printcInfo("out : \n"))
 568         if self.out == "":
 569             self.logger.write("Unable to get output\n")
 570         else:
 571             self.logger.write(self.out + "\n")
 572         self.logger.write(src.printcolors.printcInfo("err : \n"))
 573         self.logger.write(self.err + "\n")
 574
 575     def get_status(self):
 576         """Get the status of the job (used by the Gui for xml display)
 577
 578         :return: The current status of the job
 579         :rtype: String
 580         """
 581         if not self.machine.successfully_connected(self.logger):
 582             return "SSH connection KO"
 583         if not self.has_begun():
 584             return "Not launched"
 585         if self.cancelled:
 586             return "Cancelled"
 587         if self.is_running():
 588             return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
 589                                                     time.localtime(self._T0))
 590         if self.has_finished():
 591             if self.is_timeout():
 592                 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
 593                                                     time.localtime(self._Tf))
 594             return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
 595                                                      time.localtime(self._Tf))
 596
 597 class Jobs(object):
 598     '''Class to manage the jobs to be run
 599     '''
 600     def __init__(self,
 601                  runner,
 602                  logger,
 603                  job_file_path,
 604                  config_jobs,
 605                  lenght_columns = 20):
 606         # The jobs configuration
 607         self.cfg_jobs = config_jobs
 608         self.job_file_path = job_file_path
 609         # The machine that will be used today
 610         self.lmachines = []
 611         # The list of machine (hosts, port) that will be used today
 612         # (a same host can have several machine instances since there
 613         # can be several ssh parameters)
 614         self.lhosts = []
 615         # The jobs to be launched today
 616         self.ljobs = []
 617         # The jobs that will not be launched today
 618         self.ljobs_not_today = []
 619         self.runner = runner
 620         self.logger = logger
 621         self.len_columns = lenght_columns
 622
 623         # the list of jobs that have not been run yet
 624         self._l_jobs_not_started = []
 625         # the list of jobs that have already ran
 626         self._l_jobs_finished = []
 627         # the list of jobs that are running
 628         self._l_jobs_running = []
 629
 630         self.determine_jobs_and_machines()
 631
 632     def define_job(self, job_def, machine):
 633         '''Takes a pyconf job definition and a machine (from class machine)
 634            and returns the job instance corresponding to the definition.
 635
 636         :param job_def src.config.Mapping: a job definition
 637         :param machine machine: the machine on which the job will run
 638         :return: The corresponding job in a job class instance
 639         :rtype: job
 640         '''
 641         name = job_def.name
 642         cmmnds = job_def.commands
 643         if not "timeout" in job_def:
 644             timeout = 4*60*60 # default timeout = 4h
 645         else:
 646             timeout = job_def.timeout
 647         after = None
 648         if 'after' in job_def:
 649             after = job_def.after
 650         application = None
 651         if 'application' in job_def:
 652             application = job_def.application
 653         board = None
 654         if 'board' in job_def:
 655             board = job_def.board
 656
 657         return Job(name,
 658                    machine,
 659                    application,
 660                    board,
 661                    cmmnds,
 662                    timeout,
 663                    self.runner.cfg,
 664                    self.logger,
 665                    after = after)
 666
 667     def determine_jobs_and_machines(self):
 668         '''Function that reads the pyconf jobs definition and instantiates all
 669            the machines and jobs to be done today.
 670
 671         :return: Nothing
 672         :rtype: N\A
 673         '''
 674         today = datetime.date.weekday(datetime.date.today())
 675         host_list = []
 676
 677         for job_def in self.cfg_jobs.jobs :
 678
 679             if not "machine" in job_def:
 680                 msg = _('WARNING: The job "%s" do not have the key '
 681                        '"machine", this job is ignored.\n\n' % job_def.name)
 682                 self.logger.write(src.printcolors.printcWarning(msg))
 683                 continue
 684             name_machine = job_def.machine
 685
 686             a_machine = None
 687             for mach in self.lmachines:
 688                 if mach.name == name_machine:
 689                     a_machine = mach
 690                     break
 691
 692             if a_machine == None:
 693                 for machine_def in self.cfg_jobs.machines:
 694                     if machine_def.name == name_machine:
 695                         if 'host' not in machine_def:
 696                             host = self.runner.cfg.VARS.hostname
 697                         else:
 698                             host = machine_def.host
 699
 700                         if 'user' not in machine_def:
 701                             user = self.runner.cfg.VARS.user
 702                         else:
 703                             user = machine_def.user
 704
 705                         if 'port' not in machine_def:
 706                             port = 22
 707                         else:
 708                             port = machine_def.port
 709
 710                         if 'password' not in machine_def:
 711                             passwd = None
 712                         else:
 713                             passwd = machine_def.password
 714
 715                         if 'sat_path' not in machine_def:
 716                             sat_path = "salomeTools"
 717                         else:
 718                             sat_path = machine_def.sat_path
 719
 720                         a_machine = Machine(
 721                                             machine_def.name,
 722                                             host,
 723                                             user,
 724                                             port=port,
 725                                             passwd=passwd,
 726                                             sat_path=sat_path
 727                                             )
 728
 729                         self.lmachines.append(a_machine)
 730                         if (host, port) not in host_list:
 731                             host_list.append((host, port))
 732
 733                 if a_machine == None:
 734                     msg = _("WARNING: The job \"%(job_name)s\" requires the "
 735                             "machine \"%(machine_name)s\" but this machine "
 736                             "is not defined in the configuration file.\n"
 737                             "The job will not be launched")
 738                     self.logger.write(src.printcolors.printcWarning(msg))
 739
 740             a_job = self.define_job(job_def, a_machine)
 741
 742             if today in job_def.when:
 743                 self.ljobs.append(a_job)
 744             else: # today in job_def.when
 745                 self.ljobs_not_today.append(a_job)
 746
 747         self.lhosts = host_list
 748
 749     def ssh_connection_all_machines(self, pad=50):
 750         '''Function that do the ssh connection to every machine
 751            to be used today.
 752
 753         :return: Nothing
 754         :rtype: N\A
 755         '''
 756         self.logger.write(src.printcolors.printcInfo((
 757                         "Establishing connection with all the machines :\n")))
 758         for machine in self.lmachines:
 759             # little algorithm in order to display traces
 760             begin_line = (_("Connection to %s: " % machine.name))
 761             if pad - len(begin_line) < 0:
 762                 endline = " "
 763             else:
 764                 endline = (pad - len(begin_line)) * "." + " "
 765
 766             step = "SSH connection"
 767             self.logger.write( begin_line + endline + step)
 768             self.logger.flush()
 769             # the call to the method that initiate the ssh connection
 770             msg = machine.connect(self.logger)
 771
 772             # Copy salomeTools to the remote machine
 773             if machine.successfully_connected(self.logger):
 774                 step = _("Copy SAT")
 775                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 776                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 777                 self.logger.flush()
 778                 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
 779                                             self.job_file_path)
 780                 # get the remote machine distribution using a sat command
 781                 (__, out_dist, __) = machine.exec_command(
 782                                 os.path.join(machine.sat_path,
 783                                     "sat config --value VARS.dist --no_label"),
 784                                 self.logger)
 785                 machine.distribution = out_dist.read().decode().replace("\n",
 786                                                                         "")
 787                 # Print the status of the copy
 788                 if res_copy == 0:
 789                     self.logger.write('\r%s' %
 790                                 ((len(begin_line)+len(endline)+20) * " "), 3)
 791                     self.logger.write('\r%s%s%s' %
 792                         (begin_line,
 793                          endline,
 794                          src.printcolors.printc(src.OK_STATUS)), 3)
 795                 else:
 796                     self.logger.write('\r%s' %
 797                             ((len(begin_line)+len(endline)+20) * " "), 3)
 798                     self.logger.write('\r%s%s%s %s' %
 799                         (begin_line,
 800                          endline,
 801                          src.printcolors.printc(src.OK_STATUS),
 802                          _("Copy of SAT failed")), 3)
 803             else:
 804                 self.logger.write('\r%s' %
 805                                   ((len(begin_line)+len(endline)+20) * " "), 3)
 806                 self.logger.write('\r%s%s%s %s' %
 807                     (begin_line,
 808                      endline,
 809                      src.printcolors.printc(src.KO_STATUS),
 810                      msg), 3)
 811             self.logger.write("\n", 3)
 812
 813         self.logger.write("\n")
 814
 815
 816     def is_occupied(self, hostname):
 817         '''Function that returns True if a job is running on
 818            the machine defined by its host and its port.
 819
 820         :param hostname (str, int): the pair (host, port)
 821         :return: the job that is running on the host,
 822                 or false if there is no job running on the host.
 823         :rtype: job / bool
 824         '''
 825         host = hostname[0]
 826         port = hostname[1]
 827         for jb in self.ljobs:
 828             if jb.machine.host == host and jb.machine.port == port:
 829                 if jb.is_running():
 830                     return jb
 831         return False
 832
 833     def update_jobs_states_list(self):
 834         '''Function that updates the lists that store the currently
 835            running jobs and the jobs that have already finished.
 836
 837         :return: Nothing.
 838         :rtype: N\A
 839         '''
 840         jobs_finished_list = []
 841         jobs_running_list = []
 842         for jb in self.ljobs:
 843             if jb.is_running():
 844                 jobs_running_list.append(jb)
 845                 jb.check_time()
 846             if jb.has_finished():
 847                 jobs_finished_list.append(jb)
 848
 849         nb_job_finished_before = len(self._l_jobs_finished)
 850         self._l_jobs_finished = jobs_finished_list
 851         self._l_jobs_running = jobs_running_list
 852
 853         nb_job_finished_now = len(self._l_jobs_finished)
 854
 855         return nb_job_finished_now > nb_job_finished_before
 856
 857     def cancel_dependencies_of_failing_jobs(self):
 858         '''Function that cancels all the jobs that depend on a failing one.
 859
 860         :return: Nothing.
 861         :rtype: N\A
 862         '''
 863
 864         for job in self.ljobs:
 865             if job.after is None:
 866                 continue
 867             father_job = self.find_job_that_has_name(job.after)
 868             if father_job is not None and father_job.has_failed():
 869                 job.cancel()
 870
 871     def find_job_that_has_name(self, name):
 872         '''Returns the job by its name.
 873
 874         :param name str: a job name
 875         :return: the job that has the name.
 876         :rtype: job
 877         '''
 878         for jb in self.ljobs:
 879             if jb.name == name:
 880                 return jb
 881         # the following is executed only if the job was not found
 882         return None
 883
 884     def str_of_length(self, text, length):
 885         '''Takes a string text of any length and returns
 886            the most close string of length "length".
 887
 888         :param text str: any string
 889         :param length int: a length for the returned string
 890         :return: the most close string of length "length"
 891         :rtype: str
 892         '''
 893         if len(text) > length:
 894             text_out = text[:length-3] + '...'
 895         else:
 896             diff = length - len(text)
 897             before = " " * (diff//2)
 898             after = " " * (diff//2 + diff%2)
 899             text_out = before + text + after
 900
 901         return text_out
 902
 903     def display_status(self, len_col):
 904         '''Takes a lenght and construct the display of the current status
 905            of the jobs in an array that has a column for each host.
 906            It displays the job that is currently running on the host
 907            of the column.
 908
 909         :param len_col int: the size of the column
 910         :return: Nothing
 911         :rtype: N\A
 912         '''
 913
 914         display_line = ""
 915         for host_port in self.lhosts:
 916             jb = self.is_occupied(host_port)
 917             if not jb: # nothing running on the host
 918                 empty = self.str_of_length("empty", len_col)
 919                 display_line += "|" + empty
 920             else:
 921                 display_line += "|" + src.printcolors.printcInfo(
 922                                         self.str_of_length(jb.name, len_col))
 923
 924         self.logger.write("\r" + display_line + "|")
 925         self.logger.flush()
 926
 927
 928     def run_jobs(self):
 929         '''The main method. Runs all the jobs on every host.
 930            For each host, at a given time, only one job can be running.
 931            The jobs that have the field after (that contain the job that has
 932            to be run before it) are run after the previous job.
 933            This method stops when all the jobs are finished.
 934
 935         :return: Nothing
 936         :rtype: N\A
 937         '''
 938
 939         # Print header
 940         self.logger.write(src.printcolors.printcInfo(
 941                                                 _('Executing the jobs :\n')))
 942         text_line = ""
 943         for host_port in self.lhosts:
 944             host = host_port[0]
 945             port = host_port[1]
 946             if port == 22: # default value
 947                 text_line += "|" + self.str_of_length(host, self.len_columns)
 948             else:
 949                 text_line += "|" + self.str_of_length(
 950                                 "("+host+", "+str(port)+")", self.len_columns)
 951
 952         tiret_line = " " + "-"*(len(text_line)-1) + "\n"
 953         self.logger.write(tiret_line)
 954         self.logger.write(text_line + "|\n")
 955         self.logger.write(tiret_line)
 956         self.logger.flush()
 957
 958         # The infinite loop that runs the jobs
 959         l_jobs_not_started = src.deepcopy_list(self.ljobs)
 960         while len(self._l_jobs_finished) != len(self.ljobs):
 961             new_job_start = False
 962             for host_port in self.lhosts:
 963
 964                 if self.is_occupied(host_port):
 965                     continue
 966
 967                 for jb in l_jobs_not_started:
 968                     if (jb.machine.host, jb.machine.port) != host_port:
 969                         continue
 970                     if jb.after == None:
 971                         jb.run()
 972                         l_jobs_not_started.remove(jb)
 973                         new_job_start = True
 974                         break
 975                     else:
 976                         jb_before = self.find_job_that_has_name(jb.after)
 977                         if jb_before is None:
 978                             jb.cancel()
 979                             msg = _("This job was not launched because its "
 980                                     "father is not in the jobs list.")
 981                             jb.out = msg
 982                             jb.err = msg
 983                             break
 984                         if jb_before.has_finished():
 985                             jb.run()
 986                             l_jobs_not_started.remove(jb)
 987                             new_job_start = True
 988                             break
 989             self.cancel_dependencies_of_failing_jobs()
 990             new_job_finished = self.update_jobs_states_list()
 991
 992             if new_job_start or new_job_finished:
 993                 if self.gui:
 994                     self.gui.update_xml_files(self.ljobs)
 995                 # Display the current status
 996                 self.display_status(self.len_columns)
 997
 998             # Make sure that the proc is not entirely busy
 999             time.sleep(0.001)
1000
1001         self.logger.write("\n")
1002         self.logger.write(tiret_line)
1003         self.logger.write("\n\n")
1004
1005         if self.gui:
1006             self.gui.update_xml_files(self.ljobs)
1007             self.gui.last_update()
1008
1009     def write_all_results(self):
1010         '''Display all the jobs outputs.
1011
1012         :return: Nothing
1013         :rtype: N\A
1014         '''
1015
1016         for jb in self.ljobs:
1017             self.logger.write(src.printcolors.printcLabel(
1018                         "#------- Results for job %s -------#\n" % jb.name))
1019             jb.write_results()
1020             self.logger.write("\n\n")
1021
1022 class Gui(object):
1023     '''Class to manage the the xml data that can be displayed in a browser to
1024        see the jobs states
1025     '''
1026
1027     def __init__(self, xml_dir_path, l_jobs, l_jobs_not_today, file_boards=""):
1028         '''Initialization
1029
1030         :param xml_dir_path str: The path to the directory where to put
1031                                  the xml resulting files
1032         :param l_jobs List: the list of jobs that run today
1033         :param l_jobs_not_today List: the list of jobs that do not run today
1034         :param file_boards str: the file path from which to read the
1035                                    expected boards
1036         '''
1037         # The path of the csv files to read to fill the expected boards
1038         self.file_boards = file_boards
1039
1040         today = datetime.date.weekday(datetime.date.today())
1041         self.parse_csv_boards(today)
1042
1043         # The path of the global xml file
1044         self.xml_dir_path = xml_dir_path
1045         # Initialize the xml files
1046         xml_global_path = os.path.join(self.xml_dir_path, "global_report.xml")
1047         self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1048                                                          "JobsReport")
1049         # The xml files that corresponds to the boards.
1050         # {name_board : xml_object}}
1051         self.d_xml_board_files = {}
1052         # Create the lines and columns
1053         self.initialize_boards(l_jobs, l_jobs_not_today)
1054
1055         # Write the xml file
1056         self.update_xml_files(l_jobs)
1057
1058     def add_xml_board(self, name):
1059         xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1060         self.d_xml_board_files[name] =  src.xmlManager.XmlLogFile(
1061                                                     xml_board_path,
1062                                                     "JobsReport")
1063         self.d_xml_board_files[name].add_simple_node("distributions")
1064         self.d_xml_board_files[name].add_simple_node("applications")
1065         self.d_xml_board_files[name].add_simple_node("board", text=name)
1066
1067     def initialize_boards(self, l_jobs, l_jobs_not_today):
1068         '''Get all the first information needed for each file and write the
1069            first version of the files
1070         :param l_jobs List: the list of jobs that run today
1071         :param l_jobs_not_today List: the list of jobs that do not run today
1072         '''
1073         # Get the boards to fill and put it in a dictionary
1074         # {board_name : xml instance corresponding to the board}
1075         for job in l_jobs + l_jobs_not_today:
1076             board = job.board
1077             if (board is not None and
1078                                 board not in self.d_xml_board_files.keys()):
1079                 self.add_xml_board(board)
1080
1081         # Verify that the boards given as input are done
1082         for board in list(self.d_input_boards.keys()):
1083             if board not in self.d_xml_board_files:
1084                 self.add_xml_board(board)
1085             root_node = self.d_xml_board_files[board].xmlroot
1086             src.xmlManager.append_node_attrib(root_node,
1087                                               {"input_file" : self.file_boards})
1088
1089         # Loop over all jobs in order to get the lines and columns for each
1090         # xml file
1091         d_dist = {}
1092         d_application = {}
1093         for board in self.d_xml_board_files:
1094             d_dist[board] = []
1095             d_application[board] = []
1096
1097         l_hosts_ports = []
1098
1099         for job in l_jobs + l_jobs_not_today:
1100
1101             if (job.machine.host, job.machine.port) not in l_hosts_ports:
1102                 l_hosts_ports.append((job.machine.host, job.machine.port))
1103
1104             distrib = job.machine.distribution
1105             application = job.application
1106
1107             board_job = job.board
1108             if board is None:
1109                 continue
1110             for board in self.d_xml_board_files:
1111                 if board_job == board:
1112                     if distrib is not None and distrib not in d_dist[board]:
1113                         d_dist[board].append(distrib)
1114                         src.xmlManager.add_simple_node(
1115                             self.d_xml_board_files[board].xmlroot.find(
1116                                                             'distributions'),
1117                                                    "dist",
1118                                                    attrib={"name" : distrib})
1119
1120                 if board_job == board:
1121                     if (application is not None and
1122                                     application not in d_application[board]):
1123                         d_application[board].append(application)
1124                         src.xmlManager.add_simple_node(
1125                             self.d_xml_board_files[board].xmlroot.find(
1126                                                                 'applications'),
1127                                                    "application",
1128                                                    attrib={
1129                                                         "name" : application})
1130
1131         # Verify that there are no missing application or distribution in the
1132         # xml board files (regarding the input boards)
1133         for board in self.d_xml_board_files:
1134             l_dist = d_dist[board]
1135             if board not in self.d_input_boards.keys():
1136                 continue
1137             for dist in self.d_input_boards[board]["rows"]:
1138                 if dist not in l_dist:
1139                     src.xmlManager.add_simple_node(
1140                             self.d_xml_board_files[board].xmlroot.find(
1141                                                             'distributions'),
1142                                                    "dist",
1143                                                    attrib={"name" : dist})
1144             l_appli = d_application[board]
1145             for appli in self.d_input_boards[board]["columns"]:
1146                 if appli not in l_appli:
1147                     src.xmlManager.add_simple_node(
1148                             self.d_xml_board_files[board].xmlroot.find(
1149                                                                 'applications'),
1150                                                    "application",
1151                                                    attrib={"name" : appli})
1152
1153         # Initialize the hosts_ports node for the global file
1154         self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1155                                                                 "hosts_ports")
1156         for host, port in l_hosts_ports:
1157             host_port = "%s:%i" % (host, port)
1158             src.xmlManager.add_simple_node(self.xmlhosts_ports,
1159                                            "host_port",
1160                                            attrib={"name" : host_port})
1161
1162         # Initialize the jobs node in all files
1163         for xml_file in [self.xml_global_file] + list(
1164                                             self.d_xml_board_files.values()):
1165             xml_jobs = xml_file.add_simple_node("jobs")
1166             # Get the jobs present in the config file but
1167             # that will not be launched today
1168             self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1169
1170             xml_file.add_simple_node("infos",
1171                                      attrib={"name" : "last update",
1172                                              "JobsCommandStatus" : "running"})
1173
1174         # Find in each board the squares that needs to be filled regarding the
1175         # input csv files but that are not covered by a today job
1176         for board in self.d_input_boards.keys():
1177             xml_root_board = self.d_xml_board_files[board].xmlroot
1178             xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1179                                                  "missing_jobs")
1180             for row, column in self.d_input_boards[board]["jobs"]:
1181                 found = False
1182                 for job in l_jobs:
1183                     if (job.application == column and
1184                         job.machine.distribution == row):
1185                         found = True
1186                         break
1187                 if not found:
1188                     src.xmlManager.add_simple_node(xml_missing,
1189                                             "job",
1190                                             attrib={"distribution" : row,
1191                                                     "application" : column })
1192
1193     def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1194         '''Get all the first information needed for each file and write the
1195            first version of the files
1196
1197         :param xml_node_jobs etree.Element: the node corresponding to a job
1198         :param l_jobs_not_today List: the list of jobs that do not run today
1199         '''
1200         for job in l_jobs_not_today:
1201             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1202                                                  "job",
1203                                                  attrib={"name" : job.name})
1204             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1205             src.xmlManager.add_simple_node(xmlj,
1206                                            "distribution",
1207                                            job.machine.distribution)
1208             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1209             src.xmlManager.add_simple_node(xmlj,
1210                                        "commands", " ; ".join(job.commands))
1211             src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1212             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1213             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1214             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1215             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1216             src.xmlManager.add_simple_node(xmlj, "sat_path",
1217                                                         job.machine.sat_path)
1218
1219     def parse_csv_boards(self, today):
1220         """ Parse the csv file that describes the boards to produce and fill
1221             the dict d_input_boards that contain the csv file contain
1222
1223         :param today int: the current day of the week
1224         """
1225         # open the csv file and read its content
1226         l_read = []
1227         with open(self.file_boards, 'r') as f:
1228             reader = csv.reader(f,delimiter=CSV_DELIMITER)
1229             for row in reader:
1230                 l_read.append(row)
1231         # get the delimiter for the boards (empty line)
1232         boards_delimiter = [''] * len(l_read[0])
1233         # Make the list of boards, by splitting with the delimiter
1234         l_boards = [list(y) for x, y in itertools.groupby(l_read,
1235                                     lambda z: z == boards_delimiter) if not x]
1236
1237         # loop over the csv lists of lines and get the rows, columns and jobs
1238         d_boards = {}
1239         for input_board in l_boards:
1240             # get board name
1241             board_name = input_board[0][0]
1242
1243             # Get columns list
1244             columns = input_board[0][1:]
1245
1246             rows = []
1247             jobs = []
1248             for line in input_board[1:]:
1249                 row = line[0]
1250                 for i, square in enumerate(line[1:]):
1251                     if square=='':
1252                         continue
1253                     days = square.split(DAYS_SEPARATOR)
1254                     days = [int(day) for day in days]
1255                     if today in days:
1256                         if row not in rows:
1257                             rows.append(row)
1258                         job = (row, columns[i])
1259                         jobs.append(job)
1260
1261             d_boards[board_name] = {"rows" : rows,
1262                                     "columns" : columns,
1263                                     "jobs" : jobs}
1264
1265         self.d_input_boards = d_boards
1266
1267     def update_xml_files(self, l_jobs):
1268         '''Write all the xml files with updated information about the jobs
1269
1270         :param l_jobs List: the list of jobs that run today
1271         '''
1272         for xml_file in [self.xml_global_file] + list(
1273                                             self.d_xml_board_files.values()):
1274             self.update_xml_file(l_jobs, xml_file)
1275
1276         # Write the file
1277         self.write_xml_files()
1278
1279     def update_xml_file(self, l_jobs, xml_file):
1280         '''update information about the jobs for the file xml_file
1281
1282         :param l_jobs List: the list of jobs that run today
1283         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1284         '''
1285
1286         xml_node_jobs = xml_file.xmlroot.find('jobs')
1287         # Update the job names and status node
1288         for job in l_jobs:
1289             # Find the node corresponding to the job and delete it
1290             # in order to recreate it
1291             for xmljob in xml_node_jobs.findall('job'):
1292                 if xmljob.attrib['name'] == job.name:
1293                     xml_node_jobs.remove(xmljob)
1294
1295             T0 = str(job._T0)
1296             if T0 != "-1":
1297                 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1298                                        time.localtime(job._T0))
1299             Tf = str(job._Tf)
1300             if Tf != "-1":
1301                 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1302                                        time.localtime(job._Tf))
1303
1304             # recreate the job node
1305             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1306                                                   "job",
1307                                                   attrib={"name" : job.name})
1308             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1309             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1310             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1311             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1312             src.xmlManager.add_simple_node(xmlj, "sat_path",
1313                                            job.machine.sat_path)
1314             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1315             src.xmlManager.add_simple_node(xmlj, "distribution",
1316                                            job.machine.distribution)
1317             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1318             src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1319             src.xmlManager.add_simple_node(xmlj, "commands",
1320                                            " ; ".join(job.commands))
1321             src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1322             src.xmlManager.add_simple_node(xmlj, "begin", T0)
1323             src.xmlManager.add_simple_node(xmlj, "end", Tf)
1324             src.xmlManager.add_simple_node(xmlj, "out",
1325                                            src.printcolors.cleancolor(job.out))
1326             src.xmlManager.add_simple_node(xmlj, "err",
1327                                            src.printcolors.cleancolor(job.err))
1328             src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1329             if len(job.remote_log_files) > 0:
1330                 src.xmlManager.add_simple_node(xmlj,
1331                                                "remote_log_file_path",
1332                                                job.remote_log_files[0])
1333             else:
1334                 src.xmlManager.add_simple_node(xmlj,
1335                                                "remote_log_file_path",
1336                                                "nothing")
1337
1338             xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1339             # get the job father
1340             if job.after is not None:
1341                 job_father = None
1342                 for jb in l_jobs:
1343                     if jb.name == job.after:
1344                         job_father = jb
1345
1346                 if (job_father is not None and
1347                         len(job_father.remote_log_files) > 0):
1348                     link = job_father.remote_log_files[0]
1349                 else:
1350                     link = "nothing"
1351                 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1352
1353             # Verify that the job is to be done today regarding the input csv
1354             # files
1355             if job.board and job.board in self.d_input_boards.keys():
1356                 found = False
1357                 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1358                     if (job.machine.distribution == dist
1359                         and job.application == appli):
1360                         found = True
1361                         src.xmlManager.add_simple_node(xmlj,
1362                                                "extra_job",
1363                                                "no")
1364                         break
1365                 if not found:
1366                     src.xmlManager.add_simple_node(xmlj,
1367                                                "extra_job",
1368                                                "yes")
1369
1370
1371         # Update the date
1372         xml_node_infos = xml_file.xmlroot.find('infos')
1373         src.xmlManager.append_node_attrib(xml_node_infos,
1374                     attrib={"value" :
1375                     datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1376
1377
1378
1379     def last_update(self, finish_status = "finished"):
1380         '''update information about the jobs for the file xml_file
1381
1382         :param l_jobs List: the list of jobs that run today
1383         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1384         '''
1385         for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1386             xml_node_infos = xml_file.xmlroot.find('infos')
1387             src.xmlManager.append_node_attrib(xml_node_infos,
1388                         attrib={"JobsCommandStatus" : finish_status})
1389         # Write the file
1390         self.write_xml_files()
1391
1392     def write_xml_files(self):
1393         ''' Write the xml files
1394         '''
1395         self.xml_global_file.write_tree(STYLESHEET_GLOBAL)
1396         for xml_file in self.d_xml_board_files.values():
1397             xml_file.write_tree(STYLESHEET_BOARD)
1398
1399 ##
1400 # Describes the command
1401 def description():
1402     return _("The jobs command launches maintenances that are described"
1403              " in the dedicated jobs configuration file.")
1404
1405 ##
1406 # Runs the command.
1407 def run(args, runner, logger):
1408
1409     (options, args) = parser.parse_args(args)
1410
1411     l_cfg_dir = runner.cfg.PATHS.JOBPATH
1412
1413     # list option : display all the available config files
1414     if options.list:
1415         for cfg_dir in l_cfg_dir:
1416             if not options.no_label:
1417                 logger.write("------ %s\n" %
1418                                  src.printcolors.printcHeader(cfg_dir))
1419
1420             for f in sorted(os.listdir(cfg_dir)):
1421                 if not f.endswith('.pyconf'):
1422                     continue
1423                 cfilename = f[:-7]
1424                 logger.write("%s\n" % cfilename)
1425         return 0
1426
1427     # Make sure the jobs_config option has been called
1428     if not options.jobs_cfg:
1429         message = _("The option --jobs_config is required\n")
1430         src.printcolors.printcError(message)
1431         return 1
1432
1433     # Find the file in the directories
1434     found = False
1435     for cfg_dir in l_cfg_dir:
1436         file_jobs_cfg = os.path.join(cfg_dir, options.jobs_cfg)
1437         if not file_jobs_cfg.endswith('.pyconf'):
1438             file_jobs_cfg += '.pyconf'
1439
1440         if not os.path.exists(file_jobs_cfg):
1441             continue
1442         else:
1443             found = True
1444             break
1445
1446     if not found:
1447         msg = _("The file configuration %(name_file)s was not found."
1448                 "\nUse the --list option to get the possible files.")
1449         src.printcolors.printcError(msg)
1450         return 1
1451
1452     info = [
1453         (_("Platform"), runner.cfg.VARS.dist),
1454         (_("File containing the jobs configuration"), file_jobs_cfg)
1455     ]
1456     src.print_info(logger, info)
1457
1458     # Read the config that is in the file
1459     config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1460     if options.only_jobs:
1461         l_jb = src.pyconf.Sequence()
1462         for jb in config_jobs.jobs:
1463             if jb.name in options.only_jobs:
1464                 l_jb.append(jb,
1465                 "Adding a job that was given in only_jobs option parameters")
1466         config_jobs.jobs = l_jb
1467
1468     # Initialization
1469     today_jobs = Jobs(runner,
1470                       logger,
1471                       file_jobs_cfg,
1472                       config_jobs)
1473     # SSH connection to all machines
1474     today_jobs.ssh_connection_all_machines()
1475     if options.test_connection:
1476         return 0
1477
1478     gui = None
1479     if options.publish:
1480         # Copy the stylesheets in the log directory
1481         log_dir = runner.cfg.USER.log_dir
1482         xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1483         files_to_copy = []
1484         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1485         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1486         files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1487         for file_path in files_to_copy:
1488             shutil.copy2(file_path, log_dir)
1489
1490         # Instanciate the Gui in order to produce the xml files that contain all
1491         # the boards
1492         gui = Gui(runner.cfg.USER.log_dir,
1493                   today_jobs.ljobs,
1494                   today_jobs.ljobs_not_today,
1495                   file_boards = options.input_boards)
1496
1497         # Display the list of the xml files
1498         logger.write(src.printcolors.printcInfo(("Here is the list of published"
1499                                                  " files :\n")), 4)
1500         logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1501         for board in gui.d_xml_board_files.keys():
1502             file_path = gui.d_xml_board_files[board].logFile
1503             file_name = os.path.basename(file_path)
1504             logger.write("%s\n" % file_path, 4)
1505             logger.add_link(file_name, "board", 0, board)
1506
1507         logger.write("\n", 4)
1508
1509     today_jobs.gui = gui
1510
1511     interruped = False
1512     try:
1513         # Run all the jobs contained in config_jobs
1514         today_jobs.run_jobs()
1515     except KeyboardInterrupt:
1516         interruped = True
1517         logger.write("\n\n%s\n\n" %
1518                 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1519     finally:
1520         if interruped:
1521             msg = _("Killing the running jobs and trying"
1522                     " to get the corresponding logs\n")
1523             logger.write(src.printcolors.printcWarning(msg))
1524
1525         # find the potential not finished jobs and kill them
1526         for jb in today_jobs.ljobs:
1527             if not jb.has_finished():
1528                 try:
1529                     jb.kill_remote_process()
1530                 except Exception as e:
1531                     msg = _("Failed to kill job %s: %s\n" % (jb.name, e))
1532                     logger.write(src.printcolors.printcWarning(msg))
1533         if interruped:
1534             if today_jobs.gui:
1535                 today_jobs.gui.last_update(_("Forced interruption"))
1536         else:
1537             if today_jobs.gui:
1538                 today_jobs.gui.last_update()
1539         # Output the results
1540         today_jobs.write_all_results()