commands/jobs.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import datetime
  21 import time
  22 import csv
  23 import shutil
  24 import itertools
  25 import paramiko
  26
  27 import src
  28
  29 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
  30 STYLESHEET_BOARD = "jobs_board_report.xsl"
  31
  32 DAYS_SEPARATOR = ","
  33 CSV_DELIMITER = ";"
  34
  35 parser = src.options.Options()
  36
  37 parser.add_option('n', 'name', 'string', 'jobs_cfg',
  38                   _('The name of the config file that contains'
  39                   ' the jobs configuration'))
  40 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
  41                   _('Optional: the list of jobs to launch, by their name. '))
  42 parser.add_option('l', 'list', 'boolean', 'list',
  43                   _('Optional: list all available config files.'))
  44 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
  45                   _("Optional: try to connect to the machines. "
  46                     "Not executing the jobs."),
  47                   False)
  48 parser.add_option('p', 'publish', 'boolean', 'publish',
  49                   _("Optional: generate an xml file that can be read in a "
  50                     "browser to display the jobs status."),
  51                   False)
  52 parser.add_option('i', 'input_boards', 'string', 'input_boards', _("Optional: "
  53                                 "the path to csv file that contain "
  54                                 "the expected boards."),"")
  55 parser.add_option('n', 'completion', 'boolean', 'no_label',
  56                   _("Optional (internal use): do not print labels, Works only "
  57                     "with --list."),
  58                   False)
  59
  60 class Machine(object):
  61     '''Class to manage a ssh connection on a machine
  62     '''
  63     def __init__(self,
  64                  name,
  65                  host,
  66                  user,
  67                  port=22,
  68                  passwd=None,
  69                  sat_path="salomeTools"):
  70         self.name = name
  71         self.host = host
  72         self.port = port
  73         self.distribution = None # Will be filled after copying SAT on the machine
  74         self.user = user
  75         self.password = passwd
  76         self.sat_path = sat_path
  77         self.ssh = paramiko.SSHClient()
  78         self._connection_successful = None
  79
  80     def connect(self, logger):
  81         '''Initiate the ssh connection to the remote machine
  82
  83         :param logger src.logger.Logger: The logger instance
  84         :return: Nothing
  85         :rtype: N\A
  86         '''
  87
  88         self._connection_successful = False
  89         self.ssh.load_system_host_keys()
  90         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  91         try:
  92             self.ssh.connect(self.host,
  93                              port=self.port,
  94                              username=self.user,
  95                              password = self.password)
  96         except paramiko.AuthenticationException:
  97             message = src.KO_STATUS + _("Authentication failed")
  98         except paramiko.BadHostKeyException:
  99             message = (src.KO_STATUS +
 100                        _("The server's host key could not be verified"))
 101         except paramiko.SSHException:
 102             message = ( _("SSHException error connecting or "
 103                           "establishing an SSH session"))
 104         except:
 105             message = ( _("Error connecting or establishing an SSH session"))
 106         else:
 107             self._connection_successful = True
 108             message = ""
 109         return message
 110
 111     def successfully_connected(self, logger):
 112         '''Verify if the connection to the remote machine has succeed
 113
 114         :param logger src.logger.Logger: The logger instance
 115         :return: True if the connection has succeed, False if not
 116         :rtype: bool
 117         '''
 118         if self._connection_successful == None:
 119             message = _("Warning : trying to ask if the connection to "
 120             "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
 121             " no connection request" %
 122                         (self.name, self.host, self.port, self.user))
 123             logger.write( src.printcolors.printcWarning(message))
 124         return self._connection_successful
 125
 126     def copy_sat(self, sat_local_path, job_file):
 127         '''Copy salomeTools to the remote machine in self.sat_path
 128         '''
 129         res = 0
 130         try:
 131             # open a sftp connection
 132             self.sftp = self.ssh.open_sftp()
 133             # Create the sat directory on remote machine if it is not existing
 134             self.mkdir(self.sat_path, ignore_existing=True)
 135             # Put sat
 136             self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
 137             # put the job configuration file in order to make it reachable
 138             # on the remote machine
 139             self.sftp.put(job_file, os.path.join(".salomeTools",
 140                                                  "Jobs",
 141                                                  ".jobs_command_file.pyconf"))
 142         except Exception as e:
 143             res = str(e)
 144             self._connection_successful = False
 145
 146         return res
 147
 148     def put_dir(self, source, target, filters = []):
 149         ''' Uploads the contents of the source directory to the target path. The
 150             target directory needs to exists. All sub-directories in source are
 151             created under target.
 152         '''
 153         for item in os.listdir(source):
 154             if item in filters:
 155                 continue
 156             source_path = os.path.join(source, item)
 157             destination_path = os.path.join(target, item)
 158             if os.path.islink(source_path):
 159                 linkto = os.readlink(source_path)
 160                 try:
 161                     self.sftp.symlink(linkto, destination_path)
 162                     self.sftp.chmod(destination_path,
 163                                     os.stat(source_path).st_mode)
 164                 except IOError:
 165                     pass
 166             else:
 167                 if os.path.isfile(source_path):
 168                     self.sftp.put(source_path, destination_path)
 169                     self.sftp.chmod(destination_path,
 170                                     os.stat(source_path).st_mode)
 171                 else:
 172                     self.mkdir(destination_path, ignore_existing=True)
 173                     self.put_dir(source_path, destination_path)
 174
 175     def mkdir(self, path, mode=511, ignore_existing=False):
 176         ''' Augments mkdir by adding an option to not fail
 177             if the folder exists
 178         '''
 179         try:
 180             self.sftp.mkdir(path, mode)
 181         except IOError:
 182             if ignore_existing:
 183                 pass
 184             else:
 185                 raise
 186
 187     def exec_command(self, command, logger):
 188         '''Execute the command on the remote machine
 189
 190         :param command str: The command to be run
 191         :param logger src.logger.Logger: The logger instance
 192         :return: the stdin, stdout, and stderr of the executing command,
 193                  as a 3-tuple
 194         :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
 195                 paramiko.channel.ChannelFile)
 196         '''
 197         try:
 198             # Does not wait the end of the command
 199             (stdin, stdout, stderr) = self.ssh.exec_command(command)
 200         except paramiko.SSHException:
 201             message = src.KO_STATUS + _(
 202                             ": the server failed to execute the command\n")
 203             logger.write( src.printcolors.printcError(message))
 204             return (None, None, None)
 205         except:
 206             logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
 207             return (None, None, None)
 208         else:
 209             return (stdin, stdout, stderr)
 210
 211     def close(self):
 212         '''Close the ssh connection
 213
 214         :rtype: N\A
 215         '''
 216         self.ssh.close()
 217
 218     def write_info(self, logger):
 219         '''Prints the informations relative to the machine in the logger
 220            (terminal traces and log file)
 221
 222         :param logger src.logger.Logger: The logger instance
 223         :return: Nothing
 224         :rtype: N\A
 225         '''
 226         logger.write("host : " + self.host + "\n")
 227         logger.write("port : " + str(self.port) + "\n")
 228         logger.write("user : " + str(self.user) + "\n")
 229         if self.successfully_connected(logger):
 230             status = src.OK_STATUS
 231         else:
 232             status = src.KO_STATUS
 233         logger.write("Connection : " + status + "\n\n")
 234
 235
 236 class Job(object):
 237     '''Class to manage one job
 238     '''
 239     def __init__(self, name, machine, application, board,
 240                  commands, timeout, config, logger, after=None):
 241
 242         self.name = name
 243         self.machine = machine
 244         self.after = after
 245         self.timeout = timeout
 246         self.application = application
 247         self.board = board
 248         self.config = config
 249         self.logger = logger
 250         # The list of log files to download from the remote machine
 251         self.remote_log_files = []
 252
 253         # The remote command status
 254         # -1 means that it has not been launched,
 255         # 0 means success and 1 means fail
 256         self.res_job = "-1"
 257         self.cancelled = False
 258
 259         self._T0 = -1
 260         self._Tf = -1
 261         self._has_begun = False
 262         self._has_finished = False
 263         self._has_timouted = False
 264         self._stdin = None # Store the command inputs field
 265         self._stdout = None # Store the command outputs field
 266         self._stderr = None # Store the command errors field
 267
 268         self.out = ""
 269         self.err = ""
 270
 271         self.commands = commands
 272         self.command = (os.path.join(self.machine.sat_path, "sat") +
 273                         " -l " +
 274                         os.path.join(self.machine.sat_path,
 275                                      "list_log_files.txt") +
 276                         " job --jobs_config .jobs_command_file" +
 277                         " --name " +
 278                         self.name)
 279
 280     def get_pids(self):
 281         """ Get the pid(s) corresponding to the command that have been launched
 282             On the remote machine
 283
 284         :return: The list of integers corresponding to the found pids
 285         :rtype: List
 286         """
 287         pids = []
 288         cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
 289         (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
 290         pids_cmd = out_pid.readlines()
 291         pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
 292         pids+=pids_cmd
 293         return pids
 294
 295     def kill_remote_process(self, wait=1):
 296         '''Kills the process on the remote machine.
 297
 298         :return: (the output of the kill, the error of the kill)
 299         :rtype: (str, str)
 300         '''
 301
 302         pids = self.get_pids()
 303         cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
 304         (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
 305                                                             self.logger)
 306         time.sleep(wait)
 307         return (out_kill, err_kill)
 308
 309     def has_begun(self):
 310         '''Returns True if the job has already begun
 311
 312         :return: True if the job has already begun
 313         :rtype: bool
 314         '''
 315         return self._has_begun
 316
 317     def has_finished(self):
 318         '''Returns True if the job has already finished
 319            (i.e. all the commands have been executed)
 320            If it is finished, the outputs are stored in the fields out and err.
 321
 322         :return: True if the job has already finished
 323         :rtype: bool
 324         '''
 325
 326         # If the method has already been called and returned True
 327         if self._has_finished:
 328             return True
 329
 330         # If the job has not begun yet
 331         if not self.has_begun():
 332             return False
 333
 334         if self._stdout.channel.closed:
 335             self._has_finished = True
 336             # Store the result outputs
 337             self.out += self._stdout.read().decode()
 338             self.err += self._stderr.read().decode()
 339             # Put end time
 340             self._Tf = time.time()
 341             # And get the remote command status and log files
 342             self.get_log_files()
 343
 344         return self._has_finished
 345
 346     def get_log_files(self):
 347         """Get the log files produced by the command launched
 348            on the remote machine, and put it in the log directory of the user,
 349            so they can be accessible from
 350         """
 351         # Do not get the files if the command is not finished
 352         if not self.has_finished():
 353             msg = _("Trying to get log files whereas the job is not finished.")
 354             self.logger.write(src.printcolors.printcWarning(msg))
 355             return
 356
 357         # First get the file that contains the list of log files to get
 358         tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
 359         remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt")
 360         self.machine.sftp.get(
 361                     remote_path,
 362                     tmp_file_path)
 363
 364         # Read the file and get the result of the command and all the log files
 365         # to get
 366         fstream_tmp = open(tmp_file_path, "r")
 367         file_lines = fstream_tmp.readlines()
 368         file_lines = [line.replace("\n", "") for line in file_lines]
 369         fstream_tmp.close()
 370         os.remove(tmp_file_path)
 371
 372         try :
 373             # The first line is the result of the command (0 success or 1 fail)
 374             self.res_job = file_lines[0]
 375         except Exception as e:
 376             self.err += _("Unable to get status from remote file %s: %s" %
 377                                                     (remote_path, str(e)))
 378
 379         for i, job_path_remote in enumerate(file_lines[1:]):
 380             try:
 381                 # For each command, there is two files to get :
 382                 # 1- The xml file describing the command and giving the
 383                 # internal traces.
 384                 # 2- The txt file containing the system command traces (like
 385                 # traces produced by the "make" command)
 386                 # 3- In case of the test command, there is another file to get :
 387                 # the xml board that contain the test results
 388                 dirname = os.path.basename(os.path.dirname(job_path_remote))
 389                 if dirname != 'OUT' and dirname != 'TEST':
 390                     # Case 1-
 391                     local_path = os.path.join(os.path.dirname(
 392                                                         self.logger.logFilePath),
 393                                               os.path.basename(job_path_remote))
 394                     if i==0: # The first is the job command
 395                         self.logger.add_link(os.path.basename(job_path_remote),
 396                                              "job",
 397                                              self.res_job,
 398                                              self.command)
 399                 elif dirname == 'OUT':
 400                     # Case 2-
 401                     local_path = os.path.join(os.path.dirname(
 402                                                         self.logger.logFilePath),
 403                                               'OUT',
 404                                               os.path.basename(job_path_remote))
 405                 elif dirname == 'TEST':
 406                     # Case 3-
 407                     local_path = os.path.join(os.path.dirname(
 408                                                         self.logger.logFilePath),
 409                                               'TEST',
 410                                               os.path.basename(job_path_remote))
 411
 412                 # Get the file
 413                 if not os.path.exists(local_path):
 414                     self.machine.sftp.get(job_path_remote, local_path)
 415                 self.remote_log_files.append(local_path)
 416             except Exception as e:
 417                 self.err += _("Unable to get %s log file from remote: %s" %
 418                                                     (str(job_path_remote),
 419                                                      str(e)))
 420
 421     def has_failed(self):
 422         '''Returns True if the job has failed.
 423            A job is considered as failed if the machine could not be reached,
 424            if the remote command failed,
 425            or if the job finished with a time out.
 426
 427         :return: True if the job has failed
 428         :rtype: bool
 429         '''
 430         if not self.has_finished():
 431             return False
 432         if not self.machine.successfully_connected(self.logger):
 433             return True
 434         if self.is_timeout():
 435             return True
 436         if self.res_job == "1":
 437             return True
 438         return False
 439
 440     def cancel(self):
 441         """In case of a failing job, one has to cancel every job that depend
 442            on it. This method put the job as failed and will not be executed.
 443         """
 444         self._has_begun = True
 445         self._has_finished = True
 446         self.cancelled = True
 447         self.out += _("This job was not launched because its father has failed.")
 448         self.err += _("This job was not launched because its father has failed.")
 449
 450     def is_running(self):
 451         '''Returns True if the job commands are running
 452
 453         :return: True if the job is running
 454         :rtype: bool
 455         '''
 456         return self.has_begun() and not self.has_finished()
 457
 458     def is_timeout(self):
 459         '''Returns True if the job commands has finished with timeout
 460
 461         :return: True if the job has finished with timeout
 462         :rtype: bool
 463         '''
 464         return self._has_timouted
 465
 466     def time_elapsed(self):
 467         """Get the time elapsed since the job launching
 468
 469         :return: The number of seconds
 470         :rtype: int
 471         """
 472         if not self.has_begun():
 473             return -1
 474         T_now = time.time()
 475         return T_now - self._T0
 476
 477     def check_time(self):
 478         """Verify that the job has not exceeded its timeout.
 479            If it has, kill the remote command and consider the job as finished.
 480         """
 481         if not self.has_begun():
 482             return
 483         if self.time_elapsed() > self.timeout:
 484             self._has_finished = True
 485             self._has_timouted = True
 486             self._Tf = time.time()
 487             self.get_pids()
 488             (out_kill, _) = self.kill_remote_process()
 489             self.out += "TIMEOUT \n" + out_kill.read().decode()
 490             self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
 491             try:
 492                 self.get_log_files()
 493             except Exception as e:
 494                 self.err += _("Unable to get remote log files: %s" % e)
 495
 496     def total_duration(self):
 497         """Give the total duration of the job
 498
 499         :return: the total duration of the job in seconds
 500         :rtype: int
 501         """
 502         return self._Tf - self._T0
 503
 504     def run(self):
 505         """Launch the job by executing the remote command.
 506         """
 507
 508         # Prevent multiple run
 509         if self.has_begun():
 510             msg = _("Warning: A job can only be launched one time")
 511             msg2 = _("Trying to launch the job \"%s\" whereas it has "
 512                      "already been launched." % self.name)
 513             self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
 514                                                                         msg2)))
 515             return
 516
 517         # Do not execute the command if the machine could not be reached
 518         if not self.machine.successfully_connected(self.logger):
 519             self._has_finished = True
 520             self.out = "N\A"
 521             self.err += ("Connection to machine (name : %s, host: %s, port:"
 522                         " %s, user: %s) has failed\nUse the log command "
 523                         "to get more information."
 524                         % (self.machine.name,
 525                            self.machine.host,
 526                            self.machine.port,
 527                            self.machine.user))
 528         else:
 529             # Usual case : Launch the command on remote machine
 530             self._T0 = time.time()
 531             self._stdin, self._stdout, self._stderr = self.machine.exec_command(
 532                                                                   self.command,
 533                                                                   self.logger)
 534             # If the results are not initialized, finish the job
 535             if (self._stdin, self._stdout, self._stderr) == (None, None, None):
 536                 self._has_finished = True
 537                 self._Tf = time.time()
 538                 self.out += "N\A"
 539                 self.err += "The server failed to execute the command"
 540
 541         # Put the beginning flag to true.
 542         self._has_begun = True
 543
 544     def write_results(self):
 545         """Display on the terminal all the job's information
 546         """
 547         self.logger.write("name : " + self.name + "\n")
 548         if self.after:
 549             self.logger.write("after : %s\n" % self.after)
 550         self.logger.write("Time elapsed : %4imin %2is \n" %
 551                      (self.total_duration()//60 , self.total_duration()%60))
 552         if self._T0 != -1:
 553             self.logger.write("Begin time : %s\n" %
 554                          time.strftime('%Y-%m-%d %H:%M:%S',
 555                                        time.localtime(self._T0)) )
 556         if self._Tf != -1:
 557             self.logger.write("End time   : %s\n\n" %
 558                          time.strftime('%Y-%m-%d %H:%M:%S',
 559                                        time.localtime(self._Tf)) )
 560
 561         machine_head = "Informations about connection :\n"
 562         underline = (len(machine_head) - 2) * "-"
 563         self.logger.write(src.printcolors.printcInfo(
 564                                                 machine_head+underline+"\n"))
 565         self.machine.write_info(self.logger)
 566
 567         self.logger.write(src.printcolors.printcInfo("out : \n"))
 568         if self.out == "":
 569             self.logger.write("Unable to get output\n")
 570         else:
 571             self.logger.write(self.out + "\n")
 572         self.logger.write(src.printcolors.printcInfo("err : \n"))
 573         self.logger.write(self.err + "\n")
 574
 575     def get_status(self):
 576         """Get the status of the job (used by the Gui for xml display)
 577
 578         :return: The current status of the job
 579         :rtype: String
 580         """
 581         if not self.machine.successfully_connected(self.logger):
 582             return "SSH connection KO"
 583         if not self.has_begun():
 584             return "Not launched"
 585         if self.cancelled:
 586             return "Cancelled"
 587         if self.is_running():
 588             return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
 589                                                     time.localtime(self._T0))
 590         if self.has_finished():
 591             if self.is_timeout():
 592                 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
 593                                                     time.localtime(self._Tf))
 594             return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
 595                                                      time.localtime(self._Tf))
 596
 597 class Jobs(object):
 598     '''Class to manage the jobs to be run
 599     '''
 600     def __init__(self,
 601                  runner,
 602                  logger,
 603                  job_file_path,
 604                  config_jobs,
 605                  lenght_columns = 20):
 606         # The jobs configuration
 607         self.cfg_jobs = config_jobs
 608         self.job_file_path = job_file_path
 609         # The machine that will be used today
 610         self.lmachines = []
 611         # The list of machine (hosts, port) that will be used today
 612         # (a same host can have several machine instances since there
 613         # can be several ssh parameters)
 614         self.lhosts = []
 615         # The jobs to be launched today
 616         self.ljobs = []
 617         # The jobs that will not be launched today
 618         self.ljobs_not_today = []
 619         self.runner = runner
 620         self.logger = logger
 621         self.len_columns = lenght_columns
 622
 623         # the list of jobs that have not been run yet
 624         self._l_jobs_not_started = []
 625         # the list of jobs that have already ran
 626         self._l_jobs_finished = []
 627         # the list of jobs that are running
 628         self._l_jobs_running = []
 629
 630         self.determine_jobs_and_machines()
 631
 632     def define_job(self, job_def, machine):
 633         '''Takes a pyconf job definition and a machine (from class machine)
 634            and returns the job instance corresponding to the definition.
 635
 636         :param job_def src.config.Mapping: a job definition
 637         :param machine machine: the machine on which the job will run
 638         :return: The corresponding job in a job class instance
 639         :rtype: job
 640         '''
 641         name = job_def.name
 642         cmmnds = job_def.commands
 643         timeout = job_def.timeout
 644         after = None
 645         if 'after' in job_def:
 646             after = job_def.after
 647         application = None
 648         if 'application' in job_def:
 649             application = job_def.application
 650         board = None
 651         if 'board' in job_def:
 652             board = job_def.board
 653
 654         return Job(name,
 655                    machine,
 656                    application,
 657                    board,
 658                    cmmnds,
 659                    timeout,
 660                    self.runner.cfg,
 661                    self.logger,
 662                    after = after)
 663
 664     def determine_jobs_and_machines(self):
 665         '''Function that reads the pyconf jobs definition and instantiates all
 666            the machines and jobs to be done today.
 667
 668         :return: Nothing
 669         :rtype: N\A
 670         '''
 671         today = datetime.date.weekday(datetime.date.today())
 672         host_list = []
 673
 674         for job_def in self.cfg_jobs.jobs :
 675
 676             if not "machine" in job_def:
 677                 msg = _('WARNING: The job "%s" do not have the key '
 678                        '"machine", this job is ignored.\n\n' % job_def.name)
 679                 self.logger.write(src.printcolors.printcWarning(msg))
 680                 continue
 681             name_machine = job_def.machine
 682
 683             a_machine = None
 684             for mach in self.lmachines:
 685                 if mach.name == name_machine:
 686                     a_machine = mach
 687                     break
 688
 689             if a_machine == None:
 690                 for machine_def in self.cfg_jobs.machines:
 691                     if machine_def.name == name_machine:
 692                         if 'host' not in machine_def:
 693                             host = self.runner.cfg.VARS.hostname
 694                         else:
 695                             host = machine_def.host
 696
 697                         if 'user' not in machine_def:
 698                             user = self.runner.cfg.VARS.user
 699                         else:
 700                             user = machine_def.user
 701
 702                         if 'port' not in machine_def:
 703                             port = 22
 704                         else:
 705                             port = machine_def.port
 706
 707                         if 'password' not in machine_def:
 708                             passwd = None
 709                         else:
 710                             passwd = machine_def.password
 711
 712                         if 'sat_path' not in machine_def:
 713                             sat_path = "salomeTools"
 714                         else:
 715                             sat_path = machine_def.sat_path
 716
 717                         a_machine = Machine(
 718                                             machine_def.name,
 719                                             host,
 720                                             user,
 721                                             port=port,
 722                                             passwd=passwd,
 723                                             sat_path=sat_path
 724                                             )
 725
 726                         self.lmachines.append(a_machine)
 727                         if (host, port) not in host_list:
 728                             host_list.append((host, port))
 729
 730                 if a_machine == None:
 731                     msg = _("WARNING: The job \"%(job_name)s\" requires the "
 732                             "machine \"%(machine_name)s\" but this machine "
 733                             "is not defined in the configuration file.\n"
 734                             "The job will not be launched")
 735                     self.logger.write(src.printcolors.printcWarning(msg))
 736
 737             a_job = self.define_job(job_def, a_machine)
 738
 739             if today in job_def.when:
 740                 self.ljobs.append(a_job)
 741             else: # today in job_def.when
 742                 self.ljobs_not_today.append(a_job)
 743
 744         self.lhosts = host_list
 745
 746     def ssh_connection_all_machines(self, pad=50):
 747         '''Function that do the ssh connection to every machine
 748            to be used today.
 749
 750         :return: Nothing
 751         :rtype: N\A
 752         '''
 753         self.logger.write(src.printcolors.printcInfo((
 754                         "Establishing connection with all the machines :\n")))
 755         for machine in self.lmachines:
 756             # little algorithm in order to display traces
 757             begin_line = (_("Connection to %s: " % machine.name))
 758             if pad - len(begin_line) < 0:
 759                 endline = " "
 760             else:
 761                 endline = (pad - len(begin_line)) * "." + " "
 762
 763             step = "SSH connection"
 764             self.logger.write( begin_line + endline + step)
 765             self.logger.flush()
 766             # the call to the method that initiate the ssh connection
 767             msg = machine.connect(self.logger)
 768
 769             # Copy salomeTools to the remote machine
 770             if machine.successfully_connected(self.logger):
 771                 step = _("Copy SAT")
 772                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 773                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 774                 self.logger.flush()
 775                 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
 776                                             self.job_file_path)
 777                 # get the remote machine distribution using a sat command
 778                 (__, out_dist, __) = machine.exec_command(
 779                                 os.path.join(machine.sat_path,
 780                                     "sat config --value VARS.dist --no_label"),
 781                                 self.logger)
 782                 machine.distribution = out_dist.read().decode().replace("\n",
 783                                                                         "")
 784                 # Print the status of the copy
 785                 if res_copy == 0:
 786                     self.logger.write('\r%s' %
 787                                 ((len(begin_line)+len(endline)+20) * " "), 3)
 788                     self.logger.write('\r%s%s%s' %
 789                         (begin_line,
 790                          endline,
 791                          src.printcolors.printc(src.OK_STATUS)), 3)
 792                 else:
 793                     self.logger.write('\r%s' %
 794                             ((len(begin_line)+len(endline)+20) * " "), 3)
 795                     self.logger.write('\r%s%s%s %s' %
 796                         (begin_line,
 797                          endline,
 798                          src.printcolors.printc(src.OK_STATUS),
 799                          _("Copy of SAT failed")), 3)
 800             else:
 801                 self.logger.write('\r%s' %
 802                                   ((len(begin_line)+len(endline)+20) * " "), 3)
 803                 self.logger.write('\r%s%s%s %s' %
 804                     (begin_line,
 805                      endline,
 806                      src.printcolors.printc(src.KO_STATUS),
 807                      msg), 3)
 808             self.logger.write("\n", 3)
 809
 810         self.logger.write("\n")
 811
 812
 813     def is_occupied(self, hostname):
 814         '''Function that returns True if a job is running on
 815            the machine defined by its host and its port.
 816
 817         :param hostname (str, int): the pair (host, port)
 818         :return: the job that is running on the host,
 819                 or false if there is no job running on the host.
 820         :rtype: job / bool
 821         '''
 822         host = hostname[0]
 823         port = hostname[1]
 824         for jb in self.ljobs:
 825             if jb.machine.host == host and jb.machine.port == port:
 826                 if jb.is_running():
 827                     return jb
 828         return False
 829
 830     def update_jobs_states_list(self):
 831         '''Function that updates the lists that store the currently
 832            running jobs and the jobs that have already finished.
 833
 834         :return: Nothing.
 835         :rtype: N\A
 836         '''
 837         jobs_finished_list = []
 838         jobs_running_list = []
 839         for jb in self.ljobs:
 840             if jb.is_running():
 841                 jobs_running_list.append(jb)
 842                 jb.check_time()
 843             if jb.has_finished():
 844                 jobs_finished_list.append(jb)
 845
 846         nb_job_finished_before = len(self._l_jobs_finished)
 847         self._l_jobs_finished = jobs_finished_list
 848         self._l_jobs_running = jobs_running_list
 849
 850         nb_job_finished_now = len(self._l_jobs_finished)
 851
 852         return nb_job_finished_now > nb_job_finished_before
 853
 854     def cancel_dependencies_of_failing_jobs(self):
 855         '''Function that cancels all the jobs that depend on a failing one.
 856
 857         :return: Nothing.
 858         :rtype: N\A
 859         '''
 860
 861         for job in self.ljobs:
 862             if job.after is None:
 863                 continue
 864             father_job = self.find_job_that_has_name(job.after)
 865             if father_job is not None and father_job.has_failed():
 866                 job.cancel()
 867
 868     def find_job_that_has_name(self, name):
 869         '''Returns the job by its name.
 870
 871         :param name str: a job name
 872         :return: the job that has the name.
 873         :rtype: job
 874         '''
 875         for jb in self.ljobs:
 876             if jb.name == name:
 877                 return jb
 878         # the following is executed only if the job was not found
 879         return None
 880
 881     def str_of_length(self, text, length):
 882         '''Takes a string text of any length and returns
 883            the most close string of length "length".
 884
 885         :param text str: any string
 886         :param length int: a length for the returned string
 887         :return: the most close string of length "length"
 888         :rtype: str
 889         '''
 890         if len(text) > length:
 891             text_out = text[:length-3] + '...'
 892         else:
 893             diff = length - len(text)
 894             before = " " * (diff//2)
 895             after = " " * (diff//2 + diff%2)
 896             text_out = before + text + after
 897
 898         return text_out
 899
 900     def display_status(self, len_col):
 901         '''Takes a lenght and construct the display of the current status
 902            of the jobs in an array that has a column for each host.
 903            It displays the job that is currently running on the host
 904            of the column.
 905
 906         :param len_col int: the size of the column
 907         :return: Nothing
 908         :rtype: N\A
 909         '''
 910
 911         display_line = ""
 912         for host_port in self.lhosts:
 913             jb = self.is_occupied(host_port)
 914             if not jb: # nothing running on the host
 915                 empty = self.str_of_length("empty", len_col)
 916                 display_line += "|" + empty
 917             else:
 918                 display_line += "|" + src.printcolors.printcInfo(
 919                                         self.str_of_length(jb.name, len_col))
 920
 921         self.logger.write("\r" + display_line + "|")
 922         self.logger.flush()
 923
 924
 925     def run_jobs(self):
 926         '''The main method. Runs all the jobs on every host.
 927            For each host, at a given time, only one job can be running.
 928            The jobs that have the field after (that contain the job that has
 929            to be run before it) are run after the previous job.
 930            This method stops when all the jobs are finished.
 931
 932         :return: Nothing
 933         :rtype: N\A
 934         '''
 935
 936         # Print header
 937         self.logger.write(src.printcolors.printcInfo(
 938                                                 _('Executing the jobs :\n')))
 939         text_line = ""
 940         for host_port in self.lhosts:
 941             host = host_port[0]
 942             port = host_port[1]
 943             if port == 22: # default value
 944                 text_line += "|" + self.str_of_length(host, self.len_columns)
 945             else:
 946                 text_line += "|" + self.str_of_length(
 947                                 "("+host+", "+str(port)+")", self.len_columns)
 948
 949         tiret_line = " " + "-"*(len(text_line)-1) + "\n"
 950         self.logger.write(tiret_line)
 951         self.logger.write(text_line + "|\n")
 952         self.logger.write(tiret_line)
 953         self.logger.flush()
 954
 955         # The infinite loop that runs the jobs
 956         l_jobs_not_started = src.deepcopy_list(self.ljobs)
 957         while len(self._l_jobs_finished) != len(self.ljobs):
 958             new_job_start = False
 959             for host_port in self.lhosts:
 960
 961                 if self.is_occupied(host_port):
 962                     continue
 963
 964                 for jb in l_jobs_not_started:
 965                     if (jb.machine.host, jb.machine.port) != host_port:
 966                         continue
 967                     if jb.after == None:
 968                         jb.run()
 969                         l_jobs_not_started.remove(jb)
 970                         new_job_start = True
 971                         break
 972                     else:
 973                         jb_before = self.find_job_that_has_name(jb.after)
 974                         if jb_before is None:
 975                             jb.cancel()
 976                             msg = _("This job was not launched because its "
 977                                     "father is not in the jobs list.")
 978                             jb.out = msg
 979                             jb.err = msg
 980                             break
 981                         if jb_before.has_finished():
 982                             jb.run()
 983                             l_jobs_not_started.remove(jb)
 984                             new_job_start = True
 985                             break
 986             self.cancel_dependencies_of_failing_jobs()
 987             new_job_finished = self.update_jobs_states_list()
 988
 989             if new_job_start or new_job_finished:
 990                 if self.gui:
 991                     self.gui.update_xml_files(self.ljobs)
 992                 # Display the current status
 993                 self.display_status(self.len_columns)
 994
 995             # Make sure that the proc is not entirely busy
 996             time.sleep(0.001)
 997
 998         self.logger.write("\n")
 999         self.logger.write(tiret_line)
1000         self.logger.write("\n\n")
1001
1002         if self.gui:
1003             self.gui.update_xml_files(self.ljobs)
1004             self.gui.last_update()
1005
1006     def write_all_results(self):
1007         '''Display all the jobs outputs.
1008
1009         :return: Nothing
1010         :rtype: N\A
1011         '''
1012
1013         for jb in self.ljobs:
1014             self.logger.write(src.printcolors.printcLabel(
1015                         "#------- Results for job %s -------#\n" % jb.name))
1016             jb.write_results()
1017             self.logger.write("\n\n")
1018
1019 class Gui(object):
1020     '''Class to manage the the xml data that can be displayed in a browser to
1021        see the jobs states
1022     '''
1023
1024     def __init__(self, xml_dir_path, l_jobs, l_jobs_not_today, file_boards=""):
1025         '''Initialization
1026
1027         :param xml_dir_path str: The path to the directory where to put
1028                                  the xml resulting files
1029         :param l_jobs List: the list of jobs that run today
1030         :param l_jobs_not_today List: the list of jobs that do not run today
1031         :param file_boards str: the file path from which to read the
1032                                    expected boards
1033         '''
1034         # The path of the csv files to read to fill the expected boards
1035         self.file_boards = file_boards
1036
1037         today = datetime.date.weekday(datetime.date.today())
1038         self.parse_csv_boards(today)
1039
1040         # The path of the global xml file
1041         self.xml_dir_path = xml_dir_path
1042         # Initialize the xml files
1043         xml_global_path = os.path.join(self.xml_dir_path, "global_report.xml")
1044         self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1045                                                          "JobsReport")
1046         # The xml files that corresponds to the boards.
1047         # {name_board : xml_object}}
1048         self.d_xml_board_files = {}
1049         # Create the lines and columns
1050         self.initialize_boards(l_jobs, l_jobs_not_today)
1051
1052         # Write the xml file
1053         self.update_xml_files(l_jobs)
1054
1055     def add_xml_board(self, name):
1056         xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1057         self.d_xml_board_files[name] =  src.xmlManager.XmlLogFile(
1058                                                     xml_board_path,
1059                                                     "JobsReport")
1060         self.d_xml_board_files[name].add_simple_node("distributions")
1061         self.d_xml_board_files[name].add_simple_node("applications")
1062         self.d_xml_board_files[name].add_simple_node("board", text=name)
1063
1064     def initialize_boards(self, l_jobs, l_jobs_not_today):
1065         '''Get all the first information needed for each file and write the
1066            first version of the files
1067         :param l_jobs List: the list of jobs that run today
1068         :param l_jobs_not_today List: the list of jobs that do not run today
1069         '''
1070         # Get the boards to fill and put it in a dictionary
1071         # {board_name : xml instance corresponding to the board}
1072         for job in l_jobs + l_jobs_not_today:
1073             board = job.board
1074             if (board is not None and
1075                                 board not in self.d_xml_board_files.keys()):
1076                 self.add_xml_board(board)
1077
1078         # Verify that the boards given as input are done
1079         for board in list(self.d_input_boards.keys()):
1080             if board not in self.d_xml_board_files:
1081                 self.add_xml_board(board)
1082             root_node = self.d_xml_board_files[board].xmlroot
1083             src.xmlManager.append_node_attrib(root_node,
1084                                               {"input_file" : self.file_boards})
1085
1086         # Loop over all jobs in order to get the lines and columns for each
1087         # xml file
1088         d_dist = {}
1089         d_application = {}
1090         for board in self.d_xml_board_files:
1091             d_dist[board] = []
1092             d_application[board] = []
1093
1094         l_hosts_ports = []
1095
1096         for job in l_jobs + l_jobs_not_today:
1097
1098             if (job.machine.host, job.machine.port) not in l_hosts_ports:
1099                 l_hosts_ports.append((job.machine.host, job.machine.port))
1100
1101             distrib = job.machine.distribution
1102             application = job.application
1103
1104             board_job = job.board
1105             if board is None:
1106                 continue
1107             for board in self.d_xml_board_files:
1108                 if board_job == board:
1109                     if distrib is not None and distrib not in d_dist[board]:
1110                         d_dist[board].append(distrib)
1111                         src.xmlManager.add_simple_node(
1112                             self.d_xml_board_files[board].xmlroot.find(
1113                                                             'distributions'),
1114                                                    "dist",
1115                                                    attrib={"name" : distrib})
1116
1117                 if board_job == board:
1118                     if (application is not None and
1119                                     application not in d_application[board]):
1120                         d_application[board].append(application)
1121                         src.xmlManager.add_simple_node(
1122                             self.d_xml_board_files[board].xmlroot.find(
1123                                                                 'applications'),
1124                                                    "application",
1125                                                    attrib={
1126                                                         "name" : application})
1127
1128         # Verify that there are no missing application or distribution in the
1129         # xml board files (regarding the input boards)
1130         for board in self.d_xml_board_files:
1131             l_dist = d_dist[board]
1132             if board not in self.d_input_boards.keys():
1133                 continue
1134             for dist in self.d_input_boards[board]["rows"]:
1135                 if dist not in l_dist:
1136                     src.xmlManager.add_simple_node(
1137                             self.d_xml_board_files[board].xmlroot.find(
1138                                                             'distributions'),
1139                                                    "dist",
1140                                                    attrib={"name" : dist})
1141             l_appli = d_application[board]
1142             for appli in self.d_input_boards[board]["columns"]:
1143                 if appli not in l_appli:
1144                     src.xmlManager.add_simple_node(
1145                             self.d_xml_board_files[board].xmlroot.find(
1146                                                                 'applications'),
1147                                                    "application",
1148                                                    attrib={"name" : appli})
1149
1150         # Initialize the hosts_ports node for the global file
1151         self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1152                                                                 "hosts_ports")
1153         for host, port in l_hosts_ports:
1154             host_port = "%s:%i" % (host, port)
1155             src.xmlManager.add_simple_node(self.xmlhosts_ports,
1156                                            "host_port",
1157                                            attrib={"name" : host_port})
1158
1159         # Initialize the jobs node in all files
1160         for xml_file in [self.xml_global_file] + list(
1161                                             self.d_xml_board_files.values()):
1162             xml_jobs = xml_file.add_simple_node("jobs")
1163             # Get the jobs present in the config file but
1164             # that will not be launched today
1165             self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1166
1167             xml_file.add_simple_node("infos",
1168                                      attrib={"name" : "last update",
1169                                              "JobsCommandStatus" : "running"})
1170
1171         # Find in each board the squares that needs to be filled regarding the
1172         # input csv files but that are not covered by a today job
1173         for board in self.d_input_boards.keys():
1174             xml_root_board = self.d_xml_board_files[board].xmlroot
1175             xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1176                                                  "missing_jobs")
1177             for row, column in self.d_input_boards[board]["jobs"]:
1178                 found = False
1179                 for job in l_jobs:
1180                     if (job.application == column and
1181                         job.machine.distribution == row):
1182                         found = True
1183                         break
1184                 if not found:
1185                     src.xmlManager.add_simple_node(xml_missing,
1186                                             "job",
1187                                             attrib={"distribution" : row,
1188                                                     "application" : column })
1189
1190     def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1191         '''Get all the first information needed for each file and write the
1192            first version of the files
1193
1194         :param xml_node_jobs etree.Element: the node corresponding to a job
1195         :param l_jobs_not_today List: the list of jobs that do not run today
1196         '''
1197         for job in l_jobs_not_today:
1198             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1199                                                  "job",
1200                                                  attrib={"name" : job.name})
1201             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1202             src.xmlManager.add_simple_node(xmlj,
1203                                            "distribution",
1204                                            job.machine.distribution)
1205             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1206             src.xmlManager.add_simple_node(xmlj,
1207                                        "commands", " ; ".join(job.commands))
1208             src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1209             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1210             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1211             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1212             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1213             src.xmlManager.add_simple_node(xmlj, "sat_path",
1214                                                         job.machine.sat_path)
1215
1216     def parse_csv_boards(self, today):
1217         """ Parse the csv files that describes the boards to produce and fill
1218             the dict d_input_boards that contain the csv file contain
1219
1220         :param today int: the current day of the week
1221         """
1222         # loop over each csv file and read its content
1223         l_read = []
1224         with open(self.file_boards, 'r') as f:
1225             reader = csv.reader(f,delimiter=CSV_DELIMITER)
1226             for row in reader:
1227                 l_read.append(row)
1228         # get the delimiter for the boards (empty line)
1229         boards_delimiter = [''] * len(l_read[0])
1230         # Make the list of boards, by splitting with the delimiter
1231         l_boards = [list(y) for x, y in itertools.groupby(l_read,
1232                                     lambda z: z == boards_delimiter) if not x]
1233
1234         # loop over the csv lists of lines and get the rows, columns and jobs
1235         d_boards = {}
1236         for input_board in l_boards:
1237             # get board name
1238             board_name = input_board[0][0]
1239
1240             # Get columns list
1241             columns = input_board[0][1:]
1242
1243             rows = []
1244             jobs = []
1245             for line in input_board[1:]:
1246                 row = line[0]
1247                 for i, square in enumerate(line[1:]):
1248                     if square=='':
1249                         continue
1250                     days = square.split(DAYS_SEPARATOR)
1251                     days = [int(day) for day in days]
1252                     if today in days:
1253                         if row not in rows:
1254                             rows.append(row)
1255                         job = (row, columns[i])
1256                         jobs.append(job)
1257
1258             d_boards[board_name] = {"rows" : rows,
1259                                     "columns" : columns,
1260                                     "jobs" : jobs}
1261
1262         self.d_input_boards = d_boards
1263
1264     def update_xml_files(self, l_jobs):
1265         '''Write all the xml files with updated information about the jobs
1266
1267         :param l_jobs List: the list of jobs that run today
1268         '''
1269         for xml_file in [self.xml_global_file] + list(
1270                                             self.d_xml_board_files.values()):
1271             self.update_xml_file(l_jobs, xml_file)
1272
1273         # Write the file
1274         self.write_xml_files()
1275
1276     def update_xml_file(self, l_jobs, xml_file):
1277         '''update information about the jobs for the file xml_file
1278
1279         :param l_jobs List: the list of jobs that run today
1280         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1281         '''
1282
1283         xml_node_jobs = xml_file.xmlroot.find('jobs')
1284         # Update the job names and status node
1285         for job in l_jobs:
1286             # Find the node corresponding to the job and delete it
1287             # in order to recreate it
1288             for xmljob in xml_node_jobs.findall('job'):
1289                 if xmljob.attrib['name'] == job.name:
1290                     xml_node_jobs.remove(xmljob)
1291
1292             T0 = str(job._T0)
1293             if T0 != "-1":
1294                 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1295                                        time.localtime(job._T0))
1296             Tf = str(job._Tf)
1297             if Tf != "-1":
1298                 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1299                                        time.localtime(job._Tf))
1300
1301             # recreate the job node
1302             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1303                                                   "job",
1304                                                   attrib={"name" : job.name})
1305             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1306             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1307             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1308             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1309             src.xmlManager.add_simple_node(xmlj, "sat_path",
1310                                            job.machine.sat_path)
1311             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1312             src.xmlManager.add_simple_node(xmlj, "distribution",
1313                                            job.machine.distribution)
1314             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1315             src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1316             src.xmlManager.add_simple_node(xmlj, "commands",
1317                                            " ; ".join(job.commands))
1318             src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1319             src.xmlManager.add_simple_node(xmlj, "begin", T0)
1320             src.xmlManager.add_simple_node(xmlj, "end", Tf)
1321             src.xmlManager.add_simple_node(xmlj, "out",
1322                                            src.printcolors.cleancolor(job.out))
1323             src.xmlManager.add_simple_node(xmlj, "err",
1324                                            src.printcolors.cleancolor(job.err))
1325             src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1326             if len(job.remote_log_files) > 0:
1327                 src.xmlManager.add_simple_node(xmlj,
1328                                                "remote_log_file_path",
1329                                                job.remote_log_files[0])
1330             else:
1331                 src.xmlManager.add_simple_node(xmlj,
1332                                                "remote_log_file_path",
1333                                                "nothing")
1334
1335             xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1336             # get the job father
1337             if job.after is not None:
1338                 job_father = None
1339                 for jb in l_jobs:
1340                     if jb.name == job.after:
1341                         job_father = jb
1342
1343                 if (job_father is not None and
1344                         len(job_father.remote_log_files) > 0):
1345                     link = job_father.remote_log_files[0]
1346                 else:
1347                     link = "nothing"
1348                 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1349
1350             # Verify that the job is to be done today regarding the input csv
1351             # files
1352             if job.board and job.board in self.d_input_boards.keys():
1353                 found = False
1354                 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1355                     if (job.machine.distribution == dist
1356                         and job.application == appli):
1357                         found = True
1358                         src.xmlManager.add_simple_node(xmlj,
1359                                                "extra_job",
1360                                                "no")
1361                         break
1362                 if not found:
1363                     src.xmlManager.add_simple_node(xmlj,
1364                                                "extra_job",
1365                                                "yes")
1366
1367
1368         # Update the date
1369         xml_node_infos = xml_file.xmlroot.find('infos')
1370         src.xmlManager.append_node_attrib(xml_node_infos,
1371                     attrib={"value" :
1372                     datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1373
1374
1375
1376     def last_update(self, finish_status = "finished"):
1377         '''update information about the jobs for the file xml_file
1378
1379         :param l_jobs List: the list of jobs that run today
1380         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1381         '''
1382         for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1383             xml_node_infos = xml_file.xmlroot.find('infos')
1384             src.xmlManager.append_node_attrib(xml_node_infos,
1385                         attrib={"JobsCommandStatus" : finish_status})
1386         # Write the file
1387         self.write_xml_files()
1388
1389     def write_xml_files(self):
1390         ''' Write the xml files
1391         '''
1392         self.xml_global_file.write_tree(STYLESHEET_GLOBAL)
1393         for xml_file in self.d_xml_board_files.values():
1394             xml_file.write_tree(STYLESHEET_BOARD)
1395
1396 ##
1397 # Describes the command
1398 def description():
1399     return _("The jobs command launches maintenances that are described"
1400              " in the dedicated jobs configuration file.")
1401
1402 ##
1403 # Runs the command.
1404 def run(args, runner, logger):
1405
1406     (options, args) = parser.parse_args(args)
1407
1408     l_cfg_dir = runner.cfg.PATHS.JOBPATH
1409
1410     # list option : display all the available config files
1411     if options.list:
1412         for cfg_dir in l_cfg_dir:
1413             if not options.no_label:
1414                 logger.write("------ %s\n" %
1415                                  src.printcolors.printcHeader(cfg_dir))
1416
1417             for f in sorted(os.listdir(cfg_dir)):
1418                 if not f.endswith('.pyconf'):
1419                     continue
1420                 cfilename = f[:-7]
1421                 logger.write("%s\n" % cfilename)
1422         return 0
1423
1424     # Make sure the jobs_config option has been called
1425     if not options.jobs_cfg:
1426         message = _("The option --jobs_config is required\n")
1427         src.printcolors.printcError(message)
1428         return 1
1429
1430     # Find the file in the directories
1431     found = False
1432     for cfg_dir in l_cfg_dir:
1433         file_jobs_cfg = os.path.join(cfg_dir, options.jobs_cfg)
1434         if not file_jobs_cfg.endswith('.pyconf'):
1435             file_jobs_cfg += '.pyconf'
1436
1437         if not os.path.exists(file_jobs_cfg):
1438             continue
1439         else:
1440             found = True
1441             break
1442
1443     if not found:
1444         msg = _("The file configuration %(name_file)s was not found."
1445                 "\nUse the --list option to get the possible files.")
1446         src.printcolors.printcError(msg)
1447         return 1
1448
1449     info = [
1450         (_("Platform"), runner.cfg.VARS.dist),
1451         (_("File containing the jobs configuration"), file_jobs_cfg)
1452     ]
1453     src.print_info(logger, info)
1454
1455     # Read the config that is in the file
1456     config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1457     if options.only_jobs:
1458         l_jb = src.pyconf.Sequence()
1459         for jb in config_jobs.jobs:
1460             if jb.name in options.only_jobs:
1461                 l_jb.append(jb,
1462                 "Adding a job that was given in only_jobs option parameters")
1463         config_jobs.jobs = l_jb
1464
1465     # Initialization
1466     today_jobs = Jobs(runner,
1467                       logger,
1468                       file_jobs_cfg,
1469                       config_jobs)
1470     # SSH connection to all machines
1471     today_jobs.ssh_connection_all_machines()
1472     if options.test_connection:
1473         return 0
1474
1475     gui = None
1476     if options.publish:
1477         # Copy the stylesheets in the log directory
1478         log_dir = runner.cfg.SITE.log.log_dir
1479         xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1480         files_to_copy = []
1481         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1482         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1483         files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1484         for file_path in files_to_copy:
1485             shutil.copy2(file_path, log_dir)
1486
1487         # Instanciate the Gui in order to produce the xml files that contain all
1488         # the boards
1489         gui = Gui(runner.cfg.SITE.log.log_dir,
1490                   today_jobs.ljobs,
1491                   today_jobs.ljobs_not_today,
1492                   file_boards = options.input_boards)
1493
1494         # Display the list of the xml files
1495         logger.write(src.printcolors.printcInfo(("Here is the list of published"
1496                                                  " files :\n")), 4)
1497         logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1498         for board in gui.d_xml_board_files.keys():
1499             file_path = gui.d_xml_board_files[board].logFile
1500             file_name = os.path.basename(file_path)
1501             logger.write("%s\n" % file_path, 4)
1502             logger.add_link(file_name, "board", 0, board)
1503
1504         logger.write("\n", 4)
1505
1506     today_jobs.gui = gui
1507
1508     interruped = False
1509     try:
1510         # Run all the jobs contained in config_jobs
1511         today_jobs.run_jobs()
1512     except KeyboardInterrupt:
1513         interruped = True
1514         logger.write("\n\n%s\n\n" %
1515                 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1516     finally:
1517         if interruped:
1518             msg = _("Killing the running jobs and trying"
1519                     " to get the corresponding logs\n")
1520             logger.write(src.printcolors.printcWarning(msg))
1521
1522         # find the potential not finished jobs and kill them
1523         for jb in today_jobs.ljobs:
1524             if not jb.has_finished():
1525                 try:
1526                     jb.kill_remote_process()
1527                 except Exception as e:
1528                     msg = _("Failed to kill job %s: %s\n" % (jb.name, e))
1529                     logger.write(src.printcolors.printcWarning(msg))
1530         if interruped:
1531             if today_jobs.gui:
1532                 today_jobs.gui.last_update(_("Forced interruption"))
1533         else:
1534             if today_jobs.gui:
1535                 today_jobs.gui.last_update()
1536         # Output the results
1537         today_jobs.write_all_results()