commands/jobs.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import datetime
  21 import time
  22 import csv
  23 import shutil
  24 import paramiko
  25
  26 import src
  27
  28 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
  29 STYLESHEET_BOARD = "jobs_board_report.xsl"
  30 d_INT_DAY = {0 : "monday",
  31              1 : "tuesday",
  32              2 : "wednesday",
  33              3 : "thursday",
  34              4 : "friday",
  35              5 : "saturday",
  36              6 : "sunday"}
  37
  38 parser = src.options.Options()
  39
  40 parser.add_option('n', 'name', 'string', 'jobs_cfg',
  41                   _('The name of the config file that contains'
  42                   ' the jobs configuration'))
  43 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
  44                   _('Optional: the list of jobs to launch, by their name. '))
  45 parser.add_option('l', 'list', 'boolean', 'list',
  46                   _('Optional: list all available config files.'))
  47 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
  48                   _("Optional: try to connect to the machines. "
  49                     "Not executing the jobs."),
  50                   False)
  51 parser.add_option('p', 'publish', 'boolean', 'publish',
  52                   _("Optional: generate an xml file that can be read in a "
  53                     "browser to display the jobs status."),
  54                   False)
  55 parser.add_option('i', 'input_boards', 'list2', 'input_boards', _("Optional: "
  56                                 "the list of path to csv files that contain "
  57                                 "the expected boards."),[])
  58 parser.add_option('n', 'completion', 'boolean', 'no_label',
  59                   _("Optional (internal use): do not print labels, Works only "
  60                     "with --list."),
  61                   False)
  62
  63 class Machine(object):
  64     '''Class to manage a ssh connection on a machine
  65     '''
  66     def __init__(self,
  67                  name,
  68                  host,
  69                  user,
  70                  port=22,
  71                  passwd=None,
  72                  sat_path="salomeTools"):
  73         self.name = name
  74         self.host = host
  75         self.port = port
  76         self.distribution = None # Will be filled after copying SAT on the machine
  77         self.user = user
  78         self.password = passwd
  79         self.sat_path = sat_path
  80         self.ssh = paramiko.SSHClient()
  81         self._connection_successful = None
  82
  83     def connect(self, logger):
  84         '''Initiate the ssh connection to the remote machine
  85
  86         :param logger src.logger.Logger: The logger instance
  87         :return: Nothing
  88         :rtype: N\A
  89         '''
  90
  91         self._connection_successful = False
  92         self.ssh.load_system_host_keys()
  93         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  94         try:
  95             self.ssh.connect(self.host,
  96                              port=self.port,
  97                              username=self.user,
  98                              password = self.password)
  99         except paramiko.AuthenticationException:
 100             message = src.KO_STATUS + _("Authentication failed")
 101         except paramiko.BadHostKeyException:
 102             message = (src.KO_STATUS +
 103                        _("The server's host key could not be verified"))
 104         except paramiko.SSHException:
 105             message = ( _("SSHException error connecting or "
 106                           "establishing an SSH session"))
 107         except:
 108             message = ( _("Error connecting or establishing an SSH session"))
 109         else:
 110             self._connection_successful = True
 111             message = ""
 112         return message
 113
 114     def successfully_connected(self, logger):
 115         '''Verify if the connection to the remote machine has succeed
 116
 117         :param logger src.logger.Logger: The logger instance
 118         :return: True if the connection has succeed, False if not
 119         :rtype: bool
 120         '''
 121         if self._connection_successful == None:
 122             message = _("Warning : trying to ask if the connection to "
 123             "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
 124             " no connection request" %
 125                         (self.name, self.host, self.port, self.user))
 126             logger.write( src.printcolors.printcWarning(message))
 127         return self._connection_successful
 128
 129     def copy_sat(self, sat_local_path, job_file):
 130         '''Copy salomeTools to the remote machine in self.sat_path
 131         '''
 132         res = 0
 133         try:
 134             # open a sftp connection
 135             self.sftp = self.ssh.open_sftp()
 136             # Create the sat directory on remote machine if it is not existing
 137             self.mkdir(self.sat_path, ignore_existing=True)
 138             # Put sat
 139             self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
 140             # put the job configuration file in order to make it reachable
 141             # on the remote machine
 142             self.sftp.put(job_file, os.path.join(".salomeTools",
 143                                                  "Jobs",
 144                                                  ".jobs_command_file.pyconf"))
 145         except Exception as e:
 146             res = str(e)
 147             self._connection_successful = False
 148
 149         return res
 150
 151     def put_dir(self, source, target, filters = []):
 152         ''' Uploads the contents of the source directory to the target path. The
 153             target directory needs to exists. All sub-directories in source are
 154             created under target.
 155         '''
 156         for item in os.listdir(source):
 157             if item in filters:
 158                 continue
 159             source_path = os.path.join(source, item)
 160             destination_path = os.path.join(target, item)
 161             if os.path.islink(source_path):
 162                 linkto = os.readlink(source_path)
 163                 try:
 164                     self.sftp.symlink(linkto, destination_path)
 165                     self.sftp.chmod(destination_path,
 166                                     os.stat(source_path).st_mode)
 167                 except IOError:
 168                     pass
 169             else:
 170                 if os.path.isfile(source_path):
 171                     self.sftp.put(source_path, destination_path)
 172                     self.sftp.chmod(destination_path,
 173                                     os.stat(source_path).st_mode)
 174                 else:
 175                     self.mkdir(destination_path, ignore_existing=True)
 176                     self.put_dir(source_path, destination_path)
 177
 178     def mkdir(self, path, mode=511, ignore_existing=False):
 179         ''' Augments mkdir by adding an option to not fail
 180             if the folder exists
 181         '''
 182         try:
 183             self.sftp.mkdir(path, mode)
 184         except IOError:
 185             if ignore_existing:
 186                 pass
 187             else:
 188                 raise
 189
 190     def exec_command(self, command, logger):
 191         '''Execute the command on the remote machine
 192
 193         :param command str: The command to be run
 194         :param logger src.logger.Logger: The logger instance
 195         :return: the stdin, stdout, and stderr of the executing command,
 196                  as a 3-tuple
 197         :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
 198                 paramiko.channel.ChannelFile)
 199         '''
 200         try:
 201             # Does not wait the end of the command
 202             (stdin, stdout, stderr) = self.ssh.exec_command(command)
 203         except paramiko.SSHException:
 204             message = src.KO_STATUS + _(
 205                             ": the server failed to execute the command\n")
 206             logger.write( src.printcolors.printcError(message))
 207             return (None, None, None)
 208         except:
 209             logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
 210             return (None, None, None)
 211         else:
 212             return (stdin, stdout, stderr)
 213
 214     def close(self):
 215         '''Close the ssh connection
 216
 217         :rtype: N\A
 218         '''
 219         self.ssh.close()
 220
 221     def write_info(self, logger):
 222         '''Prints the informations relative to the machine in the logger
 223            (terminal traces and log file)
 224
 225         :param logger src.logger.Logger: The logger instance
 226         :return: Nothing
 227         :rtype: N\A
 228         '''
 229         logger.write("host : " + self.host + "\n")
 230         logger.write("port : " + str(self.port) + "\n")
 231         logger.write("user : " + str(self.user) + "\n")
 232         if self.successfully_connected(logger):
 233             status = src.OK_STATUS
 234         else:
 235             status = src.KO_STATUS
 236         logger.write("Connection : " + status + "\n\n")
 237
 238
 239 class Job(object):
 240     '''Class to manage one job
 241     '''
 242     def __init__(self, name, machine, application, board,
 243                  commands, timeout, config, logger, after=None):
 244
 245         self.name = name
 246         self.machine = machine
 247         self.after = after
 248         self.timeout = timeout
 249         self.application = application
 250         self.board = board
 251         self.config = config
 252         self.logger = logger
 253         # The list of log files to download from the remote machine
 254         self.remote_log_files = []
 255
 256         # The remote command status
 257         # -1 means that it has not been launched,
 258         # 0 means success and 1 means fail
 259         self.res_job = "-1"
 260         self.cancelled = False
 261
 262         self._T0 = -1
 263         self._Tf = -1
 264         self._has_begun = False
 265         self._has_finished = False
 266         self._has_timouted = False
 267         self._stdin = None # Store the command inputs field
 268         self._stdout = None # Store the command outputs field
 269         self._stderr = None # Store the command errors field
 270
 271         self.out = ""
 272         self.err = ""
 273
 274         self.commands = commands
 275         self.command = (os.path.join(self.machine.sat_path, "sat") +
 276                         " -l " +
 277                         os.path.join(self.machine.sat_path,
 278                                      "list_log_files.txt") +
 279                         " job --jobs_config .jobs_command_file" +
 280                         " --name " +
 281                         self.name)
 282
 283     def get_pids(self):
 284         """ Get the pid(s) corresponding to the command that have been launched
 285             On the remote machine
 286
 287         :return: The list of integers corresponding to the found pids
 288         :rtype: List
 289         """
 290         pids = []
 291         cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
 292         (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
 293         pids_cmd = out_pid.readlines()
 294         pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
 295         pids+=pids_cmd
 296         return pids
 297
 298     def kill_remote_process(self, wait=1):
 299         '''Kills the process on the remote machine.
 300
 301         :return: (the output of the kill, the error of the kill)
 302         :rtype: (str, str)
 303         '''
 304
 305         pids = self.get_pids()
 306         cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
 307         (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
 308                                                             self.logger)
 309         time.sleep(wait)
 310         return (out_kill, err_kill)
 311
 312     def has_begun(self):
 313         '''Returns True if the job has already begun
 314
 315         :return: True if the job has already begun
 316         :rtype: bool
 317         '''
 318         return self._has_begun
 319
 320     def has_finished(self):
 321         '''Returns True if the job has already finished
 322            (i.e. all the commands have been executed)
 323            If it is finished, the outputs are stored in the fields out and err.
 324
 325         :return: True if the job has already finished
 326         :rtype: bool
 327         '''
 328
 329         # If the method has already been called and returned True
 330         if self._has_finished:
 331             return True
 332
 333         # If the job has not begun yet
 334         if not self.has_begun():
 335             return False
 336
 337         if self._stdout.channel.closed:
 338             self._has_finished = True
 339             # Store the result outputs
 340             self.out += self._stdout.read().decode()
 341             self.err += self._stderr.read().decode()
 342             # Put end time
 343             self._Tf = time.time()
 344             # And get the remote command status and log files
 345             self.get_log_files()
 346
 347         return self._has_finished
 348
 349     def get_log_files(self):
 350         """Get the log files produced by the command launched
 351            on the remote machine, and put it in the log directory of the user,
 352            so they can be accessible from
 353         """
 354         # Do not get the files if the command is not finished
 355         if not self.has_finished():
 356             msg = _("Trying to get log files whereas the job is not finished.")
 357             self.logger.write(src.printcolors.printcWarning(msg))
 358             return
 359
 360         # First get the file that contains the list of log files to get
 361         tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
 362         remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt")
 363         self.machine.sftp.get(
 364                     remote_path,
 365                     tmp_file_path)
 366
 367         # Read the file and get the result of the command and all the log files
 368         # to get
 369         fstream_tmp = open(tmp_file_path, "r")
 370         file_lines = fstream_tmp.readlines()
 371         file_lines = [line.replace("\n", "") for line in file_lines]
 372         fstream_tmp.close()
 373         os.remove(tmp_file_path)
 374
 375         try :
 376             # The first line is the result of the command (0 success or 1 fail)
 377             self.res_job = file_lines[0]
 378         except Exception as e:
 379             self.err += _("Unable to get status from remote file %s: %s" %
 380                                                     (remote_path, str(e)))
 381
 382         for i, job_path_remote in enumerate(file_lines[1:]):
 383             try:
 384                 # For each command, there is two files to get :
 385                 # 1- The xml file describing the command and giving the
 386                 # internal traces.
 387                 # 2- The txt file containing the system command traces (like
 388                 # traces produced by the "make" command)
 389                 # 3- In case of the test command, there is another file to get :
 390                 # the xml board that contain the test results
 391                 dirname = os.path.basename(os.path.dirname(job_path_remote))
 392                 if dirname != 'OUT' and dirname != 'TEST':
 393                     # Case 1-
 394                     local_path = os.path.join(os.path.dirname(
 395                                                         self.logger.logFilePath),
 396                                               os.path.basename(job_path_remote))
 397                     if i==0: # The first is the job command
 398                         self.logger.add_link(os.path.basename(job_path_remote),
 399                                              "job",
 400                                              self.res_job,
 401                                              self.command)
 402                 elif dirname == 'OUT':
 403                     # Case 2-
 404                     local_path = os.path.join(os.path.dirname(
 405                                                         self.logger.logFilePath),
 406                                               'OUT',
 407                                               os.path.basename(job_path_remote))
 408                 elif dirname == 'TEST':
 409                     # Case 3-
 410                     local_path = os.path.join(os.path.dirname(
 411                                                         self.logger.logFilePath),
 412                                               'TEST',
 413                                               os.path.basename(job_path_remote))
 414
 415                 # Get the file
 416                 if not os.path.exists(local_path):
 417                     self.machine.sftp.get(job_path_remote, local_path)
 418                 self.remote_log_files.append(local_path)
 419             except Exception as e:
 420                 self.err += _("Unable to get %s log file from remote: %s" %
 421                                                     (job_path_remote, str(e)))
 422
 423     def has_failed(self):
 424         '''Returns True if the job has failed.
 425            A job is considered as failed if the machine could not be reached,
 426            if the remote command failed,
 427            or if the job finished with a time out.
 428
 429         :return: True if the job has failed
 430         :rtype: bool
 431         '''
 432         if not self.has_finished():
 433             return False
 434         if not self.machine.successfully_connected(self.logger):
 435             return True
 436         if self.is_timeout():
 437             return True
 438         if self.res_job == "1":
 439             return True
 440         return False
 441
 442     def cancel(self):
 443         """In case of a failing job, one has to cancel every job that depend
 444            on it. This method put the job as failed and will not be executed.
 445         """
 446         self._has_begun = True
 447         self._has_finished = True
 448         self.cancelled = True
 449         self.out += _("This job was not launched because its father has failed.")
 450         self.err += _("This job was not launched because its father has failed.")
 451
 452     def is_running(self):
 453         '''Returns True if the job commands are running
 454
 455         :return: True if the job is running
 456         :rtype: bool
 457         '''
 458         return self.has_begun() and not self.has_finished()
 459
 460     def is_timeout(self):
 461         '''Returns True if the job commands has finished with timeout
 462
 463         :return: True if the job has finished with timeout
 464         :rtype: bool
 465         '''
 466         return self._has_timouted
 467
 468     def time_elapsed(self):
 469         """Get the time elapsed since the job launching
 470
 471         :return: The number of seconds
 472         :rtype: int
 473         """
 474         if not self.has_begun():
 475             return -1
 476         T_now = time.time()
 477         return T_now - self._T0
 478
 479     def check_time(self):
 480         """Verify that the job has not exceeded its timeout.
 481            If it has, kill the remote command and consider the job as finished.
 482         """
 483         if not self.has_begun():
 484             return
 485         if self.time_elapsed() > self.timeout:
 486             self._has_finished = True
 487             self._has_timouted = True
 488             self._Tf = time.time()
 489             self.get_pids()
 490             (out_kill, _) = self.kill_remote_process()
 491             self.out += "TIMEOUT \n" + out_kill.read().decode()
 492             self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
 493             try:
 494                 self.get_log_files()
 495             except Exception as e:
 496                 self.err += _("Unable to get remote log files: %s" % e)
 497
 498     def total_duration(self):
 499         """Give the total duration of the job
 500
 501         :return: the total duration of the job in seconds
 502         :rtype: int
 503         """
 504         return self._Tf - self._T0
 505
 506     def run(self):
 507         """Launch the job by executing the remote command.
 508         """
 509
 510         # Prevent multiple run
 511         if self.has_begun():
 512             msg = _("Warning: A job can only be launched one time")
 513             msg2 = _("Trying to launch the job \"%s\" whereas it has "
 514                      "already been launched." % self.name)
 515             self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
 516                                                                         msg2)))
 517             return
 518
 519         # Do not execute the command if the machine could not be reached
 520         if not self.machine.successfully_connected(self.logger):
 521             self._has_finished = True
 522             self.out = "N\A"
 523             self.err += ("Connection to machine (name : %s, host: %s, port:"
 524                         " %s, user: %s) has failed\nUse the log command "
 525                         "to get more information."
 526                         % (self.machine.name,
 527                            self.machine.host,
 528                            self.machine.port,
 529                            self.machine.user))
 530         else:
 531             # Usual case : Launch the command on remote machine
 532             self._T0 = time.time()
 533             self._stdin, self._stdout, self._stderr = self.machine.exec_command(
 534                                                                   self.command,
 535                                                                   self.logger)
 536             # If the results are not initialized, finish the job
 537             if (self._stdin, self._stdout, self._stderr) == (None, None, None):
 538                 self._has_finished = True
 539                 self._Tf = time.time()
 540                 self.out += "N\A"
 541                 self.err += "The server failed to execute the command"
 542
 543         # Put the beginning flag to true.
 544         self._has_begun = True
 545
 546     def write_results(self):
 547         """Display on the terminal all the job's information
 548         """
 549         self.logger.write("name : " + self.name + "\n")
 550         if self.after:
 551             self.logger.write("after : %s\n" % self.after)
 552         self.logger.write("Time elapsed : %4imin %2is \n" %
 553                      (self.total_duration()//60 , self.total_duration()%60))
 554         if self._T0 != -1:
 555             self.logger.write("Begin time : %s\n" %
 556                          time.strftime('%Y-%m-%d %H:%M:%S',
 557                                        time.localtime(self._T0)) )
 558         if self._Tf != -1:
 559             self.logger.write("End time   : %s\n\n" %
 560                          time.strftime('%Y-%m-%d %H:%M:%S',
 561                                        time.localtime(self._Tf)) )
 562
 563         machine_head = "Informations about connection :\n"
 564         underline = (len(machine_head) - 2) * "-"
 565         self.logger.write(src.printcolors.printcInfo(
 566                                                 machine_head+underline+"\n"))
 567         self.machine.write_info(self.logger)
 568
 569         self.logger.write(src.printcolors.printcInfo("out : \n"))
 570         if self.out == "":
 571             self.logger.write("Unable to get output\n")
 572         else:
 573             self.logger.write(self.out + "\n")
 574         self.logger.write(src.printcolors.printcInfo("err : \n"))
 575         self.logger.write(self.err + "\n")
 576
 577     def get_status(self):
 578         """Get the status of the job (used by the Gui for xml display)
 579
 580         :return: The current status of the job
 581         :rtype: String
 582         """
 583         if not self.machine.successfully_connected(self.logger):
 584             return "SSH connection KO"
 585         if not self.has_begun():
 586             return "Not launched"
 587         if self.cancelled:
 588             return "Cancelled"
 589         if self.is_running():
 590             return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
 591                                                     time.localtime(self._T0))
 592         if self.has_finished():
 593             if self.is_timeout():
 594                 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
 595                                                     time.localtime(self._Tf))
 596             return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
 597                                                      time.localtime(self._Tf))
 598
 599 class Jobs(object):
 600     '''Class to manage the jobs to be run
 601     '''
 602     def __init__(self,
 603                  runner,
 604                  logger,
 605                  job_file_path,
 606                  config_jobs,
 607                  lenght_columns = 20):
 608         # The jobs configuration
 609         self.cfg_jobs = config_jobs
 610         self.job_file_path = job_file_path
 611         # The machine that will be used today
 612         self.lmachines = []
 613         # The list of machine (hosts, port) that will be used today
 614         # (a same host can have several machine instances since there
 615         # can be several ssh parameters)
 616         self.lhosts = []
 617         # The jobs to be launched today
 618         self.ljobs = []
 619         # The jobs that will not be launched today
 620         self.ljobs_not_today = []
 621         self.runner = runner
 622         self.logger = logger
 623         self.len_columns = lenght_columns
 624
 625         # the list of jobs that have not been run yet
 626         self._l_jobs_not_started = []
 627         # the list of jobs that have already ran
 628         self._l_jobs_finished = []
 629         # the list of jobs that are running
 630         self._l_jobs_running = []
 631
 632         self.determine_jobs_and_machines()
 633
 634     def define_job(self, job_def, machine):
 635         '''Takes a pyconf job definition and a machine (from class machine)
 636            and returns the job instance corresponding to the definition.
 637
 638         :param job_def src.config.Mapping: a job definition
 639         :param machine machine: the machine on which the job will run
 640         :return: The corresponding job in a job class instance
 641         :rtype: job
 642         '''
 643         name = job_def.name
 644         cmmnds = job_def.commands
 645         timeout = job_def.timeout
 646         after = None
 647         if 'after' in job_def:
 648             after = job_def.after
 649         application = None
 650         if 'application' in job_def:
 651             application = job_def.application
 652         board = None
 653         if 'board' in job_def:
 654             board = job_def.board
 655
 656         return Job(name,
 657                    machine,
 658                    application,
 659                    board,
 660                    cmmnds,
 661                    timeout,
 662                    self.runner.cfg,
 663                    self.logger,
 664                    after = after)
 665
 666     def determine_jobs_and_machines(self):
 667         '''Function that reads the pyconf jobs definition and instantiates all
 668            the machines and jobs to be done today.
 669
 670         :return: Nothing
 671         :rtype: N\A
 672         '''
 673         today = datetime.date.weekday(datetime.date.today())
 674         host_list = []
 675
 676         for job_def in self.cfg_jobs.jobs :
 677
 678             if not "machine" in job_def:
 679                 msg = _('WARNING: The job "%s" do not have the key '
 680                        '"machine", this job is ignored.\n\n' % job_def.name)
 681                 self.logger.write(src.printcolors.printcWarning(msg))
 682                 continue
 683             name_machine = job_def.machine
 684
 685             a_machine = None
 686             for mach in self.lmachines:
 687                 if mach.name == name_machine:
 688                     a_machine = mach
 689                     break
 690
 691             if a_machine == None:
 692                 for machine_def in self.cfg_jobs.machines:
 693                     if machine_def.name == name_machine:
 694                         if 'host' not in machine_def:
 695                             host = self.runner.cfg.VARS.hostname
 696                         else:
 697                             host = machine_def.host
 698
 699                         if 'user' not in machine_def:
 700                             user = self.runner.cfg.VARS.user
 701                         else:
 702                             user = machine_def.user
 703
 704                         if 'port' not in machine_def:
 705                             port = 22
 706                         else:
 707                             port = machine_def.port
 708
 709                         if 'password' not in machine_def:
 710                             passwd = None
 711                         else:
 712                             passwd = machine_def.password
 713
 714                         if 'sat_path' not in machine_def:
 715                             sat_path = "salomeTools"
 716                         else:
 717                             sat_path = machine_def.sat_path
 718
 719                         a_machine = Machine(
 720                                             machine_def.name,
 721                                             host,
 722                                             user,
 723                                             port=port,
 724                                             passwd=passwd,
 725                                             sat_path=sat_path
 726                                             )
 727
 728                         self.lmachines.append(a_machine)
 729                         if (host, port) not in host_list:
 730                             host_list.append((host, port))
 731
 732                 if a_machine == None:
 733                     msg = _("WARNING: The job \"%(job_name)s\" requires the "
 734                             "machine \"%(machine_name)s\" but this machine "
 735                             "is not defined in the configuration file.\n"
 736                             "The job will not be launched")
 737                     self.logger.write(src.printcolors.printcWarning(msg))
 738
 739             a_job = self.define_job(job_def, a_machine)
 740
 741             if today in job_def.when:
 742                 self.ljobs.append(a_job)
 743             else: # today in job_def.when
 744                 self.ljobs_not_today.append(a_job)
 745
 746         self.lhosts = host_list
 747
 748     def ssh_connection_all_machines(self, pad=50):
 749         '''Function that do the ssh connection to every machine
 750            to be used today.
 751
 752         :return: Nothing
 753         :rtype: N\A
 754         '''
 755         self.logger.write(src.printcolors.printcInfo((
 756                         "Establishing connection with all the machines :\n")))
 757         for machine in self.lmachines:
 758             # little algorithm in order to display traces
 759             begin_line = (_("Connection to %s: " % machine.name))
 760             if pad - len(begin_line) < 0:
 761                 endline = " "
 762             else:
 763                 endline = (pad - len(begin_line)) * "." + " "
 764
 765             step = "SSH connection"
 766             self.logger.write( begin_line + endline + step)
 767             self.logger.flush()
 768             # the call to the method that initiate the ssh connection
 769             msg = machine.connect(self.logger)
 770
 771             # Copy salomeTools to the remote machine
 772             if machine.successfully_connected(self.logger):
 773                 step = _("Copy SAT")
 774                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 775                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 776                 self.logger.flush()
 777                 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
 778                                             self.job_file_path)
 779                 # get the remote machine distribution using a sat command
 780                 (__, out_dist, __) = machine.exec_command(
 781                                 os.path.join(machine.sat_path,
 782                                     "sat config --value VARS.dist --no_label"),
 783                                 self.logger)
 784                 machine.distribution = out_dist.read().decode().replace("\n",
 785                                                                         "")
 786                 # Print the status of the copy
 787                 if res_copy == 0:
 788                     self.logger.write('\r%s' %
 789                                 ((len(begin_line)+len(endline)+20) * " "), 3)
 790                     self.logger.write('\r%s%s%s' %
 791                         (begin_line,
 792                          endline,
 793                          src.printcolors.printc(src.OK_STATUS)), 3)
 794                 else:
 795                     self.logger.write('\r%s' %
 796                             ((len(begin_line)+len(endline)+20) * " "), 3)
 797                     self.logger.write('\r%s%s%s %s' %
 798                         (begin_line,
 799                          endline,
 800                          src.printcolors.printc(src.OK_STATUS),
 801                          _("Copy of SAT failed")), 3)
 802             else:
 803                 self.logger.write('\r%s' %
 804                                   ((len(begin_line)+len(endline)+20) * " "), 3)
 805                 self.logger.write('\r%s%s%s %s' %
 806                     (begin_line,
 807                      endline,
 808                      src.printcolors.printc(src.KO_STATUS),
 809                      msg), 3)
 810             self.logger.write("\n", 3)
 811
 812         self.logger.write("\n")
 813
 814
 815     def is_occupied(self, hostname):
 816         '''Function that returns True if a job is running on
 817            the machine defined by its host and its port.
 818
 819         :param hostname (str, int): the pair (host, port)
 820         :return: the job that is running on the host,
 821                 or false if there is no job running on the host.
 822         :rtype: job / bool
 823         '''
 824         host = hostname[0]
 825         port = hostname[1]
 826         for jb in self.ljobs:
 827             if jb.machine.host == host and jb.machine.port == port:
 828                 if jb.is_running():
 829                     return jb
 830         return False
 831
 832     def update_jobs_states_list(self):
 833         '''Function that updates the lists that store the currently
 834            running jobs and the jobs that have already finished.
 835
 836         :return: Nothing.
 837         :rtype: N\A
 838         '''
 839         jobs_finished_list = []
 840         jobs_running_list = []
 841         for jb in self.ljobs:
 842             if jb.is_running():
 843                 jobs_running_list.append(jb)
 844                 jb.check_time()
 845             if jb.has_finished():
 846                 jobs_finished_list.append(jb)
 847
 848         nb_job_finished_before = len(self._l_jobs_finished)
 849         self._l_jobs_finished = jobs_finished_list
 850         self._l_jobs_running = jobs_running_list
 851
 852         nb_job_finished_now = len(self._l_jobs_finished)
 853
 854         return nb_job_finished_now > nb_job_finished_before
 855
 856     def cancel_dependencies_of_failing_jobs(self):
 857         '''Function that cancels all the jobs that depend on a failing one.
 858
 859         :return: Nothing.
 860         :rtype: N\A
 861         '''
 862
 863         for job in self.ljobs:
 864             if job.after is None:
 865                 continue
 866             father_job = self.find_job_that_has_name(job.after)
 867             if father_job is not None and father_job.has_failed():
 868                 job.cancel()
 869
 870     def find_job_that_has_name(self, name):
 871         '''Returns the job by its name.
 872
 873         :param name str: a job name
 874         :return: the job that has the name.
 875         :rtype: job
 876         '''
 877         for jb in self.ljobs:
 878             if jb.name == name:
 879                 return jb
 880         # the following is executed only if the job was not found
 881         return None
 882
 883     def str_of_length(self, text, length):
 884         '''Takes a string text of any length and returns
 885            the most close string of length "length".
 886
 887         :param text str: any string
 888         :param length int: a length for the returned string
 889         :return: the most close string of length "length"
 890         :rtype: str
 891         '''
 892         if len(text) > length:
 893             text_out = text[:length-3] + '...'
 894         else:
 895             diff = length - len(text)
 896             before = " " * (diff//2)
 897             after = " " * (diff//2 + diff%2)
 898             text_out = before + text + after
 899
 900         return text_out
 901
 902     def display_status(self, len_col):
 903         '''Takes a lenght and construct the display of the current status
 904            of the jobs in an array that has a column for each host.
 905            It displays the job that is currently running on the host
 906            of the column.
 907
 908         :param len_col int: the size of the column
 909         :return: Nothing
 910         :rtype: N\A
 911         '''
 912
 913         display_line = ""
 914         for host_port in self.lhosts:
 915             jb = self.is_occupied(host_port)
 916             if not jb: # nothing running on the host
 917                 empty = self.str_of_length("empty", len_col)
 918                 display_line += "|" + empty
 919             else:
 920                 display_line += "|" + src.printcolors.printcInfo(
 921                                         self.str_of_length(jb.name, len_col))
 922
 923         self.logger.write("\r" + display_line + "|")
 924         self.logger.flush()
 925
 926
 927     def run_jobs(self):
 928         '''The main method. Runs all the jobs on every host.
 929            For each host, at a given time, only one job can be running.
 930            The jobs that have the field after (that contain the job that has
 931            to be run before it) are run after the previous job.
 932            This method stops when all the jobs are finished.
 933
 934         :return: Nothing
 935         :rtype: N\A
 936         '''
 937
 938         # Print header
 939         self.logger.write(src.printcolors.printcInfo(
 940                                                 _('Executing the jobs :\n')))
 941         text_line = ""
 942         for host_port in self.lhosts:
 943             host = host_port[0]
 944             port = host_port[1]
 945             if port == 22: # default value
 946                 text_line += "|" + self.str_of_length(host, self.len_columns)
 947             else:
 948                 text_line += "|" + self.str_of_length(
 949                                 "("+host+", "+str(port)+")", self.len_columns)
 950
 951         tiret_line = " " + "-"*(len(text_line)-1) + "\n"
 952         self.logger.write(tiret_line)
 953         self.logger.write(text_line + "|\n")
 954         self.logger.write(tiret_line)
 955         self.logger.flush()
 956
 957         # The infinite loop that runs the jobs
 958         l_jobs_not_started = src.deepcopy_list(self.ljobs)
 959         while len(self._l_jobs_finished) != len(self.ljobs):
 960             new_job_start = False
 961             for host_port in self.lhosts:
 962
 963                 if self.is_occupied(host_port):
 964                     continue
 965
 966                 for jb in l_jobs_not_started:
 967                     if (jb.machine.host, jb.machine.port) != host_port:
 968                         continue
 969                     if jb.after == None:
 970                         jb.run()
 971                         l_jobs_not_started.remove(jb)
 972                         new_job_start = True
 973                         break
 974                     else:
 975                         jb_before = self.find_job_that_has_name(jb.after)
 976                         if jb_before is None:
 977                             jb.cancel()
 978                             msg = _("This job was not launched because its "
 979                                     "father is not in the jobs list.")
 980                             jb.out = msg
 981                             jb.err = msg
 982                             break
 983                         if jb_before.has_finished():
 984                             jb.run()
 985                             l_jobs_not_started.remove(jb)
 986                             new_job_start = True
 987                             break
 988             self.cancel_dependencies_of_failing_jobs()
 989             new_job_finished = self.update_jobs_states_list()
 990
 991             if new_job_start or new_job_finished:
 992                 if self.gui:
 993                     self.gui.update_xml_files(self.ljobs)
 994                 # Display the current status
 995                 self.display_status(self.len_columns)
 996
 997             # Make sure that the proc is not entirely busy
 998             time.sleep(0.001)
 999
1000         self.logger.write("\n")
1001         self.logger.write(tiret_line)
1002         self.logger.write("\n\n")
1003
1004         if self.gui:
1005             self.gui.update_xml_files(self.ljobs)
1006             self.gui.last_update()
1007
1008     def write_all_results(self):
1009         '''Display all the jobs outputs.
1010
1011         :return: Nothing
1012         :rtype: N\A
1013         '''
1014
1015         for jb in self.ljobs:
1016             self.logger.write(src.printcolors.printcLabel(
1017                         "#------- Results for job %s -------#\n" % jb.name))
1018             jb.write_results()
1019             self.logger.write("\n\n")
1020
1021 class Gui(object):
1022     '''Class to manage the the xml data that can be displayed in a browser to
1023        see the jobs states
1024     '''
1025
1026     def __init__(self, xml_dir_path, l_jobs, l_jobs_not_today, l_file_boards = []):
1027         '''Initialization
1028
1029         :param xml_dir_path str: The path to the directory where to put
1030                                  the xml resulting files
1031         :param l_jobs List: the list of jobs that run today
1032         :param l_jobs_not_today List: the list of jobs that do not run today
1033         :param l_file_boards List: the list of file path from which to read the
1034                                    expected boards
1035         '''
1036         # The path of the csv files to read to fill the expected boards
1037         self.l_file_boards = l_file_boards
1038
1039         today = d_INT_DAY[datetime.date.weekday(datetime.date.today())]
1040         self.parse_csv_boards(today)
1041
1042         # The path of the global xml file
1043         self.xml_dir_path = xml_dir_path
1044         # Initialize the xml files
1045         xml_global_path = os.path.join(self.xml_dir_path, "global_report.xml")
1046         self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1047                                                          "JobsReport")
1048         # The xml files that corresponds to the boards.
1049         # {name_board : xml_object}}
1050         self.d_xml_board_files = {}
1051         # Create the lines and columns
1052         self.initialize_boards(l_jobs, l_jobs_not_today)
1053
1054         # Write the xml file
1055         self.update_xml_files(l_jobs)
1056
1057     def add_xml_board(self, name):
1058         xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1059         self.d_xml_board_files[name] =  src.xmlManager.XmlLogFile(
1060                                                     xml_board_path,
1061                                                     "JobsReport")
1062         self.d_xml_board_files[name].add_simple_node("distributions")
1063         self.d_xml_board_files[name].add_simple_node("applications")
1064         self.d_xml_board_files[name].add_simple_node("board", text=name)
1065
1066     def initialize_boards(self, l_jobs, l_jobs_not_today):
1067         '''Get all the first information needed for each file and write the
1068            first version of the files
1069         :param l_jobs List: the list of jobs that run today
1070         :param l_jobs_not_today List: the list of jobs that do not run today
1071         '''
1072         # Get the boards to fill and put it in a dictionary
1073         # {board_name : xml instance corresponding to the board}
1074         for job in l_jobs + l_jobs_not_today:
1075             board = job.board
1076             if (board is not None and
1077                                 board not in self.d_xml_board_files.keys()):
1078                 self.add_xml_board(board)
1079
1080         # Verify that the boards given as input are done
1081         for board in list(self.d_input_boards.keys()):
1082             if board not in self.d_xml_board_files:
1083                 self.add_xml_board(board)
1084
1085         # Loop over all jobs in order to get the lines and columns for each
1086         # xml file
1087         d_dist = {}
1088         d_application = {}
1089         for board in self.d_xml_board_files:
1090             d_dist[board] = []
1091             d_application[board] = []
1092
1093         l_hosts_ports = []
1094
1095         for job in l_jobs + l_jobs_not_today:
1096
1097             if (job.machine.host, job.machine.port) not in l_hosts_ports:
1098                 l_hosts_ports.append((job.machine.host, job.machine.port))
1099
1100             distrib = job.machine.distribution
1101             application = job.application
1102
1103             board_job = job.board
1104             if board is None:
1105                 continue
1106             for board in self.d_xml_board_files:
1107                 if board_job == board:
1108                     if distrib is not None and distrib not in d_dist[board]:
1109                         d_dist[board].append(distrib)
1110                         src.xmlManager.add_simple_node(
1111                             self.d_xml_board_files[board].xmlroot.find(
1112                                                             'distributions'),
1113                                                    "dist",
1114                                                    attrib={"name" : distrib})
1115
1116                 if board_job == board:
1117                     if (application is not None and
1118                                     application not in d_application[board]):
1119                         d_application[board].append(application)
1120                         src.xmlManager.add_simple_node(
1121                             self.d_xml_board_files[board].xmlroot.find(
1122                                                                 'applications'),
1123                                                    "application",
1124                                                    attrib={
1125                                                         "name" : application})
1126
1127         # Verify that there are no missing application or distribution in the
1128         # xml board files (regarding the input boards)
1129         for board in self.d_xml_board_files:
1130             l_dist = d_dist[board]
1131             if board not in self.d_input_boards.keys():
1132                 continue
1133             for dist in self.d_input_boards[board]["rows"]:
1134                 if dist not in l_dist:
1135                     src.xmlManager.add_simple_node(
1136                             self.d_xml_board_files[board].xmlroot.find(
1137                                                             'distributions'),
1138                                                    "dist",
1139                                                    attrib={"name" : dist})
1140             l_appli = d_application[board]
1141             for appli in self.d_input_boards[board]["columns"]:
1142                 if appli not in l_appli:
1143                     src.xmlManager.add_simple_node(
1144                             self.d_xml_board_files[board].xmlroot.find(
1145                                                                 'applications'),
1146                                                    "application",
1147                                                    attrib={"name" : appli})
1148
1149         # Initialize the hosts_ports node for the global file
1150         self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1151                                                                 "hosts_ports")
1152         for host, port in l_hosts_ports:
1153             host_port = "%s:%i" % (host, port)
1154             src.xmlManager.add_simple_node(self.xmlhosts_ports,
1155                                            "host_port",
1156                                            attrib={"name" : host_port})
1157
1158         # Initialize the jobs node in all files
1159         for xml_file in [self.xml_global_file] + list(
1160                                             self.d_xml_board_files.values()):
1161             xml_jobs = xml_file.add_simple_node("jobs")
1162             # Get the jobs present in the config file but
1163             # that will not be launched today
1164             self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1165
1166             xml_file.add_simple_node("infos",
1167                                      attrib={"name" : "last update",
1168                                              "JobsCommandStatus" : "running"})
1169
1170         # Find in each board the squares that needs to be filled regarding the
1171         # input csv files but that are not covered by a today job
1172         for board in self.d_input_boards.keys():
1173             xml_root_board = self.d_xml_board_files[board].xmlroot
1174             xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1175                                                  "missing_jobs")
1176             for row, column in self.d_input_boards[board]["jobs"]:
1177                 found = False
1178                 for job in l_jobs:
1179                     if (job.application == column and
1180                         job.machine.distribution == row):
1181                         found = True
1182                         break
1183                 if not found:
1184                     src.xmlManager.add_simple_node(xml_missing,
1185                                             "job",
1186                                             attrib={"distribution" : row,
1187                                                     "application" : column })
1188
1189     def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1190         '''Get all the first information needed for each file and write the
1191            first version of the files
1192
1193         :param xml_node_jobs etree.Element: the node corresponding to a job
1194         :param l_jobs_not_today List: the list of jobs that do not run today
1195         '''
1196         for job in l_jobs_not_today:
1197             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1198                                                  "job",
1199                                                  attrib={"name" : job.name})
1200             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1201             src.xmlManager.add_simple_node(xmlj,
1202                                            "distribution",
1203                                            job.machine.distribution)
1204             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1205             src.xmlManager.add_simple_node(xmlj,
1206                                        "commands", " ; ".join(job.commands))
1207             src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1208             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1209             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1210             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1211             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1212             src.xmlManager.add_simple_node(xmlj, "sat_path",
1213                                                         job.machine.sat_path)
1214
1215     def parse_csv_boards(self, today):
1216         """ Parse the csv files that describes the boards to produce and fill
1217             the dict d_input_boards that contain the csv file contain
1218
1219         :param today str: the current day of the week
1220         """
1221         # loop over each csv file and read its content
1222         l_boards = []
1223         for file_path in self.l_file_boards:
1224             l_read = []
1225             with open(file_path, 'r') as f:
1226                 reader = csv.reader(f)
1227                 for row in reader:
1228                     l_read.append(row)
1229             l_boards.append(l_read)
1230
1231         # loop over the csv lists of lines and get the rows, columns and jobs
1232         d_boards = {}
1233         for input_board in l_boards:
1234             # get board name
1235             board_name = input_board[0][0]
1236
1237             # Get columns list
1238             columns = input_board[0][1:]
1239
1240             rows = []
1241             columns_out = []
1242             jobs = []
1243             for line in input_board[1:]:
1244                 row = line[0]
1245                 for i, square in enumerate(line[1:]):
1246                     if today in square:
1247                         if row not in rows:
1248                             rows.append(row)
1249                         if columns[i] not in columns_out:
1250                             columns_out.append(columns[i])
1251                         job = (row, columns[i])
1252                         jobs.append(job)
1253
1254             d_boards[board_name] = {"rows" : rows,
1255                                     "columns" : columns_out,
1256                                     "jobs" : jobs}
1257
1258         self.d_input_boards = d_boards
1259
1260     def update_xml_files(self, l_jobs):
1261         '''Write all the xml files with updated information about the jobs
1262
1263         :param l_jobs List: the list of jobs that run today
1264         '''
1265         for xml_file in [self.xml_global_file] + list(
1266                                             self.d_xml_board_files.values()):
1267             self.update_xml_file(l_jobs, xml_file)
1268
1269         # Write the file
1270         self.write_xml_files()
1271
1272     def update_xml_file(self, l_jobs, xml_file):
1273         '''update information about the jobs for the file xml_file
1274
1275         :param l_jobs List: the list of jobs that run today
1276         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1277         '''
1278
1279         xml_node_jobs = xml_file.xmlroot.find('jobs')
1280         # Update the job names and status node
1281         for job in l_jobs:
1282             # Find the node corresponding to the job and delete it
1283             # in order to recreate it
1284             for xmljob in xml_node_jobs.findall('job'):
1285                 if xmljob.attrib['name'] == job.name:
1286                     xml_node_jobs.remove(xmljob)
1287
1288             T0 = str(job._T0)
1289             if T0 != "-1":
1290                 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1291                                        time.localtime(job._T0))
1292             Tf = str(job._Tf)
1293             if Tf != "-1":
1294                 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1295                                        time.localtime(job._Tf))
1296
1297             # recreate the job node
1298             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1299                                                   "job",
1300                                                   attrib={"name" : job.name})
1301             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1302             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1303             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1304             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1305             src.xmlManager.add_simple_node(xmlj, "sat_path",
1306                                            job.machine.sat_path)
1307             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1308             src.xmlManager.add_simple_node(xmlj, "distribution",
1309                                            job.machine.distribution)
1310             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1311             src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1312             src.xmlManager.add_simple_node(xmlj, "commands",
1313                                            " ; ".join(job.commands))
1314             src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1315             src.xmlManager.add_simple_node(xmlj, "begin", T0)
1316             src.xmlManager.add_simple_node(xmlj, "end", Tf)
1317             src.xmlManager.add_simple_node(xmlj, "out",
1318                                            src.printcolors.cleancolor(job.out))
1319             src.xmlManager.add_simple_node(xmlj, "err",
1320                                            src.printcolors.cleancolor(job.err))
1321             src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1322             if len(job.remote_log_files) > 0:
1323                 src.xmlManager.add_simple_node(xmlj,
1324                                                "remote_log_file_path",
1325                                                job.remote_log_files[0])
1326             else:
1327                 src.xmlManager.add_simple_node(xmlj,
1328                                                "remote_log_file_path",
1329                                                "nothing")
1330
1331             xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1332             # get the job father
1333             if job.after is not None:
1334                 job_father = None
1335                 for jb in l_jobs:
1336                     if jb.name == job.after:
1337                         job_father = jb
1338
1339                 if (job_father is not None and
1340                         len(job_father.remote_log_files) > 0):
1341                     link = job_father.remote_log_files[0]
1342                 else:
1343                     link = "nothing"
1344                 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1345
1346             # Verify that the job is to be done today regarding the input csv
1347             # files
1348             if job.board and job.board in self.d_input_boards.keys():
1349                 found = False
1350                 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1351                     if (job.machine.distribution == dist
1352                         and job.application == appli):
1353                         found = True
1354                         src.xmlManager.add_simple_node(xmlj,
1355                                                "extra_job",
1356                                                "no")
1357                         break
1358                 if not found:
1359                     src.xmlManager.add_simple_node(xmlj,
1360                                                "extra_job",
1361                                                "yes")
1362
1363
1364         # Update the date
1365         xml_node_infos = xml_file.xmlroot.find('infos')
1366         src.xmlManager.append_node_attrib(xml_node_infos,
1367                     attrib={"value" :
1368                     datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1369
1370
1371
1372     def last_update(self, finish_status = "finished"):
1373         '''update information about the jobs for the file xml_file
1374
1375         :param l_jobs List: the list of jobs that run today
1376         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1377         '''
1378         for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1379             xml_node_infos = xml_file.xmlroot.find('infos')
1380             src.xmlManager.append_node_attrib(xml_node_infos,
1381                         attrib={"JobsCommandStatus" : finish_status})
1382         # Write the file
1383         self.write_xml_files()
1384
1385     def write_xml_files(self):
1386         ''' Write the xml files
1387         '''
1388         self.xml_global_file.write_tree(STYLESHEET_GLOBAL)
1389         for xml_file in self.d_xml_board_files.values():
1390             xml_file.write_tree(STYLESHEET_BOARD)
1391
1392 ##
1393 # Describes the command
1394 def description():
1395     return _("The jobs command launches maintenances that are described"
1396              " in the dedicated jobs configuration file.")
1397
1398 ##
1399 # Runs the command.
1400 def run(args, runner, logger):
1401
1402     (options, args) = parser.parse_args(args)
1403
1404     l_cfg_dir = runner.cfg.PATHS.JOBPATH
1405
1406     # list option : display all the available config files
1407     if options.list:
1408         for cfg_dir in l_cfg_dir:
1409             if not options.no_label:
1410                 logger.write("------ %s\n" %
1411                                  src.printcolors.printcHeader(cfg_dir))
1412
1413             for f in sorted(os.listdir(cfg_dir)):
1414                 if not f.endswith('.pyconf'):
1415                     continue
1416                 cfilename = f[:-7]
1417                 logger.write("%s\n" % cfilename)
1418         return 0
1419
1420     # Make sure the jobs_config option has been called
1421     if not options.jobs_cfg:
1422         message = _("The option --jobs_config is required\n")
1423         src.printcolors.printcError(message)
1424         return 1
1425
1426     # Find the file in the directories
1427     found = False
1428     for cfg_dir in l_cfg_dir:
1429         file_jobs_cfg = os.path.join(cfg_dir, options.jobs_cfg)
1430         if not file_jobs_cfg.endswith('.pyconf'):
1431             file_jobs_cfg += '.pyconf'
1432
1433         if not os.path.exists(file_jobs_cfg):
1434             continue
1435         else:
1436             found = True
1437             break
1438
1439     if not found:
1440         msg = _("The file configuration %(name_file)s was not found."
1441                 "\nUse the --list option to get the possible files.")
1442         src.printcolors.printcError(msg)
1443         return 1
1444
1445     info = [
1446         (_("Platform"), runner.cfg.VARS.dist),
1447         (_("File containing the jobs configuration"), file_jobs_cfg)
1448     ]
1449     src.print_info(logger, info)
1450
1451     # Read the config that is in the file
1452     config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1453     if options.only_jobs:
1454         l_jb = src.pyconf.Sequence()
1455         for jb in config_jobs.jobs:
1456             if jb.name in options.only_jobs:
1457                 l_jb.append(jb,
1458                 "Adding a job that was given in only_jobs option parameters")
1459         config_jobs.jobs = l_jb
1460
1461     # Initialization
1462     today_jobs = Jobs(runner,
1463                       logger,
1464                       file_jobs_cfg,
1465                       config_jobs)
1466     # SSH connection to all machines
1467     today_jobs.ssh_connection_all_machines()
1468     if options.test_connection:
1469         return 0
1470
1471     gui = None
1472     if options.publish:
1473         # Copy the stylesheets in the log directory
1474         log_dir = runner.cfg.SITE.log.log_dir
1475         xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1476         files_to_copy = []
1477         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1478         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1479         files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1480         for file_path in files_to_copy:
1481             shutil.copy2(file_path, log_dir)
1482
1483         # Instanciate the Gui in order to produce the xml files that contain all
1484         # the boards
1485         gui = Gui(runner.cfg.SITE.log.log_dir,
1486                   today_jobs.ljobs,
1487                   today_jobs.ljobs_not_today,
1488                   l_file_boards = options.input_boards)
1489
1490         # Display the list of the xml files
1491         logger.write(src.printcolors.printcInfo(("Here is the list of published"
1492                                                  " files :\n")), 4)
1493         logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1494         for board in gui.d_xml_board_files.keys():
1495             logger.write("%s\n" % gui.d_xml_board_files[board].logFile, 4)
1496
1497         logger.write("\n", 4)
1498
1499     today_jobs.gui = gui
1500
1501     interruped = False
1502     try:
1503         # Run all the jobs contained in config_jobs
1504         today_jobs.run_jobs()
1505     except KeyboardInterrupt:
1506         interruped = True
1507         logger.write("\n\n%s\n\n" %
1508                 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1509     finally:
1510         if interruped:
1511             msg = _("Killing the running jobs and trying"
1512                     " to get the corresponding logs\n")
1513             logger.write(src.printcolors.printcWarning(msg))
1514
1515         # find the potential not finished jobs and kill them
1516         for jb in today_jobs.ljobs:
1517             if not jb.has_finished():
1518                 try:
1519                     jb.kill_remote_process()
1520                 except Exception as e:
1521                     msg = _("Failed to kill job %s: %s\n" % (jb.name, e))
1522                     logger.write(src.printcolors.printcWarning(msg))
1523         if interruped:
1524             if today_jobs.gui:
1525                 today_jobs.gui.last_update(_("Forced interruption"))
1526         else:
1527             if today_jobs.gui:
1528                 today_jobs.gui.last_update()
1529         # Output the results
1530         today_jobs.write_all_results()