commands/jobs.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import datetime
  21 import time
  22 import csv
  23 import shutil
  24 import itertools
  25 import paramiko
  26
  27 import src
  28
  29 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
  30 STYLESHEET_BOARD = "jobs_board_report.xsl"
  31
  32 DAYS_SEPARATOR = ","
  33 CSV_DELIMITER = ";"
  34
  35 parser = src.options.Options()
  36
  37 parser.add_option('n', 'name', 'string', 'jobs_cfg',
  38                   _('The name of the config file that contains'
  39                   ' the jobs configuration'))
  40 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
  41                   _('Optional: the list of jobs to launch, by their name. '))
  42 parser.add_option('l', 'list', 'boolean', 'list',
  43                   _('Optional: list all available config files.'))
  44 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
  45                   _("Optional: try to connect to the machines. "
  46                     "Not executing the jobs."),
  47                   False)
  48 parser.add_option('p', 'publish', 'boolean', 'publish',
  49                   _("Optional: generate an xml file that can be read in a "
  50                     "browser to display the jobs status."),
  51                   False)
  52 parser.add_option('i', 'input_boards', 'string', 'input_boards', _("Optional: "
  53                                 "the path to csv file that contain "
  54                                 "the expected boards."),"")
  55 parser.add_option('n', 'completion', 'boolean', 'no_label',
  56                   _("Optional (internal use): do not print labels, Works only "
  57                     "with --list."),
  58                   False)
  59
  60 class Machine(object):
  61     '''Class to manage a ssh connection on a machine
  62     '''
  63     def __init__(self,
  64                  name,
  65                  host,
  66                  user,
  67                  port=22,
  68                  passwd=None,
  69                  sat_path="salomeTools"):
  70         self.name = name
  71         self.host = host
  72         self.port = port
  73         self.distribution = None # Will be filled after copying SAT on the machine
  74         self.user = user
  75         self.password = passwd
  76         self.sat_path = sat_path
  77         self.ssh = paramiko.SSHClient()
  78         self._connection_successful = None
  79
  80     def connect(self, logger):
  81         '''Initiate the ssh connection to the remote machine
  82
  83         :param logger src.logger.Logger: The logger instance
  84         :return: Nothing
  85         :rtype: N\A
  86         '''
  87
  88         self._connection_successful = False
  89         self.ssh.load_system_host_keys()
  90         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  91         try:
  92             self.ssh.connect(self.host,
  93                              port=self.port,
  94                              username=self.user,
  95                              password = self.password)
  96         except paramiko.AuthenticationException:
  97             message = src.KO_STATUS + _("Authentication failed")
  98         except paramiko.BadHostKeyException:
  99             message = (src.KO_STATUS +
 100                        _("The server's host key could not be verified"))
 101         except paramiko.SSHException:
 102             message = ( _("SSHException error connecting or "
 103                           "establishing an SSH session"))
 104         except:
 105             message = ( _("Error connecting or establishing an SSH session"))
 106         else:
 107             self._connection_successful = True
 108             message = ""
 109         return message
 110
 111     def successfully_connected(self, logger):
 112         '''Verify if the connection to the remote machine has succeed
 113
 114         :param logger src.logger.Logger: The logger instance
 115         :return: True if the connection has succeed, False if not
 116         :rtype: bool
 117         '''
 118         if self._connection_successful == None:
 119             message = _("Warning : trying to ask if the connection to "
 120             "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
 121             " no connection request" %
 122                         (self.name, self.host, self.port, self.user))
 123             logger.write( src.printcolors.printcWarning(message))
 124         return self._connection_successful
 125
 126     def copy_sat(self, sat_local_path, job_file):
 127         '''Copy salomeTools to the remote machine in self.sat_path
 128         '''
 129         res = 0
 130         try:
 131             # open a sftp connection
 132             self.sftp = self.ssh.open_sftp()
 133             # Create the sat directory on remote machine if it is not existing
 134             self.mkdir(self.sat_path, ignore_existing=True)
 135             # Put sat
 136             self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
 137             # put the job configuration file in order to make it reachable
 138             # on the remote machine
 139             self.sftp.put(job_file, os.path.join(".salomeTools",
 140                                                  "Jobs",
 141                                                  ".jobs_command_file.pyconf"))
 142         except Exception as e:
 143             res = str(e)
 144             self._connection_successful = False
 145
 146         return res
 147
 148     def put_dir(self, source, target, filters = []):
 149         ''' Uploads the contents of the source directory to the target path. The
 150             target directory needs to exists. All sub-directories in source are
 151             created under target.
 152         '''
 153         for item in os.listdir(source):
 154             if item in filters:
 155                 continue
 156             source_path = os.path.join(source, item)
 157             destination_path = os.path.join(target, item)
 158             if os.path.islink(source_path):
 159                 linkto = os.readlink(source_path)
 160                 try:
 161                     self.sftp.symlink(linkto, destination_path)
 162                     self.sftp.chmod(destination_path,
 163                                     os.stat(source_path).st_mode)
 164                 except IOError:
 165                     pass
 166             else:
 167                 if os.path.isfile(source_path):
 168                     self.sftp.put(source_path, destination_path)
 169                     self.sftp.chmod(destination_path,
 170                                     os.stat(source_path).st_mode)
 171                 else:
 172                     self.mkdir(destination_path, ignore_existing=True)
 173                     self.put_dir(source_path, destination_path)
 174
 175     def mkdir(self, path, mode=511, ignore_existing=False):
 176         ''' Augments mkdir by adding an option to not fail
 177             if the folder exists
 178         '''
 179         try:
 180             self.sftp.mkdir(path, mode)
 181         except IOError:
 182             if ignore_existing:
 183                 pass
 184             else:
 185                 raise
 186
 187     def exec_command(self, command, logger):
 188         '''Execute the command on the remote machine
 189
 190         :param command str: The command to be run
 191         :param logger src.logger.Logger: The logger instance
 192         :return: the stdin, stdout, and stderr of the executing command,
 193                  as a 3-tuple
 194         :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
 195                 paramiko.channel.ChannelFile)
 196         '''
 197         try:
 198             # Does not wait the end of the command
 199             (stdin, stdout, stderr) = self.ssh.exec_command(command)
 200         except paramiko.SSHException:
 201             message = src.KO_STATUS + _(
 202                             ": the server failed to execute the command\n")
 203             logger.write( src.printcolors.printcError(message))
 204             return (None, None, None)
 205         except:
 206             logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
 207             return (None, None, None)
 208         else:
 209             return (stdin, stdout, stderr)
 210
 211     def close(self):
 212         '''Close the ssh connection
 213
 214         :rtype: N\A
 215         '''
 216         self.ssh.close()
 217
 218     def write_info(self, logger):
 219         '''Prints the informations relative to the machine in the logger
 220            (terminal traces and log file)
 221
 222         :param logger src.logger.Logger: The logger instance
 223         :return: Nothing
 224         :rtype: N\A
 225         '''
 226         logger.write("host : " + self.host + "\n")
 227         logger.write("port : " + str(self.port) + "\n")
 228         logger.write("user : " + str(self.user) + "\n")
 229         if self.successfully_connected(logger):
 230             status = src.OK_STATUS
 231         else:
 232             status = src.KO_STATUS
 233         logger.write("Connection : " + status + "\n\n")
 234
 235
 236 class Job(object):
 237     '''Class to manage one job
 238     '''
 239     def __init__(self, name, machine, application, board,
 240                  commands, timeout, config, logger, after=None):
 241
 242         self.name = name
 243         self.machine = machine
 244         self.after = after
 245         self.timeout = timeout
 246         self.application = application
 247         self.board = board
 248         self.config = config
 249         self.logger = logger
 250         # The list of log files to download from the remote machine
 251         self.remote_log_files = []
 252
 253         # The remote command status
 254         # -1 means that it has not been launched,
 255         # 0 means success and 1 means fail
 256         self.res_job = "-1"
 257         self.cancelled = False
 258
 259         self._T0 = -1
 260         self._Tf = -1
 261         self._has_begun = False
 262         self._has_finished = False
 263         self._has_timouted = False
 264         self._stdin = None # Store the command inputs field
 265         self._stdout = None # Store the command outputs field
 266         self._stderr = None # Store the command errors field
 267
 268         self.out = ""
 269         self.err = ""
 270
 271         self.commands = commands
 272         self.command = (os.path.join(self.machine.sat_path, "sat") +
 273                         " -l " +
 274                         os.path.join(self.machine.sat_path,
 275                                      "list_log_files.txt") +
 276                         " job --jobs_config .jobs_command_file" +
 277                         " --name " +
 278                         self.name)
 279
 280     def get_pids(self):
 281         """ Get the pid(s) corresponding to the command that have been launched
 282             On the remote machine
 283
 284         :return: The list of integers corresponding to the found pids
 285         :rtype: List
 286         """
 287         pids = []
 288         cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
 289         (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
 290         pids_cmd = out_pid.readlines()
 291         pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
 292         pids+=pids_cmd
 293         return pids
 294
 295     def kill_remote_process(self, wait=1):
 296         '''Kills the process on the remote machine.
 297
 298         :return: (the output of the kill, the error of the kill)
 299         :rtype: (str, str)
 300         '''
 301
 302         pids = self.get_pids()
 303         cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
 304         (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
 305                                                             self.logger)
 306         time.sleep(wait)
 307         return (out_kill, err_kill)
 308
 309     def has_begun(self):
 310         '''Returns True if the job has already begun
 311
 312         :return: True if the job has already begun
 313         :rtype: bool
 314         '''
 315         return self._has_begun
 316
 317     def has_finished(self):
 318         '''Returns True if the job has already finished
 319            (i.e. all the commands have been executed)
 320            If it is finished, the outputs are stored in the fields out and err.
 321
 322         :return: True if the job has already finished
 323         :rtype: bool
 324         '''
 325
 326         # If the method has already been called and returned True
 327         if self._has_finished:
 328             return True
 329
 330         # If the job has not begun yet
 331         if not self.has_begun():
 332             return False
 333
 334         if self._stdout.channel.closed:
 335             self._has_finished = True
 336             # Store the result outputs
 337             self.out += self._stdout.read().decode()
 338             self.err += self._stderr.read().decode()
 339             # Put end time
 340             self._Tf = time.time()
 341             # And get the remote command status and log files
 342             self.get_log_files()
 343
 344         return self._has_finished
 345
 346     def get_log_files(self):
 347         """Get the log files produced by the command launched
 348            on the remote machine, and put it in the log directory of the user,
 349            so they can be accessible from
 350         """
 351         # Do not get the files if the command is not finished
 352         if not self.has_finished():
 353             msg = _("Trying to get log files whereas the job is not finished.")
 354             self.logger.write(src.printcolors.printcWarning(msg))
 355             return
 356
 357         # First get the file that contains the list of log files to get
 358         tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
 359         remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt")
 360         self.machine.sftp.get(
 361                     remote_path,
 362                     tmp_file_path)
 363
 364         # Read the file and get the result of the command and all the log files
 365         # to get
 366         fstream_tmp = open(tmp_file_path, "r")
 367         file_lines = fstream_tmp.readlines()
 368         file_lines = [line.replace("\n", "") for line in file_lines]
 369         fstream_tmp.close()
 370         os.remove(tmp_file_path)
 371
 372         try :
 373             # The first line is the result of the command (0 success or 1 fail)
 374             self.res_job = file_lines[0]
 375         except Exception as e:
 376             self.err += _("Unable to get status from remote file %s: %s" %
 377                                                     (remote_path, str(e)))
 378
 379         for i, job_path_remote in enumerate(file_lines[1:]):
 380             try:
 381                 # For each command, there is two files to get :
 382                 # 1- The xml file describing the command and giving the
 383                 # internal traces.
 384                 # 2- The txt file containing the system command traces (like
 385                 # traces produced by the "make" command)
 386                 # 3- In case of the test command, there is another file to get :
 387                 # the xml board that contain the test results
 388                 dirname = os.path.basename(os.path.dirname(job_path_remote))
 389                 if dirname != 'OUT' and dirname != 'TEST':
 390                     # Case 1-
 391                     local_path = os.path.join(os.path.dirname(
 392                                                         self.logger.logFilePath),
 393                                               os.path.basename(job_path_remote))
 394                     if i==0: # The first is the job command
 395                         self.logger.add_link(os.path.basename(job_path_remote),
 396                                              "job",
 397                                              self.res_job,
 398                                              self.command)
 399                 elif dirname == 'OUT':
 400                     # Case 2-
 401                     local_path = os.path.join(os.path.dirname(
 402                                                         self.logger.logFilePath),
 403                                               'OUT',
 404                                               os.path.basename(job_path_remote))
 405                 elif dirname == 'TEST':
 406                     # Case 3-
 407                     local_path = os.path.join(os.path.dirname(
 408                                                         self.logger.logFilePath),
 409                                               'TEST',
 410                                               os.path.basename(job_path_remote))
 411
 412                 # Get the file
 413                 if not os.path.exists(local_path):
 414                     self.machine.sftp.get(job_path_remote, local_path)
 415                 self.remote_log_files.append(local_path)
 416             except Exception as e:
 417                 self.err += _("Unable to get %s log file from remote: %s" %
 418                                                     (str(job_path_remote),
 419                                                      str(e)))
 420
 421     def has_failed(self):
 422         '''Returns True if the job has failed.
 423            A job is considered as failed if the machine could not be reached,
 424            if the remote command failed,
 425            or if the job finished with a time out.
 426
 427         :return: True if the job has failed
 428         :rtype: bool
 429         '''
 430         if not self.has_finished():
 431             return False
 432         if not self.machine.successfully_connected(self.logger):
 433             return True
 434         if self.is_timeout():
 435             return True
 436         if self.res_job == "1":
 437             return True
 438         return False
 439
 440     def cancel(self):
 441         """In case of a failing job, one has to cancel every job that depend
 442            on it. This method put the job as failed and will not be executed.
 443         """
 444         if self.cancelled:
 445             return
 446         self._has_begun = True
 447         self._has_finished = True
 448         self.cancelled = True
 449         self.out += _("This job was not launched because its father has failed.")
 450         self.err += _("This job was not launched because its father has failed.")
 451
 452     def is_running(self):
 453         '''Returns True if the job commands are running
 454
 455         :return: True if the job is running
 456         :rtype: bool
 457         '''
 458         return self.has_begun() and not self.has_finished()
 459
 460     def is_timeout(self):
 461         '''Returns True if the job commands has finished with timeout
 462
 463         :return: True if the job has finished with timeout
 464         :rtype: bool
 465         '''
 466         return self._has_timouted
 467
 468     def time_elapsed(self):
 469         """Get the time elapsed since the job launching
 470
 471         :return: The number of seconds
 472         :rtype: int
 473         """
 474         if not self.has_begun():
 475             return -1
 476         T_now = time.time()
 477         return T_now - self._T0
 478
 479     def check_time(self):
 480         """Verify that the job has not exceeded its timeout.
 481            If it has, kill the remote command and consider the job as finished.
 482         """
 483         if not self.has_begun():
 484             return
 485         if self.time_elapsed() > self.timeout:
 486             self._has_finished = True
 487             self._has_timouted = True
 488             self._Tf = time.time()
 489             self.get_pids()
 490             (out_kill, _) = self.kill_remote_process()
 491             self.out += "TIMEOUT \n" + out_kill.read().decode()
 492             self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
 493             try:
 494                 self.get_log_files()
 495             except Exception as e:
 496                 self.err += _("Unable to get remote log files: %s" % e)
 497
 498     def total_duration(self):
 499         """Give the total duration of the job
 500
 501         :return: the total duration of the job in seconds
 502         :rtype: int
 503         """
 504         return self._Tf - self._T0
 505
 506     def run(self):
 507         """Launch the job by executing the remote command.
 508         """
 509
 510         # Prevent multiple run
 511         if self.has_begun():
 512             msg = _("Warning: A job can only be launched one time")
 513             msg2 = _("Trying to launch the job \"%s\" whereas it has "
 514                      "already been launched." % self.name)
 515             self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
 516                                                                         msg2)))
 517             return
 518
 519         # Do not execute the command if the machine could not be reached
 520         if not self.machine.successfully_connected(self.logger):
 521             self._has_finished = True
 522             self.out = "N\A"
 523             self.err += ("Connection to machine (name : %s, host: %s, port:"
 524                         " %s, user: %s) has failed\nUse the log command "
 525                         "to get more information."
 526                         % (self.machine.name,
 527                            self.machine.host,
 528                            self.machine.port,
 529                            self.machine.user))
 530         else:
 531             # Usual case : Launch the command on remote machine
 532             self._T0 = time.time()
 533             self._stdin, self._stdout, self._stderr = self.machine.exec_command(
 534                                                                   self.command,
 535                                                                   self.logger)
 536             # If the results are not initialized, finish the job
 537             if (self._stdin, self._stdout, self._stderr) == (None, None, None):
 538                 self._has_finished = True
 539                 self._Tf = time.time()
 540                 self.out += "N\A"
 541                 self.err += "The server failed to execute the command"
 542
 543         # Put the beginning flag to true.
 544         self._has_begun = True
 545
 546     def write_results(self):
 547         """Display on the terminal all the job's information
 548         """
 549         self.logger.write("name : " + self.name + "\n")
 550         if self.after:
 551             self.logger.write("after : %s\n" % self.after)
 552         self.logger.write("Time elapsed : %4imin %2is \n" %
 553                      (self.total_duration()//60 , self.total_duration()%60))
 554         if self._T0 != -1:
 555             self.logger.write("Begin time : %s\n" %
 556                          time.strftime('%Y-%m-%d %H:%M:%S',
 557                                        time.localtime(self._T0)) )
 558         if self._Tf != -1:
 559             self.logger.write("End time   : %s\n\n" %
 560                          time.strftime('%Y-%m-%d %H:%M:%S',
 561                                        time.localtime(self._Tf)) )
 562
 563         machine_head = "Informations about connection :\n"
 564         underline = (len(machine_head) - 2) * "-"
 565         self.logger.write(src.printcolors.printcInfo(
 566                                                 machine_head+underline+"\n"))
 567         self.machine.write_info(self.logger)
 568
 569         self.logger.write(src.printcolors.printcInfo("out : \n"))
 570         if self.out == "":
 571             self.logger.write("Unable to get output\n")
 572         else:
 573             self.logger.write(self.out + "\n")
 574         self.logger.write(src.printcolors.printcInfo("err : \n"))
 575         self.logger.write(self.err + "\n")
 576
 577     def get_status(self):
 578         """Get the status of the job (used by the Gui for xml display)
 579
 580         :return: The current status of the job
 581         :rtype: String
 582         """
 583         if not self.machine.successfully_connected(self.logger):
 584             return "SSH connection KO"
 585         if not self.has_begun():
 586             return "Not launched"
 587         if self.cancelled:
 588             return "Cancelled"
 589         if self.is_running():
 590             return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
 591                                                     time.localtime(self._T0))
 592         if self.has_finished():
 593             if self.is_timeout():
 594                 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
 595                                                     time.localtime(self._Tf))
 596             return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
 597                                                      time.localtime(self._Tf))
 598
 599 class Jobs(object):
 600     '''Class to manage the jobs to be run
 601     '''
 602     def __init__(self,
 603                  runner,
 604                  logger,
 605                  job_file_path,
 606                  config_jobs,
 607                  lenght_columns = 20):
 608         # The jobs configuration
 609         self.cfg_jobs = config_jobs
 610         self.job_file_path = job_file_path
 611         # The machine that will be used today
 612         self.lmachines = []
 613         # The list of machine (hosts, port) that will be used today
 614         # (a same host can have several machine instances since there
 615         # can be several ssh parameters)
 616         self.lhosts = []
 617         # The jobs to be launched today
 618         self.ljobs = []
 619         # The jobs that will not be launched today
 620         self.ljobs_not_today = []
 621         self.runner = runner
 622         self.logger = logger
 623         self.len_columns = lenght_columns
 624
 625         # the list of jobs that have not been run yet
 626         self._l_jobs_not_started = []
 627         # the list of jobs that have already ran
 628         self._l_jobs_finished = []
 629         # the list of jobs that are running
 630         self._l_jobs_running = []
 631
 632         self.determine_jobs_and_machines()
 633
 634     def define_job(self, job_def, machine):
 635         '''Takes a pyconf job definition and a machine (from class machine)
 636            and returns the job instance corresponding to the definition.
 637
 638         :param job_def src.config.Mapping: a job definition
 639         :param machine machine: the machine on which the job will run
 640         :return: The corresponding job in a job class instance
 641         :rtype: job
 642         '''
 643         name = job_def.name
 644         cmmnds = job_def.commands
 645         if not "timeout" in job_def:
 646             timeout = 4*60*60 # default timeout = 4h
 647         else:
 648             timeout = job_def.timeout
 649         after = None
 650         if 'after' in job_def:
 651             after = job_def.after
 652         application = None
 653         if 'application' in job_def:
 654             application = job_def.application
 655         board = None
 656         if 'board' in job_def:
 657             board = job_def.board
 658
 659         return Job(name,
 660                    machine,
 661                    application,
 662                    board,
 663                    cmmnds,
 664                    timeout,
 665                    self.runner.cfg,
 666                    self.logger,
 667                    after = after)
 668
 669     def determine_jobs_and_machines(self):
 670         '''Function that reads the pyconf jobs definition and instantiates all
 671            the machines and jobs to be done today.
 672
 673         :return: Nothing
 674         :rtype: N\A
 675         '''
 676         today = datetime.date.weekday(datetime.date.today())
 677         host_list = []
 678
 679         for job_def in self.cfg_jobs.jobs :
 680
 681             if not "machine" in job_def:
 682                 msg = _('WARNING: The job "%s" do not have the key '
 683                        '"machine", this job is ignored.\n\n' % job_def.name)
 684                 self.logger.write(src.printcolors.printcWarning(msg))
 685                 continue
 686             name_machine = job_def.machine
 687
 688             a_machine = None
 689             for mach in self.lmachines:
 690                 if mach.name == name_machine:
 691                     a_machine = mach
 692                     break
 693
 694             if a_machine == None:
 695                 for machine_def in self.cfg_jobs.machines:
 696                     if machine_def.name == name_machine:
 697                         if 'host' not in machine_def:
 698                             host = self.runner.cfg.VARS.hostname
 699                         else:
 700                             host = machine_def.host
 701
 702                         if 'user' not in machine_def:
 703                             user = self.runner.cfg.VARS.user
 704                         else:
 705                             user = machine_def.user
 706
 707                         if 'port' not in machine_def:
 708                             port = 22
 709                         else:
 710                             port = machine_def.port
 711
 712                         if 'password' not in machine_def:
 713                             passwd = None
 714                         else:
 715                             passwd = machine_def.password
 716
 717                         if 'sat_path' not in machine_def:
 718                             sat_path = "salomeTools"
 719                         else:
 720                             sat_path = machine_def.sat_path
 721
 722                         a_machine = Machine(
 723                                             machine_def.name,
 724                                             host,
 725                                             user,
 726                                             port=port,
 727                                             passwd=passwd,
 728                                             sat_path=sat_path
 729                                             )
 730
 731                         self.lmachines.append(a_machine)
 732                         if (host, port) not in host_list:
 733                             host_list.append((host, port))
 734
 735                 if a_machine == None:
 736                     msg = _("WARNING: The job \"%(job_name)s\" requires the "
 737                             "machine \"%(machine_name)s\" but this machine "
 738                             "is not defined in the configuration file.\n"
 739                             "The job will not be launched")
 740                     self.logger.write(src.printcolors.printcWarning(msg))
 741
 742             a_job = self.define_job(job_def, a_machine)
 743
 744             if today in job_def.when:
 745                 self.ljobs.append(a_job)
 746             else: # today in job_def.when
 747                 self.ljobs_not_today.append(a_job)
 748
 749         self.lhosts = host_list
 750
 751     def ssh_connection_all_machines(self, pad=50):
 752         '''Function that do the ssh connection to every machine
 753            to be used today.
 754
 755         :return: Nothing
 756         :rtype: N\A
 757         '''
 758         self.logger.write(src.printcolors.printcInfo((
 759                         "Establishing connection with all the machines :\n")))
 760         for machine in self.lmachines:
 761             # little algorithm in order to display traces
 762             begin_line = (_("Connection to %s: " % machine.name))
 763             if pad - len(begin_line) < 0:
 764                 endline = " "
 765             else:
 766                 endline = (pad - len(begin_line)) * "." + " "
 767
 768             step = "SSH connection"
 769             self.logger.write( begin_line + endline + step)
 770             self.logger.flush()
 771             # the call to the method that initiate the ssh connection
 772             msg = machine.connect(self.logger)
 773
 774             # Copy salomeTools to the remote machine
 775             if machine.successfully_connected(self.logger):
 776                 step = _("Copy SAT")
 777                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 778                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 779                 self.logger.flush()
 780                 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
 781                                             self.job_file_path)
 782                 # get the remote machine distribution using a sat command
 783                 (__, out_dist, __) = machine.exec_command(
 784                                 os.path.join(machine.sat_path,
 785                                     "sat config --value VARS.dist --no_label"),
 786                                 self.logger)
 787                 machine.distribution = out_dist.read().decode().replace("\n",
 788                                                                         "")
 789                 # Print the status of the copy
 790                 if res_copy == 0:
 791                     self.logger.write('\r%s' %
 792                                 ((len(begin_line)+len(endline)+20) * " "), 3)
 793                     self.logger.write('\r%s%s%s' %
 794                         (begin_line,
 795                          endline,
 796                          src.printcolors.printc(src.OK_STATUS)), 3)
 797                 else:
 798                     self.logger.write('\r%s' %
 799                             ((len(begin_line)+len(endline)+20) * " "), 3)
 800                     self.logger.write('\r%s%s%s %s' %
 801                         (begin_line,
 802                          endline,
 803                          src.printcolors.printc(src.OK_STATUS),
 804                          _("Copy of SAT failed")), 3)
 805             else:
 806                 self.logger.write('\r%s' %
 807                                   ((len(begin_line)+len(endline)+20) * " "), 3)
 808                 self.logger.write('\r%s%s%s %s' %
 809                     (begin_line,
 810                      endline,
 811                      src.printcolors.printc(src.KO_STATUS),
 812                      msg), 3)
 813             self.logger.write("\n", 3)
 814
 815         self.logger.write("\n")
 816
 817
 818     def is_occupied(self, hostname):
 819         '''Function that returns True if a job is running on
 820            the machine defined by its host and its port.
 821
 822         :param hostname (str, int): the pair (host, port)
 823         :return: the job that is running on the host,
 824                 or false if there is no job running on the host.
 825         :rtype: job / bool
 826         '''
 827         host = hostname[0]
 828         port = hostname[1]
 829         for jb in self.ljobs:
 830             if jb.machine.host == host and jb.machine.port == port:
 831                 if jb.is_running():
 832                     return jb
 833         return False
 834
 835     def update_jobs_states_list(self):
 836         '''Function that updates the lists that store the currently
 837            running jobs and the jobs that have already finished.
 838
 839         :return: Nothing.
 840         :rtype: N\A
 841         '''
 842         jobs_finished_list = []
 843         jobs_running_list = []
 844         for jb in self.ljobs:
 845             if jb.is_running():
 846                 jobs_running_list.append(jb)
 847                 jb.check_time()
 848             if jb.has_finished():
 849                 jobs_finished_list.append(jb)
 850
 851         nb_job_finished_before = len(self._l_jobs_finished)
 852         self._l_jobs_finished = jobs_finished_list
 853         self._l_jobs_running = jobs_running_list
 854
 855         nb_job_finished_now = len(self._l_jobs_finished)
 856
 857         return nb_job_finished_now > nb_job_finished_before
 858
 859     def cancel_dependencies_of_failing_jobs(self):
 860         '''Function that cancels all the jobs that depend on a failing one.
 861
 862         :return: Nothing.
 863         :rtype: N\A
 864         '''
 865
 866         for job in self.ljobs:
 867             if job.after is None:
 868                 continue
 869             father_job = self.find_job_that_has_name(job.after)
 870             if father_job is not None and father_job.has_failed():
 871                 job.cancel()
 872
 873     def find_job_that_has_name(self, name):
 874         '''Returns the job by its name.
 875
 876         :param name str: a job name
 877         :return: the job that has the name.
 878         :rtype: job
 879         '''
 880         for jb in self.ljobs:
 881             if jb.name == name:
 882                 return jb
 883         # the following is executed only if the job was not found
 884         return None
 885
 886     def str_of_length(self, text, length):
 887         '''Takes a string text of any length and returns
 888            the most close string of length "length".
 889
 890         :param text str: any string
 891         :param length int: a length for the returned string
 892         :return: the most close string of length "length"
 893         :rtype: str
 894         '''
 895         if len(text) > length:
 896             text_out = text[:length-3] + '...'
 897         else:
 898             diff = length - len(text)
 899             before = " " * (diff//2)
 900             after = " " * (diff//2 + diff%2)
 901             text_out = before + text + after
 902
 903         return text_out
 904
 905     def display_status(self, len_col):
 906         '''Takes a lenght and construct the display of the current status
 907            of the jobs in an array that has a column for each host.
 908            It displays the job that is currently running on the host
 909            of the column.
 910
 911         :param len_col int: the size of the column
 912         :return: Nothing
 913         :rtype: N\A
 914         '''
 915
 916         display_line = ""
 917         for host_port in self.lhosts:
 918             jb = self.is_occupied(host_port)
 919             if not jb: # nothing running on the host
 920                 empty = self.str_of_length("empty", len_col)
 921                 display_line += "|" + empty
 922             else:
 923                 display_line += "|" + src.printcolors.printcInfo(
 924                                         self.str_of_length(jb.name, len_col))
 925
 926         self.logger.write("\r" + display_line + "|")
 927         self.logger.flush()
 928
 929
 930     def run_jobs(self):
 931         '''The main method. Runs all the jobs on every host.
 932            For each host, at a given time, only one job can be running.
 933            The jobs that have the field after (that contain the job that has
 934            to be run before it) are run after the previous job.
 935            This method stops when all the jobs are finished.
 936
 937         :return: Nothing
 938         :rtype: N\A
 939         '''
 940
 941         # Print header
 942         self.logger.write(src.printcolors.printcInfo(
 943                                                 _('Executing the jobs :\n')))
 944         text_line = ""
 945         for host_port in self.lhosts:
 946             host = host_port[0]
 947             port = host_port[1]
 948             if port == 22: # default value
 949                 text_line += "|" + self.str_of_length(host, self.len_columns)
 950             else:
 951                 text_line += "|" + self.str_of_length(
 952                                 "("+host+", "+str(port)+")", self.len_columns)
 953
 954         tiret_line = " " + "-"*(len(text_line)-1) + "\n"
 955         self.logger.write(tiret_line)
 956         self.logger.write(text_line + "|\n")
 957         self.logger.write(tiret_line)
 958         self.logger.flush()
 959
 960         # The infinite loop that runs the jobs
 961         l_jobs_not_started = src.deepcopy_list(self.ljobs)
 962         while len(self._l_jobs_finished) != len(self.ljobs):
 963             new_job_start = False
 964             for host_port in self.lhosts:
 965
 966                 if self.is_occupied(host_port):
 967                     continue
 968
 969                 for jb in l_jobs_not_started:
 970                     if (jb.machine.host, jb.machine.port) != host_port:
 971                         continue
 972                     if jb.after == None:
 973                         jb.run()
 974                         l_jobs_not_started.remove(jb)
 975                         new_job_start = True
 976                         break
 977                     else:
 978                         jb_before = self.find_job_that_has_name(jb.after)
 979                         if jb_before is None:
 980                             jb.cancel()
 981                             msg = _("This job was not launched because its "
 982                                     "father is not in the jobs list.")
 983                             jb.out = msg
 984                             jb.err = msg
 985                             break
 986                         if jb_before.has_finished():
 987                             jb.run()
 988                             l_jobs_not_started.remove(jb)
 989                             new_job_start = True
 990                             break
 991             self.cancel_dependencies_of_failing_jobs()
 992             new_job_finished = self.update_jobs_states_list()
 993
 994             if new_job_start or new_job_finished:
 995                 if self.gui:
 996                     self.gui.update_xml_files(self.ljobs)
 997                 # Display the current status
 998                 self.display_status(self.len_columns)
 999
1000             # Make sure that the proc is not entirely busy
1001             time.sleep(0.001)
1002
1003         self.logger.write("\n")
1004         self.logger.write(tiret_line)
1005         self.logger.write("\n\n")
1006
1007         if self.gui:
1008             self.gui.update_xml_files(self.ljobs)
1009             self.gui.last_update()
1010
1011     def write_all_results(self):
1012         '''Display all the jobs outputs.
1013
1014         :return: Nothing
1015         :rtype: N\A
1016         '''
1017
1018         for jb in self.ljobs:
1019             self.logger.write(src.printcolors.printcLabel(
1020                         "#------- Results for job %s -------#\n" % jb.name))
1021             jb.write_results()
1022             self.logger.write("\n\n")
1023
1024 class Gui(object):
1025     '''Class to manage the the xml data that can be displayed in a browser to
1026        see the jobs states
1027     '''
1028
1029     def __init__(self, xml_dir_path, l_jobs, l_jobs_not_today, prefix, file_boards=""):
1030         '''Initialization
1031
1032         :param xml_dir_path str: The path to the directory where to put
1033                                  the xml resulting files
1034         :param l_jobs List: the list of jobs that run today
1035         :param l_jobs_not_today List: the list of jobs that do not run today
1036         :param file_boards str: the file path from which to read the
1037                                    expected boards
1038         '''
1039         # The prefix to add to the xml files : date_hour
1040         self.prefix = prefix
1041
1042         # The path of the csv files to read to fill the expected boards
1043         self.file_boards = file_boards
1044
1045         if file_boards != "":
1046             today = datetime.date.weekday(datetime.date.today())
1047             self.parse_csv_boards(today)
1048         else:
1049             self.d_input_boards = {}
1050
1051         # The path of the global xml file
1052         self.xml_dir_path = xml_dir_path
1053         # Initialize the xml files
1054         xml_global_path = os.path.join(self.xml_dir_path, "global_report.xml")
1055         self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1056                                                          "JobsReport")
1057         # The xml files that corresponds to the boards.
1058         # {name_board : xml_object}}
1059         self.d_xml_board_files = {}
1060         # Create the lines and columns
1061         self.initialize_boards(l_jobs, l_jobs_not_today)
1062
1063         # Write the xml file
1064         self.update_xml_files(l_jobs)
1065
1066     def add_xml_board(self, name):
1067         xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1068         self.d_xml_board_files[name] =  src.xmlManager.XmlLogFile(
1069                                                     xml_board_path,
1070                                                     "JobsReport")
1071         self.d_xml_board_files[name].add_simple_node("distributions")
1072         self.d_xml_board_files[name].add_simple_node("applications")
1073         self.d_xml_board_files[name].add_simple_node("board", text=name)
1074
1075     def initialize_boards(self, l_jobs, l_jobs_not_today):
1076         '''Get all the first information needed for each file and write the
1077            first version of the files
1078         :param l_jobs List: the list of jobs that run today
1079         :param l_jobs_not_today List: the list of jobs that do not run today
1080         '''
1081         # Get the boards to fill and put it in a dictionary
1082         # {board_name : xml instance corresponding to the board}
1083         for job in l_jobs + l_jobs_not_today:
1084             board = job.board
1085             if (board is not None and
1086                                 board not in self.d_xml_board_files.keys()):
1087                 self.add_xml_board(board)
1088
1089         # Verify that the boards given as input are done
1090         for board in list(self.d_input_boards.keys()):
1091             if board not in self.d_xml_board_files:
1092                 self.add_xml_board(board)
1093             root_node = self.d_xml_board_files[board].xmlroot
1094             src.xmlManager.append_node_attrib(root_node,
1095                                               {"input_file" : self.file_boards})
1096
1097         # Loop over all jobs in order to get the lines and columns for each
1098         # xml file
1099         d_dist = {}
1100         d_application = {}
1101         for board in self.d_xml_board_files:
1102             d_dist[board] = []
1103             d_application[board] = []
1104
1105         l_hosts_ports = []
1106
1107         for job in l_jobs + l_jobs_not_today:
1108
1109             if (job.machine.host, job.machine.port) not in l_hosts_ports:
1110                 l_hosts_ports.append((job.machine.host, job.machine.port))
1111
1112             distrib = job.machine.distribution
1113             application = job.application
1114
1115             board_job = job.board
1116             if board is None:
1117                 continue
1118             for board in self.d_xml_board_files:
1119                 if board_job == board:
1120                     if distrib is not None and distrib not in d_dist[board]:
1121                         d_dist[board].append(distrib)
1122                         src.xmlManager.add_simple_node(
1123                             self.d_xml_board_files[board].xmlroot.find(
1124                                                             'distributions'),
1125                                                    "dist",
1126                                                    attrib={"name" : distrib})
1127
1128                 if board_job == board:
1129                     if (application is not None and
1130                                     application not in d_application[board]):
1131                         d_application[board].append(application)
1132                         src.xmlManager.add_simple_node(
1133                             self.d_xml_board_files[board].xmlroot.find(
1134                                                                 'applications'),
1135                                                    "application",
1136                                                    attrib={
1137                                                         "name" : application})
1138
1139         # Verify that there are no missing application or distribution in the
1140         # xml board files (regarding the input boards)
1141         for board in self.d_xml_board_files:
1142             l_dist = d_dist[board]
1143             if board not in self.d_input_boards.keys():
1144                 continue
1145             for dist in self.d_input_boards[board]["rows"]:
1146                 if dist not in l_dist:
1147                     src.xmlManager.add_simple_node(
1148                             self.d_xml_board_files[board].xmlroot.find(
1149                                                             'distributions'),
1150                                                    "dist",
1151                                                    attrib={"name" : dist})
1152             l_appli = d_application[board]
1153             for appli in self.d_input_boards[board]["columns"]:
1154                 if appli not in l_appli:
1155                     src.xmlManager.add_simple_node(
1156                             self.d_xml_board_files[board].xmlroot.find(
1157                                                                 'applications'),
1158                                                    "application",
1159                                                    attrib={"name" : appli})
1160
1161         # Initialize the hosts_ports node for the global file
1162         self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1163                                                                 "hosts_ports")
1164         for host, port in l_hosts_ports:
1165             host_port = "%s:%i" % (host, port)
1166             src.xmlManager.add_simple_node(self.xmlhosts_ports,
1167                                            "host_port",
1168                                            attrib={"name" : host_port})
1169
1170         # Initialize the jobs node in all files
1171         for xml_file in [self.xml_global_file] + list(
1172                                             self.d_xml_board_files.values()):
1173             xml_jobs = xml_file.add_simple_node("jobs")
1174             # Get the jobs present in the config file but
1175             # that will not be launched today
1176             self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1177
1178             xml_file.add_simple_node("infos",
1179                                      attrib={"name" : "last update",
1180                                              "JobsCommandStatus" : "running"})
1181
1182         # Find in each board the squares that needs to be filled regarding the
1183         # input csv files but that are not covered by a today job
1184         for board in self.d_input_boards.keys():
1185             xml_root_board = self.d_xml_board_files[board].xmlroot
1186             xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1187                                                  "missing_jobs")
1188             for row, column in self.d_input_boards[board]["jobs"]:
1189                 found = False
1190                 for job in l_jobs:
1191                     if (job.application == column and
1192                         job.machine.distribution == row):
1193                         found = True
1194                         break
1195                 if not found:
1196                     src.xmlManager.add_simple_node(xml_missing,
1197                                             "job",
1198                                             attrib={"distribution" : row,
1199                                                     "application" : column })
1200
1201     def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1202         '''Get all the first information needed for each file and write the
1203            first version of the files
1204
1205         :param xml_node_jobs etree.Element: the node corresponding to a job
1206         :param l_jobs_not_today List: the list of jobs that do not run today
1207         '''
1208         for job in l_jobs_not_today:
1209             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1210                                                  "job",
1211                                                  attrib={"name" : job.name})
1212             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1213             src.xmlManager.add_simple_node(xmlj,
1214                                            "distribution",
1215                                            job.machine.distribution)
1216             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1217             src.xmlManager.add_simple_node(xmlj,
1218                                        "commands", " ; ".join(job.commands))
1219             src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1220             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1221             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1222             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1223             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1224             src.xmlManager.add_simple_node(xmlj, "sat_path",
1225                                                         job.machine.sat_path)
1226
1227     def parse_csv_boards(self, today):
1228         """ Parse the csv file that describes the boards to produce and fill
1229             the dict d_input_boards that contain the csv file contain
1230
1231         :param today int: the current day of the week
1232         """
1233         # open the csv file and read its content
1234         l_read = []
1235         with open(self.file_boards, 'r') as f:
1236             reader = csv.reader(f,delimiter=CSV_DELIMITER)
1237             for row in reader:
1238                 l_read.append(row)
1239         # get the delimiter for the boards (empty line)
1240         boards_delimiter = [''] * len(l_read[0])
1241         # Make the list of boards, by splitting with the delimiter
1242         l_boards = [list(y) for x, y in itertools.groupby(l_read,
1243                                     lambda z: z == boards_delimiter) if not x]
1244
1245         # loop over the csv lists of lines and get the rows, columns and jobs
1246         d_boards = {}
1247         for input_board in l_boards:
1248             # get board name
1249             board_name = input_board[0][0]
1250
1251             # Get columns list
1252             columns = input_board[0][1:]
1253
1254             rows = []
1255             jobs = []
1256             for line in input_board[1:]:
1257                 row = line[0]
1258                 for i, square in enumerate(line[1:]):
1259                     if square=='':
1260                         continue
1261                     days = square.split(DAYS_SEPARATOR)
1262                     days = [int(day) for day in days]
1263                     if today in days:
1264                         if row not in rows:
1265                             rows.append(row)
1266                         job = (row, columns[i])
1267                         jobs.append(job)
1268
1269             d_boards[board_name] = {"rows" : rows,
1270                                     "columns" : columns,
1271                                     "jobs" : jobs}
1272
1273         self.d_input_boards = d_boards
1274
1275     def update_xml_files(self, l_jobs):
1276         '''Write all the xml files with updated information about the jobs
1277
1278         :param l_jobs List: the list of jobs that run today
1279         '''
1280         for xml_file in [self.xml_global_file] + list(
1281                                             self.d_xml_board_files.values()):
1282             self.update_xml_file(l_jobs, xml_file)
1283
1284         # Write the file
1285         self.write_xml_files()
1286
1287     def update_xml_file(self, l_jobs, xml_file):
1288         '''update information about the jobs for the file xml_file
1289
1290         :param l_jobs List: the list of jobs that run today
1291         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1292         '''
1293
1294         xml_node_jobs = xml_file.xmlroot.find('jobs')
1295         # Update the job names and status node
1296         for job in l_jobs:
1297             # Find the node corresponding to the job and delete it
1298             # in order to recreate it
1299             for xmljob in xml_node_jobs.findall('job'):
1300                 if xmljob.attrib['name'] == job.name:
1301                     xml_node_jobs.remove(xmljob)
1302
1303             T0 = str(job._T0)
1304             if T0 != "-1":
1305                 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1306                                        time.localtime(job._T0))
1307             Tf = str(job._Tf)
1308             if Tf != "-1":
1309                 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1310                                        time.localtime(job._Tf))
1311
1312             # recreate the job node
1313             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1314                                                   "job",
1315                                                   attrib={"name" : job.name})
1316             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1317             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1318             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1319             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1320             src.xmlManager.add_simple_node(xmlj, "sat_path",
1321                                            job.machine.sat_path)
1322             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1323             src.xmlManager.add_simple_node(xmlj, "distribution",
1324                                            job.machine.distribution)
1325             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1326             src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1327             src.xmlManager.add_simple_node(xmlj, "commands",
1328                                            " ; ".join(job.commands))
1329             src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1330             src.xmlManager.add_simple_node(xmlj, "begin", T0)
1331             src.xmlManager.add_simple_node(xmlj, "end", Tf)
1332             src.xmlManager.add_simple_node(xmlj, "out",
1333                                            src.printcolors.cleancolor(job.out))
1334             src.xmlManager.add_simple_node(xmlj, "err",
1335                                            src.printcolors.cleancolor(job.err))
1336             src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1337             if len(job.remote_log_files) > 0:
1338                 src.xmlManager.add_simple_node(xmlj,
1339                                                "remote_log_file_path",
1340                                                job.remote_log_files[0])
1341             else:
1342                 src.xmlManager.add_simple_node(xmlj,
1343                                                "remote_log_file_path",
1344                                                "nothing")
1345
1346             xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1347             # get the job father
1348             if job.after is not None:
1349                 job_father = None
1350                 for jb in l_jobs:
1351                     if jb.name == job.after:
1352                         job_father = jb
1353
1354                 if (job_father is not None and
1355                         len(job_father.remote_log_files) > 0):
1356                     link = job_father.remote_log_files[0]
1357                 else:
1358                     link = "nothing"
1359                 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1360
1361             # Verify that the job is to be done today regarding the input csv
1362             # files
1363             if job.board and job.board in self.d_input_boards.keys():
1364                 found = False
1365                 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1366                     if (job.machine.distribution == dist
1367                         and job.application == appli):
1368                         found = True
1369                         src.xmlManager.add_simple_node(xmlj,
1370                                                "extra_job",
1371                                                "no")
1372                         break
1373                 if not found:
1374                     src.xmlManager.add_simple_node(xmlj,
1375                                                "extra_job",
1376                                                "yes")
1377
1378
1379         # Update the date
1380         xml_node_infos = xml_file.xmlroot.find('infos')
1381         src.xmlManager.append_node_attrib(xml_node_infos,
1382                     attrib={"value" :
1383                     datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1384
1385
1386
1387     def last_update(self, finish_status = "finished"):
1388         '''update information about the jobs for the file xml_file
1389
1390         :param l_jobs List: the list of jobs that run today
1391         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1392         '''
1393         for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1394             xml_node_infos = xml_file.xmlroot.find('infos')
1395             src.xmlManager.append_node_attrib(xml_node_infos,
1396                         attrib={"JobsCommandStatus" : finish_status})
1397         # Write the file
1398         self.write_xml_files()
1399
1400     def write_xml_file(self, xml_file, stylesheet):
1401         ''' Write one xml file and the same file with prefix
1402         '''
1403         xml_file.write_tree(stylesheet)
1404         file_path = xml_file.logFile
1405         file_dir = os.path.dirname(file_path)
1406         file_name = os.path.basename(file_path)
1407         file_name_with_prefix = self.prefix + "_" + file_name
1408         xml_file.write_tree(stylesheet, os.path.join(file_dir,
1409                                                      file_name_with_prefix))
1410
1411     def write_xml_files(self):
1412         ''' Write the xml files
1413         '''
1414         self.write_xml_file(self.xml_global_file, STYLESHEET_GLOBAL)
1415         for xml_file in self.d_xml_board_files.values():
1416             self.write_xml_file(xml_file, STYLESHEET_BOARD)
1417
1418
1419 ##
1420 # Describes the command
1421 def description():
1422     return _("The jobs command launches maintenances that are described"
1423              " in the dedicated jobs configuration file.")
1424
1425 ##
1426 # Runs the command.
1427 def run(args, runner, logger):
1428
1429     (options, args) = parser.parse_args(args)
1430
1431     l_cfg_dir = runner.cfg.PATHS.JOBPATH
1432
1433     # list option : display all the available config files
1434     if options.list:
1435         for cfg_dir in l_cfg_dir:
1436             if not options.no_label:
1437                 logger.write("------ %s\n" %
1438                                  src.printcolors.printcHeader(cfg_dir))
1439
1440             for f in sorted(os.listdir(cfg_dir)):
1441                 if not f.endswith('.pyconf'):
1442                     continue
1443                 cfilename = f[:-7]
1444                 logger.write("%s\n" % cfilename)
1445         return 0
1446
1447     # Make sure the jobs_config option has been called
1448     if not options.jobs_cfg:
1449         message = _("The option --jobs_config is required\n")
1450         src.printcolors.printcError(message)
1451         return 1
1452
1453     # Find the file in the directories
1454     found = False
1455     for cfg_dir in l_cfg_dir:
1456         file_jobs_cfg = os.path.join(cfg_dir, options.jobs_cfg)
1457         if not file_jobs_cfg.endswith('.pyconf'):
1458             file_jobs_cfg += '.pyconf'
1459
1460         if not os.path.exists(file_jobs_cfg):
1461             continue
1462         else:
1463             found = True
1464             break
1465
1466     if not found:
1467         msg = _("The file configuration %(name_file)s was not found."
1468                 "\nUse the --list option to get the possible files.")
1469         src.printcolors.printcError(msg)
1470         return 1
1471
1472     info = [
1473         (_("Platform"), runner.cfg.VARS.dist),
1474         (_("File containing the jobs configuration"), file_jobs_cfg)
1475     ]
1476     src.print_info(logger, info)
1477
1478     # Read the config that is in the file
1479     config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1480     if options.only_jobs:
1481         l_jb = src.pyconf.Sequence()
1482         for jb in config_jobs.jobs:
1483             if jb.name in options.only_jobs:
1484                 l_jb.append(jb,
1485                 "Adding a job that was given in only_jobs option parameters")
1486         config_jobs.jobs = l_jb
1487
1488     # Initialization
1489     today_jobs = Jobs(runner,
1490                       logger,
1491                       file_jobs_cfg,
1492                       config_jobs)
1493     # SSH connection to all machines
1494     today_jobs.ssh_connection_all_machines()
1495     if options.test_connection:
1496         return 0
1497
1498     gui = None
1499     if options.publish:
1500         # Copy the stylesheets in the log directory
1501         log_dir = runner.cfg.USER.log_dir
1502         xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1503         files_to_copy = []
1504         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1505         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1506         files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1507         for file_path in files_to_copy:
1508             shutil.copy2(file_path, log_dir)
1509
1510         # Instanciate the Gui in order to produce the xml files that contain all
1511         # the boards
1512         gui = Gui(runner.cfg.USER.log_dir,
1513                   today_jobs.ljobs,
1514                   today_jobs.ljobs_not_today,
1515                   runner.cfg.VARS.datehour,
1516                   file_boards = options.input_boards)
1517
1518         # Display the list of the xml files
1519         logger.write(src.printcolors.printcInfo(("Here is the list of published"
1520                                                  " files :\n")), 4)
1521         logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1522         for board in gui.d_xml_board_files.keys():
1523             file_path = gui.d_xml_board_files[board].logFile
1524             file_name = os.path.basename(file_path)
1525             logger.write("%s\n" % file_path, 4)
1526             logger.add_link(file_name, "board", 0, board)
1527
1528         logger.write("\n", 4)
1529
1530     today_jobs.gui = gui
1531
1532     interruped = False
1533     try:
1534         # Run all the jobs contained in config_jobs
1535         today_jobs.run_jobs()
1536     except KeyboardInterrupt:
1537         interruped = True
1538         logger.write("\n\n%s\n\n" %
1539                 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1540     finally:
1541         if interruped:
1542             msg = _("Killing the running jobs and trying"
1543                     " to get the corresponding logs\n")
1544             logger.write(src.printcolors.printcWarning(msg))
1545
1546         # find the potential not finished jobs and kill them
1547         for jb in today_jobs.ljobs:
1548             if not jb.has_finished():
1549                 try:
1550                     jb.kill_remote_process()
1551                 except Exception as e:
1552                     msg = _("Failed to kill job %s: %s\n" % (jb.name, e))
1553                     logger.write(src.printcolors.printcWarning(msg))
1554         if interruped:
1555             if today_jobs.gui:
1556                 today_jobs.gui.last_update(_("Forced interruption"))
1557         else:
1558             if today_jobs.gui:
1559                 today_jobs.gui.last_update()
1560         # Output the results
1561         today_jobs.write_all_results()