commands/jobs.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import sys
  21 import tempfile
  22 import traceback
  23 import datetime
  24 import time
  25 import csv
  26 import shutil
  27 import itertools
  28 import re
  29 import paramiko
  30
  31 import src
  32 import src.ElementTree as etree
  33
  34 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
  35 STYLESHEET_BOARD = "jobs_board_report.xsl"
  36
  37 DAYS_SEPARATOR = ","
  38 CSV_DELIMITER = ";"
  39
  40 parser = src.options.Options()
  41
  42 parser.add_option('n', 'name', 'list2', 'jobs_cfg',
  43                   _('Mandatory: The name of the config file that contains'
  44                   ' the jobs configuration. Can be a list.'))
  45 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
  46                   _('Optional: the list of jobs to launch, by their name. '))
  47 parser.add_option('l', 'list', 'boolean', 'list',
  48                   _('Optional: list all available config files.'))
  49 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
  50                   _("Optional: try to connect to the machines. "
  51                     "Not executing the jobs."),
  52                   False)
  53 parser.add_option('p', 'publish', 'boolean', 'publish',
  54                   _("Optional: generate an xml file that can be read in a "
  55                     "browser to display the jobs status."),
  56                   False)
  57 parser.add_option('i', 'input_boards', 'string', 'input_boards', _("Optional: "
  58                                 "the path to csv file that contain "
  59                                 "the expected boards."),"")
  60 parser.add_option('', 'completion', 'boolean', 'no_label',
  61                   _("Optional (internal use): do not print labels, Works only "
  62                     "with --list."),
  63                   False)
  64
  65 class Machine(object):
  66     '''Class to manage a ssh connection on a machine
  67     '''
  68     def __init__(self,
  69                  name,
  70                  host,
  71                  user,
  72                  port=22,
  73                  passwd=None,
  74                  sat_path="salomeTools"):
  75         self.name = name
  76         self.host = host
  77         self.port = port
  78         self.distribution = None # Will be filled after copying SAT on the machine
  79         self.user = user
  80         self.password = passwd
  81         self.sat_path = sat_path
  82         self.ssh = paramiko.SSHClient()
  83         self._connection_successful = None
  84
  85     def connect(self, logger):
  86         '''Initiate the ssh connection to the remote machine
  87
  88         :param logger src.logger.Logger: The logger instance
  89         :return: Nothing
  90         :rtype: N\A
  91         '''
  92
  93         self._connection_successful = False
  94         self.ssh.load_system_host_keys()
  95         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  96         try:
  97             self.ssh.connect(self.host,
  98                              port=self.port,
  99                              username=self.user,
 100                              password = self.password)
 101         except paramiko.AuthenticationException:
 102             message = src.KO_STATUS + _("Authentication failed")
 103         except paramiko.BadHostKeyException:
 104             message = (src.KO_STATUS +
 105                        _("The server's host key could not be verified"))
 106         except paramiko.SSHException:
 107             message = ( _("SSHException error connecting or "
 108                           "establishing an SSH session"))
 109         except:
 110             message = ( _("Error connecting or establishing an SSH session"))
 111         else:
 112             self._connection_successful = True
 113             message = ""
 114         return message
 115
 116     def successfully_connected(self, logger):
 117         '''Verify if the connection to the remote machine has succeed
 118
 119         :param logger src.logger.Logger: The logger instance
 120         :return: True if the connection has succeed, False if not
 121         :rtype: bool
 122         '''
 123         if self._connection_successful == None:
 124             message = _("Warning : trying to ask if the connection to "
 125             "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
 126             " no connection request" %
 127                         (self.name, self.host, self.port, self.user))
 128             logger.write( src.printcolors.printcWarning(message))
 129         return self._connection_successful
 130
 131     def copy_sat(self, sat_local_path, job_file):
 132         '''Copy salomeTools to the remote machine in self.sat_path
 133         '''
 134         res = 0
 135         try:
 136             # open a sftp connection
 137             self.sftp = self.ssh.open_sftp()
 138             # Create the sat directory on remote machine if it is not existing
 139             self.mkdir(self.sat_path, ignore_existing=True)
 140             # Put sat
 141             self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
 142             # put the job configuration file in order to make it reachable
 143             # on the remote machine
 144             remote_job_file_name = ".%s" % os.path.basename(job_file)
 145             self.sftp.put(job_file, os.path.join(self.sat_path,
 146                                                  remote_job_file_name))
 147         except Exception as e:
 148             res = str(e)
 149             self._connection_successful = False
 150
 151         return res
 152
 153     def put_dir(self, source, target, filters = []):
 154         ''' Uploads the contents of the source directory to the target path. The
 155             target directory needs to exists. All sub-directories in source are
 156             created under target.
 157         '''
 158         for item in os.listdir(source):
 159             if item in filters:
 160                 continue
 161             source_path = os.path.join(source, item)
 162             destination_path = os.path.join(target, item)
 163             if os.path.islink(source_path):
 164                 linkto = os.readlink(source_path)
 165                 try:
 166                     self.sftp.symlink(linkto, destination_path)
 167                     self.sftp.chmod(destination_path,
 168                                     os.stat(source_path).st_mode)
 169                 except IOError:
 170                     pass
 171             else:
 172                 if os.path.isfile(source_path):
 173                     self.sftp.put(source_path, destination_path)
 174                     self.sftp.chmod(destination_path,
 175                                     os.stat(source_path).st_mode)
 176                 else:
 177                     self.mkdir(destination_path, ignore_existing=True)
 178                     self.put_dir(source_path, destination_path)
 179
 180     def mkdir(self, path, mode=511, ignore_existing=False):
 181         ''' Augments mkdir by adding an option to not fail
 182             if the folder exists
 183         '''
 184         try:
 185             self.sftp.mkdir(path, mode)
 186         except IOError:
 187             if ignore_existing:
 188                 pass
 189             else:
 190                 raise
 191
 192     def exec_command(self, command, logger):
 193         '''Execute the command on the remote machine
 194
 195         :param command str: The command to be run
 196         :param logger src.logger.Logger: The logger instance
 197         :return: the stdin, stdout, and stderr of the executing command,
 198                  as a 3-tuple
 199         :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
 200                 paramiko.channel.ChannelFile)
 201         '''
 202         try:
 203             # Does not wait the end of the command
 204             (stdin, stdout, stderr) = self.ssh.exec_command(command)
 205         except paramiko.SSHException:
 206             message = src.KO_STATUS + _(
 207                             ": the server failed to execute the command\n")
 208             logger.write( src.printcolors.printcError(message))
 209             return (None, None, None)
 210         except:
 211             logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
 212             return (None, None, None)
 213         else:
 214             return (stdin, stdout, stderr)
 215
 216     def close(self):
 217         '''Close the ssh connection
 218
 219         :rtype: N\A
 220         '''
 221         self.ssh.close()
 222
 223     def write_info(self, logger):
 224         '''Prints the informations relative to the machine in the logger
 225            (terminal traces and log file)
 226
 227         :param logger src.logger.Logger: The logger instance
 228         :return: Nothing
 229         :rtype: N\A
 230         '''
 231         logger.write("host : " + self.host + "\n")
 232         logger.write("port : " + str(self.port) + "\n")
 233         logger.write("user : " + str(self.user) + "\n")
 234         if self.successfully_connected(logger):
 235             status = src.OK_STATUS
 236         else:
 237             status = src.KO_STATUS
 238         logger.write("Connection : " + status + "\n\n")
 239
 240
 241 class Job(object):
 242     '''Class to manage one job
 243     '''
 244     def __init__(self,
 245                  name,
 246                  machine,
 247                  application,
 248                  board,
 249                  commands,
 250                  timeout,
 251                  config,
 252                  job_file_path,
 253                  logger,
 254                  after=None,
 255                  prefix=None):
 256
 257         self.name = name
 258         self.machine = machine
 259         self.after = after
 260         self.timeout = timeout
 261         self.application = application
 262         self.board = board
 263         self.config = config
 264         self.logger = logger
 265         # The list of log files to download from the remote machine
 266         self.remote_log_files = []
 267
 268         # The remote command status
 269         # -1 means that it has not been launched,
 270         # 0 means success and 1 means fail
 271         self.res_job = "-1"
 272         self.cancelled = False
 273
 274         self._T0 = -1
 275         self._Tf = -1
 276         self._has_begun = False
 277         self._has_finished = False
 278         self._has_timouted = False
 279         self._stdin = None # Store the command inputs field
 280         self._stdout = None # Store the command outputs field
 281         self._stderr = None # Store the command errors field
 282
 283         self.out = ""
 284         self.err = ""
 285
 286         self.name_remote_jobs_pyconf = ".%s" % os.path.basename(job_file_path)
 287         self.commands = commands
 288         self.command = (os.path.join(self.machine.sat_path, "sat") +
 289                         " -l " +
 290                         os.path.join(self.machine.sat_path,
 291                                      "list_log_files.txt") +
 292                         " job --jobs_config " +
 293                         os.path.join(self.machine.sat_path,
 294                                      self.name_remote_jobs_pyconf) +
 295                         " --name " +
 296                         self.name)
 297         if prefix:
 298             self.command = prefix + ' "' + self.command +'"'
 299
 300     def get_pids(self):
 301         """ Get the pid(s) corresponding to the command that have been launched
 302             On the remote machine
 303
 304         :return: The list of integers corresponding to the found pids
 305         :rtype: List
 306         """
 307         pids = []
 308         cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
 309         (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
 310         pids_cmd = out_pid.readlines()
 311         pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
 312         pids+=pids_cmd
 313         return pids
 314
 315     def kill_remote_process(self, wait=1):
 316         '''Kills the process on the remote machine.
 317
 318         :return: (the output of the kill, the error of the kill)
 319         :rtype: (str, str)
 320         '''
 321         try:
 322             pids = self.get_pids()
 323         except:
 324             return ("Unable to get the pid of the command.", "")
 325
 326         cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
 327         (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
 328                                                             self.logger)
 329         time.sleep(wait)
 330         return (out_kill.read().decode(), err_kill.read().decode())
 331
 332     def has_begun(self):
 333         '''Returns True if the job has already begun
 334
 335         :return: True if the job has already begun
 336         :rtype: bool
 337         '''
 338         return self._has_begun
 339
 340     def has_finished(self):
 341         '''Returns True if the job has already finished
 342            (i.e. all the commands have been executed)
 343            If it is finished, the outputs are stored in the fields out and err.
 344
 345         :return: True if the job has already finished
 346         :rtype: bool
 347         '''
 348
 349         # If the method has already been called and returned True
 350         if self._has_finished:
 351             return True
 352
 353         # If the job has not begun yet
 354         if not self.has_begun():
 355             return False
 356
 357         if self._stdout.channel.closed:
 358             self._has_finished = True
 359             # Store the result outputs
 360             self.out += self._stdout.read().decode()
 361             self.err += self._stderr.read().decode()
 362             # Put end time
 363             self._Tf = time.time()
 364             # And get the remote command status and log files
 365             try:
 366                 self.get_log_files()
 367             except Exception as e:
 368                 self.err += _("Unable to get remote log files: %s" % e)
 369
 370         return self._has_finished
 371
 372     def get_log_files(self):
 373         """Get the log files produced by the command launched
 374            on the remote machine, and put it in the log directory of the user,
 375            so they can be accessible from
 376         """
 377         # Do not get the files if the command is not finished
 378         if not self.has_finished():
 379             msg = _("Trying to get log files whereas the job is not finished.")
 380             self.logger.write(src.printcolors.printcWarning(msg))
 381             return
 382
 383         # First get the file that contains the list of log files to get
 384         tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
 385         remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt")
 386         self.machine.sftp.get(
 387                     remote_path,
 388                     tmp_file_path)
 389
 390         # Read the file and get the result of the command and all the log files
 391         # to get
 392         fstream_tmp = open(tmp_file_path, "r")
 393         file_lines = fstream_tmp.readlines()
 394         file_lines = [line.replace("\n", "") for line in file_lines]
 395         fstream_tmp.close()
 396         os.remove(tmp_file_path)
 397
 398         try :
 399             # The first line is the result of the command (0 success or 1 fail)
 400             self.res_job = file_lines[0]
 401         except Exception as e:
 402             self.err += _("Unable to get status from remote file %s: %s" %
 403                                                     (remote_path, str(e)))
 404
 405         for i, job_path_remote in enumerate(file_lines[1:]):
 406             try:
 407                 # For each command, there is two files to get :
 408                 # 1- The xml file describing the command and giving the
 409                 # internal traces.
 410                 # 2- The txt file containing the system command traces (like
 411                 # traces produced by the "make" command)
 412                 # 3- In case of the test command, there is another file to get :
 413                 # the xml board that contain the test results
 414                 dirname = os.path.basename(os.path.dirname(job_path_remote))
 415                 if dirname != 'OUT' and dirname != 'TEST':
 416                     # Case 1-
 417                     local_path = os.path.join(os.path.dirname(
 418                                                         self.logger.logFilePath),
 419                                               os.path.basename(job_path_remote))
 420                     if i==0: # The first is the job command
 421                         self.logger.add_link(os.path.basename(job_path_remote),
 422                                              "job",
 423                                              self.res_job,
 424                                              self.command)
 425                 elif dirname == 'OUT':
 426                     # Case 2-
 427                     local_path = os.path.join(os.path.dirname(
 428                                                         self.logger.logFilePath),
 429                                               'OUT',
 430                                               os.path.basename(job_path_remote))
 431                 elif dirname == 'TEST':
 432                     # Case 3-
 433                     local_path = os.path.join(os.path.dirname(
 434                                                         self.logger.logFilePath),
 435                                               'TEST',
 436                                               os.path.basename(job_path_remote))
 437
 438                 # Get the file
 439                 if not os.path.exists(local_path):
 440                     self.machine.sftp.get(job_path_remote, local_path)
 441                 self.remote_log_files.append(local_path)
 442             except Exception as e:
 443                 self.err += _("Unable to get %s log file from remote: %s" %
 444                                                     (str(job_path_remote),
 445                                                      str(e)))
 446
 447     def has_failed(self):
 448         '''Returns True if the job has failed.
 449            A job is considered as failed if the machine could not be reached,
 450            if the remote command failed,
 451            or if the job finished with a time out.
 452
 453         :return: True if the job has failed
 454         :rtype: bool
 455         '''
 456         if not self.has_finished():
 457             return False
 458         if not self.machine.successfully_connected(self.logger):
 459             return True
 460         if self.is_timeout():
 461             return True
 462         if self.res_job == "1":
 463             return True
 464         return False
 465
 466     def cancel(self):
 467         """In case of a failing job, one has to cancel every job that depend
 468            on it. This method put the job as failed and will not be executed.
 469         """
 470         if self.cancelled:
 471             return
 472         self._has_begun = True
 473         self._has_finished = True
 474         self.cancelled = True
 475         self.out += _("This job was not launched because its father has failed.")
 476         self.err += _("This job was not launched because its father has failed.")
 477
 478     def is_running(self):
 479         '''Returns True if the job commands are running
 480
 481         :return: True if the job is running
 482         :rtype: bool
 483         '''
 484         return self.has_begun() and not self.has_finished()
 485
 486     def is_timeout(self):
 487         '''Returns True if the job commands has finished with timeout
 488
 489         :return: True if the job has finished with timeout
 490         :rtype: bool
 491         '''
 492         return self._has_timouted
 493
 494     def time_elapsed(self):
 495         """Get the time elapsed since the job launching
 496
 497         :return: The number of seconds
 498         :rtype: int
 499         """
 500         if not self.has_begun():
 501             return -1
 502         T_now = time.time()
 503         return T_now - self._T0
 504
 505     def check_time(self):
 506         """Verify that the job has not exceeded its timeout.
 507            If it has, kill the remote command and consider the job as finished.
 508         """
 509         if not self.has_begun():
 510             return
 511         if self.time_elapsed() > self.timeout:
 512             self._has_finished = True
 513             self._has_timouted = True
 514             self._Tf = time.time()
 515             (out_kill, _) = self.kill_remote_process()
 516             self.out += "TIMEOUT \n" + out_kill
 517             self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
 518             try:
 519                 self.get_log_files()
 520             except Exception as e:
 521                 # The 2 following lines must be suppressed after the bug is fixed
 522                 print("The error type: ")
 523                 print(type(e))
 524                 print("The error: ")
 525                 print(e)
 526                 print("Local scope:")
 527                 print(dir())
 528                 print("The str type: ")
 529                 print(type(str))
 530                 print("str: ")
 531                 print(str)
 532                 self.err += _("Unable to get remote log files!")
 533
 534     def total_duration(self):
 535         """Give the total duration of the job
 536
 537         :return: the total duration of the job in seconds
 538         :rtype: int
 539         """
 540         return self._Tf - self._T0
 541
 542     def run(self):
 543         """Launch the job by executing the remote command.
 544         """
 545
 546         # Prevent multiple run
 547         if self.has_begun():
 548             msg = _("Warning: A job can only be launched one time")
 549             msg2 = _("Trying to launch the job \"%s\" whereas it has "
 550                      "already been launched." % self.name)
 551             self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
 552                                                                         msg2)))
 553             return
 554
 555         # Do not execute the command if the machine could not be reached
 556         if not self.machine.successfully_connected(self.logger):
 557             self._has_finished = True
 558             self.out = "N\A"
 559             self.err += ("Connection to machine (name : %s, host: %s, port:"
 560                         " %s, user: %s) has failed\nUse the log command "
 561                         "to get more information."
 562                         % (self.machine.name,
 563                            self.machine.host,
 564                            self.machine.port,
 565                            self.machine.user))
 566         else:
 567             # Usual case : Launch the command on remote machine
 568             self._T0 = time.time()
 569             self._stdin, self._stdout, self._stderr = self.machine.exec_command(
 570                                                                   self.command,
 571                                                                   self.logger)
 572             # If the results are not initialized, finish the job
 573             if (self._stdin, self._stdout, self._stderr) == (None, None, None):
 574                 self._has_finished = True
 575                 self._Tf = time.time()
 576                 self.out += "N\A"
 577                 self.err += "The server failed to execute the command"
 578
 579         # Put the beginning flag to true.
 580         self._has_begun = True
 581
 582     def write_results(self):
 583         """Display on the terminal all the job's information
 584         """
 585         self.logger.write("name : " + self.name + "\n")
 586         if self.after:
 587             self.logger.write("after : %s\n" % self.after)
 588         self.logger.write("Time elapsed : %4imin %2is \n" %
 589                      (self.total_duration()//60 , self.total_duration()%60))
 590         if self._T0 != -1:
 591             self.logger.write("Begin time : %s\n" %
 592                          time.strftime('%Y-%m-%d %H:%M:%S',
 593                                        time.localtime(self._T0)) )
 594         if self._Tf != -1:
 595             self.logger.write("End time   : %s\n\n" %
 596                          time.strftime('%Y-%m-%d %H:%M:%S',
 597                                        time.localtime(self._Tf)) )
 598
 599         machine_head = "Informations about connection :\n"
 600         underline = (len(machine_head) - 2) * "-"
 601         self.logger.write(src.printcolors.printcInfo(
 602                                                 machine_head+underline+"\n"))
 603         self.machine.write_info(self.logger)
 604
 605         self.logger.write(src.printcolors.printcInfo("out : \n"))
 606         if self.out == "":
 607             self.logger.write("Unable to get output\n")
 608         else:
 609             self.logger.write(self.out + "\n")
 610         self.logger.write(src.printcolors.printcInfo("err : \n"))
 611         self.logger.write(self.err + "\n")
 612
 613     def get_status(self):
 614         """Get the status of the job (used by the Gui for xml display)
 615
 616         :return: The current status of the job
 617         :rtype: String
 618         """
 619         if not self.machine.successfully_connected(self.logger):
 620             return "SSH connection KO"
 621         if not self.has_begun():
 622             return "Not launched"
 623         if self.cancelled:
 624             return "Cancelled"
 625         if self.is_running():
 626             return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
 627                                                     time.localtime(self._T0))
 628         if self.has_finished():
 629             if self.is_timeout():
 630                 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
 631                                                     time.localtime(self._Tf))
 632             return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
 633                                                      time.localtime(self._Tf))
 634
 635 class Jobs(object):
 636     '''Class to manage the jobs to be run
 637     '''
 638     def __init__(self,
 639                  runner,
 640                  logger,
 641                  job_file_path,
 642                  config_jobs,
 643                  lenght_columns = 20):
 644         # The jobs configuration
 645         self.cfg_jobs = config_jobs
 646         self.job_file_path = job_file_path
 647         # The machine that will be used today
 648         self.lmachines = []
 649         # The list of machine (hosts, port) that will be used today
 650         # (a same host can have several machine instances since there
 651         # can be several ssh parameters)
 652         self.lhosts = []
 653         # The jobs to be launched today
 654         self.ljobs = []
 655         # The jobs that will not be launched today
 656         self.ljobs_not_today = []
 657         self.runner = runner
 658         self.logger = logger
 659         self.len_columns = lenght_columns
 660
 661         # the list of jobs that have not been run yet
 662         self._l_jobs_not_started = []
 663         # the list of jobs that have already ran
 664         self._l_jobs_finished = []
 665         # the list of jobs that are running
 666         self._l_jobs_running = []
 667
 668         self.determine_jobs_and_machines()
 669
 670     def define_job(self, job_def, machine):
 671         '''Takes a pyconf job definition and a machine (from class machine)
 672            and returns the job instance corresponding to the definition.
 673
 674         :param job_def src.config.Mapping: a job definition
 675         :param machine machine: the machine on which the job will run
 676         :return: The corresponding job in a job class instance
 677         :rtype: job
 678         '''
 679         name = job_def.name
 680         cmmnds = job_def.commands
 681         if not "timeout" in job_def:
 682             timeout = 4*60*60 # default timeout = 4h
 683         else:
 684             timeout = job_def.timeout
 685         after = None
 686         if 'after' in job_def:
 687             after = job_def.after
 688         application = None
 689         if 'application' in job_def:
 690             application = job_def.application
 691         board = None
 692         if 'board' in job_def:
 693             board = job_def.board
 694         prefix = None
 695         if "prefix" in job_def:
 696             prefix = job_def.prefix
 697
 698         return Job(name,
 699                    machine,
 700                    application,
 701                    board,
 702                    cmmnds,
 703                    timeout,
 704                    self.runner.cfg,
 705                    self.job_file_path,
 706                    self.logger,
 707                    after = after,
 708                    prefix = prefix)
 709
 710     def determine_jobs_and_machines(self):
 711         '''Function that reads the pyconf jobs definition and instantiates all
 712            the machines and jobs to be done today.
 713
 714         :return: Nothing
 715         :rtype: N\A
 716         '''
 717         today = datetime.date.weekday(datetime.date.today())
 718         host_list = []
 719
 720         for job_def in self.cfg_jobs.jobs :
 721
 722             if not "machine" in job_def:
 723                 msg = _('WARNING: The job "%s" do not have the key '
 724                        '"machine", this job is ignored.\n\n' % job_def.name)
 725                 self.logger.write(src.printcolors.printcWarning(msg))
 726                 continue
 727             name_machine = job_def.machine
 728
 729             a_machine = None
 730             for mach in self.lmachines:
 731                 if mach.name == name_machine:
 732                     a_machine = mach
 733                     break
 734
 735             if a_machine == None:
 736                 for machine_def in self.cfg_jobs.machines:
 737                     if machine_def.name == name_machine:
 738                         if 'host' not in machine_def:
 739                             host = self.runner.cfg.VARS.hostname
 740                         else:
 741                             host = machine_def.host
 742
 743                         if 'user' not in machine_def:
 744                             user = self.runner.cfg.VARS.user
 745                         else:
 746                             user = machine_def.user
 747
 748                         if 'port' not in machine_def:
 749                             port = 22
 750                         else:
 751                             port = machine_def.port
 752
 753                         if 'password' not in machine_def:
 754                             passwd = None
 755                         else:
 756                             passwd = machine_def.password
 757
 758                         if 'sat_path' not in machine_def:
 759                             sat_path = "salomeTools"
 760                         else:
 761                             sat_path = machine_def.sat_path
 762
 763                         a_machine = Machine(
 764                                             machine_def.name,
 765                                             host,
 766                                             user,
 767                                             port=port,
 768                                             passwd=passwd,
 769                                             sat_path=sat_path
 770                                             )
 771
 772                         self.lmachines.append(a_machine)
 773                         if (host, port) not in host_list:
 774                             host_list.append((host, port))
 775
 776                 if a_machine == None:
 777                     msg = _("WARNING: The job \"%(job_name)s\" requires the "
 778                             "machine \"%(machine_name)s\" but this machine "
 779                             "is not defined in the configuration file.\n"
 780                             "The job will not be launched")
 781                     self.logger.write(src.printcolors.printcWarning(msg))
 782
 783             a_job = self.define_job(job_def, a_machine)
 784
 785             if today in job_def.when:
 786                 self.ljobs.append(a_job)
 787             else: # today in job_def.when
 788                 self.ljobs_not_today.append(a_job)
 789
 790         self.lhosts = host_list
 791
 792     def ssh_connection_all_machines(self, pad=50):
 793         '''Function that do the ssh connection to every machine
 794            to be used today.
 795
 796         :return: Nothing
 797         :rtype: N\A
 798         '''
 799         self.logger.write(src.printcolors.printcInfo((
 800                         "Establishing connection with all the machines :\n")))
 801         for machine in self.lmachines:
 802             # little algorithm in order to display traces
 803             begin_line = (_("Connection to %s: " % machine.name))
 804             if pad - len(begin_line) < 0:
 805                 endline = " "
 806             else:
 807                 endline = (pad - len(begin_line)) * "." + " "
 808
 809             step = "SSH connection"
 810             self.logger.write( begin_line + endline + step)
 811             self.logger.flush()
 812             # the call to the method that initiate the ssh connection
 813             msg = machine.connect(self.logger)
 814
 815             # Copy salomeTools to the remote machine
 816             if machine.successfully_connected(self.logger):
 817                 step = _("Remove SAT")
 818                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 819                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 820                 (__, out_dist, __) = machine.exec_command(
 821                                                 "rm -rf %s" % machine.sat_path,
 822                                                 self.logger)
 823                 out_dist.read()
 824
 825                 self.logger.flush()
 826                 step = _("Copy SAT")
 827                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 828                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 829                 self.logger.flush()
 830                 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
 831                                             self.job_file_path)
 832                 # get the remote machine distribution using a sat command
 833                 (__, out_dist, __) = machine.exec_command(
 834                                 os.path.join(machine.sat_path,
 835                                     "sat config --value VARS.dist --no_label"),
 836                                 self.logger)
 837                 machine.distribution = out_dist.read().decode().replace("\n",
 838                                                                         "")
 839                 # Print the status of the copy
 840                 if res_copy == 0:
 841                     self.logger.write('\r%s' %
 842                                 ((len(begin_line)+len(endline)+20) * " "), 3)
 843                     self.logger.write('\r%s%s%s' %
 844                         (begin_line,
 845                          endline,
 846                          src.printcolors.printc(src.OK_STATUS)), 3)
 847                 else:
 848                     self.logger.write('\r%s' %
 849                             ((len(begin_line)+len(endline)+20) * " "), 3)
 850                     self.logger.write('\r%s%s%s %s' %
 851                         (begin_line,
 852                          endline,
 853                          src.printcolors.printc(src.KO_STATUS),
 854                          _("Copy of SAT failed: %s" % res_copy)), 3)
 855             else:
 856                 self.logger.write('\r%s' %
 857                                   ((len(begin_line)+len(endline)+20) * " "), 3)
 858                 self.logger.write('\r%s%s%s %s' %
 859                     (begin_line,
 860                      endline,
 861                      src.printcolors.printc(src.KO_STATUS),
 862                      msg), 3)
 863             self.logger.write("\n", 3)
 864
 865         self.logger.write("\n")
 866
 867
 868     def is_occupied(self, hostname):
 869         '''Function that returns True if a job is running on
 870            the machine defined by its host and its port.
 871
 872         :param hostname (str, int): the pair (host, port)
 873         :return: the job that is running on the host,
 874                 or false if there is no job running on the host.
 875         :rtype: job / bool
 876         '''
 877         host = hostname[0]
 878         port = hostname[1]
 879         for jb in self.ljobs:
 880             if jb.machine.host == host and jb.machine.port == port:
 881                 if jb.is_running():
 882                     return jb
 883         return False
 884
 885     def update_jobs_states_list(self):
 886         '''Function that updates the lists that store the currently
 887            running jobs and the jobs that have already finished.
 888
 889         :return: Nothing.
 890         :rtype: N\A
 891         '''
 892         jobs_finished_list = []
 893         jobs_running_list = []
 894         for jb in self.ljobs:
 895             if jb.is_running():
 896                 jobs_running_list.append(jb)
 897                 jb.check_time()
 898             if jb.has_finished():
 899                 jobs_finished_list.append(jb)
 900
 901         nb_job_finished_before = len(self._l_jobs_finished)
 902         self._l_jobs_finished = jobs_finished_list
 903         self._l_jobs_running = jobs_running_list
 904
 905         nb_job_finished_now = len(self._l_jobs_finished)
 906
 907         return nb_job_finished_now > nb_job_finished_before
 908
 909     def cancel_dependencies_of_failing_jobs(self):
 910         '''Function that cancels all the jobs that depend on a failing one.
 911
 912         :return: Nothing.
 913         :rtype: N\A
 914         '''
 915
 916         for job in self.ljobs:
 917             if job.after is None:
 918                 continue
 919             father_job = self.find_job_that_has_name(job.after)
 920             if father_job is not None and father_job.has_failed():
 921                 job.cancel()
 922
 923     def find_job_that_has_name(self, name):
 924         '''Returns the job by its name.
 925
 926         :param name str: a job name
 927         :return: the job that has the name.
 928         :rtype: job
 929         '''
 930         for jb in self.ljobs:
 931             if jb.name == name:
 932                 return jb
 933         # the following is executed only if the job was not found
 934         return None
 935
 936     def str_of_length(self, text, length):
 937         '''Takes a string text of any length and returns
 938            the most close string of length "length".
 939
 940         :param text str: any string
 941         :param length int: a length for the returned string
 942         :return: the most close string of length "length"
 943         :rtype: str
 944         '''
 945         if len(text) > length:
 946             text_out = text[:length-3] + '...'
 947         else:
 948             diff = length - len(text)
 949             before = " " * (diff//2)
 950             after = " " * (diff//2 + diff%2)
 951             text_out = before + text + after
 952
 953         return text_out
 954
 955     def display_status(self, len_col):
 956         '''Takes a lenght and construct the display of the current status
 957            of the jobs in an array that has a column for each host.
 958            It displays the job that is currently running on the host
 959            of the column.
 960
 961         :param len_col int: the size of the column
 962         :return: Nothing
 963         :rtype: N\A
 964         '''
 965
 966         display_line = ""
 967         for host_port in self.lhosts:
 968             jb = self.is_occupied(host_port)
 969             if not jb: # nothing running on the host
 970                 empty = self.str_of_length("empty", len_col)
 971                 display_line += "|" + empty
 972             else:
 973                 display_line += "|" + src.printcolors.printcInfo(
 974                                         self.str_of_length(jb.name, len_col))
 975
 976         self.logger.write("\r" + display_line + "|")
 977         self.logger.flush()
 978
 979
 980     def run_jobs(self):
 981         '''The main method. Runs all the jobs on every host.
 982            For each host, at a given time, only one job can be running.
 983            The jobs that have the field after (that contain the job that has
 984            to be run before it) are run after the previous job.
 985            This method stops when all the jobs are finished.
 986
 987         :return: Nothing
 988         :rtype: N\A
 989         '''
 990
 991         # Print header
 992         self.logger.write(src.printcolors.printcInfo(
 993                                                 _('Executing the jobs :\n')))
 994         text_line = ""
 995         for host_port in self.lhosts:
 996             host = host_port[0]
 997             port = host_port[1]
 998             if port == 22: # default value
 999                 text_line += "|" + self.str_of_length(host, self.len_columns)
1000             else:
1001                 text_line += "|" + self.str_of_length(
1002                                 "("+host+", "+str(port)+")", self.len_columns)
1003
1004         tiret_line = " " + "-"*(len(text_line)-1) + "\n"
1005         self.logger.write(tiret_line)
1006         self.logger.write(text_line + "|\n")
1007         self.logger.write(tiret_line)
1008         self.logger.flush()
1009
1010         # The infinite loop that runs the jobs
1011         l_jobs_not_started = src.deepcopy_list(self.ljobs)
1012         while len(self._l_jobs_finished) != len(self.ljobs):
1013             new_job_start = False
1014             for host_port in self.lhosts:
1015
1016                 if self.is_occupied(host_port):
1017                     continue
1018
1019                 for jb in l_jobs_not_started:
1020                     if (jb.machine.host, jb.machine.port) != host_port:
1021                         continue
1022                     if jb.after == None:
1023                         jb.run()
1024                         l_jobs_not_started.remove(jb)
1025                         new_job_start = True
1026                         break
1027                     else:
1028                         jb_before = self.find_job_that_has_name(jb.after)
1029                         if jb_before is None:
1030                             jb.cancel()
1031                             msg = _("This job was not launched because its "
1032                                     "father is not in the jobs list.")
1033                             jb.out = msg
1034                             jb.err = msg
1035                             break
1036                         if jb_before.has_finished():
1037                             jb.run()
1038                             l_jobs_not_started.remove(jb)
1039                             new_job_start = True
1040                             break
1041             self.cancel_dependencies_of_failing_jobs()
1042             new_job_finished = self.update_jobs_states_list()
1043
1044             if new_job_start or new_job_finished:
1045                 if self.gui:
1046                     self.gui.update_xml_files(self.ljobs)
1047                 # Display the current status
1048                 self.display_status(self.len_columns)
1049
1050             # Make sure that the proc is not entirely busy
1051             time.sleep(0.001)
1052
1053         self.logger.write("\n")
1054         self.logger.write(tiret_line)
1055         self.logger.write("\n\n")
1056
1057         if self.gui:
1058             self.gui.update_xml_files(self.ljobs)
1059             self.gui.last_update()
1060
1061     def write_all_results(self):
1062         '''Display all the jobs outputs.
1063
1064         :return: Nothing
1065         :rtype: N\A
1066         '''
1067
1068         for jb in self.ljobs:
1069             self.logger.write(src.printcolors.printcLabel(
1070                         "#------- Results for job %s -------#\n" % jb.name))
1071             jb.write_results()
1072             self.logger.write("\n\n")
1073
1074 class Gui(object):
1075     '''Class to manage the the xml data that can be displayed in a browser to
1076        see the jobs states
1077     '''
1078
1079     def __init__(self,
1080                  xml_dir_path,
1081                  l_jobs,
1082                  l_jobs_not_today,
1083                  prefix,
1084                  logger,
1085                  file_boards=""):
1086         '''Initialization
1087
1088         :param xml_dir_path str: The path to the directory where to put
1089                                  the xml resulting files
1090         :param l_jobs List: the list of jobs that run today
1091         :param l_jobs_not_today List: the list of jobs that do not run today
1092         :param file_boards str: the file path from which to read the
1093                                    expected boards
1094         '''
1095         # The logging instance
1096         self.logger = logger
1097
1098         # The prefix to add to the xml files : date_hour
1099         self.prefix = prefix
1100
1101         # The path of the csv files to read to fill the expected boards
1102         self.file_boards = file_boards
1103
1104         if file_boards != "":
1105             today = datetime.date.weekday(datetime.date.today())
1106             self.parse_csv_boards(today)
1107         else:
1108             self.d_input_boards = {}
1109
1110         # The path of the global xml file
1111         self.xml_dir_path = xml_dir_path
1112         # Initialize the xml files
1113         self.global_name = "global_report"
1114         xml_global_path = os.path.join(self.xml_dir_path,
1115                                        self.global_name + ".xml")
1116         self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1117                                                          "JobsReport")
1118
1119         # Find history for each job
1120         self.history = {}
1121         self.find_history(l_jobs, l_jobs_not_today)
1122
1123         # The xml files that corresponds to the boards.
1124         # {name_board : xml_object}}
1125         self.d_xml_board_files = {}
1126
1127         # Create the lines and columns
1128         self.initialize_boards(l_jobs, l_jobs_not_today)
1129
1130         # Write the xml file
1131         self.update_xml_files(l_jobs)
1132
1133     def add_xml_board(self, name):
1134         '''Add a board to the board list
1135         :param name str: the board name
1136         '''
1137         xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1138         self.d_xml_board_files[name] =  src.xmlManager.XmlLogFile(
1139                                                     xml_board_path,
1140                                                     "JobsReport")
1141         self.d_xml_board_files[name].add_simple_node("distributions")
1142         self.d_xml_board_files[name].add_simple_node("applications")
1143         self.d_xml_board_files[name].add_simple_node("board", text=name)
1144
1145     def initialize_boards(self, l_jobs, l_jobs_not_today):
1146         '''Get all the first information needed for each file and write the
1147            first version of the files
1148         :param l_jobs List: the list of jobs that run today
1149         :param l_jobs_not_today List: the list of jobs that do not run today
1150         '''
1151         # Get the boards to fill and put it in a dictionary
1152         # {board_name : xml instance corresponding to the board}
1153         for job in l_jobs + l_jobs_not_today:
1154             board = job.board
1155             if (board is not None and
1156                                 board not in self.d_xml_board_files.keys()):
1157                 self.add_xml_board(board)
1158
1159         # Verify that the boards given as input are done
1160         for board in list(self.d_input_boards.keys()):
1161             if board not in self.d_xml_board_files:
1162                 self.add_xml_board(board)
1163             root_node = self.d_xml_board_files[board].xmlroot
1164             src.xmlManager.append_node_attrib(root_node,
1165                                               {"input_file" : self.file_boards})
1166
1167         # Loop over all jobs in order to get the lines and columns for each
1168         # xml file
1169         d_dist = {}
1170         d_application = {}
1171         for board in self.d_xml_board_files:
1172             d_dist[board] = []
1173             d_application[board] = []
1174
1175         l_hosts_ports = []
1176
1177         for job in l_jobs + l_jobs_not_today:
1178
1179             if (job.machine.host, job.machine.port) not in l_hosts_ports:
1180                 l_hosts_ports.append((job.machine.host, job.machine.port))
1181
1182             distrib = job.machine.distribution
1183             application = job.application
1184
1185             board_job = job.board
1186             if board is None:
1187                 continue
1188             for board in self.d_xml_board_files:
1189                 if board_job == board:
1190                     if distrib is not None and distrib not in d_dist[board]:
1191                         d_dist[board].append(distrib)
1192                         src.xmlManager.add_simple_node(
1193                             self.d_xml_board_files[board].xmlroot.find(
1194                                                             'distributions'),
1195                                                    "dist",
1196                                                    attrib={"name" : distrib})
1197
1198                 if board_job == board:
1199                     if (application is not None and
1200                                     application not in d_application[board]):
1201                         d_application[board].append(application)
1202                         src.xmlManager.add_simple_node(
1203                             self.d_xml_board_files[board].xmlroot.find(
1204                                                                 'applications'),
1205                                                    "application",
1206                                                    attrib={
1207                                                         "name" : application})
1208
1209         # Verify that there are no missing application or distribution in the
1210         # xml board files (regarding the input boards)
1211         for board in self.d_xml_board_files:
1212             l_dist = d_dist[board]
1213             if board not in self.d_input_boards.keys():
1214                 continue
1215             for dist in self.d_input_boards[board]["rows"]:
1216                 if dist not in l_dist:
1217                     src.xmlManager.add_simple_node(
1218                             self.d_xml_board_files[board].xmlroot.find(
1219                                                             'distributions'),
1220                                                    "dist",
1221                                                    attrib={"name" : dist})
1222             l_appli = d_application[board]
1223             for appli in self.d_input_boards[board]["columns"]:
1224                 if appli not in l_appli:
1225                     src.xmlManager.add_simple_node(
1226                             self.d_xml_board_files[board].xmlroot.find(
1227                                                                 'applications'),
1228                                                    "application",
1229                                                    attrib={"name" : appli})
1230
1231         # Initialize the hosts_ports node for the global file
1232         self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1233                                                                 "hosts_ports")
1234         for host, port in l_hosts_ports:
1235             host_port = "%s:%i" % (host, port)
1236             src.xmlManager.add_simple_node(self.xmlhosts_ports,
1237                                            "host_port",
1238                                            attrib={"name" : host_port})
1239
1240         # Initialize the jobs node in all files
1241         for xml_file in [self.xml_global_file] + list(
1242                                             self.d_xml_board_files.values()):
1243             xml_jobs = xml_file.add_simple_node("jobs")
1244             # Get the jobs present in the config file but
1245             # that will not be launched today
1246             self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1247
1248             # add also the infos node
1249             xml_file.add_simple_node("infos",
1250                                      attrib={"name" : "last update",
1251                                              "JobsCommandStatus" : "running"})
1252
1253             # and put the history node
1254             history_node = xml_file.add_simple_node("history")
1255             name_board = os.path.basename(xml_file.logFile)[:-len(".xml")]
1256             # serach for board files
1257             expression = "^[0-9]{8}_+[0-9]{6}_" + name_board + ".xml$"
1258             oExpr = re.compile(expression)
1259             # Get the list of xml borad files that are in the log directory
1260             for file_name in os.listdir(self.xml_dir_path):
1261                 if oExpr.search(file_name):
1262                     date = os.path.basename(file_name).split("_")[0]
1263                     file_path = os.path.join(self.xml_dir_path, file_name)
1264                     src.xmlManager.add_simple_node(history_node,
1265                                                    "link",
1266                                                    text=file_path,
1267                                                    attrib={"date" : date})
1268
1269
1270         # Find in each board the squares that needs to be filled regarding the
1271         # input csv files but that are not covered by a today job
1272         for board in self.d_input_boards.keys():
1273             xml_root_board = self.d_xml_board_files[board].xmlroot
1274             # Find the missing jobs for today
1275             xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1276                                                  "missing_jobs")
1277             for row, column in self.d_input_boards[board]["jobs"]:
1278                 found = False
1279                 for job in l_jobs:
1280                     if (job.application == column and
1281                         job.machine.distribution == row):
1282                         found = True
1283                         break
1284                 if not found:
1285                     src.xmlManager.add_simple_node(xml_missing,
1286                                             "job",
1287                                             attrib={"distribution" : row,
1288                                                     "application" : column })
1289             # Find the missing jobs not today
1290             xml_missing_not_today = src.xmlManager.add_simple_node(
1291                                                  xml_root_board,
1292                                                  "missing_jobs_not_today")
1293             for row, column in self.d_input_boards[board]["jobs_not_today"]:
1294                 found = False
1295                 for job in l_jobs_not_today:
1296                     if (job.application == column and
1297                         job.machine.distribution == row):
1298                         found = True
1299                         break
1300                 if not found:
1301                     src.xmlManager.add_simple_node(xml_missing_not_today,
1302                                             "job",
1303                                             attrib={"distribution" : row,
1304                                                     "application" : column })
1305
1306     def find_history(self, l_jobs, l_jobs_not_today):
1307         """find, for each job, in the existent xml boards the results for the
1308            job. Store the results in the dictionnary self.history = {name_job :
1309            list of (date, status, list links)}
1310
1311         :param l_jobs List: the list of jobs to run today
1312         :param l_jobs_not_today List: the list of jobs that do not run today
1313         """
1314         # load the all the history
1315         expression = "^[0-9]{8}_+[0-9]{6}_" + self.global_name + ".xml$"
1316         oExpr = re.compile(expression)
1317         # Get the list of global xml that are in the log directory
1318         l_globalxml = []
1319         for file_name in os.listdir(self.xml_dir_path):
1320             if oExpr.search(file_name):
1321                 file_path = os.path.join(self.xml_dir_path, file_name)
1322                 try:
1323                     global_xml = src.xmlManager.ReadXmlFile(file_path)
1324                     l_globalxml.append(global_xml)
1325                 except Exception as e:
1326                     msg = _("\nWARNING: the file %s can not be read, it will be "
1327                             "ignored\n%s" % (file_path, e))
1328                     self.logger.write("%s\n" % src.printcolors.printcWarning(
1329                                                                         msg), 5)
1330
1331         # Construct the dictionnary self.history
1332         for job in l_jobs + l_jobs_not_today:
1333             l_links = []
1334             for global_xml in l_globalxml:
1335                 date = os.path.basename(global_xml.filePath).split("_")[0]
1336                 global_root_node = global_xml.xmlroot.find("jobs")
1337                 job_node = src.xmlManager.find_node_by_attrib(
1338                                                               global_root_node,
1339                                                               "job",
1340                                                               "name",
1341                                                               job.name)
1342                 if job_node:
1343                     if job_node.find("remote_log_file_path") is not None:
1344                         link = job_node.find("remote_log_file_path").text
1345                         res_job = job_node.find("res").text
1346                         if link != "nothing":
1347                             l_links.append((date, res_job, link))
1348             l_links = sorted(l_links, reverse=True)
1349             self.history[job.name] = l_links
1350
1351     def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1352         '''Get all the first information needed for each file and write the
1353            first version of the files
1354
1355         :param xml_node_jobs etree.Element: the node corresponding to a job
1356         :param l_jobs_not_today List: the list of jobs that do not run today
1357         '''
1358         for job in l_jobs_not_today:
1359             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1360                                                  "job",
1361                                                  attrib={"name" : job.name})
1362             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1363             src.xmlManager.add_simple_node(xmlj,
1364                                            "distribution",
1365                                            job.machine.distribution)
1366             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1367             src.xmlManager.add_simple_node(xmlj,
1368                                        "commands", " ; ".join(job.commands))
1369             src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1370             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1371             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1372             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1373             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1374             src.xmlManager.add_simple_node(xmlj, "sat_path",
1375                                                         job.machine.sat_path)
1376             xml_history = src.xmlManager.add_simple_node(xmlj, "history")
1377             for i, (date, res_job, link) in enumerate(self.history[job.name]):
1378                 if i==0:
1379                     # tag the first one (the last one)
1380                     src.xmlManager.add_simple_node(xml_history,
1381                                                    "link",
1382                                                    text=link,
1383                                                    attrib={"date" : date,
1384                                                            "res" : res_job,
1385                                                            "last" : "yes"})
1386                 else:
1387                     src.xmlManager.add_simple_node(xml_history,
1388                                                    "link",
1389                                                    text=link,
1390                                                    attrib={"date" : date,
1391                                                            "res" : res_job,
1392                                                            "last" : "no"})
1393
1394     def parse_csv_boards(self, today):
1395         """ Parse the csv file that describes the boards to produce and fill
1396             the dict d_input_boards that contain the csv file contain
1397
1398         :param today int: the current day of the week
1399         """
1400         # open the csv file and read its content
1401         l_read = []
1402         with open(self.file_boards, 'r') as f:
1403             reader = csv.reader(f,delimiter=CSV_DELIMITER)
1404             for row in reader:
1405                 l_read.append(row)
1406         # get the delimiter for the boards (empty line)
1407         boards_delimiter = [''] * len(l_read[0])
1408         # Make the list of boards, by splitting with the delimiter
1409         l_boards = [list(y) for x, y in itertools.groupby(l_read,
1410                                     lambda z: z == boards_delimiter) if not x]
1411
1412         # loop over the csv lists of lines and get the rows, columns and jobs
1413         d_boards = {}
1414         for input_board in l_boards:
1415             # get board name
1416             board_name = input_board[0][0]
1417
1418             # Get columns list
1419             columns = input_board[0][1:]
1420
1421             rows = []
1422             jobs = []
1423             jobs_not_today = []
1424             for line in input_board[1:]:
1425                 row = line[0]
1426                 rows.append(row)
1427                 for i, square in enumerate(line[1:]):
1428                     if square=='':
1429                         continue
1430                     days = square.split(DAYS_SEPARATOR)
1431                     days = [int(day) for day in days]
1432                     job = (row, columns[i])
1433                     if today in days:
1434                         jobs.append(job)
1435                     else:
1436                         jobs_not_today.append(job)
1437
1438             d_boards[board_name] = {"rows" : rows,
1439                                     "columns" : columns,
1440                                     "jobs" : jobs,
1441                                     "jobs_not_today" : jobs_not_today}
1442
1443         self.d_input_boards = d_boards
1444
1445     def update_xml_files(self, l_jobs):
1446         '''Write all the xml files with updated information about the jobs
1447
1448         :param l_jobs List: the list of jobs that run today
1449         '''
1450         for xml_file in [self.xml_global_file] + list(
1451                                             self.d_xml_board_files.values()):
1452             self.update_xml_file(l_jobs, xml_file)
1453
1454         # Write the file
1455         self.write_xml_files()
1456
1457     def update_xml_file(self, l_jobs, xml_file):
1458         '''update information about the jobs for the file xml_file
1459
1460         :param l_jobs List: the list of jobs that run today
1461         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1462         '''
1463
1464         xml_node_jobs = xml_file.xmlroot.find('jobs')
1465         # Update the job names and status node
1466         for job in l_jobs:
1467             # Find the node corresponding to the job and delete it
1468             # in order to recreate it
1469             for xmljob in xml_node_jobs.findall('job'):
1470                 if xmljob.attrib['name'] == job.name:
1471                     xml_node_jobs.remove(xmljob)
1472
1473             T0 = str(job._T0)
1474             if T0 != "-1":
1475                 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1476                                        time.localtime(job._T0))
1477             Tf = str(job._Tf)
1478             if Tf != "-1":
1479                 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1480                                        time.localtime(job._Tf))
1481
1482             # recreate the job node
1483             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1484                                                   "job",
1485                                                   attrib={"name" : job.name})
1486             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1487             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1488             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1489             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1490             xml_history = src.xmlManager.add_simple_node(xmlj, "history")
1491             for date, res_job, link in self.history[job.name]:
1492                 src.xmlManager.add_simple_node(xml_history,
1493                                                "link",
1494                                                text=link,
1495                                                attrib={"date" : date,
1496                                                        "res" : res_job})
1497
1498             src.xmlManager.add_simple_node(xmlj, "sat_path",
1499                                            job.machine.sat_path)
1500             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1501             src.xmlManager.add_simple_node(xmlj, "distribution",
1502                                            job.machine.distribution)
1503             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1504             src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1505             src.xmlManager.add_simple_node(xmlj, "commands",
1506                                            " ; ".join(job.commands))
1507             src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1508             src.xmlManager.add_simple_node(xmlj, "begin", T0)
1509             src.xmlManager.add_simple_node(xmlj, "end", Tf)
1510             src.xmlManager.add_simple_node(xmlj, "out",
1511                                            src.printcolors.cleancolor(job.out))
1512             src.xmlManager.add_simple_node(xmlj, "err",
1513                                            src.printcolors.cleancolor(job.err))
1514             src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1515             if len(job.remote_log_files) > 0:
1516                 src.xmlManager.add_simple_node(xmlj,
1517                                                "remote_log_file_path",
1518                                                job.remote_log_files[0])
1519             else:
1520                 src.xmlManager.add_simple_node(xmlj,
1521                                                "remote_log_file_path",
1522                                                "nothing")
1523             # Search for the test log if there is any
1524             l_test_log_files = self.find_test_log(job.remote_log_files)
1525             xml_test = src.xmlManager.add_simple_node(xmlj,
1526                                                       "test_log_file_path")
1527             for test_log_path, res_test, nb_fails in l_test_log_files:
1528                 test_path_node = src.xmlManager.add_simple_node(xml_test,
1529                                                "path",
1530                                                test_log_path)
1531                 test_path_node.attrib["res"] = res_test
1532                 test_path_node.attrib["nb_fails"] = nb_fails
1533
1534             xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1535             # get the job father
1536             if job.after is not None:
1537                 job_father = None
1538                 for jb in l_jobs:
1539                     if jb.name == job.after:
1540                         job_father = jb
1541
1542                 if (job_father is not None and
1543                         len(job_father.remote_log_files) > 0):
1544                     link = job_father.remote_log_files[0]
1545                 else:
1546                     link = "nothing"
1547                 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1548
1549             # Verify that the job is to be done today regarding the input csv
1550             # files
1551             if job.board and job.board in self.d_input_boards.keys():
1552                 found = False
1553                 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1554                     if (job.machine.distribution == dist
1555                         and job.application == appli):
1556                         found = True
1557                         src.xmlManager.add_simple_node(xmlj,
1558                                                "extra_job",
1559                                                "no")
1560                         break
1561                 if not found:
1562                     src.xmlManager.add_simple_node(xmlj,
1563                                                "extra_job",
1564                                                "yes")
1565
1566
1567         # Update the date
1568         xml_node_infos = xml_file.xmlroot.find('infos')
1569         src.xmlManager.append_node_attrib(xml_node_infos,
1570                     attrib={"value" :
1571                     datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1572
1573
1574     def find_test_log(self, l_remote_log_files):
1575         '''Find if there is a test log (board) in the remote log files and
1576            the path to it. There can be several test command, so the result is
1577            a list.
1578
1579         :param l_remote_log_files List: the list of all remote log files
1580         :return: the list of (test log files path, res of the command)
1581         :rtype: List
1582         '''
1583         res = []
1584         for file_path in l_remote_log_files:
1585             dirname = os.path.basename(os.path.dirname(file_path))
1586             file_name = os.path.basename(file_path)
1587             regex = src.logger.log_all_command_file_expression
1588             oExpr = re.compile(regex)
1589             if dirname == "TEST" and oExpr.search(file_name):
1590                 # find the res of the command
1591                 prod_node = etree.parse(file_path).getroot().find("product")
1592                 res_test = prod_node.attrib["global_res"]
1593                 # find the number of fails
1594                 testbase_node = prod_node.find("tests").find("testbase")
1595                 nb_fails = int(testbase_node.attrib["failed"])
1596                 # put the file path, the res of the test command and the number
1597                 # of fails in the output
1598                 res.append((file_path, res_test, nb_fails))
1599
1600         return res
1601
1602     def last_update(self, finish_status = "finished"):
1603         '''update information about the jobs for the file xml_file
1604
1605         :param l_jobs List: the list of jobs that run today
1606         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1607         '''
1608         for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1609             xml_node_infos = xml_file.xmlroot.find('infos')
1610             src.xmlManager.append_node_attrib(xml_node_infos,
1611                         attrib={"JobsCommandStatus" : finish_status})
1612         # Write the file
1613         self.write_xml_files()
1614
1615     def write_xml_file(self, xml_file, stylesheet):
1616         ''' Write one xml file and the same file with prefix
1617         '''
1618         xml_file.write_tree(stylesheet)
1619         file_path = xml_file.logFile
1620         file_dir = os.path.dirname(file_path)
1621         file_name = os.path.basename(file_path)
1622         file_name_with_prefix = self.prefix + "_" + file_name
1623         xml_file.write_tree(stylesheet, os.path.join(file_dir,
1624                                                      file_name_with_prefix))
1625
1626     def write_xml_files(self):
1627         ''' Write the xml files
1628         '''
1629         self.write_xml_file(self.xml_global_file, STYLESHEET_GLOBAL)
1630         for xml_file in self.d_xml_board_files.values():
1631             self.write_xml_file(xml_file, STYLESHEET_BOARD)
1632
1633 def get_config_file_path(job_config_name, l_cfg_dir):
1634     found = False
1635     file_jobs_cfg = None
1636     if os.path.exists(job_config_name) and job_config_name.endswith(".pyconf"):
1637         found = True
1638         file_jobs_cfg = job_config_name
1639     else:
1640         for cfg_dir in l_cfg_dir:
1641             file_jobs_cfg = os.path.join(cfg_dir, job_config_name)
1642             if not file_jobs_cfg.endswith('.pyconf'):
1643                 file_jobs_cfg += '.pyconf'
1644
1645             if not os.path.exists(file_jobs_cfg):
1646                 continue
1647             else:
1648                 found = True
1649                 break
1650     return found, file_jobs_cfg
1651
1652 ##
1653 # Describes the command
1654 def description():
1655     return _("The jobs command launches maintenances that are described"
1656              " in the dedicated jobs configuration file.\n\nexample:\nsat "
1657              "jobs --name my_jobs --publish")
1658
1659 ##
1660 # Runs the command.
1661 def run(args, runner, logger):
1662
1663     (options, args) = parser.parse_args(args)
1664
1665     l_cfg_dir = runner.cfg.PATHS.JOBPATH
1666
1667     # list option : display all the available config files
1668     if options.list:
1669         for cfg_dir in l_cfg_dir:
1670             if not options.no_label:
1671                 logger.write("------ %s\n" %
1672                                  src.printcolors.printcHeader(cfg_dir))
1673             if not os.path.exists(cfg_dir):
1674                 continue
1675             for f in sorted(os.listdir(cfg_dir)):
1676                 if not f.endswith('.pyconf'):
1677                     continue
1678                 cfilename = f[:-7]
1679                 logger.write("%s\n" % cfilename)
1680         return 0
1681
1682     # Make sure the jobs_config option has been called
1683     if not options.jobs_cfg:
1684         message = _("The option --jobs_config is required\n")
1685         src.printcolors.printcError(message)
1686         return 1
1687
1688     # Find the file in the directories, unless it is a full path
1689     # merge all in a config
1690     merger = src.pyconf.ConfigMerger()
1691     config_jobs = src.pyconf.Config()
1692     l_conf_files_path = []
1693     for config_file in options.jobs_cfg:
1694         found, file_jobs_cfg = get_config_file_path(config_file, l_cfg_dir)
1695         if not found:
1696             msg = _("The file configuration %s was not found."
1697                     "\nUse the --list option to get the "
1698                     "possible files." % config_file)
1699             logger.write("%s\n" % src.printcolors.printcError(msg), 1)
1700             return 1
1701         l_conf_files_path.append(file_jobs_cfg)
1702         # Read the config that is in the file
1703         one_config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1704         merger.merge(config_jobs, one_config_jobs)
1705
1706     info = [
1707         (_("Platform"), runner.cfg.VARS.dist),
1708         (_("Files containing the jobs configuration"), l_conf_files_path)
1709     ]
1710     src.print_info(logger, info)
1711
1712     if options.only_jobs:
1713         l_jb = src.pyconf.Sequence()
1714         for jb in config_jobs.jobs:
1715             if jb.name in options.only_jobs:
1716                 l_jb.append(jb,
1717                 "Job that was given in only_jobs option parameters\n")
1718         config_jobs.jobs = l_jb
1719
1720     # Make a unique file that contain all the jobs in order to use it
1721     # on every machine
1722     name_pyconf = "_".join([os.path.basename(path)[:-len('.pyconf')]
1723                             for path in l_conf_files_path]) + ".pyconf"
1724     path_pyconf = src.get_tmp_filename(runner.cfg, name_pyconf)
1725     #Save config
1726     f = file( path_pyconf , 'w')
1727     config_jobs.__save__(f)
1728
1729     # Initialization
1730     today_jobs = Jobs(runner,
1731                       logger,
1732                       path_pyconf,
1733                       config_jobs)
1734     # SSH connection to all machines
1735     today_jobs.ssh_connection_all_machines()
1736     if options.test_connection:
1737         return 0
1738
1739     gui = None
1740     if options.publish:
1741         logger.write(src.printcolors.printcInfo(
1742                                         _("Initialize the xml boards : ")), 5)
1743         logger.flush()
1744
1745         # Copy the stylesheets in the log directory
1746         log_dir = runner.cfg.USER.log_dir
1747         xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1748         files_to_copy = []
1749         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1750         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1751         files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1752         for file_path in files_to_copy:
1753             shutil.copy2(file_path, log_dir)
1754
1755         # Instanciate the Gui in order to produce the xml files that contain all
1756         # the boards
1757         gui = Gui(runner.cfg.USER.log_dir,
1758                   today_jobs.ljobs,
1759                   today_jobs.ljobs_not_today,
1760                   runner.cfg.VARS.datehour,
1761                   logger,
1762                   file_boards = options.input_boards)
1763
1764         logger.write(src.printcolors.printcSuccess("OK"), 5)
1765         logger.write("\n\n", 5)
1766         logger.flush()
1767
1768         # Display the list of the xml files
1769         logger.write(src.printcolors.printcInfo(("Here is the list of published"
1770                                                  " files :\n")), 4)
1771         logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1772         for board in gui.d_xml_board_files.keys():
1773             file_path = gui.d_xml_board_files[board].logFile
1774             file_name = os.path.basename(file_path)
1775             logger.write("%s\n" % file_path, 4)
1776             logger.add_link(file_name, "board", 0, board)
1777
1778         logger.write("\n", 4)
1779
1780     today_jobs.gui = gui
1781
1782     interruped = False
1783     try:
1784         # Run all the jobs contained in config_jobs
1785         today_jobs.run_jobs()
1786     except KeyboardInterrupt:
1787         interruped = True
1788         logger.write("\n\n%s\n\n" %
1789                 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1790     except Exception as e:
1791         msg = _("CRITICAL ERROR: The jobs loop has been interrupted\n")
1792         logger.write("\n\n%s\n" % src.printcolors.printcError(msg) )
1793         logger.write("%s\n" % str(e))
1794         # get stack
1795         __, __, exc_traceback = sys.exc_info()
1796         fp = tempfile.TemporaryFile()
1797         traceback.print_tb(exc_traceback, file=fp)
1798         fp.seek(0)
1799         stack = fp.read()
1800         logger.write("\nTRACEBACK: %s\n" % stack.replace('"',"'"), 1)
1801
1802     finally:
1803         res = 0
1804         if interruped:
1805             res = 1
1806             msg = _("Killing the running jobs and trying"
1807                     " to get the corresponding logs\n")
1808             logger.write(src.printcolors.printcWarning(msg))
1809
1810         # find the potential not finished jobs and kill them
1811         for jb in today_jobs.ljobs:
1812             if not jb.has_finished():
1813                 res = 1
1814                 try:
1815                     jb.kill_remote_process()
1816                 except Exception as e:
1817                     msg = _("Failed to kill job %s: %s\n" % (jb.name, e))
1818                     logger.write(src.printcolors.printcWarning(msg))
1819             if jb.res_job != "0":
1820                 res = 1
1821         if interruped:
1822             if today_jobs.gui:
1823                 today_jobs.gui.last_update(_("Forced interruption"))
1824         else:
1825             if today_jobs.gui:
1826                 today_jobs.gui.last_update()
1827         # Output the results
1828         today_jobs.write_all_results()
1829         # Remove the temporary pyconf file
1830         if os.path.exists(path_pyconf):
1831             os.remove(path_pyconf)
1832         return res