commands/jobs.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import sys
  21 import tempfile
  22 import traceback
  23 import datetime
  24 import time
  25 import csv
  26 import shutil
  27 import itertools
  28 import re
  29 import paramiko
  30
  31 import src
  32 import src.ElementTree as etree
  33
  34 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
  35 STYLESHEET_BOARD = "jobs_board_report.xsl"
  36
  37 DAYS_SEPARATOR = ","
  38 CSV_DELIMITER = ";"
  39
  40 parser = src.options.Options()
  41
  42 parser.add_option('n', 'name', 'list2', 'jobs_cfg',
  43                   _('Mandatory: The name of the config file that contains'
  44                   ' the jobs configuration. Can be a list.'))
  45 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
  46                   _('Optional: the list of jobs to launch, by their name. '))
  47 parser.add_option('l', 'list', 'boolean', 'list',
  48                   _('Optional: list all available config files.'))
  49 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
  50                   _("Optional: try to connect to the machines. "
  51                     "Not executing the jobs."),
  52                   False)
  53 parser.add_option('p', 'publish', 'boolean', 'publish',
  54                   _("Optional: generate an xml file that can be read in a "
  55                     "browser to display the jobs status."),
  56                   False)
  57 parser.add_option('i', 'input_boards', 'string', 'input_boards', _("Optional: "
  58                                 "the path to csv file that contain "
  59                                 "the expected boards."),"")
  60 parser.add_option('', 'completion', 'boolean', 'no_label',
  61                   _("Optional (internal use): do not print labels, Works only "
  62                     "with --list."),
  63                   False)
  64
  65 class Machine(object):
  66     '''Class to manage a ssh connection on a machine
  67     '''
  68     def __init__(self,
  69                  name,
  70                  host,
  71                  user,
  72                  port=22,
  73                  passwd=None,
  74                  sat_path="salomeTools"):
  75         self.name = name
  76         self.host = host
  77         self.port = port
  78         self.distribution = None # Will be filled after copying SAT on the machine
  79         self.user = user
  80         self.password = passwd
  81         self.sat_path = sat_path
  82         self.ssh = paramiko.SSHClient()
  83         self._connection_successful = None
  84
  85     def connect(self, logger):
  86         '''Initiate the ssh connection to the remote machine
  87
  88         :param logger src.logger.Logger: The logger instance
  89         :return: Nothing
  90         :rtype: N\A
  91         '''
  92
  93         self._connection_successful = False
  94         self.ssh.load_system_host_keys()
  95         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  96         try:
  97             self.ssh.connect(self.host,
  98                              port=self.port,
  99                              username=self.user,
 100                              password = self.password)
 101         except paramiko.AuthenticationException:
 102             message = src.KO_STATUS + _("Authentication failed")
 103         except paramiko.BadHostKeyException:
 104             message = (src.KO_STATUS +
 105                        _("The server's host key could not be verified"))
 106         except paramiko.SSHException:
 107             message = ( _("SSHException error connecting or "
 108                           "establishing an SSH session"))
 109         except:
 110             message = ( _("Error connecting or establishing an SSH session"))
 111         else:
 112             self._connection_successful = True
 113             message = ""
 114         return message
 115
 116     def successfully_connected(self, logger):
 117         '''Verify if the connection to the remote machine has succeed
 118
 119         :param logger src.logger.Logger: The logger instance
 120         :return: True if the connection has succeed, False if not
 121         :rtype: bool
 122         '''
 123         if self._connection_successful == None:
 124             message = _("Warning : trying to ask if the connection to "
 125             "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
 126             " no connection request" %
 127                         (self.name, self.host, self.port, self.user))
 128             logger.write( src.printcolors.printcWarning(message))
 129         return self._connection_successful
 130
 131     def copy_sat(self, sat_local_path, job_file):
 132         '''Copy salomeTools to the remote machine in self.sat_path
 133         '''
 134         res = 0
 135         try:
 136             # open a sftp connection
 137             self.sftp = self.ssh.open_sftp()
 138             # Create the sat directory on remote machine if it is not existing
 139             self.mkdir(self.sat_path, ignore_existing=True)
 140             # Put sat
 141             self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
 142             # put the job configuration file in order to make it reachable
 143             # on the remote machine
 144             remote_job_file_name = ".%s" % os.path.basename(job_file)
 145             self.sftp.put(job_file, os.path.join(self.sat_path,
 146                                                  remote_job_file_name))
 147         except Exception as e:
 148             res = str(e)
 149             self._connection_successful = False
 150
 151         return res
 152
 153     def put_dir(self, source, target, filters = []):
 154         ''' Uploads the contents of the source directory to the target path. The
 155             target directory needs to exists. All sub-directories in source are
 156             created under target.
 157         '''
 158         for item in os.listdir(source):
 159             if item in filters:
 160                 continue
 161             source_path = os.path.join(source, item)
 162             destination_path = os.path.join(target, item)
 163             if os.path.islink(source_path):
 164                 linkto = os.readlink(source_path)
 165                 try:
 166                     self.sftp.symlink(linkto, destination_path)
 167                     self.sftp.chmod(destination_path,
 168                                     os.stat(source_path).st_mode)
 169                 except IOError:
 170                     pass
 171             else:
 172                 if os.path.isfile(source_path):
 173                     self.sftp.put(source_path, destination_path)
 174                     self.sftp.chmod(destination_path,
 175                                     os.stat(source_path).st_mode)
 176                 else:
 177                     self.mkdir(destination_path, ignore_existing=True)
 178                     self.put_dir(source_path, destination_path)
 179
 180     def mkdir(self, path, mode=511, ignore_existing=False):
 181         ''' Augments mkdir by adding an option to not fail
 182             if the folder exists
 183         '''
 184         try:
 185             self.sftp.mkdir(path, mode)
 186         except IOError:
 187             if ignore_existing:
 188                 pass
 189             else:
 190                 raise
 191
 192     def exec_command(self, command, logger):
 193         '''Execute the command on the remote machine
 194
 195         :param command str: The command to be run
 196         :param logger src.logger.Logger: The logger instance
 197         :return: the stdin, stdout, and stderr of the executing command,
 198                  as a 3-tuple
 199         :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
 200                 paramiko.channel.ChannelFile)
 201         '''
 202         try:
 203             # Does not wait the end of the command
 204             (stdin, stdout, stderr) = self.ssh.exec_command(command)
 205         except paramiko.SSHException:
 206             message = src.KO_STATUS + _(
 207                             ": the server failed to execute the command\n")
 208             logger.write( src.printcolors.printcError(message))
 209             return (None, None, None)
 210         except:
 211             logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
 212             return (None, None, None)
 213         else:
 214             return (stdin, stdout, stderr)
 215
 216     def close(self):
 217         '''Close the ssh connection
 218
 219         :rtype: N\A
 220         '''
 221         self.ssh.close()
 222
 223     def write_info(self, logger):
 224         '''Prints the informations relative to the machine in the logger
 225            (terminal traces and log file)
 226
 227         :param logger src.logger.Logger: The logger instance
 228         :return: Nothing
 229         :rtype: N\A
 230         '''
 231         logger.write("host : " + self.host + "\n")
 232         logger.write("port : " + str(self.port) + "\n")
 233         logger.write("user : " + str(self.user) + "\n")
 234         if self.successfully_connected(logger):
 235             status = src.OK_STATUS
 236         else:
 237             status = src.KO_STATUS
 238         logger.write("Connection : " + status + "\n\n")
 239
 240
 241 class Job(object):
 242     '''Class to manage one job
 243     '''
 244     def __init__(self,
 245                  name,
 246                  machine,
 247                  application,
 248                  board,
 249                  commands,
 250                  timeout,
 251                  config,
 252                  job_file_path,
 253                  logger,
 254                  after=None,
 255                  prefix=None):
 256
 257         self.name = name
 258         self.machine = machine
 259         self.after = after
 260         self.timeout = timeout
 261         self.application = application
 262         self.board = board
 263         self.config = config
 264         self.logger = logger
 265         # The list of log files to download from the remote machine
 266         self.remote_log_files = []
 267
 268         # The remote command status
 269         # -1 means that it has not been launched,
 270         # 0 means success and 1 means fail
 271         self.res_job = "-1"
 272         self.cancelled = False
 273
 274         self._T0 = -1
 275         self._Tf = -1
 276         self._has_begun = False
 277         self._has_finished = False
 278         self._has_timouted = False
 279         self._stdin = None # Store the command inputs field
 280         self._stdout = None # Store the command outputs field
 281         self._stderr = None # Store the command errors field
 282
 283         self.out = ""
 284         self.err = ""
 285
 286         self.name_remote_jobs_pyconf = ".%s" % os.path.basename(job_file_path)
 287         self.commands = commands
 288         self.command = (os.path.join(self.machine.sat_path, "sat") +
 289                         " -l " +
 290                         os.path.join(self.machine.sat_path,
 291                                      "list_log_files.txt") +
 292                         " job --jobs_config " +
 293                         os.path.join(self.machine.sat_path,
 294                                      self.name_remote_jobs_pyconf) +
 295                         " --name " +
 296                         self.name)
 297         if prefix:
 298             self.command = prefix + ' "' + self.command +'"'
 299
 300     def get_pids(self):
 301         """ Get the pid(s) corresponding to the command that have been launched
 302             On the remote machine
 303
 304         :return: The list of integers corresponding to the found pids
 305         :rtype: List
 306         """
 307         pids = []
 308         cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
 309         (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
 310         pids_cmd = out_pid.readlines()
 311         pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
 312         pids+=pids_cmd
 313         return pids
 314
 315     def kill_remote_process(self, wait=1):
 316         '''Kills the process on the remote machine.
 317
 318         :return: (the output of the kill, the error of the kill)
 319         :rtype: (str, str)
 320         '''
 321         try:
 322             pids = self.get_pids()
 323         except:
 324             return ("Unable to get the pid of the command.", "")
 325
 326         cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
 327         (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
 328                                                             self.logger)
 329         time.sleep(wait)
 330         return (out_kill.read().decode(), err_kill.read().decode())
 331
 332     def has_begun(self):
 333         '''Returns True if the job has already begun
 334
 335         :return: True if the job has already begun
 336         :rtype: bool
 337         '''
 338         return self._has_begun
 339
 340     def has_finished(self):
 341         '''Returns True if the job has already finished
 342            (i.e. all the commands have been executed)
 343            If it is finished, the outputs are stored in the fields out and err.
 344
 345         :return: True if the job has already finished
 346         :rtype: bool
 347         '''
 348
 349         # If the method has already been called and returned True
 350         if self._has_finished:
 351             return True
 352
 353         # If the job has not begun yet
 354         if not self.has_begun():
 355             return False
 356
 357         if self._stdout.channel.closed:
 358             self._has_finished = True
 359             # Store the result outputs
 360             self.out += self._stdout.read().decode()
 361             self.err += self._stderr.read().decode()
 362             # Put end time
 363             self._Tf = time.time()
 364             # And get the remote command status and log files
 365             try:
 366                 self.get_log_files()
 367             except Exception as e:
 368                 self.err += _("Unable to get remote log files: %s" % e)
 369
 370         return self._has_finished
 371
 372     def get_log_files(self):
 373         """Get the log files produced by the command launched
 374            on the remote machine, and put it in the log directory of the user,
 375            so they can be accessible from
 376         """
 377         # Do not get the files if the command is not finished
 378         if not self.has_finished():
 379             msg = _("Trying to get log files whereas the job is not finished.")
 380             self.logger.write(src.printcolors.printcWarning(msg))
 381             return
 382
 383         # First get the file that contains the list of log files to get
 384         tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
 385         remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt")
 386         self.machine.sftp.get(
 387                     remote_path,
 388                     tmp_file_path)
 389
 390         # Read the file and get the result of the command and all the log files
 391         # to get
 392         fstream_tmp = open(tmp_file_path, "r")
 393         file_lines = fstream_tmp.readlines()
 394         file_lines = [line.replace("\n", "") for line in file_lines]
 395         fstream_tmp.close()
 396         os.remove(tmp_file_path)
 397
 398         try :
 399             # The first line is the result of the command (0 success or 1 fail)
 400             self.res_job = file_lines[0]
 401         except Exception as e:
 402             self.err += _("Unable to get status from remote file %s: %s" %
 403                                                     (remote_path, str(e)))
 404
 405         for i, job_path_remote in enumerate(file_lines[1:]):
 406             try:
 407                 # For each command, there is two files to get :
 408                 # 1- The xml file describing the command and giving the
 409                 # internal traces.
 410                 # 2- The txt file containing the system command traces (like
 411                 # traces produced by the "make" command)
 412                 # 3- In case of the test command, there is another file to get :
 413                 # the xml board that contain the test results
 414                 dirname = os.path.basename(os.path.dirname(job_path_remote))
 415                 if dirname != 'OUT' and dirname != 'TEST':
 416                     # Case 1-
 417                     local_path = os.path.join(os.path.dirname(
 418                                                         self.logger.logFilePath),
 419                                               os.path.basename(job_path_remote))
 420                     if i==0: # The first is the job command
 421                         self.logger.add_link(os.path.basename(job_path_remote),
 422                                              "job",
 423                                              self.res_job,
 424                                              self.command)
 425                 elif dirname == 'OUT':
 426                     # Case 2-
 427                     local_path = os.path.join(os.path.dirname(
 428                                                         self.logger.logFilePath),
 429                                               'OUT',
 430                                               os.path.basename(job_path_remote))
 431                 elif dirname == 'TEST':
 432                     # Case 3-
 433                     local_path = os.path.join(os.path.dirname(
 434                                                         self.logger.logFilePath),
 435                                               'TEST',
 436                                               os.path.basename(job_path_remote))
 437
 438                 # Get the file
 439                 if not os.path.exists(local_path):
 440                     self.machine.sftp.get(job_path_remote, local_path)
 441                 self.remote_log_files.append(local_path)
 442             except Exception as e:
 443                 self.err += _("Unable to get %s log file from remote: %s" %
 444                                                     (str(job_path_remote),
 445                                                      str(e)))
 446
 447     def has_failed(self):
 448         '''Returns True if the job has failed.
 449            A job is considered as failed if the machine could not be reached,
 450            if the remote command failed,
 451            or if the job finished with a time out.
 452
 453         :return: True if the job has failed
 454         :rtype: bool
 455         '''
 456         if not self.has_finished():
 457             return False
 458         if not self.machine.successfully_connected(self.logger):
 459             return True
 460         if self.is_timeout():
 461             return True
 462         if self.res_job == "1":
 463             return True
 464         return False
 465
 466     def cancel(self):
 467         """In case of a failing job, one has to cancel every job that depend
 468            on it. This method put the job as failed and will not be executed.
 469         """
 470         if self.cancelled:
 471             return
 472         self._has_begun = True
 473         self._has_finished = True
 474         self.cancelled = True
 475         self.out += _("This job was not launched because its father has failed.")
 476         self.err += _("This job was not launched because its father has failed.")
 477
 478     def is_running(self):
 479         '''Returns True if the job commands are running
 480
 481         :return: True if the job is running
 482         :rtype: bool
 483         '''
 484         return self.has_begun() and not self.has_finished()
 485
 486     def is_timeout(self):
 487         '''Returns True if the job commands has finished with timeout
 488
 489         :return: True if the job has finished with timeout
 490         :rtype: bool
 491         '''
 492         return self._has_timouted
 493
 494     def time_elapsed(self):
 495         """Get the time elapsed since the job launching
 496
 497         :return: The number of seconds
 498         :rtype: int
 499         """
 500         if not self.has_begun():
 501             return -1
 502         T_now = time.time()
 503         return T_now - self._T0
 504
 505     def check_time(self):
 506         """Verify that the job has not exceeded its timeout.
 507            If it has, kill the remote command and consider the job as finished.
 508         """
 509         if not self.has_begun():
 510             return
 511         if self.time_elapsed() > self.timeout:
 512             self._has_finished = True
 513             self._has_timouted = True
 514             self._Tf = time.time()
 515             (out_kill, _) = self.kill_remote_process()
 516             self.out += "TIMEOUT \n" + out_kill
 517             self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
 518             try:
 519                 self.get_log_files()
 520             except Exception as e:
 521                 # The 2 following lines must be suppressed after the bug is fixed
 522                 print("The error type: ")
 523                 print(type(e))
 524                 print("The error: ")
 525                 print(e)
 526                 print("Local scope:")
 527                 print(dir())
 528                 print("The str type: ")
 529                 print(type(str))
 530                 print("str: ")
 531                 print(str)
 532                 self.err += _("Unable to get remote log files!")
 533
 534     def total_duration(self):
 535         """Give the total duration of the job
 536
 537         :return: the total duration of the job in seconds
 538         :rtype: int
 539         """
 540         return self._Tf - self._T0
 541
 542     def run(self):
 543         """Launch the job by executing the remote command.
 544         """
 545
 546         # Prevent multiple run
 547         if self.has_begun():
 548             msg = _("Warning: A job can only be launched one time")
 549             msg2 = _("Trying to launch the job \"%s\" whereas it has "
 550                      "already been launched." % self.name)
 551             self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
 552                                                                         msg2)))
 553             return
 554
 555         # Do not execute the command if the machine could not be reached
 556         if not self.machine.successfully_connected(self.logger):
 557             self._has_finished = True
 558             self.out = "N\A"
 559             self.err += ("Connection to machine (name : %s, host: %s, port:"
 560                         " %s, user: %s) has failed\nUse the log command "
 561                         "to get more information."
 562                         % (self.machine.name,
 563                            self.machine.host,
 564                            self.machine.port,
 565                            self.machine.user))
 566         else:
 567             # Usual case : Launch the command on remote machine
 568             self._T0 = time.time()
 569             self._stdin, self._stdout, self._stderr = self.machine.exec_command(
 570                                                                   self.command,
 571                                                                   self.logger)
 572             # If the results are not initialized, finish the job
 573             if (self._stdin, self._stdout, self._stderr) == (None, None, None):
 574                 self._has_finished = True
 575                 self._Tf = time.time()
 576                 self.out += "N\A"
 577                 self.err += "The server failed to execute the command"
 578
 579         # Put the beginning flag to true.
 580         self._has_begun = True
 581
 582     def write_results(self):
 583         """Display on the terminal all the job's information
 584         """
 585         self.logger.write("name : " + self.name + "\n")
 586         if self.after:
 587             self.logger.write("after : %s\n" % self.after)
 588         self.logger.write("Time elapsed : %4imin %2is \n" %
 589                      (self.total_duration()//60 , self.total_duration()%60))
 590         if self._T0 != -1:
 591             self.logger.write("Begin time : %s\n" %
 592                          time.strftime('%Y-%m-%d %H:%M:%S',
 593                                        time.localtime(self._T0)) )
 594         if self._Tf != -1:
 595             self.logger.write("End time   : %s\n\n" %
 596                          time.strftime('%Y-%m-%d %H:%M:%S',
 597                                        time.localtime(self._Tf)) )
 598
 599         machine_head = "Informations about connection :\n"
 600         underline = (len(machine_head) - 2) * "-"
 601         self.logger.write(src.printcolors.printcInfo(
 602                                                 machine_head+underline+"\n"))
 603         self.machine.write_info(self.logger)
 604
 605         self.logger.write(src.printcolors.printcInfo("out : \n"))
 606         if self.out == "":
 607             self.logger.write("Unable to get output\n")
 608         else:
 609             self.logger.write(self.out + "\n")
 610         self.logger.write(src.printcolors.printcInfo("err : \n"))
 611         self.logger.write(self.err + "\n")
 612
 613     def get_status(self):
 614         """Get the status of the job (used by the Gui for xml display)
 615
 616         :return: The current status of the job
 617         :rtype: String
 618         """
 619         if not self.machine.successfully_connected(self.logger):
 620             return "SSH connection KO"
 621         if not self.has_begun():
 622             return "Not launched"
 623         if self.cancelled:
 624             return "Cancelled"
 625         if self.is_running():
 626             return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
 627                                                     time.localtime(self._T0))
 628         if self.has_finished():
 629             if self.is_timeout():
 630                 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
 631                                                     time.localtime(self._Tf))
 632             return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
 633                                                      time.localtime(self._Tf))
 634
 635 class Jobs(object):
 636     '''Class to manage the jobs to be run
 637     '''
 638     def __init__(self,
 639                  runner,
 640                  logger,
 641                  job_file_path,
 642                  config_jobs,
 643                  lenght_columns = 20):
 644         # The jobs configuration
 645         self.cfg_jobs = config_jobs
 646         self.job_file_path = job_file_path
 647         # The machine that will be used today
 648         self.lmachines = []
 649         # The list of machine (hosts, port) that will be used today
 650         # (a same host can have several machine instances since there
 651         # can be several ssh parameters)
 652         self.lhosts = []
 653         # The jobs to be launched today
 654         self.ljobs = []
 655         # The jobs that will not be launched today
 656         self.ljobs_not_today = []
 657         self.runner = runner
 658         self.logger = logger
 659         self.len_columns = lenght_columns
 660
 661         # the list of jobs that have not been run yet
 662         self._l_jobs_not_started = []
 663         # the list of jobs that have already ran
 664         self._l_jobs_finished = []
 665         # the list of jobs that are running
 666         self._l_jobs_running = []
 667
 668         self.determine_jobs_and_machines()
 669
 670     def define_job(self, job_def, machine):
 671         '''Takes a pyconf job definition and a machine (from class machine)
 672            and returns the job instance corresponding to the definition.
 673
 674         :param job_def src.config.Mapping: a job definition
 675         :param machine machine: the machine on which the job will run
 676         :return: The corresponding job in a job class instance
 677         :rtype: job
 678         '''
 679         name = job_def.name
 680         cmmnds = job_def.commands
 681         if not "timeout" in job_def:
 682             timeout = 4*60*60 # default timeout = 4h
 683         else:
 684             timeout = job_def.timeout
 685         after = None
 686         if 'after' in job_def:
 687             after = job_def.after
 688         application = None
 689         if 'application' in job_def:
 690             application = job_def.application
 691         board = None
 692         if 'board' in job_def:
 693             board = job_def.board
 694         prefix = None
 695         if "prefix" in job_def:
 696             prefix = job_def.prefix
 697
 698         return Job(name,
 699                    machine,
 700                    application,
 701                    board,
 702                    cmmnds,
 703                    timeout,
 704                    self.runner.cfg,
 705                    self.job_file_path,
 706                    self.logger,
 707                    after = after,
 708                    prefix = prefix)
 709
 710     def determine_jobs_and_machines(self):
 711         '''Function that reads the pyconf jobs definition and instantiates all
 712            the machines and jobs to be done today.
 713
 714         :return: Nothing
 715         :rtype: N\A
 716         '''
 717         today = datetime.date.weekday(datetime.date.today())
 718         host_list = []
 719
 720         for job_def in self.cfg_jobs.jobs :
 721
 722             if not "machine" in job_def:
 723                 msg = _('WARNING: The job "%s" do not have the key '
 724                        '"machine", this job is ignored.\n\n' % job_def.name)
 725                 self.logger.write(src.printcolors.printcWarning(msg))
 726                 continue
 727             name_machine = job_def.machine
 728
 729             a_machine = None
 730             for mach in self.lmachines:
 731                 if mach.name == name_machine:
 732                     a_machine = mach
 733                     break
 734
 735             if a_machine == None:
 736                 for machine_def in self.cfg_jobs.machines:
 737                     if machine_def.name == name_machine:
 738                         if 'host' not in machine_def:
 739                             host = self.runner.cfg.VARS.hostname
 740                         else:
 741                             host = machine_def.host
 742
 743                         if 'user' not in machine_def:
 744                             user = self.runner.cfg.VARS.user
 745                         else:
 746                             user = machine_def.user
 747
 748                         if 'port' not in machine_def:
 749                             port = 22
 750                         else:
 751                             port = machine_def.port
 752
 753                         if 'password' not in machine_def:
 754                             passwd = None
 755                         else:
 756                             passwd = machine_def.password
 757
 758                         if 'sat_path' not in machine_def:
 759                             sat_path = "salomeTools"
 760                         else:
 761                             sat_path = machine_def.sat_path
 762
 763                         a_machine = Machine(
 764                                             machine_def.name,
 765                                             host,
 766                                             user,
 767                                             port=port,
 768                                             passwd=passwd,
 769                                             sat_path=sat_path
 770                                             )
 771
 772                         self.lmachines.append(a_machine)
 773                         if (host, port) not in host_list:
 774                             host_list.append((host, port))
 775
 776                 if a_machine == None:
 777                     msg = _("WARNING: The job \"%(job_name)s\" requires the "
 778                             "machine \"%(machine_name)s\" but this machine "
 779                             "is not defined in the configuration file.\n"
 780                             "The job will not be launched\n")
 781                     self.logger.write(src.printcolors.printcWarning(
 782                                         msg % {"job_name" : job_def.name,
 783                                                "machine_name" : name_machine}))
 784                     continue
 785
 786             a_job = self.define_job(job_def, a_machine)
 787
 788             if today in job_def.when:
 789                 self.ljobs.append(a_job)
 790             else: # today in job_def.when
 791                 self.ljobs_not_today.append(a_job)
 792
 793         self.lhosts = host_list
 794
 795     def ssh_connection_all_machines(self, pad=50):
 796         '''Function that do the ssh connection to every machine
 797            to be used today.
 798
 799         :return: Nothing
 800         :rtype: N\A
 801         '''
 802         self.logger.write(src.printcolors.printcInfo((
 803                         "Establishing connection with all the machines :\n")))
 804         for machine in self.lmachines:
 805             # little algorithm in order to display traces
 806             begin_line = (_("Connection to %s: " % machine.name))
 807             if pad - len(begin_line) < 0:
 808                 endline = " "
 809             else:
 810                 endline = (pad - len(begin_line)) * "." + " "
 811
 812             step = "SSH connection"
 813             self.logger.write( begin_line + endline + step)
 814             self.logger.flush()
 815             # the call to the method that initiate the ssh connection
 816             msg = machine.connect(self.logger)
 817
 818             # Copy salomeTools to the remote machine
 819             if machine.successfully_connected(self.logger):
 820                 step = _("Remove SAT")
 821                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 822                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 823                 (__, out_dist, __) = machine.exec_command(
 824                                                 "rm -rf %s" % machine.sat_path,
 825                                                 self.logger)
 826                 out_dist.read()
 827
 828                 self.logger.flush()
 829                 step = _("Copy SAT")
 830                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 831                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 832                 self.logger.flush()
 833                 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
 834                                             self.job_file_path)
 835                 # get the remote machine distribution using a sat command
 836                 (__, out_dist, __) = machine.exec_command(
 837                                 os.path.join(machine.sat_path,
 838                                     "sat config --value VARS.dist --no_label"),
 839                                 self.logger)
 840                 machine.distribution = out_dist.read().decode().replace("\n",
 841                                                                         "")
 842
 843                 # set the local settings of sat on the remote machine using
 844                 # the init command
 845                 (__, __, __) = machine.exec_command(
 846                                 os.path.join(machine.sat_path,
 847                                     "sat init --base unknown --workdir unknown"
 848                                     " --log_dir unknown"),
 849                                 self.logger)
 850
 851                 # Print the status of the copy
 852                 if res_copy == 0:
 853                     self.logger.write('\r%s' %
 854                                 ((len(begin_line)+len(endline)+20) * " "), 3)
 855                     self.logger.write('\r%s%s%s' %
 856                         (begin_line,
 857                          endline,
 858                          src.printcolors.printc(src.OK_STATUS)), 3)
 859                 else:
 860                     self.logger.write('\r%s' %
 861                             ((len(begin_line)+len(endline)+20) * " "), 3)
 862                     self.logger.write('\r%s%s%s %s' %
 863                         (begin_line,
 864                          endline,
 865                          src.printcolors.printc(src.KO_STATUS),
 866                          _("Copy of SAT failed: %s" % res_copy)), 3)
 867             else:
 868                 self.logger.write('\r%s' %
 869                                   ((len(begin_line)+len(endline)+20) * " "), 3)
 870                 self.logger.write('\r%s%s%s %s' %
 871                     (begin_line,
 872                      endline,
 873                      src.printcolors.printc(src.KO_STATUS),
 874                      msg), 3)
 875             self.logger.write("\n", 3)
 876
 877         self.logger.write("\n")
 878
 879
 880     def is_occupied(self, hostname):
 881         '''Function that returns True if a job is running on
 882            the machine defined by its host and its port.
 883
 884         :param hostname (str, int): the pair (host, port)
 885         :return: the job that is running on the host,
 886                 or false if there is no job running on the host.
 887         :rtype: job / bool
 888         '''
 889         host = hostname[0]
 890         port = hostname[1]
 891         for jb in self.ljobs:
 892             if jb.machine.host == host and jb.machine.port == port:
 893                 if jb.is_running():
 894                     return jb
 895         return False
 896
 897     def update_jobs_states_list(self):
 898         '''Function that updates the lists that store the currently
 899            running jobs and the jobs that have already finished.
 900
 901         :return: Nothing.
 902         :rtype: N\A
 903         '''
 904         jobs_finished_list = []
 905         jobs_running_list = []
 906         for jb in self.ljobs:
 907             if jb.is_running():
 908                 jobs_running_list.append(jb)
 909                 jb.check_time()
 910             if jb.has_finished():
 911                 jobs_finished_list.append(jb)
 912
 913         nb_job_finished_before = len(self._l_jobs_finished)
 914         self._l_jobs_finished = jobs_finished_list
 915         self._l_jobs_running = jobs_running_list
 916
 917         nb_job_finished_now = len(self._l_jobs_finished)
 918
 919         return nb_job_finished_now > nb_job_finished_before
 920
 921     def cancel_dependencies_of_failing_jobs(self):
 922         '''Function that cancels all the jobs that depend on a failing one.
 923
 924         :return: Nothing.
 925         :rtype: N\A
 926         '''
 927
 928         for job in self.ljobs:
 929             if job.after is None:
 930                 continue
 931             father_job = self.find_job_that_has_name(job.after)
 932             if father_job is not None and father_job.has_failed():
 933                 job.cancel()
 934
 935     def find_job_that_has_name(self, name):
 936         '''Returns the job by its name.
 937
 938         :param name str: a job name
 939         :return: the job that has the name.
 940         :rtype: job
 941         '''
 942         for jb in self.ljobs:
 943             if jb.name == name:
 944                 return jb
 945         # the following is executed only if the job was not found
 946         return None
 947
 948     def str_of_length(self, text, length):
 949         '''Takes a string text of any length and returns
 950            the most close string of length "length".
 951
 952         :param text str: any string
 953         :param length int: a length for the returned string
 954         :return: the most close string of length "length"
 955         :rtype: str
 956         '''
 957         if len(text) > length:
 958             text_out = text[:length-3] + '...'
 959         else:
 960             diff = length - len(text)
 961             before = " " * (diff//2)
 962             after = " " * (diff//2 + diff%2)
 963             text_out = before + text + after
 964
 965         return text_out
 966
 967     def display_status(self, len_col):
 968         '''Takes a lenght and construct the display of the current status
 969            of the jobs in an array that has a column for each host.
 970            It displays the job that is currently running on the host
 971            of the column.
 972
 973         :param len_col int: the size of the column
 974         :return: Nothing
 975         :rtype: N\A
 976         '''
 977
 978         display_line = ""
 979         for host_port in self.lhosts:
 980             jb = self.is_occupied(host_port)
 981             if not jb: # nothing running on the host
 982                 empty = self.str_of_length("empty", len_col)
 983                 display_line += "|" + empty
 984             else:
 985                 display_line += "|" + src.printcolors.printcInfo(
 986                                         self.str_of_length(jb.name, len_col))
 987
 988         self.logger.write("\r" + display_line + "|")
 989         self.logger.flush()
 990
 991
 992     def run_jobs(self):
 993         '''The main method. Runs all the jobs on every host.
 994            For each host, at a given time, only one job can be running.
 995            The jobs that have the field after (that contain the job that has
 996            to be run before it) are run after the previous job.
 997            This method stops when all the jobs are finished.
 998
 999         :return: Nothing
1000         :rtype: N\A
1001         '''
1002
1003         # Print header
1004         self.logger.write(src.printcolors.printcInfo(
1005                                                 _('Executing the jobs :\n')))
1006         text_line = ""
1007         for host_port in self.lhosts:
1008             host = host_port[0]
1009             port = host_port[1]
1010             if port == 22: # default value
1011                 text_line += "|" + self.str_of_length(host, self.len_columns)
1012             else:
1013                 text_line += "|" + self.str_of_length(
1014                                 "("+host+", "+str(port)+")", self.len_columns)
1015
1016         tiret_line = " " + "-"*(len(text_line)-1) + "\n"
1017         self.logger.write(tiret_line)
1018         self.logger.write(text_line + "|\n")
1019         self.logger.write(tiret_line)
1020         self.logger.flush()
1021
1022         # The infinite loop that runs the jobs
1023         l_jobs_not_started = src.deepcopy_list(self.ljobs)
1024         while len(self._l_jobs_finished) != len(self.ljobs):
1025             new_job_start = False
1026             for host_port in self.lhosts:
1027
1028                 if self.is_occupied(host_port):
1029                     continue
1030
1031                 for jb in l_jobs_not_started:
1032                     if (jb.machine.host, jb.machine.port) != host_port:
1033                         continue
1034                     if jb.after == None:
1035                         jb.run()
1036                         l_jobs_not_started.remove(jb)
1037                         new_job_start = True
1038                         break
1039                     else:
1040                         jb_before = self.find_job_that_has_name(jb.after)
1041                         if jb_before is None:
1042                             jb.cancel()
1043                             msg = _("This job was not launched because its "
1044                                     "father is not in the jobs list.")
1045                             jb.out = msg
1046                             jb.err = msg
1047                             break
1048                         if jb_before.has_finished():
1049                             jb.run()
1050                             l_jobs_not_started.remove(jb)
1051                             new_job_start = True
1052                             break
1053             self.cancel_dependencies_of_failing_jobs()
1054             new_job_finished = self.update_jobs_states_list()
1055
1056             if new_job_start or new_job_finished:
1057                 if self.gui:
1058                     self.gui.update_xml_files(self.ljobs)
1059                 # Display the current status
1060                 self.display_status(self.len_columns)
1061
1062             # Make sure that the proc is not entirely busy
1063             time.sleep(0.001)
1064
1065         self.logger.write("\n")
1066         self.logger.write(tiret_line)
1067         self.logger.write("\n\n")
1068
1069         if self.gui:
1070             self.gui.update_xml_files(self.ljobs)
1071             self.gui.last_update()
1072
1073     def write_all_results(self):
1074         '''Display all the jobs outputs.
1075
1076         :return: Nothing
1077         :rtype: N\A
1078         '''
1079
1080         for jb in self.ljobs:
1081             self.logger.write(src.printcolors.printcLabel(
1082                         "#------- Results for job %s -------#\n" % jb.name))
1083             jb.write_results()
1084             self.logger.write("\n\n")
1085
1086 class Gui(object):
1087     '''Class to manage the the xml data that can be displayed in a browser to
1088        see the jobs states
1089     '''
1090
1091     def __init__(self,
1092                  xml_dir_path,
1093                  l_jobs,
1094                  l_jobs_not_today,
1095                  prefix,
1096                  logger,
1097                  file_boards=""):
1098         '''Initialization
1099
1100         :param xml_dir_path str: The path to the directory where to put
1101                                  the xml resulting files
1102         :param l_jobs List: the list of jobs that run today
1103         :param l_jobs_not_today List: the list of jobs that do not run today
1104         :param file_boards str: the file path from which to read the
1105                                    expected boards
1106         '''
1107         # The logging instance
1108         self.logger = logger
1109
1110         # The prefix to add to the xml files : date_hour
1111         self.prefix = prefix
1112
1113         # The path of the csv files to read to fill the expected boards
1114         self.file_boards = file_boards
1115
1116         if file_boards != "":
1117             today = datetime.date.weekday(datetime.date.today())
1118             self.parse_csv_boards(today)
1119         else:
1120             self.d_input_boards = {}
1121
1122         # The path of the global xml file
1123         self.xml_dir_path = xml_dir_path
1124         # Initialize the xml files
1125         self.global_name = "global_report"
1126         xml_global_path = os.path.join(self.xml_dir_path,
1127                                        self.global_name + ".xml")
1128         self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1129                                                          "JobsReport")
1130
1131         # Find history for each job
1132         self.history = {}
1133         self.find_history(l_jobs, l_jobs_not_today)
1134
1135         # The xml files that corresponds to the boards.
1136         # {name_board : xml_object}}
1137         self.d_xml_board_files = {}
1138
1139         # Create the lines and columns
1140         self.initialize_boards(l_jobs, l_jobs_not_today)
1141
1142         # Write the xml file
1143         self.update_xml_files(l_jobs)
1144
1145     def add_xml_board(self, name):
1146         '''Add a board to the board list
1147         :param name str: the board name
1148         '''
1149         xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1150         self.d_xml_board_files[name] =  src.xmlManager.XmlLogFile(
1151                                                     xml_board_path,
1152                                                     "JobsReport")
1153         self.d_xml_board_files[name].add_simple_node("distributions")
1154         self.d_xml_board_files[name].add_simple_node("applications")
1155         self.d_xml_board_files[name].add_simple_node("board", text=name)
1156
1157     def initialize_boards(self, l_jobs, l_jobs_not_today):
1158         '''Get all the first information needed for each file and write the
1159            first version of the files
1160         :param l_jobs List: the list of jobs that run today
1161         :param l_jobs_not_today List: the list of jobs that do not run today
1162         '''
1163         # Get the boards to fill and put it in a dictionary
1164         # {board_name : xml instance corresponding to the board}
1165         for job in l_jobs + l_jobs_not_today:
1166             board = job.board
1167             if (board is not None and
1168                                 board not in self.d_xml_board_files.keys()):
1169                 self.add_xml_board(board)
1170
1171         # Verify that the boards given as input are done
1172         for board in list(self.d_input_boards.keys()):
1173             if board not in self.d_xml_board_files:
1174                 self.add_xml_board(board)
1175             root_node = self.d_xml_board_files[board].xmlroot
1176             src.xmlManager.append_node_attrib(root_node,
1177                                               {"input_file" : self.file_boards})
1178
1179         # Loop over all jobs in order to get the lines and columns for each
1180         # xml file
1181         d_dist = {}
1182         d_application = {}
1183         for board in self.d_xml_board_files:
1184             d_dist[board] = []
1185             d_application[board] = []
1186
1187         l_hosts_ports = []
1188
1189         for job in l_jobs + l_jobs_not_today:
1190
1191             if (job.machine.host, job.machine.port) not in l_hosts_ports:
1192                 l_hosts_ports.append((job.machine.host, job.machine.port))
1193
1194             distrib = job.machine.distribution
1195             application = job.application
1196
1197             board_job = job.board
1198             if board is None:
1199                 continue
1200             for board in self.d_xml_board_files:
1201                 if board_job == board:
1202                     if (distrib not in [None, ''] and
1203                                             distrib not in d_dist[board]):
1204                         d_dist[board].append(distrib)
1205                         src.xmlManager.add_simple_node(
1206                             self.d_xml_board_files[board].xmlroot.find(
1207                                                             'distributions'),
1208                                                    "dist",
1209                                                    attrib={"name" : distrib})
1210
1211                 if board_job == board:
1212                     if (application not in [None, ''] and
1213                                     application not in d_application[board]):
1214                         d_application[board].append(application)
1215                         src.xmlManager.add_simple_node(
1216                             self.d_xml_board_files[board].xmlroot.find(
1217                                                                 'applications'),
1218                                                    "application",
1219                                                    attrib={
1220                                                         "name" : application})
1221
1222         # Verify that there are no missing application or distribution in the
1223         # xml board files (regarding the input boards)
1224         for board in self.d_xml_board_files:
1225             l_dist = d_dist[board]
1226             if board not in self.d_input_boards.keys():
1227                 continue
1228             for dist in self.d_input_boards[board]["rows"]:
1229                 if dist not in l_dist:
1230                     src.xmlManager.add_simple_node(
1231                             self.d_xml_board_files[board].xmlroot.find(
1232                                                             'distributions'),
1233                                                    "dist",
1234                                                    attrib={"name" : dist})
1235             l_appli = d_application[board]
1236             for appli in self.d_input_boards[board]["columns"]:
1237                 if appli not in l_appli:
1238                     src.xmlManager.add_simple_node(
1239                             self.d_xml_board_files[board].xmlroot.find(
1240                                                                 'applications'),
1241                                                    "application",
1242                                                    attrib={"name" : appli})
1243
1244         # Initialize the hosts_ports node for the global file
1245         self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1246                                                                 "hosts_ports")
1247         for host, port in l_hosts_ports:
1248             host_port = "%s:%i" % (host, port)
1249             src.xmlManager.add_simple_node(self.xmlhosts_ports,
1250                                            "host_port",
1251                                            attrib={"name" : host_port})
1252
1253         # Initialize the jobs node in all files
1254         for xml_file in [self.xml_global_file] + list(
1255                                             self.d_xml_board_files.values()):
1256             xml_jobs = xml_file.add_simple_node("jobs")
1257             # Get the jobs present in the config file but
1258             # that will not be launched today
1259             self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1260
1261             # add also the infos node
1262             xml_file.add_simple_node("infos",
1263                                      attrib={"name" : "last update",
1264                                              "JobsCommandStatus" : "running"})
1265
1266             # and put the history node
1267             history_node = xml_file.add_simple_node("history")
1268             name_board = os.path.basename(xml_file.logFile)[:-len(".xml")]
1269             # serach for board files
1270             expression = "^[0-9]{8}_+[0-9]{6}_" + name_board + ".xml$"
1271             oExpr = re.compile(expression)
1272             # Get the list of xml borad files that are in the log directory
1273             for file_name in os.listdir(self.xml_dir_path):
1274                 if oExpr.search(file_name):
1275                     date = os.path.basename(file_name).split("_")[0]
1276                     file_path = os.path.join(self.xml_dir_path, file_name)
1277                     src.xmlManager.add_simple_node(history_node,
1278                                                    "link",
1279                                                    text=file_path,
1280                                                    attrib={"date" : date})
1281
1282
1283         # Find in each board the squares that needs to be filled regarding the
1284         # input csv files but that are not covered by a today job
1285         for board in self.d_input_boards.keys():
1286             xml_root_board = self.d_xml_board_files[board].xmlroot
1287             # Find the missing jobs for today
1288             xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1289                                                  "missing_jobs")
1290             for row, column in self.d_input_boards[board]["jobs"]:
1291                 found = False
1292                 for job in l_jobs:
1293                     if (job.application == column and
1294                         job.machine.distribution == row):
1295                         found = True
1296                         break
1297                 if not found:
1298                     src.xmlManager.add_simple_node(xml_missing,
1299                                             "job",
1300                                             attrib={"distribution" : row,
1301                                                     "application" : column })
1302             # Find the missing jobs not today
1303             xml_missing_not_today = src.xmlManager.add_simple_node(
1304                                                  xml_root_board,
1305                                                  "missing_jobs_not_today")
1306             for row, column in self.d_input_boards[board]["jobs_not_today"]:
1307                 found = False
1308                 for job in l_jobs_not_today:
1309                     if (job.application == column and
1310                         job.machine.distribution == row):
1311                         found = True
1312                         break
1313                 if not found:
1314                     src.xmlManager.add_simple_node(xml_missing_not_today,
1315                                             "job",
1316                                             attrib={"distribution" : row,
1317                                                     "application" : column })
1318
1319     def find_history(self, l_jobs, l_jobs_not_today):
1320         """find, for each job, in the existent xml boards the results for the
1321            job. Store the results in the dictionnary self.history = {name_job :
1322            list of (date, status, list links)}
1323
1324         :param l_jobs List: the list of jobs to run today
1325         :param l_jobs_not_today List: the list of jobs that do not run today
1326         """
1327         # load the all the history
1328         expression = "^[0-9]{8}_+[0-9]{6}_" + self.global_name + ".xml$"
1329         oExpr = re.compile(expression)
1330         # Get the list of global xml that are in the log directory
1331         l_globalxml = []
1332         for file_name in os.listdir(self.xml_dir_path):
1333             if oExpr.search(file_name):
1334                 file_path = os.path.join(self.xml_dir_path, file_name)
1335                 try:
1336                     global_xml = src.xmlManager.ReadXmlFile(file_path)
1337                     l_globalxml.append(global_xml)
1338                 except Exception as e:
1339                     msg = _("\nWARNING: the file %s can not be read, it will be "
1340                             "ignored\n%s" % (file_path, e))
1341                     self.logger.write("%s\n" % src.printcolors.printcWarning(
1342                                                                         msg), 5)
1343
1344         # Construct the dictionnary self.history
1345         for job in l_jobs + l_jobs_not_today:
1346             l_links = []
1347             for global_xml in l_globalxml:
1348                 date = os.path.basename(global_xml.filePath).split("_")[0]
1349                 global_root_node = global_xml.xmlroot.find("jobs")
1350                 job_node = src.xmlManager.find_node_by_attrib(
1351                                                               global_root_node,
1352                                                               "job",
1353                                                               "name",
1354                                                               job.name)
1355                 if job_node:
1356                     if job_node.find("remote_log_file_path") is not None:
1357                         link = job_node.find("remote_log_file_path").text
1358                         res_job = job_node.find("res").text
1359                         if link != "nothing":
1360                             l_links.append((date, res_job, link))
1361             l_links = sorted(l_links, reverse=True)
1362             self.history[job.name] = l_links
1363
1364     def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1365         '''Get all the first information needed for each file and write the
1366            first version of the files
1367
1368         :param xml_node_jobs etree.Element: the node corresponding to a job
1369         :param l_jobs_not_today List: the list of jobs that do not run today
1370         '''
1371         for job in l_jobs_not_today:
1372             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1373                                                  "job",
1374                                                  attrib={"name" : job.name})
1375             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1376             src.xmlManager.add_simple_node(xmlj,
1377                                            "distribution",
1378                                            job.machine.distribution)
1379             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1380             src.xmlManager.add_simple_node(xmlj,
1381                                        "commands", " ; ".join(job.commands))
1382             src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1383             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1384             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1385             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1386             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1387             src.xmlManager.add_simple_node(xmlj, "sat_path",
1388                                                         job.machine.sat_path)
1389             xml_history = src.xmlManager.add_simple_node(xmlj, "history")
1390             for i, (date, res_job, link) in enumerate(self.history[job.name]):
1391                 if i==0:
1392                     # tag the first one (the last one)
1393                     src.xmlManager.add_simple_node(xml_history,
1394                                                    "link",
1395                                                    text=link,
1396                                                    attrib={"date" : date,
1397                                                            "res" : res_job,
1398                                                            "last" : "yes"})
1399                 else:
1400                     src.xmlManager.add_simple_node(xml_history,
1401                                                    "link",
1402                                                    text=link,
1403                                                    attrib={"date" : date,
1404                                                            "res" : res_job,
1405                                                            "last" : "no"})
1406
1407     def parse_csv_boards(self, today):
1408         """ Parse the csv file that describes the boards to produce and fill
1409             the dict d_input_boards that contain the csv file contain
1410
1411         :param today int: the current day of the week
1412         """
1413         # open the csv file and read its content
1414         l_read = []
1415         with open(self.file_boards, 'r') as f:
1416             reader = csv.reader(f,delimiter=CSV_DELIMITER)
1417             for row in reader:
1418                 l_read.append(row)
1419         # get the delimiter for the boards (empty line)
1420         boards_delimiter = [''] * len(l_read[0])
1421         # Make the list of boards, by splitting with the delimiter
1422         l_boards = [list(y) for x, y in itertools.groupby(l_read,
1423                                     lambda z: z == boards_delimiter) if not x]
1424
1425         # loop over the csv lists of lines and get the rows, columns and jobs
1426         d_boards = {}
1427         for input_board in l_boards:
1428             # get board name
1429             board_name = input_board[0][0]
1430
1431             # Get columns list
1432             columns = input_board[0][1:]
1433
1434             rows = []
1435             jobs = []
1436             jobs_not_today = []
1437             for line in input_board[1:]:
1438                 row = line[0]
1439                 rows.append(row)
1440                 for i, square in enumerate(line[1:]):
1441                     if square=='':
1442                         continue
1443                     days = square.split(DAYS_SEPARATOR)
1444                     days = [int(day) for day in days]
1445                     job = (row, columns[i])
1446                     if today in days:
1447                         jobs.append(job)
1448                     else:
1449                         jobs_not_today.append(job)
1450
1451             d_boards[board_name] = {"rows" : rows,
1452                                     "columns" : columns,
1453                                     "jobs" : jobs,
1454                                     "jobs_not_today" : jobs_not_today}
1455
1456         self.d_input_boards = d_boards
1457
1458     def update_xml_files(self, l_jobs):
1459         '''Write all the xml files with updated information about the jobs
1460
1461         :param l_jobs List: the list of jobs that run today
1462         '''
1463         for xml_file in [self.xml_global_file] + list(
1464                                             self.d_xml_board_files.values()):
1465             self.update_xml_file(l_jobs, xml_file)
1466
1467         # Write the file
1468         self.write_xml_files()
1469
1470     def update_xml_file(self, l_jobs, xml_file):
1471         '''update information about the jobs for the file xml_file
1472
1473         :param l_jobs List: the list of jobs that run today
1474         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1475         '''
1476
1477         xml_node_jobs = xml_file.xmlroot.find('jobs')
1478         # Update the job names and status node
1479         for job in l_jobs:
1480             # Find the node corresponding to the job and delete it
1481             # in order to recreate it
1482             for xmljob in xml_node_jobs.findall('job'):
1483                 if xmljob.attrib['name'] == job.name:
1484                     xml_node_jobs.remove(xmljob)
1485
1486             T0 = str(job._T0)
1487             if T0 != "-1":
1488                 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1489                                        time.localtime(job._T0))
1490             Tf = str(job._Tf)
1491             if Tf != "-1":
1492                 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1493                                        time.localtime(job._Tf))
1494
1495             # recreate the job node
1496             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1497                                                   "job",
1498                                                   attrib={"name" : job.name})
1499             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1500             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1501             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1502             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1503             xml_history = src.xmlManager.add_simple_node(xmlj, "history")
1504             for date, res_job, link in self.history[job.name]:
1505                 src.xmlManager.add_simple_node(xml_history,
1506                                                "link",
1507                                                text=link,
1508                                                attrib={"date" : date,
1509                                                        "res" : res_job})
1510
1511             src.xmlManager.add_simple_node(xmlj, "sat_path",
1512                                            job.machine.sat_path)
1513             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1514             src.xmlManager.add_simple_node(xmlj, "distribution",
1515                                            job.machine.distribution)
1516             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1517             src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1518             src.xmlManager.add_simple_node(xmlj, "commands",
1519                                            " ; ".join(job.commands))
1520             src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1521             src.xmlManager.add_simple_node(xmlj, "begin", T0)
1522             src.xmlManager.add_simple_node(xmlj, "end", Tf)
1523             src.xmlManager.add_simple_node(xmlj, "out",
1524                                            src.printcolors.cleancolor(job.out))
1525             src.xmlManager.add_simple_node(xmlj, "err",
1526                                            src.printcolors.cleancolor(job.err))
1527             src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1528             if len(job.remote_log_files) > 0:
1529                 src.xmlManager.add_simple_node(xmlj,
1530                                                "remote_log_file_path",
1531                                                job.remote_log_files[0])
1532             else:
1533                 src.xmlManager.add_simple_node(xmlj,
1534                                                "remote_log_file_path",
1535                                                "nothing")
1536             # Search for the test log if there is any
1537             l_test_log_files = self.find_test_log(job.remote_log_files)
1538             xml_test = src.xmlManager.add_simple_node(xmlj,
1539                                                       "test_log_file_path")
1540             for test_log_path, res_test, nb_fails in l_test_log_files:
1541                 test_path_node = src.xmlManager.add_simple_node(xml_test,
1542                                                "path",
1543                                                test_log_path)
1544                 test_path_node.attrib["res"] = res_test
1545                 test_path_node.attrib["nb_fails"] = nb_fails
1546
1547             xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1548             # get the job father
1549             if job.after is not None:
1550                 job_father = None
1551                 for jb in l_jobs:
1552                     if jb.name == job.after:
1553                         job_father = jb
1554
1555                 if (job_father is not None and
1556                         len(job_father.remote_log_files) > 0):
1557                     link = job_father.remote_log_files[0]
1558                 else:
1559                     link = "nothing"
1560                 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1561
1562             # Verify that the job is to be done today regarding the input csv
1563             # files
1564             if job.board and job.board in self.d_input_boards.keys():
1565                 found = False
1566                 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1567                     if (job.machine.distribution == dist
1568                         and job.application == appli):
1569                         found = True
1570                         src.xmlManager.add_simple_node(xmlj,
1571                                                "extra_job",
1572                                                "no")
1573                         break
1574                 if not found:
1575                     src.xmlManager.add_simple_node(xmlj,
1576                                                "extra_job",
1577                                                "yes")
1578
1579
1580         # Update the date
1581         xml_node_infos = xml_file.xmlroot.find('infos')
1582         src.xmlManager.append_node_attrib(xml_node_infos,
1583                     attrib={"value" :
1584                     datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1585
1586
1587     def find_test_log(self, l_remote_log_files):
1588         '''Find if there is a test log (board) in the remote log files and
1589            the path to it. There can be several test command, so the result is
1590            a list.
1591
1592         :param l_remote_log_files List: the list of all remote log files
1593         :return: the list of (test log files path, res of the command)
1594         :rtype: List
1595         '''
1596         res = []
1597         for file_path in l_remote_log_files:
1598             dirname = os.path.basename(os.path.dirname(file_path))
1599             file_name = os.path.basename(file_path)
1600             regex = src.logger.log_all_command_file_expression
1601             oExpr = re.compile(regex)
1602             if dirname == "TEST" and oExpr.search(file_name):
1603                 # find the res of the command
1604                 prod_node = etree.parse(file_path).getroot().find("product")
1605                 res_test = prod_node.attrib["global_res"]
1606                 # find the number of fails
1607                 testbase_node = prod_node.find("tests").find("testbase")
1608                 nb_fails = int(testbase_node.attrib["failed"])
1609                 # put the file path, the res of the test command and the number
1610                 # of fails in the output
1611                 res.append((file_path, res_test, nb_fails))
1612
1613         return res
1614
1615     def last_update(self, finish_status = "finished"):
1616         '''update information about the jobs for the file xml_file
1617
1618         :param l_jobs List: the list of jobs that run today
1619         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1620         '''
1621         for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1622             xml_node_infos = xml_file.xmlroot.find('infos')
1623             src.xmlManager.append_node_attrib(xml_node_infos,
1624                         attrib={"JobsCommandStatus" : finish_status})
1625         # Write the file
1626         self.write_xml_files()
1627
1628     def write_xml_file(self, xml_file, stylesheet):
1629         ''' Write one xml file and the same file with prefix
1630         '''
1631         xml_file.write_tree(stylesheet)
1632         file_path = xml_file.logFile
1633         file_dir = os.path.dirname(file_path)
1634         file_name = os.path.basename(file_path)
1635         file_name_with_prefix = self.prefix + "_" + file_name
1636         xml_file.write_tree(stylesheet, os.path.join(file_dir,
1637                                                      file_name_with_prefix))
1638
1639     def write_xml_files(self):
1640         ''' Write the xml files
1641         '''
1642         self.write_xml_file(self.xml_global_file, STYLESHEET_GLOBAL)
1643         for xml_file in self.d_xml_board_files.values():
1644             self.write_xml_file(xml_file, STYLESHEET_BOARD)
1645
1646 def get_config_file_path(job_config_name, l_cfg_dir):
1647     found = False
1648     file_jobs_cfg = None
1649     if os.path.exists(job_config_name) and job_config_name.endswith(".pyconf"):
1650         found = True
1651         file_jobs_cfg = job_config_name
1652     else:
1653         for cfg_dir in l_cfg_dir:
1654             file_jobs_cfg = os.path.join(cfg_dir, job_config_name)
1655             if not file_jobs_cfg.endswith('.pyconf'):
1656                 file_jobs_cfg += '.pyconf'
1657
1658             if not os.path.exists(file_jobs_cfg):
1659                 continue
1660             else:
1661                 found = True
1662                 break
1663     return found, file_jobs_cfg
1664
1665 ##
1666 # Describes the command
1667 def description():
1668     return _("The jobs command launches maintenances that are described"
1669              " in the dedicated jobs configuration file.\n\nexample:\nsat "
1670              "jobs --name my_jobs --publish")
1671
1672 ##
1673 # Runs the command.
1674 def run(args, runner, logger):
1675
1676     (options, args) = parser.parse_args(args)
1677
1678     l_cfg_dir = runner.cfg.PATHS.JOBPATH
1679
1680     # list option : display all the available config files
1681     if options.list:
1682         for cfg_dir in l_cfg_dir:
1683             if not options.no_label:
1684                 logger.write("------ %s\n" %
1685                                  src.printcolors.printcHeader(cfg_dir))
1686             if not os.path.exists(cfg_dir):
1687                 continue
1688             for f in sorted(os.listdir(cfg_dir)):
1689                 if not f.endswith('.pyconf'):
1690                     continue
1691                 cfilename = f[:-7]
1692                 logger.write("%s\n" % cfilename)
1693         return 0
1694
1695     # Make sure the jobs_config option has been called
1696     if not options.jobs_cfg:
1697         message = _("The option --jobs_config is required\n")
1698         src.printcolors.printcError(message)
1699         return 1
1700
1701     # Find the file in the directories, unless it is a full path
1702     # merge all in a config
1703     merger = src.pyconf.ConfigMerger()
1704     config_jobs = src.pyconf.Config()
1705     l_conf_files_path = []
1706     for config_file in options.jobs_cfg:
1707         found, file_jobs_cfg = get_config_file_path(config_file, l_cfg_dir)
1708         if not found:
1709             msg = _("The file configuration %s was not found."
1710                     "\nUse the --list option to get the "
1711                     "possible files." % config_file)
1712             logger.write("%s\n" % src.printcolors.printcError(msg), 1)
1713             return 1
1714         l_conf_files_path.append(file_jobs_cfg)
1715         # Read the config that is in the file
1716         one_config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1717         merger.merge(config_jobs, one_config_jobs)
1718
1719     info = [
1720         (_("Platform"), runner.cfg.VARS.dist),
1721         (_("Files containing the jobs configuration"), l_conf_files_path)
1722     ]
1723     src.print_info(logger, info)
1724
1725     if options.only_jobs:
1726         l_jb = src.pyconf.Sequence()
1727         for jb in config_jobs.jobs:
1728             if jb.name in options.only_jobs:
1729                 l_jb.append(jb,
1730                 "Job that was given in only_jobs option parameters\n")
1731         config_jobs.jobs = l_jb
1732
1733     # Make a unique file that contain all the jobs in order to use it
1734     # on every machine
1735     name_pyconf = "_".join([os.path.basename(path)[:-len('.pyconf')]
1736                             for path in l_conf_files_path]) + ".pyconf"
1737     path_pyconf = src.get_tmp_filename(runner.cfg, name_pyconf)
1738     #Save config
1739     f = file( path_pyconf , 'w')
1740     config_jobs.__save__(f)
1741
1742     # log the paramiko problems
1743     log_dir = src.get_log_path(runner.cfg)
1744     paramiko_log_dir_path = os.path.join(log_dir, "JOBS")
1745     src.ensure_path_exists(paramiko_log_dir_path)
1746     paramiko.util.log_to_file(os.path.join(paramiko_log_dir_path,
1747                                            logger.txtFileName))
1748
1749     # Initialization
1750     today_jobs = Jobs(runner,
1751                       logger,
1752                       path_pyconf,
1753                       config_jobs)
1754     # SSH connection to all machines
1755     today_jobs.ssh_connection_all_machines()
1756     if options.test_connection:
1757         return 0
1758
1759     gui = None
1760     if options.publish:
1761         logger.write(src.printcolors.printcInfo(
1762                                         _("Initialize the xml boards : ")), 5)
1763         logger.flush()
1764
1765         # Copy the stylesheets in the log directory
1766         log_dir = log_dir
1767         xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1768         files_to_copy = []
1769         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1770         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1771         files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1772         for file_path in files_to_copy:
1773             shutil.copy2(file_path, log_dir)
1774
1775         # Instanciate the Gui in order to produce the xml files that contain all
1776         # the boards
1777         gui = Gui(log_dir,
1778                   today_jobs.ljobs,
1779                   today_jobs.ljobs_not_today,
1780                   runner.cfg.VARS.datehour,
1781                   logger,
1782                   file_boards = options.input_boards)
1783
1784         logger.write(src.printcolors.printcSuccess("OK"), 5)
1785         logger.write("\n\n", 5)
1786         logger.flush()
1787
1788         # Display the list of the xml files
1789         logger.write(src.printcolors.printcInfo(("Here is the list of published"
1790                                                  " files :\n")), 4)
1791         logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1792         for board in gui.d_xml_board_files.keys():
1793             file_path = gui.d_xml_board_files[board].logFile
1794             file_name = os.path.basename(file_path)
1795             logger.write("%s\n" % file_path, 4)
1796             logger.add_link(file_name, "board", 0, board)
1797
1798         logger.write("\n", 4)
1799
1800     today_jobs.gui = gui
1801
1802     interruped = False
1803     try:
1804         # Run all the jobs contained in config_jobs
1805         today_jobs.run_jobs()
1806     except KeyboardInterrupt:
1807         interruped = True
1808         logger.write("\n\n%s\n\n" %
1809                 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1810     except Exception as e:
1811         msg = _("CRITICAL ERROR: The jobs loop has been interrupted\n")
1812         logger.write("\n\n%s\n" % src.printcolors.printcError(msg) )
1813         logger.write("%s\n" % str(e))
1814         # get stack
1815         __, __, exc_traceback = sys.exc_info()
1816         fp = tempfile.TemporaryFile()
1817         traceback.print_tb(exc_traceback, file=fp)
1818         fp.seek(0)
1819         stack = fp.read()
1820         logger.write("\nTRACEBACK: %s\n" % stack.replace('"',"'"), 1)
1821
1822     finally:
1823         res = 0
1824         if interruped:
1825             res = 1
1826             msg = _("Killing the running jobs and trying"
1827                     " to get the corresponding logs\n")
1828             logger.write(src.printcolors.printcWarning(msg))
1829
1830         # find the potential not finished jobs and kill them
1831         for jb in today_jobs.ljobs:
1832             if not jb.has_finished():
1833                 res = 1
1834                 try:
1835                     jb.kill_remote_process()
1836                 except Exception as e:
1837                     msg = _("Failed to kill job %s: %s\n" % (jb.name, e))
1838                     logger.write(src.printcolors.printcWarning(msg))
1839             if jb.res_job != "0":
1840                 res = 1
1841         if interruped:
1842             if today_jobs.gui:
1843                 today_jobs.gui.last_update(_("Forced interruption"))
1844         else:
1845             if today_jobs.gui:
1846                 today_jobs.gui.last_update()
1847         # Output the results
1848         today_jobs.write_all_results()
1849         # Remove the temporary pyconf file
1850         if os.path.exists(path_pyconf):
1851             os.remove(path_pyconf)
1852         return res