commands/jobs.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import sys
  21 import tempfile
  22 import traceback
  23 import datetime
  24 import time
  25 import csv
  26 import shutil
  27 import itertools
  28 import re
  29 import paramiko
  30
  31 import src
  32 import src.ElementTree as etree
  33
  34 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
  35 STYLESHEET_BOARD = "jobs_board_report.xsl"
  36
  37 DAYS_SEPARATOR = ","
  38 CSV_DELIMITER = ";"
  39
  40 parser = src.options.Options()
  41
  42 parser.add_option('n', 'name', 'list2', 'jobs_cfg',
  43                   _('Mandatory: The name of the config file that contains'
  44                   ' the jobs configuration. Can be a list.'))
  45 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
  46                   _('Optional: the list of jobs to launch, by their name. '))
  47 parser.add_option('l', 'list', 'boolean', 'list',
  48                   _('Optional: list all available config files.'))
  49 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
  50                   _("Optional: try to connect to the machines. "
  51                     "Not executing the jobs."),
  52                   False)
  53 parser.add_option('p', 'publish', 'boolean', 'publish',
  54                   _("Optional: generate an xml file that can be read in a "
  55                     "browser to display the jobs status."),
  56                   False)
  57 parser.add_option('i', 'input_boards', 'string', 'input_boards', _("Optional: "
  58                                 "the path to csv file that contain "
  59                                 "the expected boards."),"")
  60 parser.add_option('', 'completion', 'boolean', 'no_label',
  61                   _("Optional (internal use): do not print labels, Works only "
  62                     "with --list."),
  63                   False)
  64
  65 class Machine(object):
  66     '''Class to manage a ssh connection on a machine
  67     '''
  68     def __init__(self,
  69                  name,
  70                  host,
  71                  user,
  72                  port=22,
  73                  passwd=None,
  74                  sat_path="salomeTools"):
  75         self.name = name
  76         self.host = host
  77         self.port = port
  78         self.distribution = None # Will be filled after copying SAT on the machine
  79         self.user = user
  80         self.password = passwd
  81         self.sat_path = sat_path
  82         self.ssh = paramiko.SSHClient()
  83         self._connection_successful = None
  84
  85     def connect(self, logger):
  86         '''Initiate the ssh connection to the remote machine
  87
  88         :param logger src.logger.Logger: The logger instance
  89         :return: Nothing
  90         :rtype: N\A
  91         '''
  92
  93         self._connection_successful = False
  94         self.ssh.load_system_host_keys()
  95         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  96         try:
  97             self.ssh.connect(self.host,
  98                              port=self.port,
  99                              username=self.user,
 100                              password = self.password)
 101         except paramiko.AuthenticationException:
 102             message = src.KO_STATUS + _("Authentication failed")
 103         except paramiko.BadHostKeyException:
 104             message = (src.KO_STATUS +
 105                        _("The server's host key could not be verified"))
 106         except paramiko.SSHException:
 107             message = ( _("SSHException error connecting or "
 108                           "establishing an SSH session"))
 109         except:
 110             message = ( _("Error connecting or establishing an SSH session"))
 111         else:
 112             self._connection_successful = True
 113             message = ""
 114         return message
 115
 116     def successfully_connected(self, logger):
 117         '''Verify if the connection to the remote machine has succeed
 118
 119         :param logger src.logger.Logger: The logger instance
 120         :return: True if the connection has succeed, False if not
 121         :rtype: bool
 122         '''
 123         if self._connection_successful == None:
 124             message = _("Warning : trying to ask if the connection to "
 125             "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
 126             " no connection request" %
 127                         (self.name, self.host, self.port, self.user))
 128             logger.write( src.printcolors.printcWarning(message))
 129         return self._connection_successful
 130
 131     def copy_sat(self, sat_local_path, job_file):
 132         '''Copy salomeTools to the remote machine in self.sat_path
 133         '''
 134         res = 0
 135         try:
 136             # open a sftp connection
 137             self.sftp = self.ssh.open_sftp()
 138             # Create the sat directory on remote machine if it is not existing
 139             self.mkdir(self.sat_path, ignore_existing=True)
 140             # Put sat
 141             self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
 142             # put the job configuration file in order to make it reachable
 143             # on the remote machine
 144             remote_job_file_name = ".%s" % os.path.basename(job_file)
 145             self.sftp.put(job_file, os.path.join(self.sat_path,
 146                                                  remote_job_file_name))
 147         except Exception as e:
 148             res = str(e)
 149             self._connection_successful = False
 150
 151         return res
 152
 153     def put_dir(self, source, target, filters = []):
 154         ''' Uploads the contents of the source directory to the target path. The
 155             target directory needs to exists. All sub-directories in source are
 156             created under target.
 157         '''
 158         for item in os.listdir(source):
 159             if item in filters:
 160                 continue
 161             source_path = os.path.join(source, item)
 162             destination_path = os.path.join(target, item)
 163             if os.path.islink(source_path):
 164                 linkto = os.readlink(source_path)
 165                 try:
 166                     self.sftp.symlink(linkto, destination_path)
 167                     self.sftp.chmod(destination_path,
 168                                     os.stat(source_path).st_mode)
 169                 except IOError:
 170                     pass
 171             else:
 172                 if os.path.isfile(source_path):
 173                     self.sftp.put(source_path, destination_path)
 174                     self.sftp.chmod(destination_path,
 175                                     os.stat(source_path).st_mode)
 176                 else:
 177                     self.mkdir(destination_path, ignore_existing=True)
 178                     self.put_dir(source_path, destination_path)
 179
 180     def mkdir(self, path, mode=511, ignore_existing=False):
 181         ''' Augments mkdir by adding an option to not fail
 182             if the folder exists
 183         '''
 184         try:
 185             self.sftp.mkdir(path, mode)
 186         except IOError:
 187             if ignore_existing:
 188                 pass
 189             else:
 190                 raise
 191
 192     def exec_command(self, command, logger):
 193         '''Execute the command on the remote machine
 194
 195         :param command str: The command to be run
 196         :param logger src.logger.Logger: The logger instance
 197         :return: the stdin, stdout, and stderr of the executing command,
 198                  as a 3-tuple
 199         :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
 200                 paramiko.channel.ChannelFile)
 201         '''
 202         try:
 203             # Does not wait the end of the command
 204             (stdin, stdout, stderr) = self.ssh.exec_command(command)
 205         except paramiko.SSHException:
 206             message = src.KO_STATUS + _(
 207                             ": the server failed to execute the command\n")
 208             logger.write( src.printcolors.printcError(message))
 209             return (None, None, None)
 210         except:
 211             logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
 212             return (None, None, None)
 213         else:
 214             return (stdin, stdout, stderr)
 215
 216     def close(self):
 217         '''Close the ssh connection
 218
 219         :rtype: N\A
 220         '''
 221         self.ssh.close()
 222
 223     def write_info(self, logger):
 224         '''Prints the informations relative to the machine in the logger
 225            (terminal traces and log file)
 226
 227         :param logger src.logger.Logger: The logger instance
 228         :return: Nothing
 229         :rtype: N\A
 230         '''
 231         logger.write("host : " + self.host + "\n")
 232         logger.write("port : " + str(self.port) + "\n")
 233         logger.write("user : " + str(self.user) + "\n")
 234         if self.successfully_connected(logger):
 235             status = src.OK_STATUS
 236         else:
 237             status = src.KO_STATUS
 238         logger.write("Connection : " + status + "\n\n")
 239
 240
 241 class Job(object):
 242     '''Class to manage one job
 243     '''
 244     def __init__(self,
 245                  name,
 246                  machine,
 247                  application,
 248                  board,
 249                  commands,
 250                  timeout,
 251                  config,
 252                  job_file_path,
 253                  logger,
 254                  after=None,
 255                  prefix=None):
 256
 257         self.name = name
 258         self.machine = machine
 259         self.after = after
 260         self.timeout = timeout
 261         self.application = application
 262         self.board = board
 263         self.config = config
 264         self.logger = logger
 265         # The list of log files to download from the remote machine
 266         self.remote_log_files = []
 267
 268         # The remote command status
 269         # -1 means that it has not been launched,
 270         # 0 means success and 1 means fail
 271         self.res_job = "-1"
 272         self.cancelled = False
 273
 274         self._T0 = -1
 275         self._Tf = -1
 276         self._has_begun = False
 277         self._has_finished = False
 278         self._has_timouted = False
 279         self._stdin = None # Store the command inputs field
 280         self._stdout = None # Store the command outputs field
 281         self._stderr = None # Store the command errors field
 282
 283         self.out = ""
 284         self.err = ""
 285
 286         self.name_remote_jobs_pyconf = ".%s" % os.path.basename(job_file_path)
 287         self.commands = commands
 288         self.command = (os.path.join(self.machine.sat_path, "sat") +
 289                         " -l " +
 290                         os.path.join(self.machine.sat_path,
 291                                      "list_log_files.txt") +
 292                         " job --jobs_config " +
 293                         os.path.join(self.machine.sat_path,
 294                                      self.name_remote_jobs_pyconf) +
 295                         " --name " +
 296                         self.name)
 297         if prefix:
 298             self.command = prefix + ' "' + self.command +'"'
 299
 300     def get_pids(self):
 301         """ Get the pid(s) corresponding to the command that have been launched
 302             On the remote machine
 303
 304         :return: The list of integers corresponding to the found pids
 305         :rtype: List
 306         """
 307         pids = []
 308         cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
 309         (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
 310         pids_cmd = out_pid.readlines()
 311         pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
 312         pids+=pids_cmd
 313         return pids
 314
 315     def kill_remote_process(self, wait=1):
 316         '''Kills the process on the remote machine.
 317
 318         :return: (the output of the kill, the error of the kill)
 319         :rtype: (str, str)
 320         '''
 321         try:
 322             pids = self.get_pids()
 323         except:
 324             return ("Unable to get the pid of the command.", "")
 325
 326         cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
 327         (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
 328                                                             self.logger)
 329         time.sleep(wait)
 330         return (out_kill.read().decode(), err_kill.read().decode())
 331
 332     def has_begun(self):
 333         '''Returns True if the job has already begun
 334
 335         :return: True if the job has already begun
 336         :rtype: bool
 337         '''
 338         return self._has_begun
 339
 340     def has_finished(self):
 341         '''Returns True if the job has already finished
 342            (i.e. all the commands have been executed)
 343            If it is finished, the outputs are stored in the fields out and err.
 344
 345         :return: True if the job has already finished
 346         :rtype: bool
 347         '''
 348
 349         # If the method has already been called and returned True
 350         if self._has_finished:
 351             return True
 352
 353         # If the job has not begun yet
 354         if not self.has_begun():
 355             return False
 356
 357         if self._stdout.channel.closed:
 358             self._has_finished = True
 359             # Store the result outputs
 360             self.out += self._stdout.read().decode()
 361             self.err += self._stderr.read().decode()
 362             # Put end time
 363             self._Tf = time.time()
 364             # And get the remote command status and log files
 365             try:
 366                 self.get_log_files()
 367             except Exception as e:
 368                 self.err += _("Unable to get remote log files: %s" % e)
 369
 370         return self._has_finished
 371
 372     def get_log_files(self):
 373         """Get the log files produced by the command launched
 374            on the remote machine, and put it in the log directory of the user,
 375            so they can be accessible from
 376         """
 377         # Do not get the files if the command is not finished
 378         if not self.has_finished():
 379             msg = _("Trying to get log files whereas the job is not finished.")
 380             self.logger.write(src.printcolors.printcWarning(msg))
 381             return
 382
 383         # First get the file that contains the list of log files to get
 384         tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
 385         remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt")
 386         self.machine.sftp.get(
 387                     remote_path,
 388                     tmp_file_path)
 389
 390         # Read the file and get the result of the command and all the log files
 391         # to get
 392         fstream_tmp = open(tmp_file_path, "r")
 393         file_lines = fstream_tmp.readlines()
 394         file_lines = [line.replace("\n", "") for line in file_lines]
 395         fstream_tmp.close()
 396         os.remove(tmp_file_path)
 397
 398         try :
 399             # The first line is the result of the command (0 success or 1 fail)
 400             self.res_job = file_lines[0]
 401         except Exception as e:
 402             self.err += _("Unable to get status from remote file %s: %s" %
 403                                                     (remote_path, str(e)))
 404
 405         for i, job_path_remote in enumerate(file_lines[1:]):
 406             try:
 407                 # For each command, there is two files to get :
 408                 # 1- The xml file describing the command and giving the
 409                 # internal traces.
 410                 # 2- The txt file containing the system command traces (like
 411                 # traces produced by the "make" command)
 412                 # 3- In case of the test command, there is another file to get :
 413                 # the xml board that contain the test results
 414                 dirname = os.path.basename(os.path.dirname(job_path_remote))
 415                 if dirname != 'OUT' and dirname != 'TEST':
 416                     # Case 1-
 417                     local_path = os.path.join(os.path.dirname(
 418                                                         self.logger.logFilePath),
 419                                               os.path.basename(job_path_remote))
 420                     if i==0: # The first is the job command
 421                         self.logger.add_link(os.path.basename(job_path_remote),
 422                                              "job",
 423                                              self.res_job,
 424                                              self.command)
 425                 elif dirname == 'OUT':
 426                     # Case 2-
 427                     local_path = os.path.join(os.path.dirname(
 428                                                         self.logger.logFilePath),
 429                                               'OUT',
 430                                               os.path.basename(job_path_remote))
 431                 elif dirname == 'TEST':
 432                     # Case 3-
 433                     local_path = os.path.join(os.path.dirname(
 434                                                         self.logger.logFilePath),
 435                                               'TEST',
 436                                               os.path.basename(job_path_remote))
 437
 438                 # Get the file
 439                 if not os.path.exists(local_path):
 440                     self.machine.sftp.get(job_path_remote, local_path)
 441                 self.remote_log_files.append(local_path)
 442             except Exception as e:
 443                 self.err += _("Unable to get %s log file from remote: %s" %
 444                                                     (str(job_path_remote),
 445                                                      str(e)))
 446
 447     def has_failed(self):
 448         '''Returns True if the job has failed.
 449            A job is considered as failed if the machine could not be reached,
 450            if the remote command failed,
 451            or if the job finished with a time out.
 452
 453         :return: True if the job has failed
 454         :rtype: bool
 455         '''
 456         if not self.has_finished():
 457             return False
 458         if not self.machine.successfully_connected(self.logger):
 459             return True
 460         if self.is_timeout():
 461             return True
 462         if self.res_job == "1":
 463             return True
 464         return False
 465
 466     def cancel(self):
 467         """In case of a failing job, one has to cancel every job that depend
 468            on it. This method put the job as failed and will not be executed.
 469         """
 470         if self.cancelled:
 471             return
 472         self._has_begun = True
 473         self._has_finished = True
 474         self.cancelled = True
 475         self.out += _("This job was not launched because its father has failed.")
 476         self.err += _("This job was not launched because its father has failed.")
 477
 478     def is_running(self):
 479         '''Returns True if the job commands are running
 480
 481         :return: True if the job is running
 482         :rtype: bool
 483         '''
 484         return self.has_begun() and not self.has_finished()
 485
 486     def is_timeout(self):
 487         '''Returns True if the job commands has finished with timeout
 488
 489         :return: True if the job has finished with timeout
 490         :rtype: bool
 491         '''
 492         return self._has_timouted
 493
 494     def time_elapsed(self):
 495         """Get the time elapsed since the job launching
 496
 497         :return: The number of seconds
 498         :rtype: int
 499         """
 500         if not self.has_begun():
 501             return -1
 502         T_now = time.time()
 503         return T_now - self._T0
 504
 505     def check_time(self):
 506         """Verify that the job has not exceeded its timeout.
 507            If it has, kill the remote command and consider the job as finished.
 508         """
 509         if not self.has_begun():
 510             return
 511         if self.time_elapsed() > self.timeout:
 512             self._has_finished = True
 513             self._has_timouted = True
 514             self._Tf = time.time()
 515             (out_kill, _) = self.kill_remote_process()
 516             self.out += "TIMEOUT \n" + out_kill
 517             self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
 518             try:
 519                 self.get_log_files()
 520             except Exception as e:
 521                 # The 2 following lines must be suppressed after the bug is fixed
 522                 print("The error type: ")
 523                 print(type(e))
 524                 print("The error: ")
 525                 print(e)
 526                 print("Local scope:")
 527                 print(dir())
 528                 print("The str type: ")
 529                 print(type(str))
 530                 print("str: ")
 531                 print(str)
 532                 if 'str' in dir():
 533                     del(str)
 534                 self.err += _("Unable to get remote log files!")
 535
 536     def total_duration(self):
 537         """Give the total duration of the job
 538
 539         :return: the total duration of the job in seconds
 540         :rtype: int
 541         """
 542         return self._Tf - self._T0
 543
 544     def run(self):
 545         """Launch the job by executing the remote command.
 546         """
 547
 548         # Prevent multiple run
 549         if self.has_begun():
 550             msg = _("Warning: A job can only be launched one time")
 551             msg2 = _("Trying to launch the job \"%s\" whereas it has "
 552                      "already been launched." % self.name)
 553             self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
 554                                                                         msg2)))
 555             return
 556
 557         # Do not execute the command if the machine could not be reached
 558         if not self.machine.successfully_connected(self.logger):
 559             self._has_finished = True
 560             self.out = "N\A"
 561             self.err += ("Connection to machine (name : %s, host: %s, port:"
 562                         " %s, user: %s) has failed\nUse the log command "
 563                         "to get more information."
 564                         % (self.machine.name,
 565                            self.machine.host,
 566                            self.machine.port,
 567                            self.machine.user))
 568         else:
 569             # Usual case : Launch the command on remote machine
 570             self._T0 = time.time()
 571             self._stdin, self._stdout, self._stderr = self.machine.exec_command(
 572                                                                   self.command,
 573                                                                   self.logger)
 574             # If the results are not initialized, finish the job
 575             if (self._stdin, self._stdout, self._stderr) == (None, None, None):
 576                 self._has_finished = True
 577                 self._Tf = time.time()
 578                 self.out += "N\A"
 579                 self.err += "The server failed to execute the command"
 580
 581         # Put the beginning flag to true.
 582         self._has_begun = True
 583
 584     def write_results(self):
 585         """Display on the terminal all the job's information
 586         """
 587         self.logger.write("name : " + self.name + "\n")
 588         if self.after:
 589             self.logger.write("after : %s\n" % self.after)
 590         self.logger.write("Time elapsed : %4imin %2is \n" %
 591                      (self.total_duration()//60 , self.total_duration()%60))
 592         if self._T0 != -1:
 593             self.logger.write("Begin time : %s\n" %
 594                          time.strftime('%Y-%m-%d %H:%M:%S',
 595                                        time.localtime(self._T0)) )
 596         if self._Tf != -1:
 597             self.logger.write("End time   : %s\n\n" %
 598                          time.strftime('%Y-%m-%d %H:%M:%S',
 599                                        time.localtime(self._Tf)) )
 600
 601         machine_head = "Informations about connection :\n"
 602         underline = (len(machine_head) - 2) * "-"
 603         self.logger.write(src.printcolors.printcInfo(
 604                                                 machine_head+underline+"\n"))
 605         self.machine.write_info(self.logger)
 606
 607         self.logger.write(src.printcolors.printcInfo("out : \n"))
 608         if self.out == "":
 609             self.logger.write("Unable to get output\n")
 610         else:
 611             self.logger.write(self.out + "\n")
 612         self.logger.write(src.printcolors.printcInfo("err : \n"))
 613         self.logger.write(self.err + "\n")
 614
 615     def get_status(self):
 616         """Get the status of the job (used by the Gui for xml display)
 617
 618         :return: The current status of the job
 619         :rtype: String
 620         """
 621         if not self.machine.successfully_connected(self.logger):
 622             return "SSH connection KO"
 623         if not self.has_begun():
 624             return "Not launched"
 625         if self.cancelled:
 626             return "Cancelled"
 627         if self.is_running():
 628             return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
 629                                                     time.localtime(self._T0))
 630         if self.has_finished():
 631             if self.is_timeout():
 632                 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
 633                                                     time.localtime(self._Tf))
 634             return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
 635                                                      time.localtime(self._Tf))
 636
 637 class Jobs(object):
 638     '''Class to manage the jobs to be run
 639     '''
 640     def __init__(self,
 641                  runner,
 642                  logger,
 643                  job_file_path,
 644                  config_jobs,
 645                  lenght_columns = 20):
 646         # The jobs configuration
 647         self.cfg_jobs = config_jobs
 648         self.job_file_path = job_file_path
 649         # The machine that will be used today
 650         self.lmachines = []
 651         # The list of machine (hosts, port) that will be used today
 652         # (a same host can have several machine instances since there
 653         # can be several ssh parameters)
 654         self.lhosts = []
 655         # The jobs to be launched today
 656         self.ljobs = []
 657         # The jobs that will not be launched today
 658         self.ljobs_not_today = []
 659         self.runner = runner
 660         self.logger = logger
 661         self.len_columns = lenght_columns
 662
 663         # the list of jobs that have not been run yet
 664         self._l_jobs_not_started = []
 665         # the list of jobs that have already ran
 666         self._l_jobs_finished = []
 667         # the list of jobs that are running
 668         self._l_jobs_running = []
 669
 670         self.determine_jobs_and_machines()
 671
 672     def define_job(self, job_def, machine):
 673         '''Takes a pyconf job definition and a machine (from class machine)
 674            and returns the job instance corresponding to the definition.
 675
 676         :param job_def src.config.Mapping: a job definition
 677         :param machine machine: the machine on which the job will run
 678         :return: The corresponding job in a job class instance
 679         :rtype: job
 680         '''
 681         name = job_def.name
 682         cmmnds = job_def.commands
 683         if not "timeout" in job_def:
 684             timeout = 4*60*60 # default timeout = 4h
 685         else:
 686             timeout = job_def.timeout
 687         after = None
 688         if 'after' in job_def:
 689             after = job_def.after
 690         application = None
 691         if 'application' in job_def:
 692             application = job_def.application
 693         board = None
 694         if 'board' in job_def:
 695             board = job_def.board
 696         prefix = None
 697         if "prefix" in job_def:
 698             prefix = job_def.prefix
 699
 700         return Job(name,
 701                    machine,
 702                    application,
 703                    board,
 704                    cmmnds,
 705                    timeout,
 706                    self.runner.cfg,
 707                    self.job_file_path,
 708                    self.logger,
 709                    after = after,
 710                    prefix = prefix)
 711
 712     def determine_jobs_and_machines(self):
 713         '''Function that reads the pyconf jobs definition and instantiates all
 714            the machines and jobs to be done today.
 715
 716         :return: Nothing
 717         :rtype: N\A
 718         '''
 719         today = datetime.date.weekday(datetime.date.today())
 720         host_list = []
 721
 722         for job_def in self.cfg_jobs.jobs :
 723
 724             if not "machine" in job_def:
 725                 msg = _('WARNING: The job "%s" do not have the key '
 726                        '"machine", this job is ignored.\n\n' % job_def.name)
 727                 self.logger.write(src.printcolors.printcWarning(msg))
 728                 continue
 729             name_machine = job_def.machine
 730
 731             a_machine = None
 732             for mach in self.lmachines:
 733                 if mach.name == name_machine:
 734                     a_machine = mach
 735                     break
 736
 737             if a_machine == None:
 738                 for machine_def in self.cfg_jobs.machines:
 739                     if machine_def.name == name_machine:
 740                         if 'host' not in machine_def:
 741                             host = self.runner.cfg.VARS.hostname
 742                         else:
 743                             host = machine_def.host
 744
 745                         if 'user' not in machine_def:
 746                             user = self.runner.cfg.VARS.user
 747                         else:
 748                             user = machine_def.user
 749
 750                         if 'port' not in machine_def:
 751                             port = 22
 752                         else:
 753                             port = machine_def.port
 754
 755                         if 'password' not in machine_def:
 756                             passwd = None
 757                         else:
 758                             passwd = machine_def.password
 759
 760                         if 'sat_path' not in machine_def:
 761                             sat_path = "salomeTools"
 762                         else:
 763                             sat_path = machine_def.sat_path
 764
 765                         a_machine = Machine(
 766                                             machine_def.name,
 767                                             host,
 768                                             user,
 769                                             port=port,
 770                                             passwd=passwd,
 771                                             sat_path=sat_path
 772                                             )
 773
 774                         self.lmachines.append(a_machine)
 775                         if (host, port) not in host_list:
 776                             host_list.append((host, port))
 777
 778                 if a_machine == None:
 779                     msg = _("WARNING: The job \"%(job_name)s\" requires the "
 780                             "machine \"%(machine_name)s\" but this machine "
 781                             "is not defined in the configuration file.\n"
 782                             "The job will not be launched")
 783                     self.logger.write(src.printcolors.printcWarning(msg))
 784
 785             a_job = self.define_job(job_def, a_machine)
 786
 787             if today in job_def.when:
 788                 self.ljobs.append(a_job)
 789             else: # today in job_def.when
 790                 self.ljobs_not_today.append(a_job)
 791
 792         self.lhosts = host_list
 793
 794     def ssh_connection_all_machines(self, pad=50):
 795         '''Function that do the ssh connection to every machine
 796            to be used today.
 797
 798         :return: Nothing
 799         :rtype: N\A
 800         '''
 801         self.logger.write(src.printcolors.printcInfo((
 802                         "Establishing connection with all the machines :\n")))
 803         for machine in self.lmachines:
 804             # little algorithm in order to display traces
 805             begin_line = (_("Connection to %s: " % machine.name))
 806             if pad - len(begin_line) < 0:
 807                 endline = " "
 808             else:
 809                 endline = (pad - len(begin_line)) * "." + " "
 810
 811             step = "SSH connection"
 812             self.logger.write( begin_line + endline + step)
 813             self.logger.flush()
 814             # the call to the method that initiate the ssh connection
 815             msg = machine.connect(self.logger)
 816
 817             # Copy salomeTools to the remote machine
 818             if machine.successfully_connected(self.logger):
 819                 step = _("Remove SAT")
 820                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 821                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 822                 (__, out_dist, __) = machine.exec_command(
 823                                                 "rm -rf %s" % machine.sat_path,
 824                                                 self.logger)
 825                 out_dist.read()
 826
 827                 self.logger.flush()
 828                 step = _("Copy SAT")
 829                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 830                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 831                 self.logger.flush()
 832                 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
 833                                             self.job_file_path)
 834                 # get the remote machine distribution using a sat command
 835                 (__, out_dist, __) = machine.exec_command(
 836                                 os.path.join(machine.sat_path,
 837                                     "sat config --value VARS.dist --no_label"),
 838                                 self.logger)
 839                 machine.distribution = out_dist.read().decode().replace("\n",
 840                                                                         "")
 841                 # Print the status of the copy
 842                 if res_copy == 0:
 843                     self.logger.write('\r%s' %
 844                                 ((len(begin_line)+len(endline)+20) * " "), 3)
 845                     self.logger.write('\r%s%s%s' %
 846                         (begin_line,
 847                          endline,
 848                          src.printcolors.printc(src.OK_STATUS)), 3)
 849                 else:
 850                     self.logger.write('\r%s' %
 851                             ((len(begin_line)+len(endline)+20) * " "), 3)
 852                     self.logger.write('\r%s%s%s %s' %
 853                         (begin_line,
 854                          endline,
 855                          src.printcolors.printc(src.KO_STATUS),
 856                          _("Copy of SAT failed: %s" % res_copy)), 3)
 857             else:
 858                 self.logger.write('\r%s' %
 859                                   ((len(begin_line)+len(endline)+20) * " "), 3)
 860                 self.logger.write('\r%s%s%s %s' %
 861                     (begin_line,
 862                      endline,
 863                      src.printcolors.printc(src.KO_STATUS),
 864                      msg), 3)
 865             self.logger.write("\n", 3)
 866
 867         self.logger.write("\n")
 868
 869
 870     def is_occupied(self, hostname):
 871         '''Function that returns True if a job is running on
 872            the machine defined by its host and its port.
 873
 874         :param hostname (str, int): the pair (host, port)
 875         :return: the job that is running on the host,
 876                 or false if there is no job running on the host.
 877         :rtype: job / bool
 878         '''
 879         host = hostname[0]
 880         port = hostname[1]
 881         for jb in self.ljobs:
 882             if jb.machine.host == host and jb.machine.port == port:
 883                 if jb.is_running():
 884                     return jb
 885         return False
 886
 887     def update_jobs_states_list(self):
 888         '''Function that updates the lists that store the currently
 889            running jobs and the jobs that have already finished.
 890
 891         :return: Nothing.
 892         :rtype: N\A
 893         '''
 894         jobs_finished_list = []
 895         jobs_running_list = []
 896         for jb in self.ljobs:
 897             if jb.is_running():
 898                 jobs_running_list.append(jb)
 899                 jb.check_time()
 900             if jb.has_finished():
 901                 jobs_finished_list.append(jb)
 902
 903         nb_job_finished_before = len(self._l_jobs_finished)
 904         self._l_jobs_finished = jobs_finished_list
 905         self._l_jobs_running = jobs_running_list
 906
 907         nb_job_finished_now = len(self._l_jobs_finished)
 908
 909         return nb_job_finished_now > nb_job_finished_before
 910
 911     def cancel_dependencies_of_failing_jobs(self):
 912         '''Function that cancels all the jobs that depend on a failing one.
 913
 914         :return: Nothing.
 915         :rtype: N\A
 916         '''
 917
 918         for job in self.ljobs:
 919             if job.after is None:
 920                 continue
 921             father_job = self.find_job_that_has_name(job.after)
 922             if father_job is not None and father_job.has_failed():
 923                 job.cancel()
 924
 925     def find_job_that_has_name(self, name):
 926         '''Returns the job by its name.
 927
 928         :param name str: a job name
 929         :return: the job that has the name.
 930         :rtype: job
 931         '''
 932         for jb in self.ljobs:
 933             if jb.name == name:
 934                 return jb
 935         # the following is executed only if the job was not found
 936         return None
 937
 938     def str_of_length(self, text, length):
 939         '''Takes a string text of any length and returns
 940            the most close string of length "length".
 941
 942         :param text str: any string
 943         :param length int: a length for the returned string
 944         :return: the most close string of length "length"
 945         :rtype: str
 946         '''
 947         if len(text) > length:
 948             text_out = text[:length-3] + '...'
 949         else:
 950             diff = length - len(text)
 951             before = " " * (diff//2)
 952             after = " " * (diff//2 + diff%2)
 953             text_out = before + text + after
 954
 955         return text_out
 956
 957     def display_status(self, len_col):
 958         '''Takes a lenght and construct the display of the current status
 959            of the jobs in an array that has a column for each host.
 960            It displays the job that is currently running on the host
 961            of the column.
 962
 963         :param len_col int: the size of the column
 964         :return: Nothing
 965         :rtype: N\A
 966         '''
 967
 968         display_line = ""
 969         for host_port in self.lhosts:
 970             jb = self.is_occupied(host_port)
 971             if not jb: # nothing running on the host
 972                 empty = self.str_of_length("empty", len_col)
 973                 display_line += "|" + empty
 974             else:
 975                 display_line += "|" + src.printcolors.printcInfo(
 976                                         self.str_of_length(jb.name, len_col))
 977
 978         self.logger.write("\r" + display_line + "|")
 979         self.logger.flush()
 980
 981
 982     def run_jobs(self):
 983         '''The main method. Runs all the jobs on every host.
 984            For each host, at a given time, only one job can be running.
 985            The jobs that have the field after (that contain the job that has
 986            to be run before it) are run after the previous job.
 987            This method stops when all the jobs are finished.
 988
 989         :return: Nothing
 990         :rtype: N\A
 991         '''
 992
 993         # Print header
 994         self.logger.write(src.printcolors.printcInfo(
 995                                                 _('Executing the jobs :\n')))
 996         text_line = ""
 997         for host_port in self.lhosts:
 998             host = host_port[0]
 999             port = host_port[1]
1000             if port == 22: # default value
1001                 text_line += "|" + self.str_of_length(host, self.len_columns)
1002             else:
1003                 text_line += "|" + self.str_of_length(
1004                                 "("+host+", "+str(port)+")", self.len_columns)
1005
1006         tiret_line = " " + "-"*(len(text_line)-1) + "\n"
1007         self.logger.write(tiret_line)
1008         self.logger.write(text_line + "|\n")
1009         self.logger.write(tiret_line)
1010         self.logger.flush()
1011
1012         # The infinite loop that runs the jobs
1013         l_jobs_not_started = src.deepcopy_list(self.ljobs)
1014         while len(self._l_jobs_finished) != len(self.ljobs):
1015             new_job_start = False
1016             for host_port in self.lhosts:
1017
1018                 if self.is_occupied(host_port):
1019                     continue
1020
1021                 for jb in l_jobs_not_started:
1022                     if (jb.machine.host, jb.machine.port) != host_port:
1023                         continue
1024                     if jb.after == None:
1025                         jb.run()
1026                         l_jobs_not_started.remove(jb)
1027                         new_job_start = True
1028                         break
1029                     else:
1030                         jb_before = self.find_job_that_has_name(jb.after)
1031                         if jb_before is None:
1032                             jb.cancel()
1033                             msg = _("This job was not launched because its "
1034                                     "father is not in the jobs list.")
1035                             jb.out = msg
1036                             jb.err = msg
1037                             break
1038                         if jb_before.has_finished():
1039                             jb.run()
1040                             l_jobs_not_started.remove(jb)
1041                             new_job_start = True
1042                             break
1043             self.cancel_dependencies_of_failing_jobs()
1044             new_job_finished = self.update_jobs_states_list()
1045
1046             if new_job_start or new_job_finished:
1047                 if self.gui:
1048                     self.gui.update_xml_files(self.ljobs)
1049                 # Display the current status
1050                 self.display_status(self.len_columns)
1051
1052             # Make sure that the proc is not entirely busy
1053             time.sleep(0.001)
1054
1055         self.logger.write("\n")
1056         self.logger.write(tiret_line)
1057         self.logger.write("\n\n")
1058
1059         if self.gui:
1060             self.gui.update_xml_files(self.ljobs)
1061             self.gui.last_update()
1062
1063     def write_all_results(self):
1064         '''Display all the jobs outputs.
1065
1066         :return: Nothing
1067         :rtype: N\A
1068         '''
1069
1070         for jb in self.ljobs:
1071             self.logger.write(src.printcolors.printcLabel(
1072                         "#------- Results for job %s -------#\n" % jb.name))
1073             jb.write_results()
1074             self.logger.write("\n\n")
1075
1076 class Gui(object):
1077     '''Class to manage the the xml data that can be displayed in a browser to
1078        see the jobs states
1079     '''
1080
1081     def __init__(self,
1082                  xml_dir_path,
1083                  l_jobs,
1084                  l_jobs_not_today,
1085                  prefix,
1086                  logger,
1087                  file_boards=""):
1088         '''Initialization
1089
1090         :param xml_dir_path str: The path to the directory where to put
1091                                  the xml resulting files
1092         :param l_jobs List: the list of jobs that run today
1093         :param l_jobs_not_today List: the list of jobs that do not run today
1094         :param file_boards str: the file path from which to read the
1095                                    expected boards
1096         '''
1097         # The logging instance
1098         self.logger = logger
1099
1100         # The prefix to add to the xml files : date_hour
1101         self.prefix = prefix
1102
1103         # The path of the csv files to read to fill the expected boards
1104         self.file_boards = file_boards
1105
1106         if file_boards != "":
1107             today = datetime.date.weekday(datetime.date.today())
1108             self.parse_csv_boards(today)
1109         else:
1110             self.d_input_boards = {}
1111
1112         # The path of the global xml file
1113         self.xml_dir_path = xml_dir_path
1114         # Initialize the xml files
1115         self.global_name = "global_report"
1116         xml_global_path = os.path.join(self.xml_dir_path,
1117                                        self.global_name + ".xml")
1118         self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1119                                                          "JobsReport")
1120
1121         # Find history for each job
1122         self.history = {}
1123         self.find_history(l_jobs, l_jobs_not_today)
1124
1125         # The xml files that corresponds to the boards.
1126         # {name_board : xml_object}}
1127         self.d_xml_board_files = {}
1128
1129         # Create the lines and columns
1130         self.initialize_boards(l_jobs, l_jobs_not_today)
1131
1132         # Write the xml file
1133         self.update_xml_files(l_jobs)
1134
1135     def add_xml_board(self, name):
1136         '''Add a board to the board list
1137         :param name str: the board name
1138         '''
1139         xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1140         self.d_xml_board_files[name] =  src.xmlManager.XmlLogFile(
1141                                                     xml_board_path,
1142                                                     "JobsReport")
1143         self.d_xml_board_files[name].add_simple_node("distributions")
1144         self.d_xml_board_files[name].add_simple_node("applications")
1145         self.d_xml_board_files[name].add_simple_node("board", text=name)
1146
1147     def initialize_boards(self, l_jobs, l_jobs_not_today):
1148         '''Get all the first information needed for each file and write the
1149            first version of the files
1150         :param l_jobs List: the list of jobs that run today
1151         :param l_jobs_not_today List: the list of jobs that do not run today
1152         '''
1153         # Get the boards to fill and put it in a dictionary
1154         # {board_name : xml instance corresponding to the board}
1155         for job in l_jobs + l_jobs_not_today:
1156             board = job.board
1157             if (board is not None and
1158                                 board not in self.d_xml_board_files.keys()):
1159                 self.add_xml_board(board)
1160
1161         # Verify that the boards given as input are done
1162         for board in list(self.d_input_boards.keys()):
1163             if board not in self.d_xml_board_files:
1164                 self.add_xml_board(board)
1165             root_node = self.d_xml_board_files[board].xmlroot
1166             src.xmlManager.append_node_attrib(root_node,
1167                                               {"input_file" : self.file_boards})
1168
1169         # Loop over all jobs in order to get the lines and columns for each
1170         # xml file
1171         d_dist = {}
1172         d_application = {}
1173         for board in self.d_xml_board_files:
1174             d_dist[board] = []
1175             d_application[board] = []
1176
1177         l_hosts_ports = []
1178
1179         for job in l_jobs + l_jobs_not_today:
1180
1181             if (job.machine.host, job.machine.port) not in l_hosts_ports:
1182                 l_hosts_ports.append((job.machine.host, job.machine.port))
1183
1184             distrib = job.machine.distribution
1185             application = job.application
1186
1187             board_job = job.board
1188             if board is None:
1189                 continue
1190             for board in self.d_xml_board_files:
1191                 if board_job == board:
1192                     if distrib is not None and distrib not in d_dist[board]:
1193                         d_dist[board].append(distrib)
1194                         src.xmlManager.add_simple_node(
1195                             self.d_xml_board_files[board].xmlroot.find(
1196                                                             'distributions'),
1197                                                    "dist",
1198                                                    attrib={"name" : distrib})
1199
1200                 if board_job == board:
1201                     if (application is not None and
1202                                     application not in d_application[board]):
1203                         d_application[board].append(application)
1204                         src.xmlManager.add_simple_node(
1205                             self.d_xml_board_files[board].xmlroot.find(
1206                                                                 'applications'),
1207                                                    "application",
1208                                                    attrib={
1209                                                         "name" : application})
1210
1211         # Verify that there are no missing application or distribution in the
1212         # xml board files (regarding the input boards)
1213         for board in self.d_xml_board_files:
1214             l_dist = d_dist[board]
1215             if board not in self.d_input_boards.keys():
1216                 continue
1217             for dist in self.d_input_boards[board]["rows"]:
1218                 if dist not in l_dist:
1219                     src.xmlManager.add_simple_node(
1220                             self.d_xml_board_files[board].xmlroot.find(
1221                                                             'distributions'),
1222                                                    "dist",
1223                                                    attrib={"name" : dist})
1224             l_appli = d_application[board]
1225             for appli in self.d_input_boards[board]["columns"]:
1226                 if appli not in l_appli:
1227                     src.xmlManager.add_simple_node(
1228                             self.d_xml_board_files[board].xmlroot.find(
1229                                                                 'applications'),
1230                                                    "application",
1231                                                    attrib={"name" : appli})
1232
1233         # Initialize the hosts_ports node for the global file
1234         self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1235                                                                 "hosts_ports")
1236         for host, port in l_hosts_ports:
1237             host_port = "%s:%i" % (host, port)
1238             src.xmlManager.add_simple_node(self.xmlhosts_ports,
1239                                            "host_port",
1240                                            attrib={"name" : host_port})
1241
1242         # Initialize the jobs node in all files
1243         for xml_file in [self.xml_global_file] + list(
1244                                             self.d_xml_board_files.values()):
1245             xml_jobs = xml_file.add_simple_node("jobs")
1246             # Get the jobs present in the config file but
1247             # that will not be launched today
1248             self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1249
1250             # add also the infos node
1251             xml_file.add_simple_node("infos",
1252                                      attrib={"name" : "last update",
1253                                              "JobsCommandStatus" : "running"})
1254
1255             # and put the history node
1256             history_node = xml_file.add_simple_node("history")
1257             name_board = os.path.basename(xml_file.logFile)[:-len(".xml")]
1258             # serach for board files
1259             expression = "^[0-9]{8}_+[0-9]{6}_" + name_board + ".xml$"
1260             oExpr = re.compile(expression)
1261             # Get the list of xml borad files that are in the log directory
1262             for file_name in os.listdir(self.xml_dir_path):
1263                 if oExpr.search(file_name):
1264                     date = os.path.basename(file_name).split("_")[0]
1265                     file_path = os.path.join(self.xml_dir_path, file_name)
1266                     src.xmlManager.add_simple_node(history_node,
1267                                                    "link",
1268                                                    text=file_path,
1269                                                    attrib={"date" : date})
1270
1271
1272         # Find in each board the squares that needs to be filled regarding the
1273         # input csv files but that are not covered by a today job
1274         for board in self.d_input_boards.keys():
1275             xml_root_board = self.d_xml_board_files[board].xmlroot
1276             # Find the missing jobs for today
1277             xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1278                                                  "missing_jobs")
1279             for row, column in self.d_input_boards[board]["jobs"]:
1280                 found = False
1281                 for job in l_jobs:
1282                     if (job.application == column and
1283                         job.machine.distribution == row):
1284                         found = True
1285                         break
1286                 if not found:
1287                     src.xmlManager.add_simple_node(xml_missing,
1288                                             "job",
1289                                             attrib={"distribution" : row,
1290                                                     "application" : column })
1291             # Find the missing jobs not today
1292             xml_missing_not_today = src.xmlManager.add_simple_node(
1293                                                  xml_root_board,
1294                                                  "missing_jobs_not_today")
1295             for row, column in self.d_input_boards[board]["jobs_not_today"]:
1296                 found = False
1297                 for job in l_jobs_not_today:
1298                     if (job.application == column and
1299                         job.machine.distribution == row):
1300                         found = True
1301                         break
1302                 if not found:
1303                     src.xmlManager.add_simple_node(xml_missing_not_today,
1304                                             "job",
1305                                             attrib={"distribution" : row,
1306                                                     "application" : column })
1307
1308     def find_history(self, l_jobs, l_jobs_not_today):
1309         """find, for each job, in the existent xml boards the results for the
1310            job. Store the results in the dictionnary self.history = {name_job :
1311            list of (date, status, list links)}
1312
1313         :param l_jobs List: the list of jobs to run today
1314         :param l_jobs_not_today List: the list of jobs that do not run today
1315         """
1316         # load the all the history
1317         expression = "^[0-9]{8}_+[0-9]{6}_" + self.global_name + ".xml$"
1318         oExpr = re.compile(expression)
1319         # Get the list of global xml that are in the log directory
1320         l_globalxml = []
1321         for file_name in os.listdir(self.xml_dir_path):
1322             if oExpr.search(file_name):
1323                 file_path = os.path.join(self.xml_dir_path, file_name)
1324                 try:
1325                     global_xml = src.xmlManager.ReadXmlFile(file_path)
1326                     l_globalxml.append(global_xml)
1327                 except Exception as e:
1328                     msg = _("\nWARNING: the file %s can not be read, it will be "
1329                             "ignored\n%s" % (file_path, e))
1330                     self.logger.write("%s\n" % src.printcolors.printcWarning(
1331                                                                         msg), 5)
1332
1333         # Construct the dictionnary self.history
1334         for job in l_jobs + l_jobs_not_today:
1335             l_links = []
1336             for global_xml in l_globalxml:
1337                 date = os.path.basename(global_xml.filePath).split("_")[0]
1338                 global_root_node = global_xml.xmlroot.find("jobs")
1339                 job_node = src.xmlManager.find_node_by_attrib(
1340                                                               global_root_node,
1341                                                               "job",
1342                                                               "name",
1343                                                               job.name)
1344                 if job_node:
1345                     if job_node.find("remote_log_file_path") is not None:
1346                         link = job_node.find("remote_log_file_path").text
1347                         res_job = job_node.find("res").text
1348                         if link != "nothing":
1349                             l_links.append((date, res_job, link))
1350             l_links = sorted(l_links, reverse=True)
1351             self.history[job.name] = l_links
1352
1353     def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1354         '''Get all the first information needed for each file and write the
1355            first version of the files
1356
1357         :param xml_node_jobs etree.Element: the node corresponding to a job
1358         :param l_jobs_not_today List: the list of jobs that do not run today
1359         '''
1360         for job in l_jobs_not_today:
1361             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1362                                                  "job",
1363                                                  attrib={"name" : job.name})
1364             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1365             src.xmlManager.add_simple_node(xmlj,
1366                                            "distribution",
1367                                            job.machine.distribution)
1368             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1369             src.xmlManager.add_simple_node(xmlj,
1370                                        "commands", " ; ".join(job.commands))
1371             src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1372             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1373             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1374             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1375             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1376             src.xmlManager.add_simple_node(xmlj, "sat_path",
1377                                                         job.machine.sat_path)
1378             xml_history = src.xmlManager.add_simple_node(xmlj, "history")
1379             for i, (date, res_job, link) in enumerate(self.history[job.name]):
1380                 if i==0:
1381                     # tag the first one (the last one)
1382                     src.xmlManager.add_simple_node(xml_history,
1383                                                    "link",
1384                                                    text=link,
1385                                                    attrib={"date" : date,
1386                                                            "res" : res_job,
1387                                                            "last" : "yes"})
1388                 else:
1389                     src.xmlManager.add_simple_node(xml_history,
1390                                                    "link",
1391                                                    text=link,
1392                                                    attrib={"date" : date,
1393                                                            "res" : res_job,
1394                                                            "last" : "no"})
1395
1396     def parse_csv_boards(self, today):
1397         """ Parse the csv file that describes the boards to produce and fill
1398             the dict d_input_boards that contain the csv file contain
1399
1400         :param today int: the current day of the week
1401         """
1402         # open the csv file and read its content
1403         l_read = []
1404         with open(self.file_boards, 'r') as f:
1405             reader = csv.reader(f,delimiter=CSV_DELIMITER)
1406             for row in reader:
1407                 l_read.append(row)
1408         # get the delimiter for the boards (empty line)
1409         boards_delimiter = [''] * len(l_read[0])
1410         # Make the list of boards, by splitting with the delimiter
1411         l_boards = [list(y) for x, y in itertools.groupby(l_read,
1412                                     lambda z: z == boards_delimiter) if not x]
1413
1414         # loop over the csv lists of lines and get the rows, columns and jobs
1415         d_boards = {}
1416         for input_board in l_boards:
1417             # get board name
1418             board_name = input_board[0][0]
1419
1420             # Get columns list
1421             columns = input_board[0][1:]
1422
1423             rows = []
1424             jobs = []
1425             jobs_not_today = []
1426             for line in input_board[1:]:
1427                 row = line[0]
1428                 rows.append(row)
1429                 for i, square in enumerate(line[1:]):
1430                     if square=='':
1431                         continue
1432                     days = square.split(DAYS_SEPARATOR)
1433                     days = [int(day) for day in days]
1434                     job = (row, columns[i])
1435                     if today in days:
1436                         jobs.append(job)
1437                     else:
1438                         jobs_not_today.append(job)
1439
1440             d_boards[board_name] = {"rows" : rows,
1441                                     "columns" : columns,
1442                                     "jobs" : jobs,
1443                                     "jobs_not_today" : jobs_not_today}
1444
1445         self.d_input_boards = d_boards
1446
1447     def update_xml_files(self, l_jobs):
1448         '''Write all the xml files with updated information about the jobs
1449
1450         :param l_jobs List: the list of jobs that run today
1451         '''
1452         for xml_file in [self.xml_global_file] + list(
1453                                             self.d_xml_board_files.values()):
1454             self.update_xml_file(l_jobs, xml_file)
1455
1456         # Write the file
1457         self.write_xml_files()
1458
1459     def update_xml_file(self, l_jobs, xml_file):
1460         '''update information about the jobs for the file xml_file
1461
1462         :param l_jobs List: the list of jobs that run today
1463         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1464         '''
1465
1466         xml_node_jobs = xml_file.xmlroot.find('jobs')
1467         # Update the job names and status node
1468         for job in l_jobs:
1469             # Find the node corresponding to the job and delete it
1470             # in order to recreate it
1471             for xmljob in xml_node_jobs.findall('job'):
1472                 if xmljob.attrib['name'] == job.name:
1473                     xml_node_jobs.remove(xmljob)
1474
1475             T0 = str(job._T0)
1476             if T0 != "-1":
1477                 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1478                                        time.localtime(job._T0))
1479             Tf = str(job._Tf)
1480             if Tf != "-1":
1481                 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1482                                        time.localtime(job._Tf))
1483
1484             # recreate the job node
1485             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1486                                                   "job",
1487                                                   attrib={"name" : job.name})
1488             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1489             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1490             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1491             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1492             xml_history = src.xmlManager.add_simple_node(xmlj, "history")
1493             for date, res_job, link in self.history[job.name]:
1494                 src.xmlManager.add_simple_node(xml_history,
1495                                                "link",
1496                                                text=link,
1497                                                attrib={"date" : date,
1498                                                        "res" : res_job})
1499
1500             src.xmlManager.add_simple_node(xmlj, "sat_path",
1501                                            job.machine.sat_path)
1502             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1503             src.xmlManager.add_simple_node(xmlj, "distribution",
1504                                            job.machine.distribution)
1505             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1506             src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1507             src.xmlManager.add_simple_node(xmlj, "commands",
1508                                            " ; ".join(job.commands))
1509             src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1510             src.xmlManager.add_simple_node(xmlj, "begin", T0)
1511             src.xmlManager.add_simple_node(xmlj, "end", Tf)
1512             src.xmlManager.add_simple_node(xmlj, "out",
1513                                            src.printcolors.cleancolor(job.out))
1514             src.xmlManager.add_simple_node(xmlj, "err",
1515                                            src.printcolors.cleancolor(job.err))
1516             src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1517             if len(job.remote_log_files) > 0:
1518                 src.xmlManager.add_simple_node(xmlj,
1519                                                "remote_log_file_path",
1520                                                job.remote_log_files[0])
1521             else:
1522                 src.xmlManager.add_simple_node(xmlj,
1523                                                "remote_log_file_path",
1524                                                "nothing")
1525             # Search for the test log if there is any
1526             l_test_log_files = self.find_test_log(job.remote_log_files)
1527             xml_test = src.xmlManager.add_simple_node(xmlj,
1528                                                       "test_log_file_path")
1529             for test_log_path, res_test, nb_fails in l_test_log_files:
1530                 test_path_node = src.xmlManager.add_simple_node(xml_test,
1531                                                "path",
1532                                                test_log_path)
1533                 test_path_node.attrib["res"] = res_test
1534                 test_path_node.attrib["nb_fails"] = nb_fails
1535
1536             xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1537             # get the job father
1538             if job.after is not None:
1539                 job_father = None
1540                 for jb in l_jobs:
1541                     if jb.name == job.after:
1542                         job_father = jb
1543
1544                 if (job_father is not None and
1545                         len(job_father.remote_log_files) > 0):
1546                     link = job_father.remote_log_files[0]
1547                 else:
1548                     link = "nothing"
1549                 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1550
1551             # Verify that the job is to be done today regarding the input csv
1552             # files
1553             if job.board and job.board in self.d_input_boards.keys():
1554                 found = False
1555                 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1556                     if (job.machine.distribution == dist
1557                         and job.application == appli):
1558                         found = True
1559                         src.xmlManager.add_simple_node(xmlj,
1560                                                "extra_job",
1561                                                "no")
1562                         break
1563                 if not found:
1564                     src.xmlManager.add_simple_node(xmlj,
1565                                                "extra_job",
1566                                                "yes")
1567
1568
1569         # Update the date
1570         xml_node_infos = xml_file.xmlroot.find('infos')
1571         src.xmlManager.append_node_attrib(xml_node_infos,
1572                     attrib={"value" :
1573                     datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1574
1575
1576     def find_test_log(self, l_remote_log_files):
1577         '''Find if there is a test log (board) in the remote log files and
1578            the path to it. There can be several test command, so the result is
1579            a list.
1580
1581         :param l_remote_log_files List: the list of all remote log files
1582         :return: the list of (test log files path, res of the command)
1583         :rtype: List
1584         '''
1585         res = []
1586         for file_path in l_remote_log_files:
1587             dirname = os.path.basename(os.path.dirname(file_path))
1588             file_name = os.path.basename(file_path)
1589             regex = src.logger.log_all_command_file_expression
1590             oExpr = re.compile(regex)
1591             if dirname == "TEST" and oExpr.search(file_name):
1592                 # find the res of the command
1593                 prod_node = etree.parse(file_path).getroot().find("product")
1594                 res_test = prod_node.attrib["global_res"]
1595                 # find the number of fails
1596                 testbase_node = prod_node.find("tests").find("testbase")
1597                 nb_fails = int(testbase_node.attrib["failed"])
1598                 # put the file path, the res of the test command and the number
1599                 # of fails in the output
1600                 res.append((file_path, res_test, nb_fails))
1601
1602         return res
1603
1604     def last_update(self, finish_status = "finished"):
1605         '''update information about the jobs for the file xml_file
1606
1607         :param l_jobs List: the list of jobs that run today
1608         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1609         '''
1610         for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1611             xml_node_infos = xml_file.xmlroot.find('infos')
1612             src.xmlManager.append_node_attrib(xml_node_infos,
1613                         attrib={"JobsCommandStatus" : finish_status})
1614         # Write the file
1615         self.write_xml_files()
1616
1617     def write_xml_file(self, xml_file, stylesheet):
1618         ''' Write one xml file and the same file with prefix
1619         '''
1620         xml_file.write_tree(stylesheet)
1621         file_path = xml_file.logFile
1622         file_dir = os.path.dirname(file_path)
1623         file_name = os.path.basename(file_path)
1624         file_name_with_prefix = self.prefix + "_" + file_name
1625         xml_file.write_tree(stylesheet, os.path.join(file_dir,
1626                                                      file_name_with_prefix))
1627
1628     def write_xml_files(self):
1629         ''' Write the xml files
1630         '''
1631         self.write_xml_file(self.xml_global_file, STYLESHEET_GLOBAL)
1632         for xml_file in self.d_xml_board_files.values():
1633             self.write_xml_file(xml_file, STYLESHEET_BOARD)
1634
1635 def get_config_file_path(job_config_name, l_cfg_dir):
1636     found = False
1637     file_jobs_cfg = None
1638     if os.path.exists(job_config_name) and job_config_name.endswith(".pyconf"):
1639         found = True
1640         file_jobs_cfg = job_config_name
1641     else:
1642         for cfg_dir in l_cfg_dir:
1643             file_jobs_cfg = os.path.join(cfg_dir, job_config_name)
1644             if not file_jobs_cfg.endswith('.pyconf'):
1645                 file_jobs_cfg += '.pyconf'
1646
1647             if not os.path.exists(file_jobs_cfg):
1648                 continue
1649             else:
1650                 found = True
1651                 break
1652     return found, file_jobs_cfg
1653
1654 ##
1655 # Describes the command
1656 def description():
1657     return _("The jobs command launches maintenances that are described"
1658              " in the dedicated jobs configuration file.\n\nexample:\nsat "
1659              "jobs --name my_jobs --publish")
1660
1661 ##
1662 # Runs the command.
1663 def run(args, runner, logger):
1664
1665     (options, args) = parser.parse_args(args)
1666
1667     l_cfg_dir = runner.cfg.PATHS.JOBPATH
1668
1669     # list option : display all the available config files
1670     if options.list:
1671         for cfg_dir in l_cfg_dir:
1672             if not options.no_label:
1673                 logger.write("------ %s\n" %
1674                                  src.printcolors.printcHeader(cfg_dir))
1675             if not os.path.exists(cfg_dir):
1676                 continue
1677             for f in sorted(os.listdir(cfg_dir)):
1678                 if not f.endswith('.pyconf'):
1679                     continue
1680                 cfilename = f[:-7]
1681                 logger.write("%s\n" % cfilename)
1682         return 0
1683
1684     # Make sure the jobs_config option has been called
1685     if not options.jobs_cfg:
1686         message = _("The option --jobs_config is required\n")
1687         src.printcolors.printcError(message)
1688         return 1
1689
1690     # Find the file in the directories, unless it is a full path
1691     # merge all in a config
1692     merger = src.pyconf.ConfigMerger()
1693     config_jobs = src.pyconf.Config()
1694     l_conf_files_path = []
1695     for config_file in options.jobs_cfg:
1696         found, file_jobs_cfg = get_config_file_path(config_file, l_cfg_dir)
1697         if not found:
1698             msg = _("The file configuration %s was not found."
1699                     "\nUse the --list option to get the "
1700                     "possible files." % config_file)
1701             logger.write("%s\n" % src.printcolors.printcError(msg), 1)
1702             return 1
1703         l_conf_files_path.append(file_jobs_cfg)
1704         # Read the config that is in the file
1705         one_config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1706         merger.merge(config_jobs, one_config_jobs)
1707
1708     info = [
1709         (_("Platform"), runner.cfg.VARS.dist),
1710         (_("Files containing the jobs configuration"), l_conf_files_path)
1711     ]
1712     src.print_info(logger, info)
1713
1714     if options.only_jobs:
1715         l_jb = src.pyconf.Sequence()
1716         for jb in config_jobs.jobs:
1717             if jb.name in options.only_jobs:
1718                 l_jb.append(jb,
1719                 "Job that was given in only_jobs option parameters\n")
1720         config_jobs.jobs = l_jb
1721
1722     # Make a unique file that contain all the jobs in order to use it
1723     # on every machine
1724     name_pyconf = "_".join([os.path.basename(path)[:-len('.pyconf')]
1725                             for path in l_conf_files_path]) + ".pyconf"
1726     path_pyconf = src.get_tmp_filename(runner.cfg, name_pyconf)
1727     #Save config
1728     f = file( path_pyconf , 'w')
1729     config_jobs.__save__(f)
1730
1731     # Initialization
1732     today_jobs = Jobs(runner,
1733                       logger,
1734                       path_pyconf,
1735                       config_jobs)
1736     # SSH connection to all machines
1737     today_jobs.ssh_connection_all_machines()
1738     if options.test_connection:
1739         return 0
1740
1741     gui = None
1742     if options.publish:
1743         logger.write(src.printcolors.printcInfo(
1744                                         _("Initialize the xml boards : ")), 5)
1745         logger.flush()
1746
1747         # Copy the stylesheets in the log directory
1748         log_dir = runner.cfg.USER.log_dir
1749         xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1750         files_to_copy = []
1751         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1752         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1753         files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1754         for file_path in files_to_copy:
1755             shutil.copy2(file_path, log_dir)
1756
1757         # Instanciate the Gui in order to produce the xml files that contain all
1758         # the boards
1759         gui = Gui(runner.cfg.USER.log_dir,
1760                   today_jobs.ljobs,
1761                   today_jobs.ljobs_not_today,
1762                   runner.cfg.VARS.datehour,
1763                   logger,
1764                   file_boards = options.input_boards)
1765
1766         logger.write(src.printcolors.printcSuccess("OK"), 5)
1767         logger.write("\n\n", 5)
1768         logger.flush()
1769
1770         # Display the list of the xml files
1771         logger.write(src.printcolors.printcInfo(("Here is the list of published"
1772                                                  " files :\n")), 4)
1773         logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1774         for board in gui.d_xml_board_files.keys():
1775             file_path = gui.d_xml_board_files[board].logFile
1776             file_name = os.path.basename(file_path)
1777             logger.write("%s\n" % file_path, 4)
1778             logger.add_link(file_name, "board", 0, board)
1779
1780         logger.write("\n", 4)
1781
1782     today_jobs.gui = gui
1783
1784     interruped = False
1785     try:
1786         # Run all the jobs contained in config_jobs
1787         today_jobs.run_jobs()
1788     except KeyboardInterrupt:
1789         interruped = True
1790         logger.write("\n\n%s\n\n" %
1791                 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1792     except Exception as e:
1793         msg = _("CRITICAL ERROR: The jobs loop has been interrupted\n")
1794         logger.write("\n\n%s\n" % src.printcolors.printcError(msg) )
1795         logger.write("%s\n" % str(e))
1796         # get stack
1797         __, __, exc_traceback = sys.exc_info()
1798         fp = tempfile.TemporaryFile()
1799         traceback.print_tb(exc_traceback, file=fp)
1800         fp.seek(0)
1801         stack = fp.read()
1802         logger.write("\nTRACEBACK: %s\n" % stack.replace('"',"'"), 1)
1803
1804     finally:
1805         res = 0
1806         if interruped:
1807             res = 1
1808             msg = _("Killing the running jobs and trying"
1809                     " to get the corresponding logs\n")
1810             logger.write(src.printcolors.printcWarning(msg))
1811
1812         # find the potential not finished jobs and kill them
1813         for jb in today_jobs.ljobs:
1814             if not jb.has_finished():
1815                 res = 1
1816                 try:
1817                     jb.kill_remote_process()
1818                 except Exception as e:
1819                     msg = _("Failed to kill job %s: %s\n" % (jb.name, e))
1820                     logger.write(src.printcolors.printcWarning(msg))
1821             if jb.res_job != "0":
1822                 res = 1
1823         if interruped:
1824             if today_jobs.gui:
1825                 today_jobs.gui.last_update(_("Forced interruption"))
1826         else:
1827             if today_jobs.gui:
1828                 today_jobs.gui.last_update()
1829         # Output the results
1830         today_jobs.write_all_results()
1831         # Remove the temporary pyconf file
1832         if os.path.exists(path_pyconf):
1833             os.remove(path_pyconf)
1834         return res