commands/jobs.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import sys
  21 import tempfile
  22 import traceback
  23 import datetime
  24 import time
  25 import csv
  26 import shutil
  27 import itertools
  28 import re
  29 import paramiko
  30
  31 import src
  32 import src.ElementTree as etree
  33
  34 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
  35 STYLESHEET_BOARD = "jobs_board_report.xsl"
  36
  37 DAYS_SEPARATOR = ","
  38 CSV_DELIMITER = ";"
  39
  40 parser = src.options.Options()
  41
  42 parser.add_option('n', 'name', 'list2', 'jobs_cfg',
  43                   _('Mandatory: The name of the config file that contains'
  44                   ' the jobs configuration. Can be a list.'))
  45 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
  46                   _('Optional: the list of jobs to launch, by their name. '))
  47 parser.add_option('l', 'list', 'boolean', 'list',
  48                   _('Optional: list all available config files.'))
  49 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
  50                   _("Optional: try to connect to the machines. "
  51                     "Not executing the jobs."),
  52                   False)
  53 parser.add_option('p', 'publish', 'boolean', 'publish',
  54                   _("Optional: generate an xml file that can be read in a "
  55                     "browser to display the jobs status."),
  56                   False)
  57 parser.add_option('i', 'input_boards', 'string', 'input_boards', _("Optional: "
  58                                 "the path to csv file that contain "
  59                                 "the expected boards."),"")
  60 parser.add_option('', 'completion', 'boolean', 'no_label',
  61                   _("Optional (internal use): do not print labels, Works only "
  62                     "with --list."),
  63                   False)
  64
  65 class Machine(object):
  66     '''Class to manage a ssh connection on a machine
  67     '''
  68     def __init__(self,
  69                  name,
  70                  host,
  71                  user,
  72                  port=22,
  73                  passwd=None,
  74                  sat_path="salomeTools"):
  75         self.name = name
  76         self.host = host
  77         self.port = port
  78         self.distribution = None # Will be filled after copying SAT on the machine
  79         self.user = user
  80         self.password = passwd
  81         self.sat_path = sat_path
  82         self.ssh = paramiko.SSHClient()
  83         self._connection_successful = None
  84
  85     def connect(self, logger):
  86         '''Initiate the ssh connection to the remote machine
  87
  88         :param logger src.logger.Logger: The logger instance
  89         :return: Nothing
  90         :rtype: N\A
  91         '''
  92
  93         self._connection_successful = False
  94         self.ssh.load_system_host_keys()
  95         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  96         try:
  97             self.ssh.connect(self.host,
  98                              port=self.port,
  99                              username=self.user,
 100                              password = self.password)
 101         except paramiko.AuthenticationException:
 102             message = src.KO_STATUS + _("Authentication failed")
 103         except paramiko.BadHostKeyException:
 104             message = (src.KO_STATUS +
 105                        _("The server's host key could not be verified"))
 106         except paramiko.SSHException:
 107             message = ( _("SSHException error connecting or "
 108                           "establishing an SSH session"))
 109         except:
 110             message = ( _("Error connecting or establishing an SSH session"))
 111         else:
 112             self._connection_successful = True
 113             message = ""
 114         return message
 115
 116     def successfully_connected(self, logger):
 117         '''Verify if the connection to the remote machine has succeed
 118
 119         :param logger src.logger.Logger: The logger instance
 120         :return: True if the connection has succeed, False if not
 121         :rtype: bool
 122         '''
 123         if self._connection_successful == None:
 124             message = _("Warning : trying to ask if the connection to "
 125             "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
 126             " no connection request" %
 127                         (self.name, self.host, self.port, self.user))
 128             logger.write( src.printcolors.printcWarning(message))
 129         return self._connection_successful
 130
 131     def copy_sat(self, sat_local_path, job_file):
 132         '''Copy salomeTools to the remote machine in self.sat_path
 133         '''
 134         res = 0
 135         try:
 136             # open a sftp connection
 137             self.sftp = self.ssh.open_sftp()
 138             # Create the sat directory on remote machine if it is not existing
 139             self.mkdir(self.sat_path, ignore_existing=True)
 140             # Put sat
 141             self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
 142             # put the job configuration file in order to make it reachable
 143             # on the remote machine
 144             remote_job_file_name = ".%s" % os.path.basename(job_file)
 145             self.sftp.put(job_file, os.path.join(self.sat_path,
 146                                                  remote_job_file_name))
 147         except Exception as e:
 148             res = str(e)
 149             self._connection_successful = False
 150
 151         return res
 152
 153     def put_dir(self, source, target, filters = []):
 154         ''' Uploads the contents of the source directory to the target path. The
 155             target directory needs to exists. All sub-directories in source are
 156             created under target.
 157         '''
 158         for item in os.listdir(source):
 159             if item in filters:
 160                 continue
 161             source_path = os.path.join(source, item)
 162             destination_path = os.path.join(target, item)
 163             if os.path.islink(source_path):
 164                 linkto = os.readlink(source_path)
 165                 try:
 166                     self.sftp.symlink(linkto, destination_path)
 167                     self.sftp.chmod(destination_path,
 168                                     os.stat(source_path).st_mode)
 169                 except IOError:
 170                     pass
 171             else:
 172                 if os.path.isfile(source_path):
 173                     self.sftp.put(source_path, destination_path)
 174                     self.sftp.chmod(destination_path,
 175                                     os.stat(source_path).st_mode)
 176                 else:
 177                     self.mkdir(destination_path, ignore_existing=True)
 178                     self.put_dir(source_path, destination_path)
 179
 180     def mkdir(self, path, mode=511, ignore_existing=False):
 181         ''' Augments mkdir by adding an option to not fail
 182             if the folder exists
 183         '''
 184         try:
 185             self.sftp.mkdir(path, mode)
 186         except IOError:
 187             if ignore_existing:
 188                 pass
 189             else:
 190                 raise
 191
 192     def exec_command(self, command, logger):
 193         '''Execute the command on the remote machine
 194
 195         :param command str: The command to be run
 196         :param logger src.logger.Logger: The logger instance
 197         :return: the stdin, stdout, and stderr of the executing command,
 198                  as a 3-tuple
 199         :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
 200                 paramiko.channel.ChannelFile)
 201         '''
 202         try:
 203             # Does not wait the end of the command
 204             (stdin, stdout, stderr) = self.ssh.exec_command(command)
 205         except paramiko.SSHException:
 206             message = src.KO_STATUS + _(
 207                             ": the server failed to execute the command\n")
 208             logger.write( src.printcolors.printcError(message))
 209             return (None, None, None)
 210         except:
 211             logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
 212             return (None, None, None)
 213         else:
 214             return (stdin, stdout, stderr)
 215
 216     def close(self):
 217         '''Close the ssh connection
 218
 219         :rtype: N\A
 220         '''
 221         self.ssh.close()
 222
 223     def write_info(self, logger):
 224         '''Prints the informations relative to the machine in the logger
 225            (terminal traces and log file)
 226
 227         :param logger src.logger.Logger: The logger instance
 228         :return: Nothing
 229         :rtype: N\A
 230         '''
 231         logger.write("host : " + self.host + "\n")
 232         logger.write("port : " + str(self.port) + "\n")
 233         logger.write("user : " + str(self.user) + "\n")
 234         if self.successfully_connected(logger):
 235             status = src.OK_STATUS
 236         else:
 237             status = src.KO_STATUS
 238         logger.write("Connection : " + status + "\n\n")
 239
 240
 241 class Job(object):
 242     '''Class to manage one job
 243     '''
 244     def __init__(self,
 245                  name,
 246                  machine,
 247                  application,
 248                  board,
 249                  commands,
 250                  timeout,
 251                  config,
 252                  job_file_path,
 253                  logger,
 254                  after=None,
 255                  prefix=None):
 256
 257         self.name = name
 258         self.machine = machine
 259         self.after = after
 260         self.timeout = timeout
 261         self.application = application
 262         self.board = board
 263         self.config = config
 264         self.logger = logger
 265         # The list of log files to download from the remote machine
 266         self.remote_log_files = []
 267
 268         # The remote command status
 269         # -1 means that it has not been launched,
 270         # 0 means success and 1 means fail
 271         self.res_job = "-1"
 272         self.cancelled = False
 273
 274         self._T0 = -1
 275         self._Tf = -1
 276         self._has_begun = False
 277         self._has_finished = False
 278         self._has_timouted = False
 279         self._stdin = None # Store the command inputs field
 280         self._stdout = None # Store the command outputs field
 281         self._stderr = None # Store the command errors field
 282
 283         self.out = ""
 284         self.err = ""
 285
 286         self.name_remote_jobs_pyconf = ".%s" % os.path.basename(job_file_path)
 287         self.commands = commands
 288         self.command = (os.path.join(self.machine.sat_path, "sat") +
 289                         " -l " +
 290                         os.path.join(self.machine.sat_path,
 291                                      "list_log_files.txt") +
 292                         " job --jobs_config " +
 293                         os.path.join(self.machine.sat_path,
 294                                      self.name_remote_jobs_pyconf) +
 295                         " --name " +
 296                         self.name)
 297         if prefix:
 298             self.command = prefix + ' "' + self.command +'"'
 299
 300     def get_pids(self):
 301         """ Get the pid(s) corresponding to the command that have been launched
 302             On the remote machine
 303
 304         :return: The list of integers corresponding to the found pids
 305         :rtype: List
 306         """
 307         pids = []
 308         cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
 309         (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
 310         pids_cmd = out_pid.readlines()
 311         pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
 312         pids+=pids_cmd
 313         return pids
 314
 315     def kill_remote_process(self, wait=1):
 316         '''Kills the process on the remote machine.
 317
 318         :return: (the output of the kill, the error of the kill)
 319         :rtype: (str, str)
 320         '''
 321
 322         pids = self.get_pids()
 323         cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
 324         (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
 325                                                             self.logger)
 326         time.sleep(wait)
 327         return (out_kill, err_kill)
 328
 329     def has_begun(self):
 330         '''Returns True if the job has already begun
 331
 332         :return: True if the job has already begun
 333         :rtype: bool
 334         '''
 335         return self._has_begun
 336
 337     def has_finished(self):
 338         '''Returns True if the job has already finished
 339            (i.e. all the commands have been executed)
 340            If it is finished, the outputs are stored in the fields out and err.
 341
 342         :return: True if the job has already finished
 343         :rtype: bool
 344         '''
 345
 346         # If the method has already been called and returned True
 347         if self._has_finished:
 348             return True
 349
 350         # If the job has not begun yet
 351         if not self.has_begun():
 352             return False
 353
 354         if self._stdout.channel.closed:
 355             self._has_finished = True
 356             # Store the result outputs
 357             self.out += self._stdout.read().decode()
 358             self.err += self._stderr.read().decode()
 359             # Put end time
 360             self._Tf = time.time()
 361             # And get the remote command status and log files
 362             try:
 363                 self.get_log_files()
 364             except Exception as e:
 365                 self.err += _("Unable to get remote log files: %s" % e)
 366
 367         return self._has_finished
 368
 369     def get_log_files(self):
 370         """Get the log files produced by the command launched
 371            on the remote machine, and put it in the log directory of the user,
 372            so they can be accessible from
 373         """
 374         # Do not get the files if the command is not finished
 375         if not self.has_finished():
 376             msg = _("Trying to get log files whereas the job is not finished.")
 377             self.logger.write(src.printcolors.printcWarning(msg))
 378             return
 379
 380         # First get the file that contains the list of log files to get
 381         tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
 382         remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt")
 383         self.machine.sftp.get(
 384                     remote_path,
 385                     tmp_file_path)
 386
 387         # Read the file and get the result of the command and all the log files
 388         # to get
 389         fstream_tmp = open(tmp_file_path, "r")
 390         file_lines = fstream_tmp.readlines()
 391         file_lines = [line.replace("\n", "") for line in file_lines]
 392         fstream_tmp.close()
 393         os.remove(tmp_file_path)
 394
 395         try :
 396             # The first line is the result of the command (0 success or 1 fail)
 397             self.res_job = file_lines[0]
 398         except Exception as e:
 399             self.err += _("Unable to get status from remote file %s: %s" %
 400                                                     (remote_path, str(e)))
 401
 402         for i, job_path_remote in enumerate(file_lines[1:]):
 403             try:
 404                 # For each command, there is two files to get :
 405                 # 1- The xml file describing the command and giving the
 406                 # internal traces.
 407                 # 2- The txt file containing the system command traces (like
 408                 # traces produced by the "make" command)
 409                 # 3- In case of the test command, there is another file to get :
 410                 # the xml board that contain the test results
 411                 dirname = os.path.basename(os.path.dirname(job_path_remote))
 412                 if dirname != 'OUT' and dirname != 'TEST':
 413                     # Case 1-
 414                     local_path = os.path.join(os.path.dirname(
 415                                                         self.logger.logFilePath),
 416                                               os.path.basename(job_path_remote))
 417                     if i==0: # The first is the job command
 418                         self.logger.add_link(os.path.basename(job_path_remote),
 419                                              "job",
 420                                              self.res_job,
 421                                              self.command)
 422                 elif dirname == 'OUT':
 423                     # Case 2-
 424                     local_path = os.path.join(os.path.dirname(
 425                                                         self.logger.logFilePath),
 426                                               'OUT',
 427                                               os.path.basename(job_path_remote))
 428                 elif dirname == 'TEST':
 429                     # Case 3-
 430                     local_path = os.path.join(os.path.dirname(
 431                                                         self.logger.logFilePath),
 432                                               'TEST',
 433                                               os.path.basename(job_path_remote))
 434
 435                 # Get the file
 436                 if not os.path.exists(local_path):
 437                     self.machine.sftp.get(job_path_remote, local_path)
 438                 self.remote_log_files.append(local_path)
 439             except Exception as e:
 440                 self.err += _("Unable to get %s log file from remote: %s" %
 441                                                     (str(job_path_remote),
 442                                                      str(e)))
 443
 444     def has_failed(self):
 445         '''Returns True if the job has failed.
 446            A job is considered as failed if the machine could not be reached,
 447            if the remote command failed,
 448            or if the job finished with a time out.
 449
 450         :return: True if the job has failed
 451         :rtype: bool
 452         '''
 453         if not self.has_finished():
 454             return False
 455         if not self.machine.successfully_connected(self.logger):
 456             return True
 457         if self.is_timeout():
 458             return True
 459         if self.res_job == "1":
 460             return True
 461         return False
 462
 463     def cancel(self):
 464         """In case of a failing job, one has to cancel every job that depend
 465            on it. This method put the job as failed and will not be executed.
 466         """
 467         if self.cancelled:
 468             return
 469         self._has_begun = True
 470         self._has_finished = True
 471         self.cancelled = True
 472         self.out += _("This job was not launched because its father has failed.")
 473         self.err += _("This job was not launched because its father has failed.")
 474
 475     def is_running(self):
 476         '''Returns True if the job commands are running
 477
 478         :return: True if the job is running
 479         :rtype: bool
 480         '''
 481         return self.has_begun() and not self.has_finished()
 482
 483     def is_timeout(self):
 484         '''Returns True if the job commands has finished with timeout
 485
 486         :return: True if the job has finished with timeout
 487         :rtype: bool
 488         '''
 489         return self._has_timouted
 490
 491     def time_elapsed(self):
 492         """Get the time elapsed since the job launching
 493
 494         :return: The number of seconds
 495         :rtype: int
 496         """
 497         if not self.has_begun():
 498             return -1
 499         T_now = time.time()
 500         return T_now - self._T0
 501
 502     def check_time(self):
 503         """Verify that the job has not exceeded its timeout.
 504            If it has, kill the remote command and consider the job as finished.
 505         """
 506         if not self.has_begun():
 507             return
 508         if self.time_elapsed() > self.timeout:
 509             self._has_finished = True
 510             self._has_timouted = True
 511             self._Tf = time.time()
 512             self.get_pids()
 513             (out_kill, _) = self.kill_remote_process()
 514             self.out += "TIMEOUT \n" + out_kill.read().decode()
 515             self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
 516             try:
 517                 self.get_log_files()
 518             except Exception as e:
 519                 self.err += _("Unable to get remote log files: %s" % e)
 520
 521     def total_duration(self):
 522         """Give the total duration of the job
 523
 524         :return: the total duration of the job in seconds
 525         :rtype: int
 526         """
 527         return self._Tf - self._T0
 528
 529     def run(self):
 530         """Launch the job by executing the remote command.
 531         """
 532
 533         # Prevent multiple run
 534         if self.has_begun():
 535             msg = _("Warning: A job can only be launched one time")
 536             msg2 = _("Trying to launch the job \"%s\" whereas it has "
 537                      "already been launched." % self.name)
 538             self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
 539                                                                         msg2)))
 540             return
 541
 542         # Do not execute the command if the machine could not be reached
 543         if not self.machine.successfully_connected(self.logger):
 544             self._has_finished = True
 545             self.out = "N\A"
 546             self.err += ("Connection to machine (name : %s, host: %s, port:"
 547                         " %s, user: %s) has failed\nUse the log command "
 548                         "to get more information."
 549                         % (self.machine.name,
 550                            self.machine.host,
 551                            self.machine.port,
 552                            self.machine.user))
 553         else:
 554             # Usual case : Launch the command on remote machine
 555             self._T0 = time.time()
 556             self._stdin, self._stdout, self._stderr = self.machine.exec_command(
 557                                                                   self.command,
 558                                                                   self.logger)
 559             # If the results are not initialized, finish the job
 560             if (self._stdin, self._stdout, self._stderr) == (None, None, None):
 561                 self._has_finished = True
 562                 self._Tf = time.time()
 563                 self.out += "N\A"
 564                 self.err += "The server failed to execute the command"
 565
 566         # Put the beginning flag to true.
 567         self._has_begun = True
 568
 569     def write_results(self):
 570         """Display on the terminal all the job's information
 571         """
 572         self.logger.write("name : " + self.name + "\n")
 573         if self.after:
 574             self.logger.write("after : %s\n" % self.after)
 575         self.logger.write("Time elapsed : %4imin %2is \n" %
 576                      (self.total_duration()//60 , self.total_duration()%60))
 577         if self._T0 != -1:
 578             self.logger.write("Begin time : %s\n" %
 579                          time.strftime('%Y-%m-%d %H:%M:%S',
 580                                        time.localtime(self._T0)) )
 581         if self._Tf != -1:
 582             self.logger.write("End time   : %s\n\n" %
 583                          time.strftime('%Y-%m-%d %H:%M:%S',
 584                                        time.localtime(self._Tf)) )
 585
 586         machine_head = "Informations about connection :\n"
 587         underline = (len(machine_head) - 2) * "-"
 588         self.logger.write(src.printcolors.printcInfo(
 589                                                 machine_head+underline+"\n"))
 590         self.machine.write_info(self.logger)
 591
 592         self.logger.write(src.printcolors.printcInfo("out : \n"))
 593         if self.out == "":
 594             self.logger.write("Unable to get output\n")
 595         else:
 596             self.logger.write(self.out + "\n")
 597         self.logger.write(src.printcolors.printcInfo("err : \n"))
 598         self.logger.write(self.err + "\n")
 599
 600     def get_status(self):
 601         """Get the status of the job (used by the Gui for xml display)
 602
 603         :return: The current status of the job
 604         :rtype: String
 605         """
 606         if not self.machine.successfully_connected(self.logger):
 607             return "SSH connection KO"
 608         if not self.has_begun():
 609             return "Not launched"
 610         if self.cancelled:
 611             return "Cancelled"
 612         if self.is_running():
 613             return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
 614                                                     time.localtime(self._T0))
 615         if self.has_finished():
 616             if self.is_timeout():
 617                 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
 618                                                     time.localtime(self._Tf))
 619             return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
 620                                                      time.localtime(self._Tf))
 621
 622 class Jobs(object):
 623     '''Class to manage the jobs to be run
 624     '''
 625     def __init__(self,
 626                  runner,
 627                  logger,
 628                  job_file_path,
 629                  config_jobs,
 630                  lenght_columns = 20):
 631         # The jobs configuration
 632         self.cfg_jobs = config_jobs
 633         self.job_file_path = job_file_path
 634         # The machine that will be used today
 635         self.lmachines = []
 636         # The list of machine (hosts, port) that will be used today
 637         # (a same host can have several machine instances since there
 638         # can be several ssh parameters)
 639         self.lhosts = []
 640         # The jobs to be launched today
 641         self.ljobs = []
 642         # The jobs that will not be launched today
 643         self.ljobs_not_today = []
 644         self.runner = runner
 645         self.logger = logger
 646         self.len_columns = lenght_columns
 647
 648         # the list of jobs that have not been run yet
 649         self._l_jobs_not_started = []
 650         # the list of jobs that have already ran
 651         self._l_jobs_finished = []
 652         # the list of jobs that are running
 653         self._l_jobs_running = []
 654
 655         self.determine_jobs_and_machines()
 656
 657     def define_job(self, job_def, machine):
 658         '''Takes a pyconf job definition and a machine (from class machine)
 659            and returns the job instance corresponding to the definition.
 660
 661         :param job_def src.config.Mapping: a job definition
 662         :param machine machine: the machine on which the job will run
 663         :return: The corresponding job in a job class instance
 664         :rtype: job
 665         '''
 666         name = job_def.name
 667         cmmnds = job_def.commands
 668         if not "timeout" in job_def:
 669             timeout = 4*60*60 # default timeout = 4h
 670         else:
 671             timeout = job_def.timeout
 672         after = None
 673         if 'after' in job_def:
 674             after = job_def.after
 675         application = None
 676         if 'application' in job_def:
 677             application = job_def.application
 678         board = None
 679         if 'board' in job_def:
 680             board = job_def.board
 681         prefix = None
 682         if "prefix" in job_def:
 683             prefix = job_def.prefix
 684
 685         return Job(name,
 686                    machine,
 687                    application,
 688                    board,
 689                    cmmnds,
 690                    timeout,
 691                    self.runner.cfg,
 692                    self.job_file_path,
 693                    self.logger,
 694                    after = after,
 695                    prefix = prefix)
 696
 697     def determine_jobs_and_machines(self):
 698         '''Function that reads the pyconf jobs definition and instantiates all
 699            the machines and jobs to be done today.
 700
 701         :return: Nothing
 702         :rtype: N\A
 703         '''
 704         today = datetime.date.weekday(datetime.date.today())
 705         host_list = []
 706
 707         for job_def in self.cfg_jobs.jobs :
 708
 709             if not "machine" in job_def:
 710                 msg = _('WARNING: The job "%s" do not have the key '
 711                        '"machine", this job is ignored.\n\n' % job_def.name)
 712                 self.logger.write(src.printcolors.printcWarning(msg))
 713                 continue
 714             name_machine = job_def.machine
 715
 716             a_machine = None
 717             for mach in self.lmachines:
 718                 if mach.name == name_machine:
 719                     a_machine = mach
 720                     break
 721
 722             if a_machine == None:
 723                 for machine_def in self.cfg_jobs.machines:
 724                     if machine_def.name == name_machine:
 725                         if 'host' not in machine_def:
 726                             host = self.runner.cfg.VARS.hostname
 727                         else:
 728                             host = machine_def.host
 729
 730                         if 'user' not in machine_def:
 731                             user = self.runner.cfg.VARS.user
 732                         else:
 733                             user = machine_def.user
 734
 735                         if 'port' not in machine_def:
 736                             port = 22
 737                         else:
 738                             port = machine_def.port
 739
 740                         if 'password' not in machine_def:
 741                             passwd = None
 742                         else:
 743                             passwd = machine_def.password
 744
 745                         if 'sat_path' not in machine_def:
 746                             sat_path = "salomeTools"
 747                         else:
 748                             sat_path = machine_def.sat_path
 749
 750                         a_machine = Machine(
 751                                             machine_def.name,
 752                                             host,
 753                                             user,
 754                                             port=port,
 755                                             passwd=passwd,
 756                                             sat_path=sat_path
 757                                             )
 758
 759                         self.lmachines.append(a_machine)
 760                         if (host, port) not in host_list:
 761                             host_list.append((host, port))
 762
 763                 if a_machine == None:
 764                     msg = _("WARNING: The job \"%(job_name)s\" requires the "
 765                             "machine \"%(machine_name)s\" but this machine "
 766                             "is not defined in the configuration file.\n"
 767                             "The job will not be launched")
 768                     self.logger.write(src.printcolors.printcWarning(msg))
 769
 770             a_job = self.define_job(job_def, a_machine)
 771
 772             if today in job_def.when:
 773                 self.ljobs.append(a_job)
 774             else: # today in job_def.when
 775                 self.ljobs_not_today.append(a_job)
 776
 777         self.lhosts = host_list
 778
 779     def ssh_connection_all_machines(self, pad=50):
 780         '''Function that do the ssh connection to every machine
 781            to be used today.
 782
 783         :return: Nothing
 784         :rtype: N\A
 785         '''
 786         self.logger.write(src.printcolors.printcInfo((
 787                         "Establishing connection with all the machines :\n")))
 788         for machine in self.lmachines:
 789             # little algorithm in order to display traces
 790             begin_line = (_("Connection to %s: " % machine.name))
 791             if pad - len(begin_line) < 0:
 792                 endline = " "
 793             else:
 794                 endline = (pad - len(begin_line)) * "." + " "
 795
 796             step = "SSH connection"
 797             self.logger.write( begin_line + endline + step)
 798             self.logger.flush()
 799             # the call to the method that initiate the ssh connection
 800             msg = machine.connect(self.logger)
 801
 802             # Copy salomeTools to the remote machine
 803             if machine.successfully_connected(self.logger):
 804                 step = _("Remove SAT")
 805                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 806                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 807                 (__, out_dist, __) = machine.exec_command(
 808                                                 "rm -rf %s" % machine.sat_path,
 809                                                 self.logger)
 810                 out_dist.read()
 811
 812                 self.logger.flush()
 813                 step = _("Copy SAT")
 814                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 815                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 816                 self.logger.flush()
 817                 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
 818                                             self.job_file_path)
 819                 # get the remote machine distribution using a sat command
 820                 (__, out_dist, __) = machine.exec_command(
 821                                 os.path.join(machine.sat_path,
 822                                     "sat config --value VARS.dist --no_label"),
 823                                 self.logger)
 824                 machine.distribution = out_dist.read().decode().replace("\n",
 825                                                                         "")
 826                 # Print the status of the copy
 827                 if res_copy == 0:
 828                     self.logger.write('\r%s' %
 829                                 ((len(begin_line)+len(endline)+20) * " "), 3)
 830                     self.logger.write('\r%s%s%s' %
 831                         (begin_line,
 832                          endline,
 833                          src.printcolors.printc(src.OK_STATUS)), 3)
 834                 else:
 835                     self.logger.write('\r%s' %
 836                             ((len(begin_line)+len(endline)+20) * " "), 3)
 837                     self.logger.write('\r%s%s%s %s' %
 838                         (begin_line,
 839                          endline,
 840                          src.printcolors.printc(src.KO_STATUS),
 841                          _("Copy of SAT failed: %s" % res_copy)), 3)
 842             else:
 843                 self.logger.write('\r%s' %
 844                                   ((len(begin_line)+len(endline)+20) * " "), 3)
 845                 self.logger.write('\r%s%s%s %s' %
 846                     (begin_line,
 847                      endline,
 848                      src.printcolors.printc(src.KO_STATUS),
 849                      msg), 3)
 850             self.logger.write("\n", 3)
 851
 852         self.logger.write("\n")
 853
 854
 855     def is_occupied(self, hostname):
 856         '''Function that returns True if a job is running on
 857            the machine defined by its host and its port.
 858
 859         :param hostname (str, int): the pair (host, port)
 860         :return: the job that is running on the host,
 861                 or false if there is no job running on the host.
 862         :rtype: job / bool
 863         '''
 864         host = hostname[0]
 865         port = hostname[1]
 866         for jb in self.ljobs:
 867             if jb.machine.host == host and jb.machine.port == port:
 868                 if jb.is_running():
 869                     return jb
 870         return False
 871
 872     def update_jobs_states_list(self):
 873         '''Function that updates the lists that store the currently
 874            running jobs and the jobs that have already finished.
 875
 876         :return: Nothing.
 877         :rtype: N\A
 878         '''
 879         jobs_finished_list = []
 880         jobs_running_list = []
 881         for jb in self.ljobs:
 882             if jb.is_running():
 883                 jobs_running_list.append(jb)
 884                 jb.check_time()
 885             if jb.has_finished():
 886                 jobs_finished_list.append(jb)
 887
 888         nb_job_finished_before = len(self._l_jobs_finished)
 889         self._l_jobs_finished = jobs_finished_list
 890         self._l_jobs_running = jobs_running_list
 891
 892         nb_job_finished_now = len(self._l_jobs_finished)
 893
 894         return nb_job_finished_now > nb_job_finished_before
 895
 896     def cancel_dependencies_of_failing_jobs(self):
 897         '''Function that cancels all the jobs that depend on a failing one.
 898
 899         :return: Nothing.
 900         :rtype: N\A
 901         '''
 902
 903         for job in self.ljobs:
 904             if job.after is None:
 905                 continue
 906             father_job = self.find_job_that_has_name(job.after)
 907             if father_job is not None and father_job.has_failed():
 908                 job.cancel()
 909
 910     def find_job_that_has_name(self, name):
 911         '''Returns the job by its name.
 912
 913         :param name str: a job name
 914         :return: the job that has the name.
 915         :rtype: job
 916         '''
 917         for jb in self.ljobs:
 918             if jb.name == name:
 919                 return jb
 920         # the following is executed only if the job was not found
 921         return None
 922
 923     def str_of_length(self, text, length):
 924         '''Takes a string text of any length and returns
 925            the most close string of length "length".
 926
 927         :param text str: any string
 928         :param length int: a length for the returned string
 929         :return: the most close string of length "length"
 930         :rtype: str
 931         '''
 932         if len(text) > length:
 933             text_out = text[:length-3] + '...'
 934         else:
 935             diff = length - len(text)
 936             before = " " * (diff//2)
 937             after = " " * (diff//2 + diff%2)
 938             text_out = before + text + after
 939
 940         return text_out
 941
 942     def display_status(self, len_col):
 943         '''Takes a lenght and construct the display of the current status
 944            of the jobs in an array that has a column for each host.
 945            It displays the job that is currently running on the host
 946            of the column.
 947
 948         :param len_col int: the size of the column
 949         :return: Nothing
 950         :rtype: N\A
 951         '''
 952
 953         display_line = ""
 954         for host_port in self.lhosts:
 955             jb = self.is_occupied(host_port)
 956             if not jb: # nothing running on the host
 957                 empty = self.str_of_length("empty", len_col)
 958                 display_line += "|" + empty
 959             else:
 960                 display_line += "|" + src.printcolors.printcInfo(
 961                                         self.str_of_length(jb.name, len_col))
 962
 963         self.logger.write("\r" + display_line + "|")
 964         self.logger.flush()
 965
 966
 967     def run_jobs(self):
 968         '''The main method. Runs all the jobs on every host.
 969            For each host, at a given time, only one job can be running.
 970            The jobs that have the field after (that contain the job that has
 971            to be run before it) are run after the previous job.
 972            This method stops when all the jobs are finished.
 973
 974         :return: Nothing
 975         :rtype: N\A
 976         '''
 977
 978         # Print header
 979         self.logger.write(src.printcolors.printcInfo(
 980                                                 _('Executing the jobs :\n')))
 981         text_line = ""
 982         for host_port in self.lhosts:
 983             host = host_port[0]
 984             port = host_port[1]
 985             if port == 22: # default value
 986                 text_line += "|" + self.str_of_length(host, self.len_columns)
 987             else:
 988                 text_line += "|" + self.str_of_length(
 989                                 "("+host+", "+str(port)+")", self.len_columns)
 990
 991         tiret_line = " " + "-"*(len(text_line)-1) + "\n"
 992         self.logger.write(tiret_line)
 993         self.logger.write(text_line + "|\n")
 994         self.logger.write(tiret_line)
 995         self.logger.flush()
 996
 997         # The infinite loop that runs the jobs
 998         l_jobs_not_started = src.deepcopy_list(self.ljobs)
 999         while len(self._l_jobs_finished) != len(self.ljobs):
1000             new_job_start = False
1001             for host_port in self.lhosts:
1002
1003                 if self.is_occupied(host_port):
1004                     continue
1005
1006                 for jb in l_jobs_not_started:
1007                     if (jb.machine.host, jb.machine.port) != host_port:
1008                         continue
1009                     if jb.after == None:
1010                         jb.run()
1011                         l_jobs_not_started.remove(jb)
1012                         new_job_start = True
1013                         break
1014                     else:
1015                         jb_before = self.find_job_that_has_name(jb.after)
1016                         if jb_before is None:
1017                             jb.cancel()
1018                             msg = _("This job was not launched because its "
1019                                     "father is not in the jobs list.")
1020                             jb.out = msg
1021                             jb.err = msg
1022                             break
1023                         if jb_before.has_finished():
1024                             jb.run()
1025                             l_jobs_not_started.remove(jb)
1026                             new_job_start = True
1027                             break
1028             self.cancel_dependencies_of_failing_jobs()
1029             new_job_finished = self.update_jobs_states_list()
1030
1031             if new_job_start or new_job_finished:
1032                 if self.gui:
1033                     self.gui.update_xml_files(self.ljobs)
1034                 # Display the current status
1035                 self.display_status(self.len_columns)
1036
1037             # Make sure that the proc is not entirely busy
1038             time.sleep(0.001)
1039
1040         self.logger.write("\n")
1041         self.logger.write(tiret_line)
1042         self.logger.write("\n\n")
1043
1044         if self.gui:
1045             self.gui.update_xml_files(self.ljobs)
1046             self.gui.last_update()
1047
1048     def write_all_results(self):
1049         '''Display all the jobs outputs.
1050
1051         :return: Nothing
1052         :rtype: N\A
1053         '''
1054
1055         for jb in self.ljobs:
1056             self.logger.write(src.printcolors.printcLabel(
1057                         "#------- Results for job %s -------#\n" % jb.name))
1058             jb.write_results()
1059             self.logger.write("\n\n")
1060
1061 class Gui(object):
1062     '''Class to manage the the xml data that can be displayed in a browser to
1063        see the jobs states
1064     '''
1065
1066     def __init__(self,
1067                  xml_dir_path,
1068                  l_jobs,
1069                  l_jobs_not_today,
1070                  prefix,
1071                  logger,
1072                  file_boards=""):
1073         '''Initialization
1074
1075         :param xml_dir_path str: The path to the directory where to put
1076                                  the xml resulting files
1077         :param l_jobs List: the list of jobs that run today
1078         :param l_jobs_not_today List: the list of jobs that do not run today
1079         :param file_boards str: the file path from which to read the
1080                                    expected boards
1081         '''
1082         # The logging instance
1083         self.logger = logger
1084
1085         # The prefix to add to the xml files : date_hour
1086         self.prefix = prefix
1087
1088         # The path of the csv files to read to fill the expected boards
1089         self.file_boards = file_boards
1090
1091         if file_boards != "":
1092             today = datetime.date.weekday(datetime.date.today())
1093             self.parse_csv_boards(today)
1094         else:
1095             self.d_input_boards = {}
1096
1097         # The path of the global xml file
1098         self.xml_dir_path = xml_dir_path
1099         # Initialize the xml files
1100         self.global_name = "global_report"
1101         xml_global_path = os.path.join(self.xml_dir_path,
1102                                        self.global_name + ".xml")
1103         self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1104                                                          "JobsReport")
1105
1106         # Find history for each job
1107         self.history = {}
1108         self.find_history(l_jobs, l_jobs_not_today)
1109
1110         # The xml files that corresponds to the boards.
1111         # {name_board : xml_object}}
1112         self.d_xml_board_files = {}
1113
1114         # Create the lines and columns
1115         self.initialize_boards(l_jobs, l_jobs_not_today)
1116
1117         # Write the xml file
1118         self.update_xml_files(l_jobs)
1119
1120     def add_xml_board(self, name):
1121         '''Add a board to the board list
1122         :param name str: the board name
1123         '''
1124         xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1125         self.d_xml_board_files[name] =  src.xmlManager.XmlLogFile(
1126                                                     xml_board_path,
1127                                                     "JobsReport")
1128         self.d_xml_board_files[name].add_simple_node("distributions")
1129         self.d_xml_board_files[name].add_simple_node("applications")
1130         self.d_xml_board_files[name].add_simple_node("board", text=name)
1131
1132     def initialize_boards(self, l_jobs, l_jobs_not_today):
1133         '''Get all the first information needed for each file and write the
1134            first version of the files
1135         :param l_jobs List: the list of jobs that run today
1136         :param l_jobs_not_today List: the list of jobs that do not run today
1137         '''
1138         # Get the boards to fill and put it in a dictionary
1139         # {board_name : xml instance corresponding to the board}
1140         for job in l_jobs + l_jobs_not_today:
1141             board = job.board
1142             if (board is not None and
1143                                 board not in self.d_xml_board_files.keys()):
1144                 self.add_xml_board(board)
1145
1146         # Verify that the boards given as input are done
1147         for board in list(self.d_input_boards.keys()):
1148             if board not in self.d_xml_board_files:
1149                 self.add_xml_board(board)
1150             root_node = self.d_xml_board_files[board].xmlroot
1151             src.xmlManager.append_node_attrib(root_node,
1152                                               {"input_file" : self.file_boards})
1153
1154         # Loop over all jobs in order to get the lines and columns for each
1155         # xml file
1156         d_dist = {}
1157         d_application = {}
1158         for board in self.d_xml_board_files:
1159             d_dist[board] = []
1160             d_application[board] = []
1161
1162         l_hosts_ports = []
1163
1164         for job in l_jobs + l_jobs_not_today:
1165
1166             if (job.machine.host, job.machine.port) not in l_hosts_ports:
1167                 l_hosts_ports.append((job.machine.host, job.machine.port))
1168
1169             distrib = job.machine.distribution
1170             application = job.application
1171
1172             board_job = job.board
1173             if board is None:
1174                 continue
1175             for board in self.d_xml_board_files:
1176                 if board_job == board:
1177                     if distrib is not None and distrib not in d_dist[board]:
1178                         d_dist[board].append(distrib)
1179                         src.xmlManager.add_simple_node(
1180                             self.d_xml_board_files[board].xmlroot.find(
1181                                                             'distributions'),
1182                                                    "dist",
1183                                                    attrib={"name" : distrib})
1184
1185                 if board_job == board:
1186                     if (application is not None and
1187                                     application not in d_application[board]):
1188                         d_application[board].append(application)
1189                         src.xmlManager.add_simple_node(
1190                             self.d_xml_board_files[board].xmlroot.find(
1191                                                                 'applications'),
1192                                                    "application",
1193                                                    attrib={
1194                                                         "name" : application})
1195
1196         # Verify that there are no missing application or distribution in the
1197         # xml board files (regarding the input boards)
1198         for board in self.d_xml_board_files:
1199             l_dist = d_dist[board]
1200             if board not in self.d_input_boards.keys():
1201                 continue
1202             for dist in self.d_input_boards[board]["rows"]:
1203                 if dist not in l_dist:
1204                     src.xmlManager.add_simple_node(
1205                             self.d_xml_board_files[board].xmlroot.find(
1206                                                             'distributions'),
1207                                                    "dist",
1208                                                    attrib={"name" : dist})
1209             l_appli = d_application[board]
1210             for appli in self.d_input_boards[board]["columns"]:
1211                 if appli not in l_appli:
1212                     src.xmlManager.add_simple_node(
1213                             self.d_xml_board_files[board].xmlroot.find(
1214                                                                 'applications'),
1215                                                    "application",
1216                                                    attrib={"name" : appli})
1217
1218         # Initialize the hosts_ports node for the global file
1219         self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1220                                                                 "hosts_ports")
1221         for host, port in l_hosts_ports:
1222             host_port = "%s:%i" % (host, port)
1223             src.xmlManager.add_simple_node(self.xmlhosts_ports,
1224                                            "host_port",
1225                                            attrib={"name" : host_port})
1226
1227         # Initialize the jobs node in all files
1228         for xml_file in [self.xml_global_file] + list(
1229                                             self.d_xml_board_files.values()):
1230             xml_jobs = xml_file.add_simple_node("jobs")
1231             # Get the jobs present in the config file but
1232             # that will not be launched today
1233             self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1234
1235             # add also the infos node
1236             xml_file.add_simple_node("infos",
1237                                      attrib={"name" : "last update",
1238                                              "JobsCommandStatus" : "running"})
1239
1240             # and put the history node
1241             history_node = xml_file.add_simple_node("history")
1242             name_board = os.path.basename(xml_file.logFile)[:-len(".xml")]
1243             # serach for board files
1244             expression = "^[0-9]{8}_+[0-9]{6}_" + name_board + ".xml$"
1245             oExpr = re.compile(expression)
1246             # Get the list of xml borad files that are in the log directory
1247             for file_name in os.listdir(self.xml_dir_path):
1248                 if oExpr.search(file_name):
1249                     date = os.path.basename(file_name).split("_")[0]
1250                     file_path = os.path.join(self.xml_dir_path, file_name)
1251                     src.xmlManager.add_simple_node(history_node,
1252                                                    "link",
1253                                                    text=file_path,
1254                                                    attrib={"date" : date})
1255
1256
1257         # Find in each board the squares that needs to be filled regarding the
1258         # input csv files but that are not covered by a today job
1259         for board in self.d_input_boards.keys():
1260             xml_root_board = self.d_xml_board_files[board].xmlroot
1261             # Find the missing jobs for today
1262             xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1263                                                  "missing_jobs")
1264             for row, column in self.d_input_boards[board]["jobs"]:
1265                 found = False
1266                 for job in l_jobs:
1267                     if (job.application == column and
1268                         job.machine.distribution == row):
1269                         found = True
1270                         break
1271                 if not found:
1272                     src.xmlManager.add_simple_node(xml_missing,
1273                                             "job",
1274                                             attrib={"distribution" : row,
1275                                                     "application" : column })
1276             # Find the missing jobs not today
1277             xml_missing_not_today = src.xmlManager.add_simple_node(
1278                                                  xml_root_board,
1279                                                  "missing_jobs_not_today")
1280             for row, column in self.d_input_boards[board]["jobs_not_today"]:
1281                 found = False
1282                 for job in l_jobs_not_today:
1283                     if (job.application == column and
1284                         job.machine.distribution == row):
1285                         found = True
1286                         break
1287                 if not found:
1288                     src.xmlManager.add_simple_node(xml_missing_not_today,
1289                                             "job",
1290                                             attrib={"distribution" : row,
1291                                                     "application" : column })
1292
1293     def find_history(self, l_jobs, l_jobs_not_today):
1294         """find, for each job, in the existent xml boards the results for the
1295            job. Store the results in the dictionnary self.history = {name_job :
1296            list of (date, status, list links)}
1297
1298         :param l_jobs List: the list of jobs to run today
1299         :param l_jobs_not_today List: the list of jobs that do not run today
1300         """
1301         # load the all the history
1302         expression = "^[0-9]{8}_+[0-9]{6}_" + self.global_name + ".xml$"
1303         oExpr = re.compile(expression)
1304         # Get the list of global xml that are in the log directory
1305         l_globalxml = []
1306         for file_name in os.listdir(self.xml_dir_path):
1307             if oExpr.search(file_name):
1308                 file_path = os.path.join(self.xml_dir_path, file_name)
1309                 try:
1310                     global_xml = src.xmlManager.ReadXmlFile(file_path)
1311                     l_globalxml.append(global_xml)
1312                 except Exception as e:
1313                     msg = _("\nWARNING: the file %s can not be read, it will be "
1314                             "ignored\n%s" % (file_path, e))
1315                     self.logger.write("%s\n" % src.printcolors.printcWarning(
1316                                                                         msg), 5)
1317
1318
1319         # Construct the dictionnary self.history
1320         for job in l_jobs + l_jobs_not_today:
1321             l_links = []
1322             for global_xml in l_globalxml:
1323                 date = os.path.basename(global_xml.filePath).split("_")[0]
1324                 global_root_node = global_xml.xmlroot.find("jobs")
1325                 job_node = src.xmlManager.find_node_by_attrib(
1326                                                               global_root_node,
1327                                                               "job",
1328                                                               "name",
1329                                                               job.name)
1330                 if job_node:
1331                     if job_node.find("remote_log_file_path") is not None:
1332                         link = job_node.find("remote_log_file_path").text
1333                         res_job = job_node.find("res").text
1334                         if link != "nothing":
1335                             l_links.append((date, res_job, link))
1336             l_links = sorted(l_links, reverse=True)
1337             self.history[job.name] = l_links
1338
1339     def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1340         '''Get all the first information needed for each file and write the
1341            first version of the files
1342
1343         :param xml_node_jobs etree.Element: the node corresponding to a job
1344         :param l_jobs_not_today List: the list of jobs that do not run today
1345         '''
1346         for job in l_jobs_not_today:
1347             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1348                                                  "job",
1349                                                  attrib={"name" : job.name})
1350             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1351             src.xmlManager.add_simple_node(xmlj,
1352                                            "distribution",
1353                                            job.machine.distribution)
1354             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1355             src.xmlManager.add_simple_node(xmlj,
1356                                        "commands", " ; ".join(job.commands))
1357             src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1358             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1359             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1360             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1361             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1362             src.xmlManager.add_simple_node(xmlj, "sat_path",
1363                                                         job.machine.sat_path)
1364             xml_history = src.xmlManager.add_simple_node(xmlj, "history")
1365             for i, (date, res_job, link) in enumerate(self.history[job.name]):
1366                 if i==0:
1367                     # tag the first one (the last one)
1368                     src.xmlManager.add_simple_node(xml_history,
1369                                                    "link",
1370                                                    text=link,
1371                                                    attrib={"date" : date,
1372                                                            "res" : res_job,
1373                                                            "last" : "yes"})
1374                 else:
1375                     src.xmlManager.add_simple_node(xml_history,
1376                                                    "link",
1377                                                    text=link,
1378                                                    attrib={"date" : date,
1379                                                            "res" : res_job,
1380                                                            "last" : "no"})
1381
1382     def parse_csv_boards(self, today):
1383         """ Parse the csv file that describes the boards to produce and fill
1384             the dict d_input_boards that contain the csv file contain
1385
1386         :param today int: the current day of the week
1387         """
1388         # open the csv file and read its content
1389         l_read = []
1390         with open(self.file_boards, 'r') as f:
1391             reader = csv.reader(f,delimiter=CSV_DELIMITER)
1392             for row in reader:
1393                 l_read.append(row)
1394         # get the delimiter for the boards (empty line)
1395         boards_delimiter = [''] * len(l_read[0])
1396         # Make the list of boards, by splitting with the delimiter
1397         l_boards = [list(y) for x, y in itertools.groupby(l_read,
1398                                     lambda z: z == boards_delimiter) if not x]
1399
1400         # loop over the csv lists of lines and get the rows, columns and jobs
1401         d_boards = {}
1402         for input_board in l_boards:
1403             # get board name
1404             board_name = input_board[0][0]
1405
1406             # Get columns list
1407             columns = input_board[0][1:]
1408
1409             rows = []
1410             jobs = []
1411             jobs_not_today = []
1412             for line in input_board[1:]:
1413                 row = line[0]
1414                 rows.append(row)
1415                 for i, square in enumerate(line[1:]):
1416                     if square=='':
1417                         continue
1418                     days = square.split(DAYS_SEPARATOR)
1419                     days = [int(day) for day in days]
1420                     job = (row, columns[i])
1421                     if today in days:
1422                         jobs.append(job)
1423                     else:
1424                         jobs_not_today.append(job)
1425
1426             d_boards[board_name] = {"rows" : rows,
1427                                     "columns" : columns,
1428                                     "jobs" : jobs,
1429                                     "jobs_not_today" : jobs_not_today}
1430
1431         self.d_input_boards = d_boards
1432
1433     def update_xml_files(self, l_jobs):
1434         '''Write all the xml files with updated information about the jobs
1435
1436         :param l_jobs List: the list of jobs that run today
1437         '''
1438         for xml_file in [self.xml_global_file] + list(
1439                                             self.d_xml_board_files.values()):
1440             self.update_xml_file(l_jobs, xml_file)
1441
1442         # Write the file
1443         self.write_xml_files()
1444
1445     def update_xml_file(self, l_jobs, xml_file):
1446         '''update information about the jobs for the file xml_file
1447
1448         :param l_jobs List: the list of jobs that run today
1449         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1450         '''
1451
1452         xml_node_jobs = xml_file.xmlroot.find('jobs')
1453         # Update the job names and status node
1454         for job in l_jobs:
1455             # Find the node corresponding to the job and delete it
1456             # in order to recreate it
1457             for xmljob in xml_node_jobs.findall('job'):
1458                 if xmljob.attrib['name'] == job.name:
1459                     xml_node_jobs.remove(xmljob)
1460
1461             T0 = str(job._T0)
1462             if T0 != "-1":
1463                 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1464                                        time.localtime(job._T0))
1465             Tf = str(job._Tf)
1466             if Tf != "-1":
1467                 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1468                                        time.localtime(job._Tf))
1469
1470             # recreate the job node
1471             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1472                                                   "job",
1473                                                   attrib={"name" : job.name})
1474             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1475             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1476             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1477             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1478             xml_history = src.xmlManager.add_simple_node(xmlj, "history")
1479             for date, res_job, link in self.history[job.name]:
1480                 src.xmlManager.add_simple_node(xml_history,
1481                                                "link",
1482                                                text=link,
1483                                                attrib={"date" : date,
1484                                                        "res" : res_job})
1485
1486             src.xmlManager.add_simple_node(xmlj, "sat_path",
1487                                            job.machine.sat_path)
1488             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1489             src.xmlManager.add_simple_node(xmlj, "distribution",
1490                                            job.machine.distribution)
1491             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1492             src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1493             src.xmlManager.add_simple_node(xmlj, "commands",
1494                                            " ; ".join(job.commands))
1495             src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1496             src.xmlManager.add_simple_node(xmlj, "begin", T0)
1497             src.xmlManager.add_simple_node(xmlj, "end", Tf)
1498             src.xmlManager.add_simple_node(xmlj, "out",
1499                                            src.printcolors.cleancolor(job.out))
1500             src.xmlManager.add_simple_node(xmlj, "err",
1501                                            src.printcolors.cleancolor(job.err))
1502             src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1503             if len(job.remote_log_files) > 0:
1504                 src.xmlManager.add_simple_node(xmlj,
1505                                                "remote_log_file_path",
1506                                                job.remote_log_files[0])
1507             else:
1508                 src.xmlManager.add_simple_node(xmlj,
1509                                                "remote_log_file_path",
1510                                                "nothing")
1511             # Search for the test log if there is any
1512             l_test_log_files = self.find_test_log(job.remote_log_files)
1513             xml_test = src.xmlManager.add_simple_node(xmlj,
1514                                                       "test_log_file_path")
1515             for test_log_path, res_test, nb_fails in l_test_log_files:
1516                 test_path_node = src.xmlManager.add_simple_node(xml_test,
1517                                                "path",
1518                                                test_log_path)
1519                 test_path_node.attrib["res"] = res_test
1520                 test_path_node.attrib["nb_fails"] = nb_fails
1521
1522             xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1523             # get the job father
1524             if job.after is not None:
1525                 job_father = None
1526                 for jb in l_jobs:
1527                     if jb.name == job.after:
1528                         job_father = jb
1529
1530                 if (job_father is not None and
1531                         len(job_father.remote_log_files) > 0):
1532                     link = job_father.remote_log_files[0]
1533                 else:
1534                     link = "nothing"
1535                 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1536
1537             # Verify that the job is to be done today regarding the input csv
1538             # files
1539             if job.board and job.board in self.d_input_boards.keys():
1540                 found = False
1541                 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1542                     if (job.machine.distribution == dist
1543                         and job.application == appli):
1544                         found = True
1545                         src.xmlManager.add_simple_node(xmlj,
1546                                                "extra_job",
1547                                                "no")
1548                         break
1549                 if not found:
1550                     src.xmlManager.add_simple_node(xmlj,
1551                                                "extra_job",
1552                                                "yes")
1553
1554
1555         # Update the date
1556         xml_node_infos = xml_file.xmlroot.find('infos')
1557         src.xmlManager.append_node_attrib(xml_node_infos,
1558                     attrib={"value" :
1559                     datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1560
1561
1562     def find_test_log(self, l_remote_log_files):
1563         '''Find if there is a test log (board) in the remote log files and
1564            the path to it. There can be several test command, so the result is
1565            a list.
1566
1567         :param l_remote_log_files List: the list of all remote log files
1568         :return: the list of (test log files path, res of the command)
1569         :rtype: List
1570         '''
1571         res = []
1572         for file_path in l_remote_log_files:
1573             dirname = os.path.basename(os.path.dirname(file_path))
1574             file_name = os.path.basename(file_path)
1575             regex = src.logger.log_all_command_file_expression
1576             oExpr = re.compile(regex)
1577             if dirname == "TEST" and oExpr.search(file_name):
1578                 # find the res of the command
1579                 prod_node = etree.parse(file_path).getroot().find("product")
1580                 res_test = prod_node.attrib["global_res"]
1581                 # find the number of fails
1582                 testbase_node = prod_node.find("tests").find("testbase")
1583                 nb_fails = int(testbase_node.attrib["failed"])
1584                 # put the file path, the res of the test command and the number
1585                 # of fails in the output
1586                 res.append((file_path, res_test, nb_fails))
1587
1588         return res
1589
1590     def last_update(self, finish_status = "finished"):
1591         '''update information about the jobs for the file xml_file
1592
1593         :param l_jobs List: the list of jobs that run today
1594         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1595         '''
1596         for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1597             xml_node_infos = xml_file.xmlroot.find('infos')
1598             src.xmlManager.append_node_attrib(xml_node_infos,
1599                         attrib={"JobsCommandStatus" : finish_status})
1600         # Write the file
1601         self.write_xml_files()
1602
1603     def write_xml_file(self, xml_file, stylesheet):
1604         ''' Write one xml file and the same file with prefix
1605         '''
1606         xml_file.write_tree(stylesheet)
1607         file_path = xml_file.logFile
1608         file_dir = os.path.dirname(file_path)
1609         file_name = os.path.basename(file_path)
1610         file_name_with_prefix = self.prefix + "_" + file_name
1611         xml_file.write_tree(stylesheet, os.path.join(file_dir,
1612                                                      file_name_with_prefix))
1613
1614     def write_xml_files(self):
1615         ''' Write the xml files
1616         '''
1617         self.write_xml_file(self.xml_global_file, STYLESHEET_GLOBAL)
1618         for xml_file in self.d_xml_board_files.values():
1619             self.write_xml_file(xml_file, STYLESHEET_BOARD)
1620
1621 def get_config_file_path(job_config_name, l_cfg_dir):
1622     found = False
1623     file_jobs_cfg = None
1624     if os.path.exists(job_config_name) and job_config_name.endswith(".pyconf"):
1625         found = True
1626         file_jobs_cfg = job_config_name
1627     else:
1628         for cfg_dir in l_cfg_dir:
1629             file_jobs_cfg = os.path.join(cfg_dir, job_config_name)
1630             if not file_jobs_cfg.endswith('.pyconf'):
1631                 file_jobs_cfg += '.pyconf'
1632
1633             if not os.path.exists(file_jobs_cfg):
1634                 continue
1635             else:
1636                 found = True
1637                 break
1638     return found, file_jobs_cfg
1639
1640 ##
1641 # Describes the command
1642 def description():
1643     return _("The jobs command launches maintenances that are described"
1644              " in the dedicated jobs configuration file.\n\nexample:\nsat "
1645              "jobs --name my_jobs --publish")
1646
1647 ##
1648 # Runs the command.
1649 def run(args, runner, logger):
1650
1651     (options, args) = parser.parse_args(args)
1652
1653     l_cfg_dir = runner.cfg.PATHS.JOBPATH
1654
1655     # list option : display all the available config files
1656     if options.list:
1657         for cfg_dir in l_cfg_dir:
1658             if not options.no_label:
1659                 logger.write("------ %s\n" %
1660                                  src.printcolors.printcHeader(cfg_dir))
1661             if not os.path.exists(cfg_dir):
1662                 continue
1663             for f in sorted(os.listdir(cfg_dir)):
1664                 if not f.endswith('.pyconf'):
1665                     continue
1666                 cfilename = f[:-7]
1667                 logger.write("%s\n" % cfilename)
1668         return 0
1669
1670     # Make sure the jobs_config option has been called
1671     if not options.jobs_cfg:
1672         message = _("The option --jobs_config is required\n")
1673         src.printcolors.printcError(message)
1674         return 1
1675
1676     # Find the file in the directories, unless it is a full path
1677     # merge all in a config
1678     merger = src.pyconf.ConfigMerger()
1679     config_jobs = src.pyconf.Config()
1680     l_conf_files_path = []
1681     for config_file in options.jobs_cfg:
1682         found, file_jobs_cfg = get_config_file_path(config_file, l_cfg_dir)
1683         if not found:
1684             msg = _("The file configuration %(name_file)s was not found."
1685                     "\nUse the --list option to get the "
1686                     "possible files." % config_file)
1687             src.printcolors.printcError(msg)
1688             return 1
1689         l_conf_files_path.append(file_jobs_cfg)
1690         # Read the config that is in the file
1691         one_config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1692         merger.merge(config_jobs, one_config_jobs)
1693
1694     info = [
1695         (_("Platform"), runner.cfg.VARS.dist),
1696         (_("Files containing the jobs configuration"), l_conf_files_path)
1697     ]
1698     src.print_info(logger, info)
1699
1700     if options.only_jobs:
1701         l_jb = src.pyconf.Sequence()
1702         for jb in config_jobs.jobs:
1703             if jb.name in options.only_jobs:
1704                 l_jb.append(jb,
1705                 "Job that was given in only_jobs option parameters\n")
1706         config_jobs.jobs = l_jb
1707
1708     # Make a unique file that contain all the jobs in order to use it
1709     # on every machine
1710     name_pyconf = "_".join([os.path.basename(path)[:-len('.pyconf')]
1711                             for path in l_conf_files_path]) + ".pyconf"
1712     path_pyconf = src.get_tmp_filename(runner.cfg, name_pyconf)
1713     #Save config
1714     f = file( path_pyconf , 'w')
1715     config_jobs.__save__(f)
1716
1717     # Initialization
1718     today_jobs = Jobs(runner,
1719                       logger,
1720                       path_pyconf,
1721                       config_jobs)
1722     # SSH connection to all machines
1723     today_jobs.ssh_connection_all_machines()
1724     if options.test_connection:
1725         return 0
1726
1727     gui = None
1728     if options.publish:
1729         logger.write(src.printcolors.printcInfo(
1730                                         _("Initialize the xml boards : ")), 5)
1731         logger.flush()
1732
1733         # Copy the stylesheets in the log directory
1734         log_dir = runner.cfg.USER.log_dir
1735         xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1736         files_to_copy = []
1737         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1738         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1739         files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1740         for file_path in files_to_copy:
1741             shutil.copy2(file_path, log_dir)
1742
1743         # Instanciate the Gui in order to produce the xml files that contain all
1744         # the boards
1745         gui = Gui(runner.cfg.USER.log_dir,
1746                   today_jobs.ljobs,
1747                   today_jobs.ljobs_not_today,
1748                   runner.cfg.VARS.datehour,
1749                   logger,
1750                   file_boards = options.input_boards)
1751
1752         logger.write(src.printcolors.printcSuccess("OK"), 5)
1753         logger.write("\n\n", 5)
1754         logger.flush()
1755
1756         # Display the list of the xml files
1757         logger.write(src.printcolors.printcInfo(("Here is the list of published"
1758                                                  " files :\n")), 4)
1759         logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1760         for board in gui.d_xml_board_files.keys():
1761             file_path = gui.d_xml_board_files[board].logFile
1762             file_name = os.path.basename(file_path)
1763             logger.write("%s\n" % file_path, 4)
1764             logger.add_link(file_name, "board", 0, board)
1765
1766         logger.write("\n", 4)
1767
1768     today_jobs.gui = gui
1769
1770     interruped = False
1771     try:
1772         # Run all the jobs contained in config_jobs
1773         today_jobs.run_jobs()
1774     except KeyboardInterrupt:
1775         interruped = True
1776         logger.write("\n\n%s\n\n" %
1777                 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1778     except Exception as e:
1779         msg = _("CRITICAL ERROR: The jobs loop has been interrupted\n")
1780         logger.write("\n\n%s\n" % src.printcolors.printcError(msg) )
1781         logger.write("%s\n" % str(e))
1782         # get stack
1783         __, __, exc_traceback = sys.exc_info()
1784         fp = tempfile.TemporaryFile()
1785         traceback.print_tb(exc_traceback, file=fp)
1786         fp.seek(0)
1787         stack = fp.read()
1788         logger.write("\nTRACEBACK: %s\n" % stack.replace('"',"'"), 1)
1789
1790     finally:
1791         res = 0
1792         if interruped:
1793             res = 1
1794             msg = _("Killing the running jobs and trying"
1795                     " to get the corresponding logs\n")
1796             logger.write(src.printcolors.printcWarning(msg))
1797
1798         # find the potential not finished jobs and kill them
1799         for jb in today_jobs.ljobs:
1800             if not jb.has_finished():
1801                 res = 1
1802                 try:
1803                     jb.kill_remote_process()
1804                 except Exception as e:
1805                     msg = _("Failed to kill job %s: %s\n" % (jb.name, e))
1806                     logger.write(src.printcolors.printcWarning(msg))
1807             if jb.res_job != "0":
1808                 res = 1
1809         if interruped:
1810             if today_jobs.gui:
1811                 today_jobs.gui.last_update(_("Forced interruption"))
1812         else:
1813             if today_jobs.gui:
1814                 today_jobs.gui.last_update()
1815         # Output the results
1816         today_jobs.write_all_results()
1817         # Remove the temporary pyconf file
1818         if os.path.exists(path_pyconf):
1819             os.remove(path_pyconf)
1820         return res