commands/jobs.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import datetime
  21 import time
  22 import csv
  23 import shutil
  24 import itertools
  25 import re
  26 import paramiko
  27
  28 import src
  29
  30 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
  31 STYLESHEET_BOARD = "jobs_board_report.xsl"
  32
  33 DAYS_SEPARATOR = ","
  34 CSV_DELIMITER = ";"
  35
  36 parser = src.options.Options()
  37
  38 parser.add_option('n', 'name', 'string', 'jobs_cfg',
  39                   _('Mandatory: The name of the config file that contains'
  40                   ' the jobs configuration'))
  41 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
  42                   _('Optional: the list of jobs to launch, by their name. '))
  43 parser.add_option('l', 'list', 'boolean', 'list',
  44                   _('Optional: list all available config files.'))
  45 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
  46                   _("Optional: try to connect to the machines. "
  47                     "Not executing the jobs."),
  48                   False)
  49 parser.add_option('p', 'publish', 'boolean', 'publish',
  50                   _("Optional: generate an xml file that can be read in a "
  51                     "browser to display the jobs status."),
  52                   False)
  53 parser.add_option('i', 'input_boards', 'string', 'input_boards', _("Optional: "
  54                                 "the path to csv file that contain "
  55                                 "the expected boards."),"")
  56 parser.add_option('', 'completion', 'boolean', 'no_label',
  57                   _("Optional (internal use): do not print labels, Works only "
  58                     "with --list."),
  59                   False)
  60
  61 class Machine(object):
  62     '''Class to manage a ssh connection on a machine
  63     '''
  64     def __init__(self,
  65                  name,
  66                  host,
  67                  user,
  68                  port=22,
  69                  passwd=None,
  70                  sat_path="salomeTools"):
  71         self.name = name
  72         self.host = host
  73         self.port = port
  74         self.distribution = None # Will be filled after copying SAT on the machine
  75         self.user = user
  76         self.password = passwd
  77         self.sat_path = sat_path
  78         self.ssh = paramiko.SSHClient()
  79         self._connection_successful = None
  80
  81     def connect(self, logger):
  82         '''Initiate the ssh connection to the remote machine
  83
  84         :param logger src.logger.Logger: The logger instance
  85         :return: Nothing
  86         :rtype: N\A
  87         '''
  88
  89         self._connection_successful = False
  90         self.ssh.load_system_host_keys()
  91         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  92         try:
  93             self.ssh.connect(self.host,
  94                              port=self.port,
  95                              username=self.user,
  96                              password = self.password)
  97         except paramiko.AuthenticationException:
  98             message = src.KO_STATUS + _("Authentication failed")
  99         except paramiko.BadHostKeyException:
 100             message = (src.KO_STATUS +
 101                        _("The server's host key could not be verified"))
 102         except paramiko.SSHException:
 103             message = ( _("SSHException error connecting or "
 104                           "establishing an SSH session"))
 105         except:
 106             message = ( _("Error connecting or establishing an SSH session"))
 107         else:
 108             self._connection_successful = True
 109             message = ""
 110         return message
 111
 112     def successfully_connected(self, logger):
 113         '''Verify if the connection to the remote machine has succeed
 114
 115         :param logger src.logger.Logger: The logger instance
 116         :return: True if the connection has succeed, False if not
 117         :rtype: bool
 118         '''
 119         if self._connection_successful == None:
 120             message = _("Warning : trying to ask if the connection to "
 121             "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
 122             " no connection request" %
 123                         (self.name, self.host, self.port, self.user))
 124             logger.write( src.printcolors.printcWarning(message))
 125         return self._connection_successful
 126
 127     def copy_sat(self, sat_local_path, job_file):
 128         '''Copy salomeTools to the remote machine in self.sat_path
 129         '''
 130         res = 0
 131         try:
 132             # open a sftp connection
 133             self.sftp = self.ssh.open_sftp()
 134             # Create the sat directory on remote machine if it is not existing
 135             self.mkdir(self.sat_path, ignore_existing=True)
 136             # Put sat
 137             self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
 138             # put the job configuration file in order to make it reachable
 139             # on the remote machine
 140             self.sftp.put(job_file, os.path.join(".salomeTools",
 141                                                  "Jobs",
 142                                                  ".jobs_command_file.pyconf"))
 143         except Exception as e:
 144             res = str(e)
 145             self._connection_successful = False
 146
 147         return res
 148
 149     def put_dir(self, source, target, filters = []):
 150         ''' Uploads the contents of the source directory to the target path. The
 151             target directory needs to exists. All sub-directories in source are
 152             created under target.
 153         '''
 154         for item in os.listdir(source):
 155             if item in filters:
 156                 continue
 157             source_path = os.path.join(source, item)
 158             destination_path = os.path.join(target, item)
 159             if os.path.islink(source_path):
 160                 linkto = os.readlink(source_path)
 161                 try:
 162                     self.sftp.symlink(linkto, destination_path)
 163                     self.sftp.chmod(destination_path,
 164                                     os.stat(source_path).st_mode)
 165                 except IOError:
 166                     pass
 167             else:
 168                 if os.path.isfile(source_path):
 169                     self.sftp.put(source_path, destination_path)
 170                     self.sftp.chmod(destination_path,
 171                                     os.stat(source_path).st_mode)
 172                 else:
 173                     self.mkdir(destination_path, ignore_existing=True)
 174                     self.put_dir(source_path, destination_path)
 175
 176     def mkdir(self, path, mode=511, ignore_existing=False):
 177         ''' Augments mkdir by adding an option to not fail
 178             if the folder exists
 179         '''
 180         try:
 181             self.sftp.mkdir(path, mode)
 182         except IOError:
 183             if ignore_existing:
 184                 pass
 185             else:
 186                 raise
 187
 188     def exec_command(self, command, logger):
 189         '''Execute the command on the remote machine
 190
 191         :param command str: The command to be run
 192         :param logger src.logger.Logger: The logger instance
 193         :return: the stdin, stdout, and stderr of the executing command,
 194                  as a 3-tuple
 195         :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
 196                 paramiko.channel.ChannelFile)
 197         '''
 198         try:
 199             # Does not wait the end of the command
 200             (stdin, stdout, stderr) = self.ssh.exec_command(command)
 201         except paramiko.SSHException:
 202             message = src.KO_STATUS + _(
 203                             ": the server failed to execute the command\n")
 204             logger.write( src.printcolors.printcError(message))
 205             return (None, None, None)
 206         except:
 207             logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
 208             return (None, None, None)
 209         else:
 210             return (stdin, stdout, stderr)
 211
 212     def close(self):
 213         '''Close the ssh connection
 214
 215         :rtype: N\A
 216         '''
 217         self.ssh.close()
 218
 219     def write_info(self, logger):
 220         '''Prints the informations relative to the machine in the logger
 221            (terminal traces and log file)
 222
 223         :param logger src.logger.Logger: The logger instance
 224         :return: Nothing
 225         :rtype: N\A
 226         '''
 227         logger.write("host : " + self.host + "\n")
 228         logger.write("port : " + str(self.port) + "\n")
 229         logger.write("user : " + str(self.user) + "\n")
 230         if self.successfully_connected(logger):
 231             status = src.OK_STATUS
 232         else:
 233             status = src.KO_STATUS
 234         logger.write("Connection : " + status + "\n\n")
 235
 236
 237 class Job(object):
 238     '''Class to manage one job
 239     '''
 240     def __init__(self, name, machine, application, board,
 241                  commands, timeout, config, logger, after=None):
 242
 243         self.name = name
 244         self.machine = machine
 245         self.after = after
 246         self.timeout = timeout
 247         self.application = application
 248         self.board = board
 249         self.config = config
 250         self.logger = logger
 251         # The list of log files to download from the remote machine
 252         self.remote_log_files = []
 253
 254         # The remote command status
 255         # -1 means that it has not been launched,
 256         # 0 means success and 1 means fail
 257         self.res_job = "-1"
 258         self.cancelled = False
 259
 260         self._T0 = -1
 261         self._Tf = -1
 262         self._has_begun = False
 263         self._has_finished = False
 264         self._has_timouted = False
 265         self._stdin = None # Store the command inputs field
 266         self._stdout = None # Store the command outputs field
 267         self._stderr = None # Store the command errors field
 268
 269         self.out = ""
 270         self.err = ""
 271
 272         self.commands = commands
 273         self.command = (os.path.join(self.machine.sat_path, "sat") +
 274                         " -l " +
 275                         os.path.join(self.machine.sat_path,
 276                                      "list_log_files.txt") +
 277                         " job --jobs_config .jobs_command_file" +
 278                         " --name " +
 279                         self.name)
 280
 281     def get_pids(self):
 282         """ Get the pid(s) corresponding to the command that have been launched
 283             On the remote machine
 284
 285         :return: The list of integers corresponding to the found pids
 286         :rtype: List
 287         """
 288         pids = []
 289         cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
 290         (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
 291         pids_cmd = out_pid.readlines()
 292         pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
 293         pids+=pids_cmd
 294         return pids
 295
 296     def kill_remote_process(self, wait=1):
 297         '''Kills the process on the remote machine.
 298
 299         :return: (the output of the kill, the error of the kill)
 300         :rtype: (str, str)
 301         '''
 302
 303         pids = self.get_pids()
 304         cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
 305         (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
 306                                                             self.logger)
 307         time.sleep(wait)
 308         return (out_kill, err_kill)
 309
 310     def has_begun(self):
 311         '''Returns True if the job has already begun
 312
 313         :return: True if the job has already begun
 314         :rtype: bool
 315         '''
 316         return self._has_begun
 317
 318     def has_finished(self):
 319         '''Returns True if the job has already finished
 320            (i.e. all the commands have been executed)
 321            If it is finished, the outputs are stored in the fields out and err.
 322
 323         :return: True if the job has already finished
 324         :rtype: bool
 325         '''
 326
 327         # If the method has already been called and returned True
 328         if self._has_finished:
 329             return True
 330
 331         # If the job has not begun yet
 332         if not self.has_begun():
 333             return False
 334
 335         if self._stdout.channel.closed:
 336             self._has_finished = True
 337             # Store the result outputs
 338             self.out += self._stdout.read().decode()
 339             self.err += self._stderr.read().decode()
 340             # Put end time
 341             self._Tf = time.time()
 342             # And get the remote command status and log files
 343             self.get_log_files()
 344
 345         return self._has_finished
 346
 347     def get_log_files(self):
 348         """Get the log files produced by the command launched
 349            on the remote machine, and put it in the log directory of the user,
 350            so they can be accessible from
 351         """
 352         # Do not get the files if the command is not finished
 353         if not self.has_finished():
 354             msg = _("Trying to get log files whereas the job is not finished.")
 355             self.logger.write(src.printcolors.printcWarning(msg))
 356             return
 357
 358         # First get the file that contains the list of log files to get
 359         tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
 360         remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt")
 361         self.machine.sftp.get(
 362                     remote_path,
 363                     tmp_file_path)
 364
 365         # Read the file and get the result of the command and all the log files
 366         # to get
 367         fstream_tmp = open(tmp_file_path, "r")
 368         file_lines = fstream_tmp.readlines()
 369         file_lines = [line.replace("\n", "") for line in file_lines]
 370         fstream_tmp.close()
 371         os.remove(tmp_file_path)
 372
 373         try :
 374             # The first line is the result of the command (0 success or 1 fail)
 375             self.res_job = file_lines[0]
 376         except Exception as e:
 377             self.err += _("Unable to get status from remote file %s: %s" %
 378                                                     (remote_path, str(e)))
 379
 380         for i, job_path_remote in enumerate(file_lines[1:]):
 381             try:
 382                 # For each command, there is two files to get :
 383                 # 1- The xml file describing the command and giving the
 384                 # internal traces.
 385                 # 2- The txt file containing the system command traces (like
 386                 # traces produced by the "make" command)
 387                 # 3- In case of the test command, there is another file to get :
 388                 # the xml board that contain the test results
 389                 dirname = os.path.basename(os.path.dirname(job_path_remote))
 390                 if dirname != 'OUT' and dirname != 'TEST':
 391                     # Case 1-
 392                     local_path = os.path.join(os.path.dirname(
 393                                                         self.logger.logFilePath),
 394                                               os.path.basename(job_path_remote))
 395                     if i==0: # The first is the job command
 396                         self.logger.add_link(os.path.basename(job_path_remote),
 397                                              "job",
 398                                              self.res_job,
 399                                              self.command)
 400                 elif dirname == 'OUT':
 401                     # Case 2-
 402                     local_path = os.path.join(os.path.dirname(
 403                                                         self.logger.logFilePath),
 404                                               'OUT',
 405                                               os.path.basename(job_path_remote))
 406                 elif dirname == 'TEST':
 407                     # Case 3-
 408                     local_path = os.path.join(os.path.dirname(
 409                                                         self.logger.logFilePath),
 410                                               'TEST',
 411                                               os.path.basename(job_path_remote))
 412
 413                 # Get the file
 414                 if not os.path.exists(local_path):
 415                     self.machine.sftp.get(job_path_remote, local_path)
 416                 self.remote_log_files.append(local_path)
 417             except Exception as e:
 418                 self.err += _("Unable to get %s log file from remote: %s" %
 419                                                     (str(job_path_remote),
 420                                                      str(e)))
 421
 422     def has_failed(self):
 423         '''Returns True if the job has failed.
 424            A job is considered as failed if the machine could not be reached,
 425            if the remote command failed,
 426            or if the job finished with a time out.
 427
 428         :return: True if the job has failed
 429         :rtype: bool
 430         '''
 431         if not self.has_finished():
 432             return False
 433         if not self.machine.successfully_connected(self.logger):
 434             return True
 435         if self.is_timeout():
 436             return True
 437         if self.res_job == "1":
 438             return True
 439         return False
 440
 441     def cancel(self):
 442         """In case of a failing job, one has to cancel every job that depend
 443            on it. This method put the job as failed and will not be executed.
 444         """
 445         if self.cancelled:
 446             return
 447         self._has_begun = True
 448         self._has_finished = True
 449         self.cancelled = True
 450         self.out += _("This job was not launched because its father has failed.")
 451         self.err += _("This job was not launched because its father has failed.")
 452
 453     def is_running(self):
 454         '''Returns True if the job commands are running
 455
 456         :return: True if the job is running
 457         :rtype: bool
 458         '''
 459         return self.has_begun() and not self.has_finished()
 460
 461     def is_timeout(self):
 462         '''Returns True if the job commands has finished with timeout
 463
 464         :return: True if the job has finished with timeout
 465         :rtype: bool
 466         '''
 467         return self._has_timouted
 468
 469     def time_elapsed(self):
 470         """Get the time elapsed since the job launching
 471
 472         :return: The number of seconds
 473         :rtype: int
 474         """
 475         if not self.has_begun():
 476             return -1
 477         T_now = time.time()
 478         return T_now - self._T0
 479
 480     def check_time(self):
 481         """Verify that the job has not exceeded its timeout.
 482            If it has, kill the remote command and consider the job as finished.
 483         """
 484         if not self.has_begun():
 485             return
 486         if self.time_elapsed() > self.timeout:
 487             self._has_finished = True
 488             self._has_timouted = True
 489             self._Tf = time.time()
 490             self.get_pids()
 491             (out_kill, _) = self.kill_remote_process()
 492             self.out += "TIMEOUT \n" + out_kill.read().decode()
 493             self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
 494             try:
 495                 self.get_log_files()
 496             except Exception as e:
 497                 self.err += _("Unable to get remote log files: %s" % e)
 498
 499     def total_duration(self):
 500         """Give the total duration of the job
 501
 502         :return: the total duration of the job in seconds
 503         :rtype: int
 504         """
 505         return self._Tf - self._T0
 506
 507     def run(self):
 508         """Launch the job by executing the remote command.
 509         """
 510
 511         # Prevent multiple run
 512         if self.has_begun():
 513             msg = _("Warning: A job can only be launched one time")
 514             msg2 = _("Trying to launch the job \"%s\" whereas it has "
 515                      "already been launched." % self.name)
 516             self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
 517                                                                         msg2)))
 518             return
 519
 520         # Do not execute the command if the machine could not be reached
 521         if not self.machine.successfully_connected(self.logger):
 522             self._has_finished = True
 523             self.out = "N\A"
 524             self.err += ("Connection to machine (name : %s, host: %s, port:"
 525                         " %s, user: %s) has failed\nUse the log command "
 526                         "to get more information."
 527                         % (self.machine.name,
 528                            self.machine.host,
 529                            self.machine.port,
 530                            self.machine.user))
 531         else:
 532             # Usual case : Launch the command on remote machine
 533             self._T0 = time.time()
 534             self._stdin, self._stdout, self._stderr = self.machine.exec_command(
 535                                                                   self.command,
 536                                                                   self.logger)
 537             # If the results are not initialized, finish the job
 538             if (self._stdin, self._stdout, self._stderr) == (None, None, None):
 539                 self._has_finished = True
 540                 self._Tf = time.time()
 541                 self.out += "N\A"
 542                 self.err += "The server failed to execute the command"
 543
 544         # Put the beginning flag to true.
 545         self._has_begun = True
 546
 547     def write_results(self):
 548         """Display on the terminal all the job's information
 549         """
 550         self.logger.write("name : " + self.name + "\n")
 551         if self.after:
 552             self.logger.write("after : %s\n" % self.after)
 553         self.logger.write("Time elapsed : %4imin %2is \n" %
 554                      (self.total_duration()//60 , self.total_duration()%60))
 555         if self._T0 != -1:
 556             self.logger.write("Begin time : %s\n" %
 557                          time.strftime('%Y-%m-%d %H:%M:%S',
 558                                        time.localtime(self._T0)) )
 559         if self._Tf != -1:
 560             self.logger.write("End time   : %s\n\n" %
 561                          time.strftime('%Y-%m-%d %H:%M:%S',
 562                                        time.localtime(self._Tf)) )
 563
 564         machine_head = "Informations about connection :\n"
 565         underline = (len(machine_head) - 2) * "-"
 566         self.logger.write(src.printcolors.printcInfo(
 567                                                 machine_head+underline+"\n"))
 568         self.machine.write_info(self.logger)
 569
 570         self.logger.write(src.printcolors.printcInfo("out : \n"))
 571         if self.out == "":
 572             self.logger.write("Unable to get output\n")
 573         else:
 574             self.logger.write(self.out + "\n")
 575         self.logger.write(src.printcolors.printcInfo("err : \n"))
 576         self.logger.write(self.err + "\n")
 577
 578     def get_status(self):
 579         """Get the status of the job (used by the Gui for xml display)
 580
 581         :return: The current status of the job
 582         :rtype: String
 583         """
 584         if not self.machine.successfully_connected(self.logger):
 585             return "SSH connection KO"
 586         if not self.has_begun():
 587             return "Not launched"
 588         if self.cancelled:
 589             return "Cancelled"
 590         if self.is_running():
 591             return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
 592                                                     time.localtime(self._T0))
 593         if self.has_finished():
 594             if self.is_timeout():
 595                 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
 596                                                     time.localtime(self._Tf))
 597             return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
 598                                                      time.localtime(self._Tf))
 599
 600 class Jobs(object):
 601     '''Class to manage the jobs to be run
 602     '''
 603     def __init__(self,
 604                  runner,
 605                  logger,
 606                  job_file_path,
 607                  config_jobs,
 608                  lenght_columns = 20):
 609         # The jobs configuration
 610         self.cfg_jobs = config_jobs
 611         self.job_file_path = job_file_path
 612         # The machine that will be used today
 613         self.lmachines = []
 614         # The list of machine (hosts, port) that will be used today
 615         # (a same host can have several machine instances since there
 616         # can be several ssh parameters)
 617         self.lhosts = []
 618         # The jobs to be launched today
 619         self.ljobs = []
 620         # The jobs that will not be launched today
 621         self.ljobs_not_today = []
 622         self.runner = runner
 623         self.logger = logger
 624         self.len_columns = lenght_columns
 625
 626         # the list of jobs that have not been run yet
 627         self._l_jobs_not_started = []
 628         # the list of jobs that have already ran
 629         self._l_jobs_finished = []
 630         # the list of jobs that are running
 631         self._l_jobs_running = []
 632
 633         self.determine_jobs_and_machines()
 634
 635     def define_job(self, job_def, machine):
 636         '''Takes a pyconf job definition and a machine (from class machine)
 637            and returns the job instance corresponding to the definition.
 638
 639         :param job_def src.config.Mapping: a job definition
 640         :param machine machine: the machine on which the job will run
 641         :return: The corresponding job in a job class instance
 642         :rtype: job
 643         '''
 644         name = job_def.name
 645         cmmnds = job_def.commands
 646         if not "timeout" in job_def:
 647             timeout = 4*60*60 # default timeout = 4h
 648         else:
 649             timeout = job_def.timeout
 650         after = None
 651         if 'after' in job_def:
 652             after = job_def.after
 653         application = None
 654         if 'application' in job_def:
 655             application = job_def.application
 656         board = None
 657         if 'board' in job_def:
 658             board = job_def.board
 659
 660         return Job(name,
 661                    machine,
 662                    application,
 663                    board,
 664                    cmmnds,
 665                    timeout,
 666                    self.runner.cfg,
 667                    self.logger,
 668                    after = after)
 669
 670     def determine_jobs_and_machines(self):
 671         '''Function that reads the pyconf jobs definition and instantiates all
 672            the machines and jobs to be done today.
 673
 674         :return: Nothing
 675         :rtype: N\A
 676         '''
 677         today = datetime.date.weekday(datetime.date.today())
 678         host_list = []
 679
 680         for job_def in self.cfg_jobs.jobs :
 681
 682             if not "machine" in job_def:
 683                 msg = _('WARNING: The job "%s" do not have the key '
 684                        '"machine", this job is ignored.\n\n' % job_def.name)
 685                 self.logger.write(src.printcolors.printcWarning(msg))
 686                 continue
 687             name_machine = job_def.machine
 688
 689             a_machine = None
 690             for mach in self.lmachines:
 691                 if mach.name == name_machine:
 692                     a_machine = mach
 693                     break
 694
 695             if a_machine == None:
 696                 for machine_def in self.cfg_jobs.machines:
 697                     if machine_def.name == name_machine:
 698                         if 'host' not in machine_def:
 699                             host = self.runner.cfg.VARS.hostname
 700                         else:
 701                             host = machine_def.host
 702
 703                         if 'user' not in machine_def:
 704                             user = self.runner.cfg.VARS.user
 705                         else:
 706                             user = machine_def.user
 707
 708                         if 'port' not in machine_def:
 709                             port = 22
 710                         else:
 711                             port = machine_def.port
 712
 713                         if 'password' not in machine_def:
 714                             passwd = None
 715                         else:
 716                             passwd = machine_def.password
 717
 718                         if 'sat_path' not in machine_def:
 719                             sat_path = "salomeTools"
 720                         else:
 721                             sat_path = machine_def.sat_path
 722
 723                         a_machine = Machine(
 724                                             machine_def.name,
 725                                             host,
 726                                             user,
 727                                             port=port,
 728                                             passwd=passwd,
 729                                             sat_path=sat_path
 730                                             )
 731
 732                         self.lmachines.append(a_machine)
 733                         if (host, port) not in host_list:
 734                             host_list.append((host, port))
 735
 736                 if a_machine == None:
 737                     msg = _("WARNING: The job \"%(job_name)s\" requires the "
 738                             "machine \"%(machine_name)s\" but this machine "
 739                             "is not defined in the configuration file.\n"
 740                             "The job will not be launched")
 741                     self.logger.write(src.printcolors.printcWarning(msg))
 742
 743             a_job = self.define_job(job_def, a_machine)
 744
 745             if today in job_def.when:
 746                 self.ljobs.append(a_job)
 747             else: # today in job_def.when
 748                 self.ljobs_not_today.append(a_job)
 749
 750         self.lhosts = host_list
 751
 752     def ssh_connection_all_machines(self, pad=50):
 753         '''Function that do the ssh connection to every machine
 754            to be used today.
 755
 756         :return: Nothing
 757         :rtype: N\A
 758         '''
 759         self.logger.write(src.printcolors.printcInfo((
 760                         "Establishing connection with all the machines :\n")))
 761         for machine in self.lmachines:
 762             # little algorithm in order to display traces
 763             begin_line = (_("Connection to %s: " % machine.name))
 764             if pad - len(begin_line) < 0:
 765                 endline = " "
 766             else:
 767                 endline = (pad - len(begin_line)) * "." + " "
 768
 769             step = "SSH connection"
 770             self.logger.write( begin_line + endline + step)
 771             self.logger.flush()
 772             # the call to the method that initiate the ssh connection
 773             msg = machine.connect(self.logger)
 774
 775             # Copy salomeTools to the remote machine
 776             if machine.successfully_connected(self.logger):
 777                 step = _("Copy SAT")
 778                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 779                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 780                 self.logger.flush()
 781                 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
 782                                             self.job_file_path)
 783                 # get the remote machine distribution using a sat command
 784                 (__, out_dist, __) = machine.exec_command(
 785                                 os.path.join(machine.sat_path,
 786                                     "sat config --value VARS.dist --no_label"),
 787                                 self.logger)
 788                 machine.distribution = out_dist.read().decode().replace("\n",
 789                                                                         "")
 790                 # Print the status of the copy
 791                 if res_copy == 0:
 792                     self.logger.write('\r%s' %
 793                                 ((len(begin_line)+len(endline)+20) * " "), 3)
 794                     self.logger.write('\r%s%s%s' %
 795                         (begin_line,
 796                          endline,
 797                          src.printcolors.printc(src.OK_STATUS)), 3)
 798                 else:
 799                     self.logger.write('\r%s' %
 800                             ((len(begin_line)+len(endline)+20) * " "), 3)
 801                     self.logger.write('\r%s%s%s %s' %
 802                         (begin_line,
 803                          endline,
 804                          src.printcolors.printc(src.KO_STATUS),
 805                          _("Copy of SAT failed: %s" % res_copy)), 3)
 806             else:
 807                 self.logger.write('\r%s' %
 808                                   ((len(begin_line)+len(endline)+20) * " "), 3)
 809                 self.logger.write('\r%s%s%s %s' %
 810                     (begin_line,
 811                      endline,
 812                      src.printcolors.printc(src.KO_STATUS),
 813                      msg), 3)
 814             self.logger.write("\n", 3)
 815
 816         self.logger.write("\n")
 817
 818
 819     def is_occupied(self, hostname):
 820         '''Function that returns True if a job is running on
 821            the machine defined by its host and its port.
 822
 823         :param hostname (str, int): the pair (host, port)
 824         :return: the job that is running on the host,
 825                 or false if there is no job running on the host.
 826         :rtype: job / bool
 827         '''
 828         host = hostname[0]
 829         port = hostname[1]
 830         for jb in self.ljobs:
 831             if jb.machine.host == host and jb.machine.port == port:
 832                 if jb.is_running():
 833                     return jb
 834         return False
 835
 836     def update_jobs_states_list(self):
 837         '''Function that updates the lists that store the currently
 838            running jobs and the jobs that have already finished.
 839
 840         :return: Nothing.
 841         :rtype: N\A
 842         '''
 843         jobs_finished_list = []
 844         jobs_running_list = []
 845         for jb in self.ljobs:
 846             if jb.is_running():
 847                 jobs_running_list.append(jb)
 848                 jb.check_time()
 849             if jb.has_finished():
 850                 jobs_finished_list.append(jb)
 851
 852         nb_job_finished_before = len(self._l_jobs_finished)
 853         self._l_jobs_finished = jobs_finished_list
 854         self._l_jobs_running = jobs_running_list
 855
 856         nb_job_finished_now = len(self._l_jobs_finished)
 857
 858         return nb_job_finished_now > nb_job_finished_before
 859
 860     def cancel_dependencies_of_failing_jobs(self):
 861         '''Function that cancels all the jobs that depend on a failing one.
 862
 863         :return: Nothing.
 864         :rtype: N\A
 865         '''
 866
 867         for job in self.ljobs:
 868             if job.after is None:
 869                 continue
 870             father_job = self.find_job_that_has_name(job.after)
 871             if father_job is not None and father_job.has_failed():
 872                 job.cancel()
 873
 874     def find_job_that_has_name(self, name):
 875         '''Returns the job by its name.
 876
 877         :param name str: a job name
 878         :return: the job that has the name.
 879         :rtype: job
 880         '''
 881         for jb in self.ljobs:
 882             if jb.name == name:
 883                 return jb
 884         # the following is executed only if the job was not found
 885         return None
 886
 887     def str_of_length(self, text, length):
 888         '''Takes a string text of any length and returns
 889            the most close string of length "length".
 890
 891         :param text str: any string
 892         :param length int: a length for the returned string
 893         :return: the most close string of length "length"
 894         :rtype: str
 895         '''
 896         if len(text) > length:
 897             text_out = text[:length-3] + '...'
 898         else:
 899             diff = length - len(text)
 900             before = " " * (diff//2)
 901             after = " " * (diff//2 + diff%2)
 902             text_out = before + text + after
 903
 904         return text_out
 905
 906     def display_status(self, len_col):
 907         '''Takes a lenght and construct the display of the current status
 908            of the jobs in an array that has a column for each host.
 909            It displays the job that is currently running on the host
 910            of the column.
 911
 912         :param len_col int: the size of the column
 913         :return: Nothing
 914         :rtype: N\A
 915         '''
 916
 917         display_line = ""
 918         for host_port in self.lhosts:
 919             jb = self.is_occupied(host_port)
 920             if not jb: # nothing running on the host
 921                 empty = self.str_of_length("empty", len_col)
 922                 display_line += "|" + empty
 923             else:
 924                 display_line += "|" + src.printcolors.printcInfo(
 925                                         self.str_of_length(jb.name, len_col))
 926
 927         self.logger.write("\r" + display_line + "|")
 928         self.logger.flush()
 929
 930
 931     def run_jobs(self):
 932         '''The main method. Runs all the jobs on every host.
 933            For each host, at a given time, only one job can be running.
 934            The jobs that have the field after (that contain the job that has
 935            to be run before it) are run after the previous job.
 936            This method stops when all the jobs are finished.
 937
 938         :return: Nothing
 939         :rtype: N\A
 940         '''
 941
 942         # Print header
 943         self.logger.write(src.printcolors.printcInfo(
 944                                                 _('Executing the jobs :\n')))
 945         text_line = ""
 946         for host_port in self.lhosts:
 947             host = host_port[0]
 948             port = host_port[1]
 949             if port == 22: # default value
 950                 text_line += "|" + self.str_of_length(host, self.len_columns)
 951             else:
 952                 text_line += "|" + self.str_of_length(
 953                                 "("+host+", "+str(port)+")", self.len_columns)
 954
 955         tiret_line = " " + "-"*(len(text_line)-1) + "\n"
 956         self.logger.write(tiret_line)
 957         self.logger.write(text_line + "|\n")
 958         self.logger.write(tiret_line)
 959         self.logger.flush()
 960
 961         # The infinite loop that runs the jobs
 962         l_jobs_not_started = src.deepcopy_list(self.ljobs)
 963         while len(self._l_jobs_finished) != len(self.ljobs):
 964             new_job_start = False
 965             for host_port in self.lhosts:
 966
 967                 if self.is_occupied(host_port):
 968                     continue
 969
 970                 for jb in l_jobs_not_started:
 971                     if (jb.machine.host, jb.machine.port) != host_port:
 972                         continue
 973                     if jb.after == None:
 974                         jb.run()
 975                         l_jobs_not_started.remove(jb)
 976                         new_job_start = True
 977                         break
 978                     else:
 979                         jb_before = self.find_job_that_has_name(jb.after)
 980                         if jb_before is None:
 981                             jb.cancel()
 982                             msg = _("This job was not launched because its "
 983                                     "father is not in the jobs list.")
 984                             jb.out = msg
 985                             jb.err = msg
 986                             break
 987                         if jb_before.has_finished():
 988                             jb.run()
 989                             l_jobs_not_started.remove(jb)
 990                             new_job_start = True
 991                             break
 992             self.cancel_dependencies_of_failing_jobs()
 993             new_job_finished = self.update_jobs_states_list()
 994
 995             if new_job_start or new_job_finished:
 996                 if self.gui:
 997                     self.gui.update_xml_files(self.ljobs)
 998                 # Display the current status
 999                 self.display_status(self.len_columns)
1000
1001             # Make sure that the proc is not entirely busy
1002             time.sleep(0.001)
1003
1004         self.logger.write("\n")
1005         self.logger.write(tiret_line)
1006         self.logger.write("\n\n")
1007
1008         if self.gui:
1009             self.gui.update_xml_files(self.ljobs)
1010             self.gui.last_update()
1011
1012     def write_all_results(self):
1013         '''Display all the jobs outputs.
1014
1015         :return: Nothing
1016         :rtype: N\A
1017         '''
1018
1019         for jb in self.ljobs:
1020             self.logger.write(src.printcolors.printcLabel(
1021                         "#------- Results for job %s -------#\n" % jb.name))
1022             jb.write_results()
1023             self.logger.write("\n\n")
1024
1025 class Gui(object):
1026     '''Class to manage the the xml data that can be displayed in a browser to
1027        see the jobs states
1028     '''
1029
1030     def __init__(self, xml_dir_path, l_jobs, l_jobs_not_today, prefix, file_boards=""):
1031         '''Initialization
1032
1033         :param xml_dir_path str: The path to the directory where to put
1034                                  the xml resulting files
1035         :param l_jobs List: the list of jobs that run today
1036         :param l_jobs_not_today List: the list of jobs that do not run today
1037         :param file_boards str: the file path from which to read the
1038                                    expected boards
1039         '''
1040         # The prefix to add to the xml files : date_hour
1041         self.prefix = prefix
1042
1043         # The path of the csv files to read to fill the expected boards
1044         self.file_boards = file_boards
1045
1046         if file_boards != "":
1047             today = datetime.date.weekday(datetime.date.today())
1048             self.parse_csv_boards(today)
1049         else:
1050             self.d_input_boards = {}
1051
1052         # The path of the global xml file
1053         self.xml_dir_path = xml_dir_path
1054         # Initialize the xml files
1055         self.global_name = "global_report"
1056         xml_global_path = os.path.join(self.xml_dir_path,
1057                                        self.global_name + ".xml")
1058         self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1059                                                          "JobsReport")
1060
1061         # Find history for each job
1062         self.history = {}
1063         self.find_history(l_jobs, l_jobs_not_today)
1064
1065         # The xml files that corresponds to the boards.
1066         # {name_board : xml_object}}
1067         self.d_xml_board_files = {}
1068
1069         # Create the lines and columns
1070         self.initialize_boards(l_jobs, l_jobs_not_today)
1071
1072         # Write the xml file
1073         self.update_xml_files(l_jobs)
1074
1075     def add_xml_board(self, name):
1076         '''Add a board to the board list
1077         :param name str: the board name
1078         '''
1079         xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1080         self.d_xml_board_files[name] =  src.xmlManager.XmlLogFile(
1081                                                     xml_board_path,
1082                                                     "JobsReport")
1083         self.d_xml_board_files[name].add_simple_node("distributions")
1084         self.d_xml_board_files[name].add_simple_node("applications")
1085         self.d_xml_board_files[name].add_simple_node("board", text=name)
1086
1087     def initialize_boards(self, l_jobs, l_jobs_not_today):
1088         '''Get all the first information needed for each file and write the
1089            first version of the files
1090         :param l_jobs List: the list of jobs that run today
1091         :param l_jobs_not_today List: the list of jobs that do not run today
1092         '''
1093         # Get the boards to fill and put it in a dictionary
1094         # {board_name : xml instance corresponding to the board}
1095         for job in l_jobs + l_jobs_not_today:
1096             board = job.board
1097             if (board is not None and
1098                                 board not in self.d_xml_board_files.keys()):
1099                 self.add_xml_board(board)
1100
1101         # Verify that the boards given as input are done
1102         for board in list(self.d_input_boards.keys()):
1103             if board not in self.d_xml_board_files:
1104                 self.add_xml_board(board)
1105             root_node = self.d_xml_board_files[board].xmlroot
1106             src.xmlManager.append_node_attrib(root_node,
1107                                               {"input_file" : self.file_boards})
1108
1109         # Loop over all jobs in order to get the lines and columns for each
1110         # xml file
1111         d_dist = {}
1112         d_application = {}
1113         for board in self.d_xml_board_files:
1114             d_dist[board] = []
1115             d_application[board] = []
1116
1117         l_hosts_ports = []
1118
1119         for job in l_jobs + l_jobs_not_today:
1120
1121             if (job.machine.host, job.machine.port) not in l_hosts_ports:
1122                 l_hosts_ports.append((job.machine.host, job.machine.port))
1123
1124             distrib = job.machine.distribution
1125             application = job.application
1126
1127             board_job = job.board
1128             if board is None:
1129                 continue
1130             for board in self.d_xml_board_files:
1131                 if board_job == board:
1132                     if distrib is not None and distrib not in d_dist[board]:
1133                         d_dist[board].append(distrib)
1134                         src.xmlManager.add_simple_node(
1135                             self.d_xml_board_files[board].xmlroot.find(
1136                                                             'distributions'),
1137                                                    "dist",
1138                                                    attrib={"name" : distrib})
1139
1140                 if board_job == board:
1141                     if (application is not None and
1142                                     application not in d_application[board]):
1143                         d_application[board].append(application)
1144                         src.xmlManager.add_simple_node(
1145                             self.d_xml_board_files[board].xmlroot.find(
1146                                                                 'applications'),
1147                                                    "application",
1148                                                    attrib={
1149                                                         "name" : application})
1150
1151         # Verify that there are no missing application or distribution in the
1152         # xml board files (regarding the input boards)
1153         for board in self.d_xml_board_files:
1154             l_dist = d_dist[board]
1155             if board not in self.d_input_boards.keys():
1156                 continue
1157             for dist in self.d_input_boards[board]["rows"]:
1158                 if dist not in l_dist:
1159                     src.xmlManager.add_simple_node(
1160                             self.d_xml_board_files[board].xmlroot.find(
1161                                                             'distributions'),
1162                                                    "dist",
1163                                                    attrib={"name" : dist})
1164             l_appli = d_application[board]
1165             for appli in self.d_input_boards[board]["columns"]:
1166                 if appli not in l_appli:
1167                     src.xmlManager.add_simple_node(
1168                             self.d_xml_board_files[board].xmlroot.find(
1169                                                                 'applications'),
1170                                                    "application",
1171                                                    attrib={"name" : appli})
1172
1173         # Initialize the hosts_ports node for the global file
1174         self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1175                                                                 "hosts_ports")
1176         for host, port in l_hosts_ports:
1177             host_port = "%s:%i" % (host, port)
1178             src.xmlManager.add_simple_node(self.xmlhosts_ports,
1179                                            "host_port",
1180                                            attrib={"name" : host_port})
1181
1182         # Initialize the jobs node in all files
1183         for xml_file in [self.xml_global_file] + list(
1184                                             self.d_xml_board_files.values()):
1185             xml_jobs = xml_file.add_simple_node("jobs")
1186             # Get the jobs present in the config file but
1187             # that will not be launched today
1188             self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1189
1190             # add also the infos node
1191             xml_file.add_simple_node("infos",
1192                                      attrib={"name" : "last update",
1193                                              "JobsCommandStatus" : "running"})
1194
1195             # and put the history node
1196             history_node = xml_file.add_simple_node("history")
1197             name_board = os.path.basename(xml_file.logFile)[:-len(".xml")]
1198             # serach for board files
1199             expression = "^[0-9]{8}_+[0-9]{6}_" + name_board + ".xml$"
1200             oExpr = re.compile(expression)
1201             # Get the list of xml borad files that are in the log directory
1202             for file_name in os.listdir(self.xml_dir_path):
1203                 if oExpr.search(file_name):
1204                     date = os.path.basename(file_name).split("_")[0]
1205                     file_path = os.path.join(self.xml_dir_path, file_name)
1206                     src.xmlManager.add_simple_node(history_node,
1207                                                    "link",
1208                                                    text=file_path,
1209                                                    attrib={"date" : date})
1210
1211
1212         # Find in each board the squares that needs to be filled regarding the
1213         # input csv files but that are not covered by a today job
1214         for board in self.d_input_boards.keys():
1215             xml_root_board = self.d_xml_board_files[board].xmlroot
1216             # Find the missing jobs for today
1217             xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1218                                                  "missing_jobs")
1219             for row, column in self.d_input_boards[board]["jobs"]:
1220                 found = False
1221                 for job in l_jobs:
1222                     if (job.application == column and
1223                         job.machine.distribution == row):
1224                         found = True
1225                         break
1226                 if not found:
1227                     src.xmlManager.add_simple_node(xml_missing,
1228                                             "job",
1229                                             attrib={"distribution" : row,
1230                                                     "application" : column })
1231             # Find the missing jobs not today
1232             xml_missing_not_today = src.xmlManager.add_simple_node(
1233                                                  xml_root_board,
1234                                                  "missing_jobs_not_today")
1235             for row, column in self.d_input_boards[board]["jobs_not_today"]:
1236                 found = False
1237                 for job in l_jobs_not_today:
1238                     if (job.application == column and
1239                         job.machine.distribution == row):
1240                         found = True
1241                         break
1242                 if not found:
1243                     src.xmlManager.add_simple_node(xml_missing_not_today,
1244                                             "job",
1245                                             attrib={"distribution" : row,
1246                                                     "application" : column })
1247
1248     def find_history(self, l_jobs, l_jobs_not_today):
1249         """find, for each job, in the existent xml boards the results for the
1250            job. Store the results in the dictionnary self.history = {name_job :
1251            list of (date, status, list links)}
1252
1253         :param l_jobs List: the list of jobs to run today
1254         :param l_jobs_not_today List: the list of jobs that do not run today
1255         """
1256         # load the all the history
1257         expression = "^[0-9]{8}_+[0-9]{6}_" + self.global_name + ".xml$"
1258         oExpr = re.compile(expression)
1259         # Get the list of global xml that are in the log directory
1260         l_globalxml = []
1261         for file_name in os.listdir(self.xml_dir_path):
1262             if oExpr.search(file_name):
1263                 file_path = os.path.join(self.xml_dir_path, file_name)
1264                 global_xml = src.xmlManager.ReadXmlFile(file_path)
1265                 l_globalxml.append(global_xml)
1266
1267         # Construct the dictionnary self.history
1268         for job in l_jobs + l_jobs_not_today:
1269             l_links = []
1270             for global_xml in l_globalxml:
1271                 date = os.path.basename(global_xml.filePath).split("_")[0]
1272                 global_root_node = global_xml.xmlroot.find("jobs")
1273                 job_node = src.xmlManager.find_node_by_attrib(
1274                                                               global_root_node,
1275                                                               "job",
1276                                                               "name",
1277                                                               job.name)
1278                 if job_node:
1279                     if job_node.find("remote_log_file_path") is not None:
1280                         link = job_node.find("remote_log_file_path").text
1281                         res_job = job_node.find("res").text
1282                         if link != "nothing":
1283                             l_links.append((date, res_job, link))
1284
1285             self.history[job.name] = l_links
1286
1287     def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1288         '''Get all the first information needed for each file and write the
1289            first version of the files
1290
1291         :param xml_node_jobs etree.Element: the node corresponding to a job
1292         :param l_jobs_not_today List: the list of jobs that do not run today
1293         '''
1294         for job in l_jobs_not_today:
1295             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1296                                                  "job",
1297                                                  attrib={"name" : job.name})
1298             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1299             src.xmlManager.add_simple_node(xmlj,
1300                                            "distribution",
1301                                            job.machine.distribution)
1302             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1303             src.xmlManager.add_simple_node(xmlj,
1304                                        "commands", " ; ".join(job.commands))
1305             src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1306             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1307             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1308             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1309             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1310             src.xmlManager.add_simple_node(xmlj, "sat_path",
1311                                                         job.machine.sat_path)
1312             xml_history = src.xmlManager.add_simple_node(xmlj, "history")
1313             for date, res_job, link in self.history[job.name]:
1314                 src.xmlManager.add_simple_node(xml_history,
1315                                                "link",
1316                                                text=link,
1317                                                attrib={"date" : date,
1318                                                        "res" : res_job})
1319
1320     def parse_csv_boards(self, today):
1321         """ Parse the csv file that describes the boards to produce and fill
1322             the dict d_input_boards that contain the csv file contain
1323
1324         :param today int: the current day of the week
1325         """
1326         # open the csv file and read its content
1327         l_read = []
1328         with open(self.file_boards, 'r') as f:
1329             reader = csv.reader(f,delimiter=CSV_DELIMITER)
1330             for row in reader:
1331                 l_read.append(row)
1332         # get the delimiter for the boards (empty line)
1333         boards_delimiter = [''] * len(l_read[0])
1334         # Make the list of boards, by splitting with the delimiter
1335         l_boards = [list(y) for x, y in itertools.groupby(l_read,
1336                                     lambda z: z == boards_delimiter) if not x]
1337
1338         # loop over the csv lists of lines and get the rows, columns and jobs
1339         d_boards = {}
1340         for input_board in l_boards:
1341             # get board name
1342             board_name = input_board[0][0]
1343
1344             # Get columns list
1345             columns = input_board[0][1:]
1346
1347             rows = []
1348             jobs = []
1349             jobs_not_today = []
1350             for line in input_board[1:]:
1351                 row = line[0]
1352                 rows.append(row)
1353                 for i, square in enumerate(line[1:]):
1354                     if square=='':
1355                         continue
1356                     days = square.split(DAYS_SEPARATOR)
1357                     days = [int(day) for day in days]
1358                     job = (row, columns[i])
1359                     if today in days:
1360                         jobs.append(job)
1361                     else:
1362                         jobs_not_today.append(job)
1363
1364             d_boards[board_name] = {"rows" : rows,
1365                                     "columns" : columns,
1366                                     "jobs" : jobs,
1367                                     "jobs_not_today" : jobs_not_today}
1368
1369         self.d_input_boards = d_boards
1370
1371     def update_xml_files(self, l_jobs):
1372         '''Write all the xml files with updated information about the jobs
1373
1374         :param l_jobs List: the list of jobs that run today
1375         '''
1376         for xml_file in [self.xml_global_file] + list(
1377                                             self.d_xml_board_files.values()):
1378             self.update_xml_file(l_jobs, xml_file)
1379
1380         # Write the file
1381         self.write_xml_files()
1382
1383     def update_xml_file(self, l_jobs, xml_file):
1384         '''update information about the jobs for the file xml_file
1385
1386         :param l_jobs List: the list of jobs that run today
1387         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1388         '''
1389
1390         xml_node_jobs = xml_file.xmlroot.find('jobs')
1391         # Update the job names and status node
1392         for job in l_jobs:
1393             # Find the node corresponding to the job and delete it
1394             # in order to recreate it
1395             for xmljob in xml_node_jobs.findall('job'):
1396                 if xmljob.attrib['name'] == job.name:
1397                     xml_node_jobs.remove(xmljob)
1398
1399             T0 = str(job._T0)
1400             if T0 != "-1":
1401                 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1402                                        time.localtime(job._T0))
1403             Tf = str(job._Tf)
1404             if Tf != "-1":
1405                 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1406                                        time.localtime(job._Tf))
1407
1408             # recreate the job node
1409             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1410                                                   "job",
1411                                                   attrib={"name" : job.name})
1412             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1413             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1414             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1415             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1416             xml_history = src.xmlManager.add_simple_node(xmlj, "history")
1417             for date, res_job, link in self.history[job.name]:
1418                 src.xmlManager.add_simple_node(xml_history,
1419                                                "link",
1420                                                text=link,
1421                                                attrib={"date" : date,
1422                                                        "res" : res_job})
1423
1424             src.xmlManager.add_simple_node(xmlj, "sat_path",
1425                                            job.machine.sat_path)
1426             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1427             src.xmlManager.add_simple_node(xmlj, "distribution",
1428                                            job.machine.distribution)
1429             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1430             src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1431             src.xmlManager.add_simple_node(xmlj, "commands",
1432                                            " ; ".join(job.commands))
1433             src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1434             src.xmlManager.add_simple_node(xmlj, "begin", T0)
1435             src.xmlManager.add_simple_node(xmlj, "end", Tf)
1436             src.xmlManager.add_simple_node(xmlj, "out",
1437                                            src.printcolors.cleancolor(job.out))
1438             src.xmlManager.add_simple_node(xmlj, "err",
1439                                            src.printcolors.cleancolor(job.err))
1440             src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1441             if len(job.remote_log_files) > 0:
1442                 src.xmlManager.add_simple_node(xmlj,
1443                                                "remote_log_file_path",
1444                                                job.remote_log_files[0])
1445             else:
1446                 src.xmlManager.add_simple_node(xmlj,
1447                                                "remote_log_file_path",
1448                                                "nothing")
1449
1450             xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1451             # get the job father
1452             if job.after is not None:
1453                 job_father = None
1454                 for jb in l_jobs:
1455                     if jb.name == job.after:
1456                         job_father = jb
1457
1458                 if (job_father is not None and
1459                         len(job_father.remote_log_files) > 0):
1460                     link = job_father.remote_log_files[0]
1461                 else:
1462                     link = "nothing"
1463                 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1464
1465             # Verify that the job is to be done today regarding the input csv
1466             # files
1467             if job.board and job.board in self.d_input_boards.keys():
1468                 found = False
1469                 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1470                     if (job.machine.distribution == dist
1471                         and job.application == appli):
1472                         found = True
1473                         src.xmlManager.add_simple_node(xmlj,
1474                                                "extra_job",
1475                                                "no")
1476                         break
1477                 if not found:
1478                     src.xmlManager.add_simple_node(xmlj,
1479                                                "extra_job",
1480                                                "yes")
1481
1482
1483         # Update the date
1484         xml_node_infos = xml_file.xmlroot.find('infos')
1485         src.xmlManager.append_node_attrib(xml_node_infos,
1486                     attrib={"value" :
1487                     datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1488
1489
1490
1491     def last_update(self, finish_status = "finished"):
1492         '''update information about the jobs for the file xml_file
1493
1494         :param l_jobs List: the list of jobs that run today
1495         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1496         '''
1497         for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1498             xml_node_infos = xml_file.xmlroot.find('infos')
1499             src.xmlManager.append_node_attrib(xml_node_infos,
1500                         attrib={"JobsCommandStatus" : finish_status})
1501         # Write the file
1502         self.write_xml_files()
1503
1504     def write_xml_file(self, xml_file, stylesheet):
1505         ''' Write one xml file and the same file with prefix
1506         '''
1507         xml_file.write_tree(stylesheet)
1508         file_path = xml_file.logFile
1509         file_dir = os.path.dirname(file_path)
1510         file_name = os.path.basename(file_path)
1511         file_name_with_prefix = self.prefix + "_" + file_name
1512         xml_file.write_tree(stylesheet, os.path.join(file_dir,
1513                                                      file_name_with_prefix))
1514
1515     def write_xml_files(self):
1516         ''' Write the xml files
1517         '''
1518         self.write_xml_file(self.xml_global_file, STYLESHEET_GLOBAL)
1519         for xml_file in self.d_xml_board_files.values():
1520             self.write_xml_file(xml_file, STYLESHEET_BOARD)
1521
1522
1523 ##
1524 # Describes the command
1525 def description():
1526     return _("The jobs command launches maintenances that are described"
1527              " in the dedicated jobs configuration file.\n\nexample:\nsat "
1528              "jobs --name my_jobs --publish")
1529
1530 ##
1531 # Runs the command.
1532 def run(args, runner, logger):
1533
1534     (options, args) = parser.parse_args(args)
1535
1536     l_cfg_dir = runner.cfg.PATHS.JOBPATH
1537
1538     # list option : display all the available config files
1539     if options.list:
1540         for cfg_dir in l_cfg_dir:
1541             if not options.no_label:
1542                 logger.write("------ %s\n" %
1543                                  src.printcolors.printcHeader(cfg_dir))
1544
1545             for f in sorted(os.listdir(cfg_dir)):
1546                 if not f.endswith('.pyconf'):
1547                     continue
1548                 cfilename = f[:-7]
1549                 logger.write("%s\n" % cfilename)
1550         return 0
1551
1552     # Make sure the jobs_config option has been called
1553     if not options.jobs_cfg:
1554         message = _("The option --jobs_config is required\n")
1555         src.printcolors.printcError(message)
1556         return 1
1557
1558     # Find the file in the directories
1559     found = False
1560     for cfg_dir in l_cfg_dir:
1561         file_jobs_cfg = os.path.join(cfg_dir, options.jobs_cfg)
1562         if not file_jobs_cfg.endswith('.pyconf'):
1563             file_jobs_cfg += '.pyconf'
1564
1565         if not os.path.exists(file_jobs_cfg):
1566             continue
1567         else:
1568             found = True
1569             break
1570
1571     if not found:
1572         msg = _("The file configuration %(name_file)s was not found."
1573                 "\nUse the --list option to get the possible files.")
1574         src.printcolors.printcError(msg)
1575         return 1
1576
1577     info = [
1578         (_("Platform"), runner.cfg.VARS.dist),
1579         (_("File containing the jobs configuration"), file_jobs_cfg)
1580     ]
1581     src.print_info(logger, info)
1582
1583     # Read the config that is in the file
1584     config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1585     if options.only_jobs:
1586         l_jb = src.pyconf.Sequence()
1587         for jb in config_jobs.jobs:
1588             if jb.name in options.only_jobs:
1589                 l_jb.append(jb,
1590                 "Adding a job that was given in only_jobs option parameters")
1591         config_jobs.jobs = l_jb
1592
1593     # Initialization
1594     today_jobs = Jobs(runner,
1595                       logger,
1596                       file_jobs_cfg,
1597                       config_jobs)
1598     # SSH connection to all machines
1599     today_jobs.ssh_connection_all_machines()
1600     if options.test_connection:
1601         return 0
1602
1603     gui = None
1604     if options.publish:
1605         # Copy the stylesheets in the log directory
1606         log_dir = runner.cfg.USER.log_dir
1607         xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1608         files_to_copy = []
1609         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1610         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1611         files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1612         for file_path in files_to_copy:
1613             shutil.copy2(file_path, log_dir)
1614
1615         # Instanciate the Gui in order to produce the xml files that contain all
1616         # the boards
1617         gui = Gui(runner.cfg.USER.log_dir,
1618                   today_jobs.ljobs,
1619                   today_jobs.ljobs_not_today,
1620                   runner.cfg.VARS.datehour,
1621                   file_boards = options.input_boards)
1622
1623         # Display the list of the xml files
1624         logger.write(src.printcolors.printcInfo(("Here is the list of published"
1625                                                  " files :\n")), 4)
1626         logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1627         for board in gui.d_xml_board_files.keys():
1628             file_path = gui.d_xml_board_files[board].logFile
1629             file_name = os.path.basename(file_path)
1630             logger.write("%s\n" % file_path, 4)
1631             logger.add_link(file_name, "board", 0, board)
1632
1633         logger.write("\n", 4)
1634
1635     today_jobs.gui = gui
1636
1637     interruped = False
1638     try:
1639         # Run all the jobs contained in config_jobs
1640         today_jobs.run_jobs()
1641     except KeyboardInterrupt:
1642         interruped = True
1643         logger.write("\n\n%s\n\n" %
1644                 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1645     finally:
1646         if interruped:
1647             msg = _("Killing the running jobs and trying"
1648                     " to get the corresponding logs\n")
1649             logger.write(src.printcolors.printcWarning(msg))
1650
1651         # find the potential not finished jobs and kill them
1652         for jb in today_jobs.ljobs:
1653             if not jb.has_finished():
1654                 try:
1655                     jb.kill_remote_process()
1656                 except Exception as e:
1657                     msg = _("Failed to kill job %s: %s\n" % (jb.name, e))
1658                     logger.write(src.printcolors.printcWarning(msg))
1659         if interruped:
1660             if today_jobs.gui:
1661                 today_jobs.gui.last_update(_("Forced interruption"))
1662         else:
1663             if today_jobs.gui:
1664                 today_jobs.gui.last_update()
1665         # Output the results
1666         today_jobs.write_all_results()