commands/jobs.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import datetime
  21 import time
  22 import csv
  23 import shutil
  24 import itertools
  25 import re
  26 import paramiko
  27
  28 import src
  29
  30 STYLESHEET_GLOBAL = "jobs_global_report.xsl"
  31 STYLESHEET_BOARD = "jobs_board_report.xsl"
  32
  33 DAYS_SEPARATOR = ","
  34 CSV_DELIMITER = ";"
  35
  36 parser = src.options.Options()
  37
  38 parser.add_option('n', 'name', 'string', 'jobs_cfg',
  39                   _('Mandatory: The name of the config file that contains'
  40                   ' the jobs configuration'))
  41 parser.add_option('o', 'only_jobs', 'list2', 'only_jobs',
  42                   _('Optional: the list of jobs to launch, by their name. '))
  43 parser.add_option('l', 'list', 'boolean', 'list',
  44                   _('Optional: list all available config files.'))
  45 parser.add_option('t', 'test_connection', 'boolean', 'test_connection',
  46                   _("Optional: try to connect to the machines. "
  47                     "Not executing the jobs."),
  48                   False)
  49 parser.add_option('p', 'publish', 'boolean', 'publish',
  50                   _("Optional: generate an xml file that can be read in a "
  51                     "browser to display the jobs status."),
  52                   False)
  53 parser.add_option('i', 'input_boards', 'string', 'input_boards', _("Optional: "
  54                                 "the path to csv file that contain "
  55                                 "the expected boards."),"")
  56 parser.add_option('', 'completion', 'boolean', 'no_label',
  57                   _("Optional (internal use): do not print labels, Works only "
  58                     "with --list."),
  59                   False)
  60
  61 class Machine(object):
  62     '''Class to manage a ssh connection on a machine
  63     '''
  64     def __init__(self,
  65                  name,
  66                  host,
  67                  user,
  68                  port=22,
  69                  passwd=None,
  70                  sat_path="salomeTools"):
  71         self.name = name
  72         self.host = host
  73         self.port = port
  74         self.distribution = None # Will be filled after copying SAT on the machine
  75         self.user = user
  76         self.password = passwd
  77         self.sat_path = sat_path
  78         self.ssh = paramiko.SSHClient()
  79         self._connection_successful = None
  80
  81     def connect(self, logger):
  82         '''Initiate the ssh connection to the remote machine
  83
  84         :param logger src.logger.Logger: The logger instance
  85         :return: Nothing
  86         :rtype: N\A
  87         '''
  88
  89         self._connection_successful = False
  90         self.ssh.load_system_host_keys()
  91         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
  92         try:
  93             self.ssh.connect(self.host,
  94                              port=self.port,
  95                              username=self.user,
  96                              password = self.password)
  97         except paramiko.AuthenticationException:
  98             message = src.KO_STATUS + _("Authentication failed")
  99         except paramiko.BadHostKeyException:
 100             message = (src.KO_STATUS +
 101                        _("The server's host key could not be verified"))
 102         except paramiko.SSHException:
 103             message = ( _("SSHException error connecting or "
 104                           "establishing an SSH session"))
 105         except:
 106             message = ( _("Error connecting or establishing an SSH session"))
 107         else:
 108             self._connection_successful = True
 109             message = ""
 110         return message
 111
 112     def successfully_connected(self, logger):
 113         '''Verify if the connection to the remote machine has succeed
 114
 115         :param logger src.logger.Logger: The logger instance
 116         :return: True if the connection has succeed, False if not
 117         :rtype: bool
 118         '''
 119         if self._connection_successful == None:
 120             message = _("Warning : trying to ask if the connection to "
 121             "(name: %s host: %s, port: %s, user: %s) is OK whereas there were"
 122             " no connection request" %
 123                         (self.name, self.host, self.port, self.user))
 124             logger.write( src.printcolors.printcWarning(message))
 125         return self._connection_successful
 126
 127     def copy_sat(self, sat_local_path, job_file):
 128         '''Copy salomeTools to the remote machine in self.sat_path
 129         '''
 130         res = 0
 131         try:
 132             # open a sftp connection
 133             self.sftp = self.ssh.open_sftp()
 134             # Create the sat directory on remote machine if it is not existing
 135             self.mkdir(self.sat_path, ignore_existing=True)
 136             # Put sat
 137             self.put_dir(sat_local_path, self.sat_path, filters = ['.git'])
 138             # put the job configuration file in order to make it reachable
 139             # on the remote machine
 140             self.sftp.put(job_file, os.path.join(self.sat_path,
 141                                                  ".jobs_command_file.pyconf"))
 142         except Exception as e:
 143             res = str(e)
 144             self._connection_successful = False
 145
 146         return res
 147
 148     def put_dir(self, source, target, filters = []):
 149         ''' Uploads the contents of the source directory to the target path. The
 150             target directory needs to exists. All sub-directories in source are
 151             created under target.
 152         '''
 153         for item in os.listdir(source):
 154             if item in filters:
 155                 continue
 156             source_path = os.path.join(source, item)
 157             destination_path = os.path.join(target, item)
 158             if os.path.islink(source_path):
 159                 linkto = os.readlink(source_path)
 160                 try:
 161                     self.sftp.symlink(linkto, destination_path)
 162                     self.sftp.chmod(destination_path,
 163                                     os.stat(source_path).st_mode)
 164                 except IOError:
 165                     pass
 166             else:
 167                 if os.path.isfile(source_path):
 168                     self.sftp.put(source_path, destination_path)
 169                     self.sftp.chmod(destination_path,
 170                                     os.stat(source_path).st_mode)
 171                 else:
 172                     self.mkdir(destination_path, ignore_existing=True)
 173                     self.put_dir(source_path, destination_path)
 174
 175     def mkdir(self, path, mode=511, ignore_existing=False):
 176         ''' Augments mkdir by adding an option to not fail
 177             if the folder exists
 178         '''
 179         try:
 180             self.sftp.mkdir(path, mode)
 181         except IOError:
 182             if ignore_existing:
 183                 pass
 184             else:
 185                 raise
 186
 187     def exec_command(self, command, logger):
 188         '''Execute the command on the remote machine
 189
 190         :param command str: The command to be run
 191         :param logger src.logger.Logger: The logger instance
 192         :return: the stdin, stdout, and stderr of the executing command,
 193                  as a 3-tuple
 194         :rtype: (paramiko.channel.ChannelFile, paramiko.channel.ChannelFile,
 195                 paramiko.channel.ChannelFile)
 196         '''
 197         try:
 198             # Does not wait the end of the command
 199             (stdin, stdout, stderr) = self.ssh.exec_command(command)
 200         except paramiko.SSHException:
 201             message = src.KO_STATUS + _(
 202                             ": the server failed to execute the command\n")
 203             logger.write( src.printcolors.printcError(message))
 204             return (None, None, None)
 205         except:
 206             logger.write( src.printcolors.printcError(src.KO_STATUS + '\n'))
 207             return (None, None, None)
 208         else:
 209             return (stdin, stdout, stderr)
 210
 211     def close(self):
 212         '''Close the ssh connection
 213
 214         :rtype: N\A
 215         '''
 216         self.ssh.close()
 217
 218     def write_info(self, logger):
 219         '''Prints the informations relative to the machine in the logger
 220            (terminal traces and log file)
 221
 222         :param logger src.logger.Logger: The logger instance
 223         :return: Nothing
 224         :rtype: N\A
 225         '''
 226         logger.write("host : " + self.host + "\n")
 227         logger.write("port : " + str(self.port) + "\n")
 228         logger.write("user : " + str(self.user) + "\n")
 229         if self.successfully_connected(logger):
 230             status = src.OK_STATUS
 231         else:
 232             status = src.KO_STATUS
 233         logger.write("Connection : " + status + "\n\n")
 234
 235
 236 class Job(object):
 237     '''Class to manage one job
 238     '''
 239     def __init__(self, name, machine, application, board,
 240                  commands, timeout, config, logger, after=None, prefix=None):
 241
 242         self.name = name
 243         self.machine = machine
 244         self.after = after
 245         self.timeout = timeout
 246         self.application = application
 247         self.board = board
 248         self.config = config
 249         self.logger = logger
 250         # The list of log files to download from the remote machine
 251         self.remote_log_files = []
 252
 253         # The remote command status
 254         # -1 means that it has not been launched,
 255         # 0 means success and 1 means fail
 256         self.res_job = "-1"
 257         self.cancelled = False
 258
 259         self._T0 = -1
 260         self._Tf = -1
 261         self._has_begun = False
 262         self._has_finished = False
 263         self._has_timouted = False
 264         self._stdin = None # Store the command inputs field
 265         self._stdout = None # Store the command outputs field
 266         self._stderr = None # Store the command errors field
 267
 268         self.out = ""
 269         self.err = ""
 270
 271         self.commands = commands
 272         self.command = (os.path.join(self.machine.sat_path, "sat") +
 273                         " -l " +
 274                         os.path.join(self.machine.sat_path,
 275                                      "list_log_files.txt") +
 276                         " job --jobs_config " +
 277                         os.path.join(self.machine.sat_path,
 278                                      ".jobs_command_file.pyconf") +
 279                         " --name " +
 280                         self.name)
 281         if prefix:
 282             self.command = prefix + ' "' + self.command +'"'
 283
 284     def get_pids(self):
 285         """ Get the pid(s) corresponding to the command that have been launched
 286             On the remote machine
 287
 288         :return: The list of integers corresponding to the found pids
 289         :rtype: List
 290         """
 291         pids = []
 292         cmd_pid = 'ps aux | grep "' + self.command + '" | awk \'{print $2}\''
 293         (_, out_pid, _) = self.machine.exec_command(cmd_pid, self.logger)
 294         pids_cmd = out_pid.readlines()
 295         pids_cmd = [str(src.only_numbers(pid)) for pid in pids_cmd]
 296         pids+=pids_cmd
 297         return pids
 298
 299     def kill_remote_process(self, wait=1):
 300         '''Kills the process on the remote machine.
 301
 302         :return: (the output of the kill, the error of the kill)
 303         :rtype: (str, str)
 304         '''
 305
 306         pids = self.get_pids()
 307         cmd_kill = " ; ".join([("kill -2 " + pid) for pid in pids])
 308         (_, out_kill, err_kill) = self.machine.exec_command(cmd_kill,
 309                                                             self.logger)
 310         time.sleep(wait)
 311         return (out_kill, err_kill)
 312
 313     def has_begun(self):
 314         '''Returns True if the job has already begun
 315
 316         :return: True if the job has already begun
 317         :rtype: bool
 318         '''
 319         return self._has_begun
 320
 321     def has_finished(self):
 322         '''Returns True if the job has already finished
 323            (i.e. all the commands have been executed)
 324            If it is finished, the outputs are stored in the fields out and err.
 325
 326         :return: True if the job has already finished
 327         :rtype: bool
 328         '''
 329
 330         # If the method has already been called and returned True
 331         if self._has_finished:
 332             return True
 333
 334         # If the job has not begun yet
 335         if not self.has_begun():
 336             return False
 337
 338         if self._stdout.channel.closed:
 339             self._has_finished = True
 340             # Store the result outputs
 341             self.out += self._stdout.read().decode()
 342             self.err += self._stderr.read().decode()
 343             # Put end time
 344             self._Tf = time.time()
 345             # And get the remote command status and log files
 346             self.get_log_files()
 347
 348         return self._has_finished
 349
 350     def get_log_files(self):
 351         """Get the log files produced by the command launched
 352            on the remote machine, and put it in the log directory of the user,
 353            so they can be accessible from
 354         """
 355         # Do not get the files if the command is not finished
 356         if not self.has_finished():
 357             msg = _("Trying to get log files whereas the job is not finished.")
 358             self.logger.write(src.printcolors.printcWarning(msg))
 359             return
 360
 361         # First get the file that contains the list of log files to get
 362         tmp_file_path = src.get_tmp_filename(self.config, "list_log_files.txt")
 363         remote_path = os.path.join(self.machine.sat_path, "list_log_files.txt")
 364         self.machine.sftp.get(
 365                     remote_path,
 366                     tmp_file_path)
 367
 368         # Read the file and get the result of the command and all the log files
 369         # to get
 370         fstream_tmp = open(tmp_file_path, "r")
 371         file_lines = fstream_tmp.readlines()
 372         file_lines = [line.replace("\n", "") for line in file_lines]
 373         fstream_tmp.close()
 374         os.remove(tmp_file_path)
 375
 376         try :
 377             # The first line is the result of the command (0 success or 1 fail)
 378             self.res_job = file_lines[0]
 379         except Exception as e:
 380             self.err += _("Unable to get status from remote file %s: %s" %
 381                                                     (remote_path, str(e)))
 382
 383         for i, job_path_remote in enumerate(file_lines[1:]):
 384             try:
 385                 # For each command, there is two files to get :
 386                 # 1- The xml file describing the command and giving the
 387                 # internal traces.
 388                 # 2- The txt file containing the system command traces (like
 389                 # traces produced by the "make" command)
 390                 # 3- In case of the test command, there is another file to get :
 391                 # the xml board that contain the test results
 392                 dirname = os.path.basename(os.path.dirname(job_path_remote))
 393                 if dirname != 'OUT' and dirname != 'TEST':
 394                     # Case 1-
 395                     local_path = os.path.join(os.path.dirname(
 396                                                         self.logger.logFilePath),
 397                                               os.path.basename(job_path_remote))
 398                     if i==0: # The first is the job command
 399                         self.logger.add_link(os.path.basename(job_path_remote),
 400                                              "job",
 401                                              self.res_job,
 402                                              self.command)
 403                 elif dirname == 'OUT':
 404                     # Case 2-
 405                     local_path = os.path.join(os.path.dirname(
 406                                                         self.logger.logFilePath),
 407                                               'OUT',
 408                                               os.path.basename(job_path_remote))
 409                 elif dirname == 'TEST':
 410                     # Case 3-
 411                     local_path = os.path.join(os.path.dirname(
 412                                                         self.logger.logFilePath),
 413                                               'TEST',
 414                                               os.path.basename(job_path_remote))
 415
 416                 # Get the file
 417                 if not os.path.exists(local_path):
 418                     self.machine.sftp.get(job_path_remote, local_path)
 419                 self.remote_log_files.append(local_path)
 420             except Exception as e:
 421                 self.err += _("Unable to get %s log file from remote: %s" %
 422                                                     (str(job_path_remote),
 423                                                      str(e)))
 424
 425     def has_failed(self):
 426         '''Returns True if the job has failed.
 427            A job is considered as failed if the machine could not be reached,
 428            if the remote command failed,
 429            or if the job finished with a time out.
 430
 431         :return: True if the job has failed
 432         :rtype: bool
 433         '''
 434         if not self.has_finished():
 435             return False
 436         if not self.machine.successfully_connected(self.logger):
 437             return True
 438         if self.is_timeout():
 439             return True
 440         if self.res_job == "1":
 441             return True
 442         return False
 443
 444     def cancel(self):
 445         """In case of a failing job, one has to cancel every job that depend
 446            on it. This method put the job as failed and will not be executed.
 447         """
 448         if self.cancelled:
 449             return
 450         self._has_begun = True
 451         self._has_finished = True
 452         self.cancelled = True
 453         self.out += _("This job was not launched because its father has failed.")
 454         self.err += _("This job was not launched because its father has failed.")
 455
 456     def is_running(self):
 457         '''Returns True if the job commands are running
 458
 459         :return: True if the job is running
 460         :rtype: bool
 461         '''
 462         return self.has_begun() and not self.has_finished()
 463
 464     def is_timeout(self):
 465         '''Returns True if the job commands has finished with timeout
 466
 467         :return: True if the job has finished with timeout
 468         :rtype: bool
 469         '''
 470         return self._has_timouted
 471
 472     def time_elapsed(self):
 473         """Get the time elapsed since the job launching
 474
 475         :return: The number of seconds
 476         :rtype: int
 477         """
 478         if not self.has_begun():
 479             return -1
 480         T_now = time.time()
 481         return T_now - self._T0
 482
 483     def check_time(self):
 484         """Verify that the job has not exceeded its timeout.
 485            If it has, kill the remote command and consider the job as finished.
 486         """
 487         if not self.has_begun():
 488             return
 489         if self.time_elapsed() > self.timeout:
 490             self._has_finished = True
 491             self._has_timouted = True
 492             self._Tf = time.time()
 493             self.get_pids()
 494             (out_kill, _) = self.kill_remote_process()
 495             self.out += "TIMEOUT \n" + out_kill.read().decode()
 496             self.err += "TIMEOUT : %s seconds elapsed\n" % str(self.timeout)
 497             try:
 498                 self.get_log_files()
 499             except Exception as e:
 500                 self.err += _("Unable to get remote log files: %s" % e)
 501
 502     def total_duration(self):
 503         """Give the total duration of the job
 504
 505         :return: the total duration of the job in seconds
 506         :rtype: int
 507         """
 508         return self._Tf - self._T0
 509
 510     def run(self):
 511         """Launch the job by executing the remote command.
 512         """
 513
 514         # Prevent multiple run
 515         if self.has_begun():
 516             msg = _("Warning: A job can only be launched one time")
 517             msg2 = _("Trying to launch the job \"%s\" whereas it has "
 518                      "already been launched." % self.name)
 519             self.logger.write(src.printcolors.printcWarning("%s\n%s\n" % (msg,
 520                                                                         msg2)))
 521             return
 522
 523         # Do not execute the command if the machine could not be reached
 524         if not self.machine.successfully_connected(self.logger):
 525             self._has_finished = True
 526             self.out = "N\A"
 527             self.err += ("Connection to machine (name : %s, host: %s, port:"
 528                         " %s, user: %s) has failed\nUse the log command "
 529                         "to get more information."
 530                         % (self.machine.name,
 531                            self.machine.host,
 532                            self.machine.port,
 533                            self.machine.user))
 534         else:
 535             # Usual case : Launch the command on remote machine
 536             self._T0 = time.time()
 537             self._stdin, self._stdout, self._stderr = self.machine.exec_command(
 538                                                                   self.command,
 539                                                                   self.logger)
 540             # If the results are not initialized, finish the job
 541             if (self._stdin, self._stdout, self._stderr) == (None, None, None):
 542                 self._has_finished = True
 543                 self._Tf = time.time()
 544                 self.out += "N\A"
 545                 self.err += "The server failed to execute the command"
 546
 547         # Put the beginning flag to true.
 548         self._has_begun = True
 549
 550     def write_results(self):
 551         """Display on the terminal all the job's information
 552         """
 553         self.logger.write("name : " + self.name + "\n")
 554         if self.after:
 555             self.logger.write("after : %s\n" % self.after)
 556         self.logger.write("Time elapsed : %4imin %2is \n" %
 557                      (self.total_duration()//60 , self.total_duration()%60))
 558         if self._T0 != -1:
 559             self.logger.write("Begin time : %s\n" %
 560                          time.strftime('%Y-%m-%d %H:%M:%S',
 561                                        time.localtime(self._T0)) )
 562         if self._Tf != -1:
 563             self.logger.write("End time   : %s\n\n" %
 564                          time.strftime('%Y-%m-%d %H:%M:%S',
 565                                        time.localtime(self._Tf)) )
 566
 567         machine_head = "Informations about connection :\n"
 568         underline = (len(machine_head) - 2) * "-"
 569         self.logger.write(src.printcolors.printcInfo(
 570                                                 machine_head+underline+"\n"))
 571         self.machine.write_info(self.logger)
 572
 573         self.logger.write(src.printcolors.printcInfo("out : \n"))
 574         if self.out == "":
 575             self.logger.write("Unable to get output\n")
 576         else:
 577             self.logger.write(self.out + "\n")
 578         self.logger.write(src.printcolors.printcInfo("err : \n"))
 579         self.logger.write(self.err + "\n")
 580
 581     def get_status(self):
 582         """Get the status of the job (used by the Gui for xml display)
 583
 584         :return: The current status of the job
 585         :rtype: String
 586         """
 587         if not self.machine.successfully_connected(self.logger):
 588             return "SSH connection KO"
 589         if not self.has_begun():
 590             return "Not launched"
 591         if self.cancelled:
 592             return "Cancelled"
 593         if self.is_running():
 594             return "running since " + time.strftime('%Y-%m-%d %H:%M:%S',
 595                                                     time.localtime(self._T0))
 596         if self.has_finished():
 597             if self.is_timeout():
 598                 return "Timeout since " + time.strftime('%Y-%m-%d %H:%M:%S',
 599                                                     time.localtime(self._Tf))
 600             return "Finished since " + time.strftime('%Y-%m-%d %H:%M:%S',
 601                                                      time.localtime(self._Tf))
 602
 603 class Jobs(object):
 604     '''Class to manage the jobs to be run
 605     '''
 606     def __init__(self,
 607                  runner,
 608                  logger,
 609                  job_file_path,
 610                  config_jobs,
 611                  lenght_columns = 20):
 612         # The jobs configuration
 613         self.cfg_jobs = config_jobs
 614         self.job_file_path = job_file_path
 615         # The machine that will be used today
 616         self.lmachines = []
 617         # The list of machine (hosts, port) that will be used today
 618         # (a same host can have several machine instances since there
 619         # can be several ssh parameters)
 620         self.lhosts = []
 621         # The jobs to be launched today
 622         self.ljobs = []
 623         # The jobs that will not be launched today
 624         self.ljobs_not_today = []
 625         self.runner = runner
 626         self.logger = logger
 627         self.len_columns = lenght_columns
 628
 629         # the list of jobs that have not been run yet
 630         self._l_jobs_not_started = []
 631         # the list of jobs that have already ran
 632         self._l_jobs_finished = []
 633         # the list of jobs that are running
 634         self._l_jobs_running = []
 635
 636         self.determine_jobs_and_machines()
 637
 638     def define_job(self, job_def, machine):
 639         '''Takes a pyconf job definition and a machine (from class machine)
 640            and returns the job instance corresponding to the definition.
 641
 642         :param job_def src.config.Mapping: a job definition
 643         :param machine machine: the machine on which the job will run
 644         :return: The corresponding job in a job class instance
 645         :rtype: job
 646         '''
 647         name = job_def.name
 648         cmmnds = job_def.commands
 649         if not "timeout" in job_def:
 650             timeout = 4*60*60 # default timeout = 4h
 651         else:
 652             timeout = job_def.timeout
 653         after = None
 654         if 'after' in job_def:
 655             after = job_def.after
 656         application = None
 657         if 'application' in job_def:
 658             application = job_def.application
 659         board = None
 660         if 'board' in job_def:
 661             board = job_def.board
 662         prefix = None
 663         if "prefix" in job_def:
 664             prefix = job_def.prefix
 665
 666         return Job(name,
 667                    machine,
 668                    application,
 669                    board,
 670                    cmmnds,
 671                    timeout,
 672                    self.runner.cfg,
 673                    self.logger,
 674                    after = after,
 675                    prefix = prefix)
 676
 677     def determine_jobs_and_machines(self):
 678         '''Function that reads the pyconf jobs definition and instantiates all
 679            the machines and jobs to be done today.
 680
 681         :return: Nothing
 682         :rtype: N\A
 683         '''
 684         today = datetime.date.weekday(datetime.date.today())
 685         host_list = []
 686
 687         for job_def in self.cfg_jobs.jobs :
 688
 689             if not "machine" in job_def:
 690                 msg = _('WARNING: The job "%s" do not have the key '
 691                        '"machine", this job is ignored.\n\n' % job_def.name)
 692                 self.logger.write(src.printcolors.printcWarning(msg))
 693                 continue
 694             name_machine = job_def.machine
 695
 696             a_machine = None
 697             for mach in self.lmachines:
 698                 if mach.name == name_machine:
 699                     a_machine = mach
 700                     break
 701
 702             if a_machine == None:
 703                 for machine_def in self.cfg_jobs.machines:
 704                     if machine_def.name == name_machine:
 705                         if 'host' not in machine_def:
 706                             host = self.runner.cfg.VARS.hostname
 707                         else:
 708                             host = machine_def.host
 709
 710                         if 'user' not in machine_def:
 711                             user = self.runner.cfg.VARS.user
 712                         else:
 713                             user = machine_def.user
 714
 715                         if 'port' not in machine_def:
 716                             port = 22
 717                         else:
 718                             port = machine_def.port
 719
 720                         if 'password' not in machine_def:
 721                             passwd = None
 722                         else:
 723                             passwd = machine_def.password
 724
 725                         if 'sat_path' not in machine_def:
 726                             sat_path = "salomeTools"
 727                         else:
 728                             sat_path = machine_def.sat_path
 729
 730                         a_machine = Machine(
 731                                             machine_def.name,
 732                                             host,
 733                                             user,
 734                                             port=port,
 735                                             passwd=passwd,
 736                                             sat_path=sat_path
 737                                             )
 738
 739                         self.lmachines.append(a_machine)
 740                         if (host, port) not in host_list:
 741                             host_list.append((host, port))
 742
 743                 if a_machine == None:
 744                     msg = _("WARNING: The job \"%(job_name)s\" requires the "
 745                             "machine \"%(machine_name)s\" but this machine "
 746                             "is not defined in the configuration file.\n"
 747                             "The job will not be launched")
 748                     self.logger.write(src.printcolors.printcWarning(msg))
 749
 750             a_job = self.define_job(job_def, a_machine)
 751
 752             if today in job_def.when:
 753                 self.ljobs.append(a_job)
 754             else: # today in job_def.when
 755                 self.ljobs_not_today.append(a_job)
 756
 757         self.lhosts = host_list
 758
 759     def ssh_connection_all_machines(self, pad=50):
 760         '''Function that do the ssh connection to every machine
 761            to be used today.
 762
 763         :return: Nothing
 764         :rtype: N\A
 765         '''
 766         self.logger.write(src.printcolors.printcInfo((
 767                         "Establishing connection with all the machines :\n")))
 768         for machine in self.lmachines:
 769             # little algorithm in order to display traces
 770             begin_line = (_("Connection to %s: " % machine.name))
 771             if pad - len(begin_line) < 0:
 772                 endline = " "
 773             else:
 774                 endline = (pad - len(begin_line)) * "." + " "
 775
 776             step = "SSH connection"
 777             self.logger.write( begin_line + endline + step)
 778             self.logger.flush()
 779             # the call to the method that initiate the ssh connection
 780             msg = machine.connect(self.logger)
 781
 782             # Copy salomeTools to the remote machine
 783             if machine.successfully_connected(self.logger):
 784                 step = _("Remove SAT")
 785                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 786                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 787                 (__, out_dist, __) = machine.exec_command(
 788                                                 "rm -rf %s" % machine.sat_path,
 789                                                 self.logger)
 790                 out_dist.read()
 791
 792                 self.logger.flush()
 793                 step = _("Copy SAT")
 794                 self.logger.write('\r%s%s%s' % (begin_line, endline, 20 * " "),3)
 795                 self.logger.write('\r%s%s%s' % (begin_line, endline, step), 3)
 796                 self.logger.flush()
 797                 res_copy = machine.copy_sat(self.runner.cfg.VARS.salometoolsway,
 798                                             self.job_file_path)
 799                 # get the remote machine distribution using a sat command
 800                 (__, out_dist, __) = machine.exec_command(
 801                                 os.path.join(machine.sat_path,
 802                                     "sat config --value VARS.dist --no_label"),
 803                                 self.logger)
 804                 machine.distribution = out_dist.read().decode().replace("\n",
 805                                                                         "")
 806                 # Print the status of the copy
 807                 if res_copy == 0:
 808                     self.logger.write('\r%s' %
 809                                 ((len(begin_line)+len(endline)+20) * " "), 3)
 810                     self.logger.write('\r%s%s%s' %
 811                         (begin_line,
 812                          endline,
 813                          src.printcolors.printc(src.OK_STATUS)), 3)
 814                 else:
 815                     self.logger.write('\r%s' %
 816                             ((len(begin_line)+len(endline)+20) * " "), 3)
 817                     self.logger.write('\r%s%s%s %s' %
 818                         (begin_line,
 819                          endline,
 820                          src.printcolors.printc(src.KO_STATUS),
 821                          _("Copy of SAT failed: %s" % res_copy)), 3)
 822             else:
 823                 self.logger.write('\r%s' %
 824                                   ((len(begin_line)+len(endline)+20) * " "), 3)
 825                 self.logger.write('\r%s%s%s %s' %
 826                     (begin_line,
 827                      endline,
 828                      src.printcolors.printc(src.KO_STATUS),
 829                      msg), 3)
 830             self.logger.write("\n", 3)
 831
 832         self.logger.write("\n")
 833
 834
 835     def is_occupied(self, hostname):
 836         '''Function that returns True if a job is running on
 837            the machine defined by its host and its port.
 838
 839         :param hostname (str, int): the pair (host, port)
 840         :return: the job that is running on the host,
 841                 or false if there is no job running on the host.
 842         :rtype: job / bool
 843         '''
 844         host = hostname[0]
 845         port = hostname[1]
 846         for jb in self.ljobs:
 847             if jb.machine.host == host and jb.machine.port == port:
 848                 if jb.is_running():
 849                     return jb
 850         return False
 851
 852     def update_jobs_states_list(self):
 853         '''Function that updates the lists that store the currently
 854            running jobs and the jobs that have already finished.
 855
 856         :return: Nothing.
 857         :rtype: N\A
 858         '''
 859         jobs_finished_list = []
 860         jobs_running_list = []
 861         for jb in self.ljobs:
 862             if jb.is_running():
 863                 jobs_running_list.append(jb)
 864                 jb.check_time()
 865             if jb.has_finished():
 866                 jobs_finished_list.append(jb)
 867
 868         nb_job_finished_before = len(self._l_jobs_finished)
 869         self._l_jobs_finished = jobs_finished_list
 870         self._l_jobs_running = jobs_running_list
 871
 872         nb_job_finished_now = len(self._l_jobs_finished)
 873
 874         return nb_job_finished_now > nb_job_finished_before
 875
 876     def cancel_dependencies_of_failing_jobs(self):
 877         '''Function that cancels all the jobs that depend on a failing one.
 878
 879         :return: Nothing.
 880         :rtype: N\A
 881         '''
 882
 883         for job in self.ljobs:
 884             if job.after is None:
 885                 continue
 886             father_job = self.find_job_that_has_name(job.after)
 887             if father_job is not None and father_job.has_failed():
 888                 job.cancel()
 889
 890     def find_job_that_has_name(self, name):
 891         '''Returns the job by its name.
 892
 893         :param name str: a job name
 894         :return: the job that has the name.
 895         :rtype: job
 896         '''
 897         for jb in self.ljobs:
 898             if jb.name == name:
 899                 return jb
 900         # the following is executed only if the job was not found
 901         return None
 902
 903     def str_of_length(self, text, length):
 904         '''Takes a string text of any length and returns
 905            the most close string of length "length".
 906
 907         :param text str: any string
 908         :param length int: a length for the returned string
 909         :return: the most close string of length "length"
 910         :rtype: str
 911         '''
 912         if len(text) > length:
 913             text_out = text[:length-3] + '...'
 914         else:
 915             diff = length - len(text)
 916             before = " " * (diff//2)
 917             after = " " * (diff//2 + diff%2)
 918             text_out = before + text + after
 919
 920         return text_out
 921
 922     def display_status(self, len_col):
 923         '''Takes a lenght and construct the display of the current status
 924            of the jobs in an array that has a column for each host.
 925            It displays the job that is currently running on the host
 926            of the column.
 927
 928         :param len_col int: the size of the column
 929         :return: Nothing
 930         :rtype: N\A
 931         '''
 932
 933         display_line = ""
 934         for host_port in self.lhosts:
 935             jb = self.is_occupied(host_port)
 936             if not jb: # nothing running on the host
 937                 empty = self.str_of_length("empty", len_col)
 938                 display_line += "|" + empty
 939             else:
 940                 display_line += "|" + src.printcolors.printcInfo(
 941                                         self.str_of_length(jb.name, len_col))
 942
 943         self.logger.write("\r" + display_line + "|")
 944         self.logger.flush()
 945
 946
 947     def run_jobs(self):
 948         '''The main method. Runs all the jobs on every host.
 949            For each host, at a given time, only one job can be running.
 950            The jobs that have the field after (that contain the job that has
 951            to be run before it) are run after the previous job.
 952            This method stops when all the jobs are finished.
 953
 954         :return: Nothing
 955         :rtype: N\A
 956         '''
 957
 958         # Print header
 959         self.logger.write(src.printcolors.printcInfo(
 960                                                 _('Executing the jobs :\n')))
 961         text_line = ""
 962         for host_port in self.lhosts:
 963             host = host_port[0]
 964             port = host_port[1]
 965             if port == 22: # default value
 966                 text_line += "|" + self.str_of_length(host, self.len_columns)
 967             else:
 968                 text_line += "|" + self.str_of_length(
 969                                 "("+host+", "+str(port)+")", self.len_columns)
 970
 971         tiret_line = " " + "-"*(len(text_line)-1) + "\n"
 972         self.logger.write(tiret_line)
 973         self.logger.write(text_line + "|\n")
 974         self.logger.write(tiret_line)
 975         self.logger.flush()
 976
 977         # The infinite loop that runs the jobs
 978         l_jobs_not_started = src.deepcopy_list(self.ljobs)
 979         while len(self._l_jobs_finished) != len(self.ljobs):
 980             new_job_start = False
 981             for host_port in self.lhosts:
 982
 983                 if self.is_occupied(host_port):
 984                     continue
 985
 986                 for jb in l_jobs_not_started:
 987                     if (jb.machine.host, jb.machine.port) != host_port:
 988                         continue
 989                     if jb.after == None:
 990                         jb.run()
 991                         l_jobs_not_started.remove(jb)
 992                         new_job_start = True
 993                         break
 994                     else:
 995                         jb_before = self.find_job_that_has_name(jb.after)
 996                         if jb_before is None:
 997                             jb.cancel()
 998                             msg = _("This job was not launched because its "
 999                                     "father is not in the jobs list.")
1000                             jb.out = msg
1001                             jb.err = msg
1002                             break
1003                         if jb_before.has_finished():
1004                             jb.run()
1005                             l_jobs_not_started.remove(jb)
1006                             new_job_start = True
1007                             break
1008             self.cancel_dependencies_of_failing_jobs()
1009             new_job_finished = self.update_jobs_states_list()
1010
1011             if new_job_start or new_job_finished:
1012                 if self.gui:
1013                     self.gui.update_xml_files(self.ljobs)
1014                 # Display the current status
1015                 self.display_status(self.len_columns)
1016
1017             # Make sure that the proc is not entirely busy
1018             time.sleep(0.001)
1019
1020         self.logger.write("\n")
1021         self.logger.write(tiret_line)
1022         self.logger.write("\n\n")
1023
1024         if self.gui:
1025             self.gui.update_xml_files(self.ljobs)
1026             self.gui.last_update()
1027
1028     def write_all_results(self):
1029         '''Display all the jobs outputs.
1030
1031         :return: Nothing
1032         :rtype: N\A
1033         '''
1034
1035         for jb in self.ljobs:
1036             self.logger.write(src.printcolors.printcLabel(
1037                         "#------- Results for job %s -------#\n" % jb.name))
1038             jb.write_results()
1039             self.logger.write("\n\n")
1040
1041 class Gui(object):
1042     '''Class to manage the the xml data that can be displayed in a browser to
1043        see the jobs states
1044     '''
1045
1046     def __init__(self, xml_dir_path, l_jobs, l_jobs_not_today, prefix, file_boards=""):
1047         '''Initialization
1048
1049         :param xml_dir_path str: The path to the directory where to put
1050                                  the xml resulting files
1051         :param l_jobs List: the list of jobs that run today
1052         :param l_jobs_not_today List: the list of jobs that do not run today
1053         :param file_boards str: the file path from which to read the
1054                                    expected boards
1055         '''
1056         # The prefix to add to the xml files : date_hour
1057         self.prefix = prefix
1058
1059         # The path of the csv files to read to fill the expected boards
1060         self.file_boards = file_boards
1061
1062         if file_boards != "":
1063             today = datetime.date.weekday(datetime.date.today())
1064             self.parse_csv_boards(today)
1065         else:
1066             self.d_input_boards = {}
1067
1068         # The path of the global xml file
1069         self.xml_dir_path = xml_dir_path
1070         # Initialize the xml files
1071         self.global_name = "global_report"
1072         xml_global_path = os.path.join(self.xml_dir_path,
1073                                        self.global_name + ".xml")
1074         self.xml_global_file = src.xmlManager.XmlLogFile(xml_global_path,
1075                                                          "JobsReport")
1076
1077         # Find history for each job
1078         self.history = {}
1079         self.find_history(l_jobs, l_jobs_not_today)
1080
1081         # The xml files that corresponds to the boards.
1082         # {name_board : xml_object}}
1083         self.d_xml_board_files = {}
1084
1085         # Create the lines and columns
1086         self.initialize_boards(l_jobs, l_jobs_not_today)
1087
1088         # Write the xml file
1089         self.update_xml_files(l_jobs)
1090
1091     def add_xml_board(self, name):
1092         '''Add a board to the board list
1093         :param name str: the board name
1094         '''
1095         xml_board_path = os.path.join(self.xml_dir_path, name + ".xml")
1096         self.d_xml_board_files[name] =  src.xmlManager.XmlLogFile(
1097                                                     xml_board_path,
1098                                                     "JobsReport")
1099         self.d_xml_board_files[name].add_simple_node("distributions")
1100         self.d_xml_board_files[name].add_simple_node("applications")
1101         self.d_xml_board_files[name].add_simple_node("board", text=name)
1102
1103     def initialize_boards(self, l_jobs, l_jobs_not_today):
1104         '''Get all the first information needed for each file and write the
1105            first version of the files
1106         :param l_jobs List: the list of jobs that run today
1107         :param l_jobs_not_today List: the list of jobs that do not run today
1108         '''
1109         # Get the boards to fill and put it in a dictionary
1110         # {board_name : xml instance corresponding to the board}
1111         for job in l_jobs + l_jobs_not_today:
1112             board = job.board
1113             if (board is not None and
1114                                 board not in self.d_xml_board_files.keys()):
1115                 self.add_xml_board(board)
1116
1117         # Verify that the boards given as input are done
1118         for board in list(self.d_input_boards.keys()):
1119             if board not in self.d_xml_board_files:
1120                 self.add_xml_board(board)
1121             root_node = self.d_xml_board_files[board].xmlroot
1122             src.xmlManager.append_node_attrib(root_node,
1123                                               {"input_file" : self.file_boards})
1124
1125         # Loop over all jobs in order to get the lines and columns for each
1126         # xml file
1127         d_dist = {}
1128         d_application = {}
1129         for board in self.d_xml_board_files:
1130             d_dist[board] = []
1131             d_application[board] = []
1132
1133         l_hosts_ports = []
1134
1135         for job in l_jobs + l_jobs_not_today:
1136
1137             if (job.machine.host, job.machine.port) not in l_hosts_ports:
1138                 l_hosts_ports.append((job.machine.host, job.machine.port))
1139
1140             distrib = job.machine.distribution
1141             application = job.application
1142
1143             board_job = job.board
1144             if board is None:
1145                 continue
1146             for board in self.d_xml_board_files:
1147                 if board_job == board:
1148                     if distrib is not None and distrib not in d_dist[board]:
1149                         d_dist[board].append(distrib)
1150                         src.xmlManager.add_simple_node(
1151                             self.d_xml_board_files[board].xmlroot.find(
1152                                                             'distributions'),
1153                                                    "dist",
1154                                                    attrib={"name" : distrib})
1155
1156                 if board_job == board:
1157                     if (application is not None and
1158                                     application not in d_application[board]):
1159                         d_application[board].append(application)
1160                         src.xmlManager.add_simple_node(
1161                             self.d_xml_board_files[board].xmlroot.find(
1162                                                                 'applications'),
1163                                                    "application",
1164                                                    attrib={
1165                                                         "name" : application})
1166
1167         # Verify that there are no missing application or distribution in the
1168         # xml board files (regarding the input boards)
1169         for board in self.d_xml_board_files:
1170             l_dist = d_dist[board]
1171             if board not in self.d_input_boards.keys():
1172                 continue
1173             for dist in self.d_input_boards[board]["rows"]:
1174                 if dist not in l_dist:
1175                     src.xmlManager.add_simple_node(
1176                             self.d_xml_board_files[board].xmlroot.find(
1177                                                             'distributions'),
1178                                                    "dist",
1179                                                    attrib={"name" : dist})
1180             l_appli = d_application[board]
1181             for appli in self.d_input_boards[board]["columns"]:
1182                 if appli not in l_appli:
1183                     src.xmlManager.add_simple_node(
1184                             self.d_xml_board_files[board].xmlroot.find(
1185                                                                 'applications'),
1186                                                    "application",
1187                                                    attrib={"name" : appli})
1188
1189         # Initialize the hosts_ports node for the global file
1190         self.xmlhosts_ports = self.xml_global_file.add_simple_node(
1191                                                                 "hosts_ports")
1192         for host, port in l_hosts_ports:
1193             host_port = "%s:%i" % (host, port)
1194             src.xmlManager.add_simple_node(self.xmlhosts_ports,
1195                                            "host_port",
1196                                            attrib={"name" : host_port})
1197
1198         # Initialize the jobs node in all files
1199         for xml_file in [self.xml_global_file] + list(
1200                                             self.d_xml_board_files.values()):
1201             xml_jobs = xml_file.add_simple_node("jobs")
1202             # Get the jobs present in the config file but
1203             # that will not be launched today
1204             self.put_jobs_not_today(l_jobs_not_today, xml_jobs)
1205
1206             # add also the infos node
1207             xml_file.add_simple_node("infos",
1208                                      attrib={"name" : "last update",
1209                                              "JobsCommandStatus" : "running"})
1210
1211             # and put the history node
1212             history_node = xml_file.add_simple_node("history")
1213             name_board = os.path.basename(xml_file.logFile)[:-len(".xml")]
1214             # serach for board files
1215             expression = "^[0-9]{8}_+[0-9]{6}_" + name_board + ".xml$"
1216             oExpr = re.compile(expression)
1217             # Get the list of xml borad files that are in the log directory
1218             for file_name in os.listdir(self.xml_dir_path):
1219                 if oExpr.search(file_name):
1220                     date = os.path.basename(file_name).split("_")[0]
1221                     file_path = os.path.join(self.xml_dir_path, file_name)
1222                     src.xmlManager.add_simple_node(history_node,
1223                                                    "link",
1224                                                    text=file_path,
1225                                                    attrib={"date" : date})
1226
1227
1228         # Find in each board the squares that needs to be filled regarding the
1229         # input csv files but that are not covered by a today job
1230         for board in self.d_input_boards.keys():
1231             xml_root_board = self.d_xml_board_files[board].xmlroot
1232             # Find the missing jobs for today
1233             xml_missing = src.xmlManager.add_simple_node(xml_root_board,
1234                                                  "missing_jobs")
1235             for row, column in self.d_input_boards[board]["jobs"]:
1236                 found = False
1237                 for job in l_jobs:
1238                     if (job.application == column and
1239                         job.machine.distribution == row):
1240                         found = True
1241                         break
1242                 if not found:
1243                     src.xmlManager.add_simple_node(xml_missing,
1244                                             "job",
1245                                             attrib={"distribution" : row,
1246                                                     "application" : column })
1247             # Find the missing jobs not today
1248             xml_missing_not_today = src.xmlManager.add_simple_node(
1249                                                  xml_root_board,
1250                                                  "missing_jobs_not_today")
1251             for row, column in self.d_input_boards[board]["jobs_not_today"]:
1252                 found = False
1253                 for job in l_jobs_not_today:
1254                     if (job.application == column and
1255                         job.machine.distribution == row):
1256                         found = True
1257                         break
1258                 if not found:
1259                     src.xmlManager.add_simple_node(xml_missing_not_today,
1260                                             "job",
1261                                             attrib={"distribution" : row,
1262                                                     "application" : column })
1263
1264     def find_history(self, l_jobs, l_jobs_not_today):
1265         """find, for each job, in the existent xml boards the results for the
1266            job. Store the results in the dictionnary self.history = {name_job :
1267            list of (date, status, list links)}
1268
1269         :param l_jobs List: the list of jobs to run today
1270         :param l_jobs_not_today List: the list of jobs that do not run today
1271         """
1272         # load the all the history
1273         expression = "^[0-9]{8}_+[0-9]{6}_" + self.global_name + ".xml$"
1274         oExpr = re.compile(expression)
1275         # Get the list of global xml that are in the log directory
1276         l_globalxml = []
1277         for file_name in os.listdir(self.xml_dir_path):
1278             if oExpr.search(file_name):
1279                 file_path = os.path.join(self.xml_dir_path, file_name)
1280                 global_xml = src.xmlManager.ReadXmlFile(file_path)
1281                 l_globalxml.append(global_xml)
1282
1283         # Construct the dictionnary self.history
1284         for job in l_jobs + l_jobs_not_today:
1285             l_links = []
1286             for global_xml in l_globalxml:
1287                 date = os.path.basename(global_xml.filePath).split("_")[0]
1288                 global_root_node = global_xml.xmlroot.find("jobs")
1289                 job_node = src.xmlManager.find_node_by_attrib(
1290                                                               global_root_node,
1291                                                               "job",
1292                                                               "name",
1293                                                               job.name)
1294                 if job_node:
1295                     if job_node.find("remote_log_file_path") is not None:
1296                         link = job_node.find("remote_log_file_path").text
1297                         res_job = job_node.find("res").text
1298                         if link != "nothing":
1299                             l_links.append((date, res_job, link))
1300             l_links = sorted(l_links, reverse=True)
1301             self.history[job.name] = l_links
1302
1303     def put_jobs_not_today(self, l_jobs_not_today, xml_node_jobs):
1304         '''Get all the first information needed for each file and write the
1305            first version of the files
1306
1307         :param xml_node_jobs etree.Element: the node corresponding to a job
1308         :param l_jobs_not_today List: the list of jobs that do not run today
1309         '''
1310         for job in l_jobs_not_today:
1311             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1312                                                  "job",
1313                                                  attrib={"name" : job.name})
1314             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1315             src.xmlManager.add_simple_node(xmlj,
1316                                            "distribution",
1317                                            job.machine.distribution)
1318             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1319             src.xmlManager.add_simple_node(xmlj,
1320                                        "commands", " ; ".join(job.commands))
1321             src.xmlManager.add_simple_node(xmlj, "state", "Not today")
1322             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1323             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1324             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1325             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1326             src.xmlManager.add_simple_node(xmlj, "sat_path",
1327                                                         job.machine.sat_path)
1328             xml_history = src.xmlManager.add_simple_node(xmlj, "history")
1329             for i, (date, res_job, link) in enumerate(self.history[job.name]):
1330                 if i==0:
1331                     # tag the first one (the last one)
1332                     src.xmlManager.add_simple_node(xml_history,
1333                                                    "link",
1334                                                    text=link,
1335                                                    attrib={"date" : date,
1336                                                            "res" : res_job,
1337                                                            "last" : "yes"})
1338                 else:
1339                     src.xmlManager.add_simple_node(xml_history,
1340                                                    "link",
1341                                                    text=link,
1342                                                    attrib={"date" : date,
1343                                                            "res" : res_job,
1344                                                            "last" : "no"})
1345
1346     def parse_csv_boards(self, today):
1347         """ Parse the csv file that describes the boards to produce and fill
1348             the dict d_input_boards that contain the csv file contain
1349
1350         :param today int: the current day of the week
1351         """
1352         # open the csv file and read its content
1353         l_read = []
1354         with open(self.file_boards, 'r') as f:
1355             reader = csv.reader(f,delimiter=CSV_DELIMITER)
1356             for row in reader:
1357                 l_read.append(row)
1358         # get the delimiter for the boards (empty line)
1359         boards_delimiter = [''] * len(l_read[0])
1360         # Make the list of boards, by splitting with the delimiter
1361         l_boards = [list(y) for x, y in itertools.groupby(l_read,
1362                                     lambda z: z == boards_delimiter) if not x]
1363
1364         # loop over the csv lists of lines and get the rows, columns and jobs
1365         d_boards = {}
1366         for input_board in l_boards:
1367             # get board name
1368             board_name = input_board[0][0]
1369
1370             # Get columns list
1371             columns = input_board[0][1:]
1372
1373             rows = []
1374             jobs = []
1375             jobs_not_today = []
1376             for line in input_board[1:]:
1377                 row = line[0]
1378                 rows.append(row)
1379                 for i, square in enumerate(line[1:]):
1380                     if square=='':
1381                         continue
1382                     days = square.split(DAYS_SEPARATOR)
1383                     days = [int(day) for day in days]
1384                     job = (row, columns[i])
1385                     if today in days:
1386                         jobs.append(job)
1387                     else:
1388                         jobs_not_today.append(job)
1389
1390             d_boards[board_name] = {"rows" : rows,
1391                                     "columns" : columns,
1392                                     "jobs" : jobs,
1393                                     "jobs_not_today" : jobs_not_today}
1394
1395         self.d_input_boards = d_boards
1396
1397     def update_xml_files(self, l_jobs):
1398         '''Write all the xml files with updated information about the jobs
1399
1400         :param l_jobs List: the list of jobs that run today
1401         '''
1402         for xml_file in [self.xml_global_file] + list(
1403                                             self.d_xml_board_files.values()):
1404             self.update_xml_file(l_jobs, xml_file)
1405
1406         # Write the file
1407         self.write_xml_files()
1408
1409     def update_xml_file(self, l_jobs, xml_file):
1410         '''update information about the jobs for the file xml_file
1411
1412         :param l_jobs List: the list of jobs that run today
1413         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1414         '''
1415
1416         xml_node_jobs = xml_file.xmlroot.find('jobs')
1417         # Update the job names and status node
1418         for job in l_jobs:
1419             # Find the node corresponding to the job and delete it
1420             # in order to recreate it
1421             for xmljob in xml_node_jobs.findall('job'):
1422                 if xmljob.attrib['name'] == job.name:
1423                     xml_node_jobs.remove(xmljob)
1424
1425             T0 = str(job._T0)
1426             if T0 != "-1":
1427                 T0 = time.strftime('%Y-%m-%d %H:%M:%S',
1428                                        time.localtime(job._T0))
1429             Tf = str(job._Tf)
1430             if Tf != "-1":
1431                 Tf = time.strftime('%Y-%m-%d %H:%M:%S',
1432                                        time.localtime(job._Tf))
1433
1434             # recreate the job node
1435             xmlj = src.xmlManager.add_simple_node(xml_node_jobs,
1436                                                   "job",
1437                                                   attrib={"name" : job.name})
1438             src.xmlManager.add_simple_node(xmlj, "machine", job.machine.name)
1439             src.xmlManager.add_simple_node(xmlj, "host", job.machine.host)
1440             src.xmlManager.add_simple_node(xmlj, "port", str(job.machine.port))
1441             src.xmlManager.add_simple_node(xmlj, "user", job.machine.user)
1442             xml_history = src.xmlManager.add_simple_node(xmlj, "history")
1443             for date, res_job, link in self.history[job.name]:
1444                 src.xmlManager.add_simple_node(xml_history,
1445                                                "link",
1446                                                text=link,
1447                                                attrib={"date" : date,
1448                                                        "res" : res_job})
1449
1450             src.xmlManager.add_simple_node(xmlj, "sat_path",
1451                                            job.machine.sat_path)
1452             src.xmlManager.add_simple_node(xmlj, "application", job.application)
1453             src.xmlManager.add_simple_node(xmlj, "distribution",
1454                                            job.machine.distribution)
1455             src.xmlManager.add_simple_node(xmlj, "board", job.board)
1456             src.xmlManager.add_simple_node(xmlj, "timeout", str(job.timeout))
1457             src.xmlManager.add_simple_node(xmlj, "commands",
1458                                            " ; ".join(job.commands))
1459             src.xmlManager.add_simple_node(xmlj, "state", job.get_status())
1460             src.xmlManager.add_simple_node(xmlj, "begin", T0)
1461             src.xmlManager.add_simple_node(xmlj, "end", Tf)
1462             src.xmlManager.add_simple_node(xmlj, "out",
1463                                            src.printcolors.cleancolor(job.out))
1464             src.xmlManager.add_simple_node(xmlj, "err",
1465                                            src.printcolors.cleancolor(job.err))
1466             src.xmlManager.add_simple_node(xmlj, "res", str(job.res_job))
1467             if len(job.remote_log_files) > 0:
1468                 src.xmlManager.add_simple_node(xmlj,
1469                                                "remote_log_file_path",
1470                                                job.remote_log_files[0])
1471             else:
1472                 src.xmlManager.add_simple_node(xmlj,
1473                                                "remote_log_file_path",
1474                                                "nothing")
1475
1476             xmlafter = src.xmlManager.add_simple_node(xmlj, "after", job.after)
1477             # get the job father
1478             if job.after is not None:
1479                 job_father = None
1480                 for jb in l_jobs:
1481                     if jb.name == job.after:
1482                         job_father = jb
1483
1484                 if (job_father is not None and
1485                         len(job_father.remote_log_files) > 0):
1486                     link = job_father.remote_log_files[0]
1487                 else:
1488                     link = "nothing"
1489                 src.xmlManager.append_node_attrib(xmlafter, {"link" : link})
1490
1491             # Verify that the job is to be done today regarding the input csv
1492             # files
1493             if job.board and job.board in self.d_input_boards.keys():
1494                 found = False
1495                 for dist, appli in self.d_input_boards[job.board]["jobs"]:
1496                     if (job.machine.distribution == dist
1497                         and job.application == appli):
1498                         found = True
1499                         src.xmlManager.add_simple_node(xmlj,
1500                                                "extra_job",
1501                                                "no")
1502                         break
1503                 if not found:
1504                     src.xmlManager.add_simple_node(xmlj,
1505                                                "extra_job",
1506                                                "yes")
1507
1508
1509         # Update the date
1510         xml_node_infos = xml_file.xmlroot.find('infos')
1511         src.xmlManager.append_node_attrib(xml_node_infos,
1512                     attrib={"value" :
1513                     datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
1514
1515
1516
1517     def last_update(self, finish_status = "finished"):
1518         '''update information about the jobs for the file xml_file
1519
1520         :param l_jobs List: the list of jobs that run today
1521         :param xml_file xmlManager.XmlLogFile: the xml instance to update
1522         '''
1523         for xml_file in [self.xml_global_file] + list(self.d_xml_board_files.values()):
1524             xml_node_infos = xml_file.xmlroot.find('infos')
1525             src.xmlManager.append_node_attrib(xml_node_infos,
1526                         attrib={"JobsCommandStatus" : finish_status})
1527         # Write the file
1528         self.write_xml_files()
1529
1530     def write_xml_file(self, xml_file, stylesheet):
1531         ''' Write one xml file and the same file with prefix
1532         '''
1533         xml_file.write_tree(stylesheet)
1534         file_path = xml_file.logFile
1535         file_dir = os.path.dirname(file_path)
1536         file_name = os.path.basename(file_path)
1537         file_name_with_prefix = self.prefix + "_" + file_name
1538         xml_file.write_tree(stylesheet, os.path.join(file_dir,
1539                                                      file_name_with_prefix))
1540
1541     def write_xml_files(self):
1542         ''' Write the xml files
1543         '''
1544         self.write_xml_file(self.xml_global_file, STYLESHEET_GLOBAL)
1545         for xml_file in self.d_xml_board_files.values():
1546             self.write_xml_file(xml_file, STYLESHEET_BOARD)
1547
1548
1549 ##
1550 # Describes the command
1551 def description():
1552     return _("The jobs command launches maintenances that are described"
1553              " in the dedicated jobs configuration file.\n\nexample:\nsat "
1554              "jobs --name my_jobs --publish")
1555
1556 ##
1557 # Runs the command.
1558 def run(args, runner, logger):
1559
1560     (options, args) = parser.parse_args(args)
1561
1562     l_cfg_dir = runner.cfg.PATHS.JOBPATH
1563
1564     # list option : display all the available config files
1565     if options.list:
1566         for cfg_dir in l_cfg_dir:
1567             if not options.no_label:
1568                 logger.write("------ %s\n" %
1569                                  src.printcolors.printcHeader(cfg_dir))
1570
1571             for f in sorted(os.listdir(cfg_dir)):
1572                 if not f.endswith('.pyconf'):
1573                     continue
1574                 cfilename = f[:-7]
1575                 logger.write("%s\n" % cfilename)
1576         return 0
1577
1578     # Make sure the jobs_config option has been called
1579     if not options.jobs_cfg:
1580         message = _("The option --jobs_config is required\n")
1581         src.printcolors.printcError(message)
1582         return 1
1583
1584     # Find the file in the directories, unless it is a full path
1585     found = False
1586     if os.path.exists(options.jobs_cfg):
1587         found = True
1588         file_jobs_cfg = options.jobs_cfg
1589     else:
1590         for cfg_dir in l_cfg_dir:
1591             file_jobs_cfg = os.path.join(cfg_dir, options.jobs_cfg)
1592             if not file_jobs_cfg.endswith('.pyconf'):
1593                 file_jobs_cfg += '.pyconf'
1594
1595             if not os.path.exists(file_jobs_cfg):
1596                 continue
1597             else:
1598                 found = True
1599                 break
1600
1601     if not found:
1602         msg = _("The file configuration %(name_file)s was not found."
1603                 "\nUse the --list option to get the possible files.")
1604         src.printcolors.printcError(msg)
1605         return 1
1606
1607     info = [
1608         (_("Platform"), runner.cfg.VARS.dist),
1609         (_("File containing the jobs configuration"), file_jobs_cfg)
1610     ]
1611     src.print_info(logger, info)
1612
1613     # Read the config that is in the file
1614     config_jobs = src.read_config_from_a_file(file_jobs_cfg)
1615     if options.only_jobs:
1616         l_jb = src.pyconf.Sequence()
1617         for jb in config_jobs.jobs:
1618             if jb.name in options.only_jobs:
1619                 l_jb.append(jb,
1620                 "Adding a job that was given in only_jobs option parameters")
1621         config_jobs.jobs = l_jb
1622
1623     # Initialization
1624     today_jobs = Jobs(runner,
1625                       logger,
1626                       file_jobs_cfg,
1627                       config_jobs)
1628     # SSH connection to all machines
1629     today_jobs.ssh_connection_all_machines()
1630     if options.test_connection:
1631         return 0
1632
1633     gui = None
1634     if options.publish:
1635         logger.write(src.printcolors.printcInfo(
1636                                         _("Initialize the xml boards : ")), 5)
1637         logger.flush()
1638
1639         # Copy the stylesheets in the log directory
1640         log_dir = runner.cfg.USER.log_dir
1641         xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl')
1642         files_to_copy = []
1643         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL))
1644         files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_BOARD))
1645         files_to_copy.append(os.path.join(xsl_dir, "running.gif"))
1646         for file_path in files_to_copy:
1647             shutil.copy2(file_path, log_dir)
1648
1649         # Instanciate the Gui in order to produce the xml files that contain all
1650         # the boards
1651         gui = Gui(runner.cfg.USER.log_dir,
1652                   today_jobs.ljobs,
1653                   today_jobs.ljobs_not_today,
1654                   runner.cfg.VARS.datehour,
1655                   file_boards = options.input_boards)
1656
1657         logger.write(src.printcolors.printcSuccess("OK"), 5)
1658         logger.write("\n\n", 5)
1659         logger.flush()
1660
1661         # Display the list of the xml files
1662         logger.write(src.printcolors.printcInfo(("Here is the list of published"
1663                                                  " files :\n")), 4)
1664         logger.write("%s\n" % gui.xml_global_file.logFile, 4)
1665         for board in gui.d_xml_board_files.keys():
1666             file_path = gui.d_xml_board_files[board].logFile
1667             file_name = os.path.basename(file_path)
1668             logger.write("%s\n" % file_path, 4)
1669             logger.add_link(file_name, "board", 0, board)
1670
1671         logger.write("\n", 4)
1672
1673     today_jobs.gui = gui
1674
1675     interruped = False
1676     try:
1677         # Run all the jobs contained in config_jobs
1678         today_jobs.run_jobs()
1679     except KeyboardInterrupt:
1680         interruped = True
1681         logger.write("\n\n%s\n\n" %
1682                 (src.printcolors.printcWarning(_("Forced interruption"))), 1)
1683     finally:
1684         res = 0
1685         if interruped:
1686             res = 1
1687             msg = _("Killing the running jobs and trying"
1688                     " to get the corresponding logs\n")
1689             logger.write(src.printcolors.printcWarning(msg))
1690
1691         # find the potential not finished jobs and kill them
1692         for jb in today_jobs.ljobs:
1693             if not jb.has_finished():
1694                 res = 1
1695                 try:
1696                     jb.kill_remote_process()
1697                 except Exception as e:
1698                     msg = _("Failed to kill job %s: %s\n" % (jb.name, e))
1699                     logger.write(src.printcolors.printcWarning(msg))
1700             if jb.res_job != "0":
1701                 res = 1
1702         if interruped:
1703             if today_jobs.gui:
1704                 today_jobs.gui.last_update(_("Forced interruption"))
1705         else:
1706             if today_jobs.gui:
1707                 today_jobs.gui.last_update()
1708         # Output the results
1709         today_jobs.write_all_results()
1710         return res