commands/test.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2012  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import sys
  21 import shutil
  22 import subprocess
  23 import datetime
  24 import gzip
  25
  26 try:
  27     from hashlib import sha1
  28 except ImportError:
  29     from sha import sha as sha1
  30
  31 import src
  32 import src.ElementTree as etree
  33 from src.xmlManager import add_simple_node
  34
  35 # Define all possible option for the test command :  sat test <options>
  36 parser = src.options.Options()
  37 parser.add_option('b', 'base', 'string', 'base',
  38     _("Indicate the name of the test base to use.\n\tThis name has to be "
  39       "registered in your application and in a project.\n\tA path to a test "
  40       "base can also be used."))
  41 parser.add_option('l', 'launcher', 'string', 'launcher',
  42     _("Use this option to specify the path to a SALOME launcher to use to "
  43       "launch the test scripts of the test base."))
  44 parser.add_option('g', 'grid', 'list', 'grids',
  45     _('Indicate which grid(s) to test (subdirectory of the test base).'))
  46 parser.add_option('s', 'session', 'list', 'sessions',
  47     _('Indicate which session(s) to test (subdirectory of the grid).'))
  48 parser.add_option('', 'display', 'string', 'display',
  49     _("Set the display where to launch SALOME."
  50 "\tIf value is NO then option --show-desktop=0 will be used to launch SALOME."))
  51
  52 def description():
  53     '''method that is called when salomeTools is called with --help option.
  54
  55     :return: The text to display for the test command description.
  56     :rtype: str
  57     '''
  58     return _("The test command runs a test base on a SALOME installation.")
  59
  60 def parse_option(args, config):
  61     """ Parse the options and do some verifications about it
  62
  63     :param args List: The list of arguments of the command
  64     :param config Config: The global configuration
  65     :return: the options of the current command launch and the full arguments
  66     :rtype: Tuple (options, args)
  67     """
  68     (options, args) = parser.parse_args(args)
  69
  70     if not options.launcher:
  71         options.launcher = ""
  72     elif not os.path.isabs(options.launcher):
  73         if not src.config_has_application(config):
  74             raise src.SatException(_("An application is required to use a "
  75                                      "relative path with option --appli"))
  76         options.launcher = os.path.join(config.APPLICATION.workdir,
  77                                         options.launcher)
  78
  79         if not os.path.exists(options.launcher):
  80             raise src.SatException(_("Launcher not found: %s") %
  81                                    options.launcher)
  82
  83     return (options, args)
  84
  85 def ask_a_path():
  86     """
  87     """
  88     path = raw_input("enter a path where to save the result: ")
  89     if path == "":
  90         result = raw_input("the result will be not save. Are you sure to "
  91                            "continue ? [y/n] ")
  92         if result == "y":
  93             return path
  94         else:
  95             return ask_a_path()
  96
  97     elif os.path.exists(path):
  98         result = raw_input("Warning, the content of %s will be deleted. Are you"
  99                            " sure to continue ? [y/n] " % path)
 100         if result == "y":
 101             return path
 102         else:
 103             return ask_a_path()
 104     else:
 105         return path
 106
 107 def save_file(filename, base):
 108     f = open(filename, 'r')
 109     content = f.read()
 110     f.close()
 111
 112     objectname = sha1(content).hexdigest()
 113
 114     f = gzip.open(os.path.join(base, '.objects', objectname), 'w')
 115     f.write(content)
 116     f.close()
 117     return objectname
 118
 119 def move_test_results(in_dir, what, out_dir, logger):
 120     if out_dir == in_dir:
 121         return
 122
 123     finalPath = out_dir
 124     pathIsOk = False
 125     while not pathIsOk:
 126         try:
 127             # create test results directory if necessary
 128             #logger.write("FINAL = %s\n" % finalPath, 5)
 129             if not os.access(finalPath, os.F_OK):
 130                 #shutil.rmtree(finalPath)
 131                 os.makedirs(finalPath)
 132             pathIsOk = True
 133         except:
 134             logger.error(_("%s cannot be created.") % finalPath)
 135             finalPath = ask_a_path()
 136
 137     if finalPath != "":
 138         os.makedirs(os.path.join(finalPath, what, 'BASES'))
 139
 140         # check if .objects directory exists
 141         if not os.access(os.path.join(finalPath, '.objects'), os.F_OK):
 142             os.makedirs(os.path.join(finalPath, '.objects'))
 143
 144         logger.write(_('copy tests results to %s ... ') % finalPath, 3)
 145         logger.flush()
 146         #logger.write("\n", 5)
 147
 148         # copy env_info.py
 149         shutil.copy2(os.path.join(in_dir, what, 'env_info.py'),
 150                      os.path.join(finalPath, what, 'env_info.py'))
 151
 152         # for all sub directory (ie testbase) in the BASES directory
 153         for testbase in os.listdir(os.path.join(in_dir, what, 'BASES')):
 154             outtestbase = os.path.join(finalPath, what, 'BASES', testbase)
 155             intestbase = os.path.join(in_dir, what, 'BASES', testbase)
 156
 157             # ignore files in root dir
 158             if not os.path.isdir(intestbase):
 159                 continue
 160
 161             os.makedirs(outtestbase)
 162             #logger.write("  copy testbase %s\n" % testbase, 5)
 163
 164             for grid_ in [m for m in os.listdir(intestbase) if os.path.isdir(
 165                                                 os.path.join(intestbase, m))]:
 166                 # ignore source configuration directories
 167                 if grid_[:4] == '.git' or grid_ == 'CVS':
 168                     continue
 169
 170                 outgrid = os.path.join(outtestbase, grid_)
 171                 ingrid = os.path.join(intestbase, grid_)
 172                 os.makedirs(outgrid)
 173                 #logger.write("    copy grid %s\n" % grid_, 5)
 174
 175                 if grid_ == 'RESSOURCES':
 176                     for file_name in os.listdir(ingrid):
 177                         if not os.path.isfile(os.path.join(ingrid,
 178                                                            file_name)):
 179                             continue
 180                         f = open(os.path.join(outgrid, file_name), "w")
 181                         f.write(save_file(os.path.join(ingrid, file_name),
 182                                           finalPath))
 183                         f.close()
 184                 else:
 185                     for session_name in [t for t in os.listdir(ingrid) if
 186                                       os.path.isdir(os.path.join(ingrid, t))]:
 187                         outsession = os.path.join(outgrid, session_name)
 188                         insession = os.path.join(ingrid, session_name)
 189                         os.makedirs(outsession)
 190
 191                         for file_name in os.listdir(insession):
 192                             if not os.path.isfile(os.path.join(insession,
 193                                                                file_name)):
 194                                 continue
 195                             if file_name.endswith('result.py'):
 196                                 shutil.copy2(os.path.join(insession, file_name),
 197                                              os.path.join(outsession, file_name))
 198                             else:
 199                                 f = open(os.path.join(outsession, file_name), "w")
 200                                 f.write(save_file(os.path.join(insession,
 201                                                                file_name),
 202                                                   finalPath))
 203                                 f.close()
 204
 205     logger.write(src.printcolors.printc("OK"), 3, False)
 206     logger.write("\n", 3, False)
 207
 208 def check_remote_machine(machine_name, logger):
 209     logger.write(_("\ncheck the display on %s\n" % machine_name), 4)
 210     ssh_cmd = 'ssh -o "StrictHostKeyChecking no" %s "ls"' % machine_name
 211     logger.write(_("Executing the command : %s " % ssh_cmd), 4)
 212     p = subprocess.Popen(ssh_cmd,
 213                          shell=True,
 214                          stdin =subprocess.PIPE,
 215                          stdout=subprocess.PIPE,
 216                          stderr=subprocess.PIPE)
 217     p.wait()
 218     if p.returncode != 0:
 219         logger.write(src.printcolors.printc(src.KO_STATUS) + "\n", 1)
 220         logger.write("    " + src.printcolors.printcError(p.stderr.read()), 2)
 221         logger.write(src.printcolors.printcWarning((
 222                                     "No ssh access to the display machine.")),1)
 223     else:
 224         logger.write(src.printcolors.printcSuccess(src.OK_STATUS) + "\n\n", 4)
 225
 226 ##
 227 # Creates the XML report for a product.
 228 def create_test_report(config, dest_path, stylesheet, xmlname=""):
 229     application_name = config.VARS.application
 230     withappli = src.config_has_application(config)
 231
 232     root = etree.Element("salome")
 233     prod_node = etree.Element("product", name=application_name, build=xmlname)
 234     root.append(prod_node)
 235
 236     if withappli:
 237
 238         add_simple_node(prod_node, "version_to_download",
 239                         config.APPLICATION.name)
 240
 241         add_simple_node(prod_node, "out_dir", config.APPLICATION.workdir)
 242
 243     # add environment
 244     exec_node = add_simple_node(prod_node, "exec")
 245     exec_node.append(etree.Element("env", name="Host", value=config.VARS.node))
 246     exec_node.append(etree.Element("env", name="Architecture",
 247                                    value=config.VARS.dist))
 248     exec_node.append(etree.Element("env", name="Number of processors",
 249                                    value=str(config.VARS.nb_proc)))
 250     exec_node.append(etree.Element("env", name="Begin date",
 251                                    value=src.parse_date(config.VARS.datehour)))
 252     exec_node.append(etree.Element("env", name="Command",
 253                                    value=config.VARS.command))
 254     exec_node.append(etree.Element("env", name="sat version",
 255                                    value=config.INTERNAL.sat_version))
 256
 257     if 'TESTS' in config:
 258         tests = add_simple_node(prod_node, "tests")
 259         known_errors = add_simple_node(prod_node, "known_errors")
 260         new_errors = add_simple_node(prod_node, "new_errors")
 261         amend = add_simple_node(prod_node, "amend")
 262         tt = {}
 263         for test in config.TESTS:
 264             if not tt.has_key(test.testbase):
 265                 tt[test.testbase] = [test]
 266             else:
 267                 tt[test.testbase].append(test)
 268
 269         for testbase in tt.keys():
 270             gn = add_simple_node(tests, "testbase")
 271             gn.attrib['name'] = testbase
 272             nb, nb_pass, nb_failed, nb_timeout, nb_not_run = 0, 0, 0, 0, 0
 273             grids = {}
 274             sessions = {}
 275             for test in tt[testbase]:
 276                 #print test.grid
 277                 if not grids.has_key(test.grid):
 278                     mn = add_simple_node(gn, "grid")
 279                     mn.attrib['name'] = test.grid
 280                     grids[test.grid] = mn
 281
 282                 if not sessions.has_key("%s/%s" % (test.grid, test.session)):
 283                     tyn = add_simple_node(mn, "session")
 284                     tyn.attrib['name'] = test.session
 285                     sessions["%s/%s" % (test.grid, test.session)] = tyn
 286
 287                 for script in test.script:
 288                     tn = add_simple_node(sessions[
 289                                     "%s/%s" % (test.grid, test.session)], "test")
 290                     tn.attrib['session'] = test.session
 291                     tn.attrib['script'] = script.name
 292                     if 'callback' in script:
 293                         try:
 294                             cnode = add_simple_node(tn, "callback")
 295                             if src.architecture.is_windows():
 296                                 import string
 297                                 cnode.text = filter(
 298                                                 lambda x: x in string.printable,
 299                                                 script.callback)
 300                             else:
 301                                 cnode.text = script.callback.decode(
 302                                                                 'string_escape')
 303                         except UnicodeDecodeError as exc:
 304                             zz = (script.callback[:exc.start] +
 305                                   '?' +
 306                                   script.callback[exc.end-2:])
 307                             cnode = add_simple_node(tn, "callback")
 308                             cnode.text = zz.decode("UTF-8")
 309
 310                     # Add the script content
 311                     cnode = add_simple_node(tn, "content")
 312                     cnode.text = script.content
 313
 314                     # Add the script execution log
 315                     cnode = add_simple_node(tn, "out")
 316                     cnode.text = script.out
 317
 318                     if 'amend' in script:
 319                         cnode = add_simple_node(tn, "amend")
 320                         cnode.text = script.amend.decode("UTF-8")
 321
 322                     if script.time < 0:
 323                         tn.attrib['exec_time'] = "?"
 324                     else:
 325                         tn.attrib['exec_time'] = "%.3f" % script.time
 326                     tn.attrib['res'] = script.res
 327
 328                     if "amend" in script:
 329                         amend_test = add_simple_node(amend, "atest")
 330                         amend_test.attrib['name'] = os.path.join(test.grid,
 331                                                                  test.session,
 332                                                                  script.name)
 333                         amend_test.attrib['reason'] = script.amend.decode(
 334                                                                         "UTF-8")
 335
 336                     # calculate status
 337                     nb += 1
 338                     if script.res == src.OK_STATUS: nb_pass += 1
 339                     elif script.res == src.TIMEOUT_STATUS: nb_timeout += 1
 340                     elif script.res == src.KO_STATUS: nb_failed += 1
 341                     else: nb_not_run += 1
 342
 343                     if "known_error" in script:
 344                         kf_script = add_simple_node(known_errors, "error")
 345                         kf_script.attrib['name'] = os.path.join(test.grid,
 346                                                                 test.session,
 347                                                                 script.name)
 348                         kf_script.attrib['date'] = script.known_error.date
 349                         kf_script.attrib[
 350                                     'expected'] = script.known_error.expected
 351                         kf_script.attrib[
 352                          'comment'] = script.known_error.comment.decode("UTF-8")
 353                         kf_script.attrib['fixed'] = str(
 354                                                        script.known_error.fixed)
 355                         overdue = datetime.datetime.today().strftime("%Y-%m-"
 356                                             "%d") > script.known_error.expected
 357                         if overdue:
 358                             kf_script.attrib['overdue'] = str(overdue)
 359
 360                     elif script.res == src.KO_STATUS:
 361                         new_err = add_simple_node(new_errors, "new_error")
 362                         script_path = os.path.join(test.grid,
 363                                                    test.session, script.name)
 364                         new_err.attrib['name'] = script_path
 365                         new_err.attrib['cmd'] = ("sat testerror %s -s %s -c 'my"
 366                                                  " comment' -p %s" % \
 367                             (application_name, script_path, config.VARS.dist))
 368
 369
 370             gn.attrib['total'] = str(nb)
 371             gn.attrib['pass'] = str(nb_pass)
 372             gn.attrib['failed'] = str(nb_failed)
 373             gn.attrib['timeout'] = str(nb_timeout)
 374             gn.attrib['not_run'] = str(nb_not_run)
 375
 376     if len(xmlname) == 0:
 377         xmlname = application_name
 378     if not xmlname.endswith(".xml"):
 379         xmlname += ".xml"
 380
 381     src.xmlManager.write_report(os.path.join(dest_path, xmlname),
 382                                 root,
 383                                 stylesheet)
 384     return src.OK_STATUS
 385
 386 def run(args, runner, logger):
 387     '''method that is called when salomeTools is called with test parameter.
 388     '''
 389     (options, args) = parse_option(args, runner.cfg)
 390
 391     with_application = False
 392     if runner.cfg.VARS.application != 'None':
 393         logger.write(_('Running tests on application %s\n') %
 394                             src.printcolors.printcLabel(
 395                                                 runner.cfg.VARS.application), 1)
 396         with_application = True
 397     elif not options.base:
 398         raise src.SatException(_('A test base is required. Use the --base '
 399                                  'option'))
 400
 401     if with_application:
 402         # check if environment is loaded
 403         if 'KERNEL_ROOT_DIR' in os.environ:
 404             logger.write(src.printcolors.printcWarning(_("WARNING: "
 405                             "SALOME environment already sourced")) + "\n", 1)
 406
 407
 408     elif options.launcher:
 409         logger.write(src.printcolors.printcWarning(_("Running SALOME "
 410                                                 "application.")) + "\n\n", 1)
 411     else:
 412         msg = _("Impossible to find any launcher.\nPlease specify an "
 413                 "application or a launcher")
 414         logger.write(src.printcolors.printcError(msg))
 415         logger.write("\n")
 416         return 1
 417
 418     # set the display
 419     show_desktop = (options.display and options.display.upper() == "NO")
 420     if options.display and options.display != "NO":
 421         remote_name = options.display.split(':')[0]
 422         if remote_name != "":
 423             check_remote_machine(remote_name, logger)
 424         # if explicitly set use user choice
 425         os.environ['DISPLAY'] = options.display
 426     elif 'DISPLAY' not in os.environ:
 427         # if no display set
 428         if ('display' in runner.cfg.SITE.test and
 429                                         len(runner.cfg.SITE.test.display) > 0):
 430             # use default value for test tool
 431             os.environ['DISPLAY'] = runner.cfg.SITE.test.display
 432         else:
 433             os.environ['DISPLAY'] = "localhost:0.0"
 434
 435     # initialization
 436     #################
 437     if with_application:
 438         tmp_dir = runner.cfg.SITE.test.tmp_dir_with_application
 439     else:
 440         tmp_dir = runner.cfg.SITE.test.tmp_dir
 441
 442     # remove previous tmp dir
 443     if os.access(tmp_dir, os.F_OK):
 444         try:
 445             shutil.rmtree(tmp_dir)
 446         except:
 447             logger.error(_("error removing TT_TMP_RESULT %s\n")
 448                                 % tmp_dir)
 449
 450     lines = []
 451     lines.append("date = '%s'" % runner.cfg.VARS.date)
 452     lines.append("hour = '%s'" % runner.cfg.VARS.hour)
 453     lines.append("node = '%s'" % runner.cfg.VARS.node)
 454     lines.append("arch = '%s'" % runner.cfg.VARS.dist)
 455
 456     if 'APPLICATION' in runner.cfg:
 457         lines.append("application_info = {}")
 458         lines.append("application_info['name'] = '%s'" %
 459                      runner.cfg.APPLICATION.name)
 460         lines.append("application_info['tag'] = '%s'" %
 461                      runner.cfg.APPLICATION.tag)
 462         lines.append("application_info['products'] = %s" %
 463                      str(runner.cfg.APPLICATION.products))
 464
 465     content = "\n".join(lines)
 466
 467     # create hash from context information
 468     dirname = sha1(content.encode()).hexdigest()
 469     base_dir = os.path.join(tmp_dir, dirname)
 470     os.makedirs(base_dir)
 471     os.environ['TT_TMP_RESULT'] = base_dir
 472
 473     # create env_info file
 474     f = open(os.path.join(base_dir, 'env_info.py'), "w")
 475     f.write(content)
 476     f.close()
 477
 478     # create working dir and bases dir
 479     working_dir = os.path.join(base_dir, 'WORK')
 480     os.makedirs(working_dir)
 481     os.makedirs(os.path.join(base_dir, 'BASES'))
 482     os.chdir(working_dir)
 483
 484     if 'PYTHONPATH' not in os.environ:
 485         os.environ['PYTHONPATH'] = ''
 486     else:
 487         for var in os.environ['PYTHONPATH'].split(':'):
 488             if var not in sys.path:
 489                 sys.path.append(var)
 490
 491     # launch of the tests
 492     #####################
 493     test_base = ""
 494     if options.base:
 495         test_base = options.base
 496     elif with_application and "test_base" in runner.cfg.APPLICATION:
 497         test_base = runner.cfg.APPLICATION.test_base.name
 498
 499     src.printcolors.print_value(logger, _('Display'), os.environ['DISPLAY'], 2)
 500     src.printcolors.print_value(logger, _('Timeout'),
 501                                 runner.cfg.SITE.test.timeout, 2)
 502     src.printcolors.print_value(logger, _("Working dir"), base_dir, 3)
 503
 504     # create the test object
 505     test_runner = src.test_module.Test(runner.cfg,
 506                                   logger,
 507                                   base_dir,
 508                                   testbase=test_base,
 509                                   grids=options.grids,
 510                                   sessions=options.sessions,
 511                                   launcher=options.launcher,
 512                                   show_desktop=show_desktop)
 513
 514     if not test_runner.test_base_found:
 515         # Fail
 516         return 1
 517
 518     # run the test
 519     logger.allowPrintLevel = False
 520     retcode = test_runner.run_all_tests()
 521     logger.allowPrintLevel = True
 522
 523     logger.write(_("Tests finished"), 1)
 524     logger.write("\n", 2, False)
 525
 526     logger.write(_("\nGenerate the specific test log\n"), 5)
 527     out_dir = os.path.join(runner.cfg.USER.log_dir, "TEST")
 528     src.ensure_path_exists(out_dir)
 529     name_xml_board = logger.logFileName.split(".")[0] + "board" + ".xml"
 530     create_test_report(runner.cfg,
 531                        out_dir,
 532                        "test.xsl",
 533                        xmlname = name_xml_board)
 534     xml_board_path = os.path.join(out_dir, name_xml_board)
 535     logger.l_logFiles.append(xml_board_path)
 536     logger.add_link(os.path.join("TEST", name_xml_board),
 537                     "board",
 538                     retcode,
 539                     "Click on the link to get the detailed test results")
 540
 541     return retcode
 542