commands/package.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2012  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import stat
  21 import shutil
  22 import datetime
  23 import tarfile
  24 import codecs
  25 import string
  26 import glob
  27 import pprint as PP
  28
  29 import src
  30
  31 from application import get_SALOME_modules
  32 import src.debug as DBG
  33
  34 BINARY = "binary"
  35 SOURCE = "Source"
  36 PROJECT = "Project"
  37 SAT = "Sat"
  38
  39 ARCHIVE_DIR = "ARCHIVES"
  40 PROJECT_DIR = "PROJECT"
  41
  42 IGNORED_DIRS = [".git", ".svn"]
  43 IGNORED_EXTENSIONS = []
  44
  45 PROJECT_TEMPLATE = """#!/usr/bin/env python
  46 #-*- coding:utf-8 -*-
  47
  48 # The path to the archive root directory
  49 root_path : $PWD + "/../"
  50 # path to the PROJECT
  51 project_path : $PWD + "/"
  52
  53 # Where to search the archives of the products
  54 ARCHIVEPATH : $root_path + "ARCHIVES"
  55 # Where to search the pyconf of the applications
  56 APPLICATIONPATH : $project_path + "applications/"
  57 # Where to search the pyconf of the products
  58 PRODUCTPATH : $project_path + "products/"
  59 # Where to search the pyconf of the jobs of the project
  60 JOBPATH : $project_path + "jobs/"
  61 # Where to search the pyconf of the machines of the project
  62 MACHINEPATH : $project_path + "machines/"
  63 """
  64
  65 LOCAL_TEMPLATE = ("""#!/usr/bin/env python
  66 #-*- coding:utf-8 -*-
  67
  68   LOCAL :
  69   {
  70     base : 'default'
  71     workdir : 'default'
  72     log_dir : 'default'
  73     archive_dir : 'default'
  74     VCS : None
  75     tag : None
  76   }
  77
  78 PROJECTS :
  79 {
  80 project_file_paths : [$VARS.salometoolsway + $VARS.sep + \"..\" + $VARS.sep"""
  81 """ + \"""" + PROJECT_DIR + """\" + $VARS.sep + "project.pyconf"]
  82 }
  83 """)
  84
  85 # Define all possible option for the package command :  sat package <options>
  86 parser = src.options.Options()
  87 parser.add_option('b', 'binaries', 'boolean', 'binaries',
  88     _('Optional: Produce a binary package.'), False)
  89 parser.add_option('f', 'force_creation', 'boolean', 'force_creation',
  90     _('Optional: Only binary package: produce the archive even if '
  91       'there are some missing products.'), False)
  92 parser.add_option('s', 'sources', 'boolean', 'sources',
  93     _('Optional: Produce a compilable archive of the sources of the '
  94       'application.'), False)
  95 parser.add_option('', 'with_vcs', 'boolean', 'with_vcs',
  96     _('Optional: Do not make archive for products in VCS mode (git, cvs, svn). '
  97       'Sat prepare will use VCS mode instead to retrieve them'),
  98     False)
  99 parser.add_option('', 'ftp', 'boolean', 'ftp',
 100     _('Optional: Do not embed archives for products in archive mode.'
 101     'Sat prepare will use ftp instead to retrieve them'),
 102     False)
 103 parser.add_option('p', 'project', 'string', 'project',
 104     _('Optional: Produce an archive that contains a project.'), "")
 105 parser.add_option('t', 'salometools', 'boolean', 'sat',
 106     _('Optional: Produce an archive that contains salomeTools.'), False)
 107 parser.add_option('n', 'name', 'string', 'name',
 108     _('Optional: The name or full path of the archive.'), None)
 109 parser.add_option('', 'add_files', 'list2', 'add_files',
 110     _('Optional: The list of additional files to add to the archive.'), [])
 111 parser.add_option('', 'without_properties', 'properties', 'without_properties',
 112     _('Optional: Filter the products by their properties.\n\tSyntax: '
 113       '--without_properties <property>:<value>'))
 114
 115
 116 def add_files(tar, name_archive, d_content, logger, f_exclude=None):
 117     '''Create an archive containing all directories and files that are given in
 118        the d_content argument.
 119
 120     :param tar tarfile: The tarfile instance used to make the archive.
 121     :param name_archive str: The name of the archive to make.
 122     :param d_content dict: The dictionary that contain all directories and files
 123                            to add in the archive.
 124                            d_content[label] =
 125                                         (path_on_local_machine, path_in_archive)
 126     :param logger Logger: the logging instance
 127     :param f_exclude Function: the function that filters
 128     :return: 0 if success, 1 if not.
 129     :rtype: int
 130     '''
 131     # get the max length of the messages in order to make the display
 132     max_len = len(max(d_content.keys(), key=len))
 133
 134     success = 0
 135     # loop over each directory or file stored in the d_content dictionary
 136     names = sorted(d_content.keys())
 137     DBG.write("add tar names", names)
 138
 139     # used to avoid duplications (for pip install in python, or single_install_dir cases)
 140     already_added=set()
 141     for name in names:
 142         # display information
 143         len_points = max_len - len(name) + 3
 144         local_path, archive_path = d_content[name]
 145         in_archive = os.path.join(name_archive, archive_path)
 146         logger.write(name + " " + len_points * "." + " "+ in_archive + " ", 3)
 147         # Get the local path and the path in archive
 148         # of the directory or file to add
 149         # Add it in the archive
 150         try:
 151             key=local_path+"->"+in_archive
 152             if key not in already_added:
 153                 tar.add(local_path, arcname=in_archive, exclude=f_exclude)
 154                 already_added.add(key)
 155             logger.write(src.printcolors.printcSuccess(_("OK")), 3)
 156         except Exception as e:
 157             logger.write(src.printcolors.printcError(_("KO ")), 3)
 158             logger.write(str(e), 3)
 159             success = 1
 160         logger.write("\n", 3)
 161     return success
 162
 163 def exclude_VCS_and_extensions(filename):
 164     ''' The function that is used to exclude from package the link to the
 165         VCS repositories (like .git)
 166
 167     :param filename Str: The filname to exclude (or not).
 168     :return: True if the file has to be exclude
 169     :rtype: Boolean
 170     '''
 171     for dir_name in IGNORED_DIRS:
 172         if dir_name in filename:
 173             return True
 174     for extension in IGNORED_EXTENSIONS:
 175         if filename.endswith(extension):
 176             return True
 177     return False
 178
 179 def produce_relative_launcher(config,
 180                               logger,
 181                               file_dir,
 182                               file_name,
 183                               binaries_dir_name):
 184     '''Create a specific SALOME launcher for the binary package. This launcher
 185        uses relative paths.
 186
 187     :param config Config: The global configuration.
 188     :param logger Logger: the logging instance
 189     :param file_dir str: the directory where to put the launcher
 190     :param file_name str: The launcher name
 191     :param binaries_dir_name str: the name of the repository where the binaries
 192                                   are, in the archive.
 193     :return: the path of the produced launcher
 194     :rtype: str
 195     '''
 196
 197     # get KERNEL installation path
 198     kernel_root_dir = os.path.join(binaries_dir_name, "KERNEL")
 199
 200     # set kernel bin dir (considering fhs property)
 201     kernel_cfg = src.product.get_product_config(config, "KERNEL")
 202     if src.get_property_in_product_cfg(kernel_cfg, "fhs"):
 203         bin_kernel_install_dir = os.path.join(kernel_root_dir,"bin")
 204     else:
 205         bin_kernel_install_dir = os.path.join(kernel_root_dir,"bin","salome")
 206
 207     # check if the application contains an application module
 208     # check also if the application has a distene product,
 209     # in this case get its licence file name
 210     l_product_info = src.product.get_products_infos(config.APPLICATION.products.keys(), config)
 211     salome_application_name="Not defined"
 212     distene_licence_file_name=False
 213     for prod_name, prod_info in l_product_info:
 214         # look for a "salome application" and a distene product
 215         if src.get_property_in_product_cfg(prod_info, "is_distene") == "yes":
 216             distene_licence_file_name = src.product.product_has_licence(prod_info,
 217                                             config.PATHS.LICENCEPATH)
 218         if src.get_property_in_product_cfg(prod_info, "is_salome_application") == "yes":
 219             salome_application_name=prod_info.name
 220
 221     # if the application contains an application module, we set ABSOLUTE_APPLI_PATH to it
 222     # if not we set it to KERNEL_INSTALL_DIR, which is sufficient, except for salome test
 223     if salome_application_name == "Not defined":
 224         app_root_dir=kernel_root_dir
 225     else:
 226         app_root_dir=os.path.join(binaries_dir_name, salome_application_name)
 227
 228     additional_env={}
 229     additional_env['sat_bin_kernel_install_dir'] = "out_dir_Path + " +\
 230                                                    config.VARS.sep + bin_kernel_install_dir
 231     if "python3" in config.APPLICATION and config.APPLICATION.python3 == "yes":
 232         additional_env['sat_python_version'] = 3
 233     else:
 234         additional_env['sat_python_version'] = 2
 235
 236     additional_env['ABSOLUTE_APPLI_PATH'] = "out_dir_Path" + config.VARS.sep + app_root_dir
 237
 238     # create an environment file writer
 239     writer = src.environment.FileEnvWriter(config,
 240                                            logger,
 241                                            file_dir,
 242                                            src_root=None,
 243                                            env_info=None)
 244
 245     filepath = os.path.join(file_dir, file_name)
 246     # Write
 247     writer.write_env_file(filepath,
 248                           False,  # for launch
 249                           "cfgForPy",
 250                           additional_env=additional_env,
 251                           no_path_init="False",
 252                           for_package = binaries_dir_name)
 253
 254     # Little hack to put out_dir_Path outside the strings
 255     src.replace_in_file(filepath, 'r"out_dir_Path', 'out_dir_Path + r"' )
 256     src.replace_in_file(filepath, "'out_dir_Path + ", "out_dir_Path + '" )
 257
 258     # A hack to put a call to a file for distene licence.
 259     # It does nothing to an application that has no distene product
 260     if distene_licence_file_name:
 261         logger.write("Application has a distene licence file! We use it in package launcher", 5)
 262         hack_for_distene_licence(filepath, distene_licence_file_name)
 263
 264     # change the rights in order to make the file executable for everybody
 265     os.chmod(filepath,
 266              stat.S_IRUSR |
 267              stat.S_IRGRP |
 268              stat.S_IROTH |
 269              stat.S_IWUSR |
 270              stat.S_IXUSR |
 271              stat.S_IXGRP |
 272              stat.S_IXOTH)
 273
 274     return filepath
 275
 276 def hack_for_distene_licence(filepath, licence_file):
 277     '''Replace the distene licence env variable by a call to a file.
 278
 279     :param filepath Str: The path to the launcher to modify.
 280     '''
 281     shutil.move(filepath, filepath + "_old")
 282     fileout= filepath
 283     filein = filepath + "_old"
 284     fin = open(filein, "r")
 285     fout = open(fileout, "w")
 286     text = fin.readlines()
 287     # Find the Distene section
 288     num_line = -1
 289     for i,line in enumerate(text):
 290         if "# Set DISTENE License" in line:
 291             num_line = i
 292             break
 293     if num_line == -1:
 294         # No distene product, there is nothing to do
 295         fin.close()
 296         for line in text:
 297             fout.write(line)
 298         fout.close()
 299         return
 300     del text[num_line +1]
 301     del text[num_line +1]
 302     text_to_insert ="""    try:
 303         distene_licence_file="%s"
 304         if sys.version_info[0] >= 3 and sys.version_info[1] >= 5:
 305             import importlib.util
 306             spec_dist = importlib.util.spec_from_file_location("distene_licence", distene_licence_file)
 307             distene=importlib.util.module_from_spec(spec_dist)
 308             spec_dist.loader.exec_module(distene)
 309         else:
 310             import imp
 311             distene = imp.load_source('distene_licence', distene_licence_file)
 312         distene.set_distene_variables(context)
 313     except:
 314         pass\n"""  % licence_file
 315     text.insert(num_line + 1, text_to_insert)
 316     for line in text:
 317         fout.write(line)
 318     fin.close()
 319     fout.close()
 320     return
 321
 322 def produce_relative_env_files(config,
 323                               logger,
 324                               file_dir,
 325                               binaries_dir_name):
 326     '''Create some specific environment files for the binary package. These
 327        files use relative paths.
 328
 329     :param config Config: The global configuration.
 330     :param logger Logger: the logging instance
 331     :param file_dir str: the directory where to put the files
 332     :param binaries_dir_name str: the name of the repository where the binaries
 333                                   are, in the archive.
 334     :return: the list of path of the produced environment files
 335     :rtype: List
 336     '''
 337     # create an environment file writer
 338     writer = src.environment.FileEnvWriter(config,
 339                                            logger,
 340                                            file_dir,
 341                                            src_root=None)
 342
 343     if src.architecture.is_windows():
 344       shell = "bat"
 345       filename  = "env_launch.bat"
 346     else:
 347       shell = "bash"
 348       filename  = "env_launch.sh"
 349
 350     # Write
 351     filepath = writer.write_env_file(filename,
 352                           False, # for launch
 353                           shell,
 354                           for_package = binaries_dir_name)
 355
 356     # Little hack to put out_dir_Path as environment variable
 357     src.replace_in_file(filepath, '"out_dir_Path', '"${out_dir_Path}' )
 358
 359     # change the rights in order to make the file executable for everybody
 360     os.chmod(filepath,
 361              stat.S_IRUSR |
 362              stat.S_IRGRP |
 363              stat.S_IROTH |
 364              stat.S_IWUSR |
 365              stat.S_IXUSR |
 366              stat.S_IXGRP |
 367              stat.S_IXOTH)
 368
 369     return filepath
 370
 371 def produce_install_bin_file(config,
 372                              logger,
 373                              file_dir,
 374                              d_sub,
 375                              file_name):
 376     '''Create a bash shell script which do substitutions in BIRARIES dir
 377        in order to use it for extra compilations.
 378
 379     :param config Config: The global configuration.
 380     :param logger Logger: the logging instance
 381     :param file_dir str: the directory where to put the files
 382     :param d_sub, dict: the dictionnary that contains the substitutions to be done
 383     :param file_name str: the name of the install script file
 384     :return: the produced file
 385     :rtype: str
 386     '''
 387     # Write
 388     filepath = os.path.join(file_dir, file_name)
 389     # open the file and write into it
 390     # use codec utf-8 as sat variables are in unicode
 391     with codecs.open(filepath, "w", 'utf-8') as installbin_file:
 392         installbin_template_path = os.path.join(config.VARS.internal_dir,
 393                                         "INSTALL_BIN.template")
 394
 395         # build the name of the directory that will contain the binaries
 396         binaries_dir_name = config.INTERNAL.config.binary_dir + config.VARS.dist
 397         # build the substitution loop
 398         loop_cmd = "for f in $(grep -RIl"
 399         for key in d_sub:
 400             loop_cmd += " -e "+ key
 401         loop_cmd += ' ' + config.INTERNAL.config.install_dir +\
 402                     '); do\n     sed -i "\n'
 403         for key in d_sub:
 404             loop_cmd += "        s?" + key + "?$(pwd)/" + d_sub[key] + "?g\n"
 405         loop_cmd += '            " $f\ndone'
 406
 407         d={}
 408         d["BINARIES_DIR"] = binaries_dir_name
 409         d["SUBSTITUTION_LOOP"]=loop_cmd
 410         d["INSTALL_DIR"]=config.INTERNAL.config.install_dir
 411
 412         # substitute the template and write it in file
 413         content=src.template.substitute(installbin_template_path, d)
 414         installbin_file.write(content)
 415         # change the rights in order to make the file executable for everybody
 416         os.chmod(filepath,
 417                  stat.S_IRUSR |
 418                  stat.S_IRGRP |
 419                  stat.S_IROTH |
 420                  stat.S_IWUSR |
 421                  stat.S_IXUSR |
 422                  stat.S_IXGRP |
 423                  stat.S_IXOTH)
 424
 425     return filepath
 426
 427 def product_appli_creation_script(config,
 428                                   logger,
 429                                   file_dir,
 430                                   binaries_dir_name):
 431     '''Create a script that can produce an application (EDF style) in the binary
 432        package.
 433
 434     :param config Config: The global configuration.
 435     :param logger Logger: the logging instance
 436     :param file_dir str: the directory where to put the file
 437     :param binaries_dir_name str: the name of the repository where the binaries
 438                                   are, in the archive.
 439     :return: the path of the produced script file
 440     :rtype: Str
 441     '''
 442     template_name = "create_appli.py.for_bin_packages.template"
 443     template_path = os.path.join(config.VARS.internal_dir, template_name)
 444     text_to_fill = open(template_path, "r").read()
 445     text_to_fill = text_to_fill.replace("TO BE FILLED 1",
 446                                         '"' + binaries_dir_name + '"')
 447
 448     text_to_add = ""
 449     for product_name in get_SALOME_modules(config):
 450         product_info = src.product.get_product_config(config, product_name)
 451
 452         if src.product.product_is_smesh_plugin(product_info):
 453             continue
 454
 455         if 'install_dir' in product_info and bool(product_info.install_dir):
 456             if src.product.product_is_cpp(product_info):
 457                 # cpp module
 458                 for cpp_name in src.product.get_product_components(product_info):
 459                     line_to_add = ("<module name=\"" +
 460                                    cpp_name +
 461                                    "\" gui=\"yes\" path=\"''' + "
 462                                    "os.path.join(dir_bin_name, \"" +
 463                                    cpp_name + "\") + '''\"/>")
 464             else:
 465                 # regular module
 466                 line_to_add = ("<module name=\"" +
 467                                product_name +
 468                                "\" gui=\"yes\" path=\"''' + "
 469                                "os.path.join(dir_bin_name, \"" +
 470                                product_name + "\") + '''\"/>")
 471             text_to_add += line_to_add + "\n"
 472
 473     filled_text = text_to_fill.replace("TO BE FILLED 2", text_to_add)
 474
 475     tmp_file_path = os.path.join(file_dir, "create_appli.py")
 476     ff = open(tmp_file_path, "w")
 477     ff.write(filled_text)
 478     ff.close()
 479
 480     # change the rights in order to make the file executable for everybody
 481     os.chmod(tmp_file_path,
 482              stat.S_IRUSR |
 483              stat.S_IRGRP |
 484              stat.S_IROTH |
 485              stat.S_IWUSR |
 486              stat.S_IXUSR |
 487              stat.S_IXGRP |
 488              stat.S_IXOTH)
 489
 490     return tmp_file_path
 491
 492 def binary_package(config, logger, options, tmp_working_dir):
 493     '''Prepare a dictionary that stores all the needed directories and files to
 494        add in a binary package.
 495
 496     :param config Config: The global configuration.
 497     :param logger Logger: the logging instance
 498     :param options OptResult: the options of the launched command
 499     :param tmp_working_dir str: The temporary local directory containing some
 500                                 specific directories or files needed in the
 501                                 binary package
 502     :return: the dictionary that stores all the needed directories and files to
 503              add in a binary package.
 504              {label : (path_on_local_machine, path_in_archive)}
 505     :rtype: dict
 506     '''
 507
 508     # Get the list of product installation to add to the archive
 509     l_products_name = sorted(config.APPLICATION.products.keys())
 510     l_product_info = src.product.get_products_infos(l_products_name,
 511                                                     config)
 512     l_install_dir = []
 513     l_source_dir = []
 514     l_not_installed = []
 515     l_sources_not_present = []
 516     generate_mesa_launcher = False  # a flag to know if we generate a mesa launcher
 517     if ("APPLICATION" in config  and
 518         "properties"  in config.APPLICATION  and
 519         "mesa_launcher_in_package"    in config.APPLICATION.properties  and
 520         config.APPLICATION.properties.mesa_launcher_in_package == "yes") :
 521             generate_mesa_launcher=True
 522
 523     for prod_name, prod_info in l_product_info:
 524         # skip product with property not_in_package set to yes
 525         if src.get_property_in_product_cfg(prod_info, "not_in_package") == "yes":
 526             continue
 527
 528         # Add the sources of the products that have the property
 529         # sources_in_package : "yes"
 530         if src.get_property_in_product_cfg(prod_info,
 531                                            "sources_in_package") == "yes":
 532             if os.path.exists(prod_info.source_dir):
 533                 l_source_dir.append((prod_name, prod_info.source_dir))
 534             else:
 535                 l_sources_not_present.append(prod_name)
 536
 537         # ignore the native and fixed products for install directories
 538         if (src.product.product_is_native(prod_info)
 539                 or src.product.product_is_fixed(prod_info)
 540                 or not src.product.product_compiles(prod_info)):
 541             continue
 542         if src.product.check_installation(config, prod_info):
 543             l_install_dir.append((prod_name, prod_info.install_dir))
 544         else:
 545             l_not_installed.append(prod_name)
 546
 547         # Add also the cpp generated modules (if any)
 548         if src.product.product_is_cpp(prod_info):
 549             # cpp module
 550             for name_cpp in src.product.get_product_components(prod_info):
 551                 install_dir = os.path.join(config.APPLICATION.workdir,
 552                                            config.INTERNAL.config.install_dir,
 553                                            name_cpp)
 554                 if os.path.exists(install_dir):
 555                     l_install_dir.append((name_cpp, install_dir))
 556                 else:
 557                     l_not_installed.append(name_cpp)
 558
 559     # check the name of the directory that (could) contains the binaries
 560     # from previous detar
 561     binaries_from_detar = os.path.join(
 562                               config.APPLICATION.workdir,
 563                               config.INTERNAL.config.binary_dir + config.VARS.dist)
 564     if os.path.exists(binaries_from_detar):
 565          logger.write("""
 566 WARNING: existing binaries directory from previous detar installation:
 567          %s
 568          To make new package from this, you have to:
 569          1) install binaries in INSTALL directory with the script "install_bin.sh"
 570             see README file for more details
 571          2) or recompile everything in INSTALL with "sat compile" command
 572             this step is long, and requires some linux packages to be installed
 573             on your system\n
 574 """ % binaries_from_detar)
 575
 576     # Print warning or error if there are some missing products
 577     if len(l_not_installed) > 0:
 578         text_missing_prods = ""
 579         for p_name in l_not_installed:
 580             text_missing_prods += "-" + p_name + "\n"
 581         if not options.force_creation:
 582             msg = _("ERROR: there are missing products installations:")
 583             logger.write("%s\n%s" % (src.printcolors.printcError(msg),
 584                                      text_missing_prods),
 585                          1)
 586             return None
 587         else:
 588             msg = _("WARNING: there are missing products installations:")
 589             logger.write("%s\n%s" % (src.printcolors.printcWarning(msg),
 590                                      text_missing_prods),
 591                          1)
 592
 593     # Do the same for sources
 594     if len(l_sources_not_present) > 0:
 595         text_missing_prods = ""
 596         for p_name in l_sources_not_present:
 597             text_missing_prods += "-" + p_name + "\n"
 598         if not options.force_creation:
 599             msg = _("ERROR: there are missing products sources:")
 600             logger.write("%s\n%s" % (src.printcolors.printcError(msg),
 601                                      text_missing_prods),
 602                          1)
 603             return None
 604         else:
 605             msg = _("WARNING: there are missing products sources:")
 606             logger.write("%s\n%s" % (src.printcolors.printcWarning(msg),
 607                                      text_missing_prods),
 608                          1)
 609
 610     # construct the name of the directory that will contain the binaries
 611     binaries_dir_name = config.INTERNAL.config.binary_dir + config.VARS.dist
 612
 613     # construct the correlation table between the product names, there
 614     # actual install directories and there install directory in archive
 615     d_products = {}
 616     for prod_name, install_dir in l_install_dir:
 617         path_in_archive = os.path.join(binaries_dir_name, os.path.basename(install_dir))
 618         d_products[prod_name + " (bin)"] = (install_dir, path_in_archive)
 619
 620     for prod_name, source_dir in l_source_dir:
 621         path_in_archive = os.path.join("SOURCES", prod_name)
 622         d_products[prod_name + " (sources)"] = (source_dir, path_in_archive)
 623
 624     # for packages of SALOME applications including KERNEL,
 625     # we produce a salome launcher or a virtual application (depending on salome version)
 626     if 'KERNEL' in config.APPLICATION.products:
 627         VersionSalome = src.get_salome_version(config)
 628         # Case where SALOME has the launcher that uses the SalomeContext API
 629         if VersionSalome >= 730:
 630             # create the relative launcher and add it to the files to add
 631             launcher_name = src.get_launcher_name(config)
 632             launcher_package = produce_relative_launcher(config,
 633                                                  logger,
 634                                                  tmp_working_dir,
 635                                                  launcher_name,
 636                                                  binaries_dir_name)
 637             d_products["launcher"] = (launcher_package, launcher_name)
 638
 639             # if the application contains mesa products, we generate in addition to the
 640             # classical salome launcher a launcher using mesa and called mesa_salome
 641             # (the mesa launcher will be used for remote usage through ssh).
 642             if generate_mesa_launcher:
 643                 #if there is one : store the use_mesa property
 644                 restore_use_mesa_option=None
 645                 if ('properties' in config.APPLICATION and
 646                     'use_mesa' in config.APPLICATION.properties):
 647                     restore_use_mesa_option = config.APPLICATION.properties.use_mesa
 648
 649                 # activate mesa property, and generate a mesa launcher
 650                 src.activate_mesa_property(config)  #activate use_mesa property
 651                 launcher_mesa_name="mesa_"+launcher_name
 652                 launcher_package_mesa = produce_relative_launcher(config,
 653                                                      logger,
 654                                                      tmp_working_dir,
 655                                                      launcher_mesa_name,
 656                                                      binaries_dir_name)
 657                 d_products["launcher (mesa)"] = (launcher_package_mesa, launcher_mesa_name)
 658
 659                 # if there was a use_mesa value, we restore it
 660                 # else we set it to the default value "no"
 661                 if restore_use_mesa_option != None:
 662                     config.APPLICATION.properties.use_mesa=restore_use_mesa_option
 663                 else:
 664                     config.APPLICATION.properties.use_mesa="no"
 665
 666             if options.sources:
 667                 # if we mix binaries and sources, we add a copy of the launcher,
 668                 # prefixed  with "bin",in order to avoid clashes
 669                 d_products["launcher (copy)"] = (launcher_package, "bin"+launcher_name)
 670         else:
 671             # Provide a script for the creation of an application EDF style
 672             appli_script = product_appli_creation_script(config,
 673                                                         logger,
 674                                                         tmp_working_dir,
 675                                                         binaries_dir_name)
 676
 677             d_products["appli script"] = (appli_script, "create_appli.py")
 678
 679     # Put also the environment file
 680     env_file = produce_relative_env_files(config,
 681                                            logger,
 682                                            tmp_working_dir,
 683                                            binaries_dir_name)
 684
 685     if src.architecture.is_windows():
 686       filename  = "env_launch.bat"
 687     else:
 688       filename  = "env_launch.sh"
 689     d_products["environment file"] = (env_file, filename)
 690
 691     return d_products
 692
 693 def source_package(sat, config, logger, options, tmp_working_dir):
 694     '''Prepare a dictionary that stores all the needed directories and files to
 695        add in a source package.
 696
 697     :param config Config: The global configuration.
 698     :param logger Logger: the logging instance
 699     :param options OptResult: the options of the launched command
 700     :param tmp_working_dir str: The temporary local directory containing some
 701                                 specific directories or files needed in the
 702                                 binary package
 703     :return: the dictionary that stores all the needed directories and files to
 704              add in a source package.
 705              {label : (path_on_local_machine, path_in_archive)}
 706     :rtype: dict
 707     '''
 708
 709     d_archives={}
 710     # Get all the products that are prepared using an archive
 711     # unless ftp mode is specified (in this case the user of the
 712     # archive will get the sources through the ftp mode of sat prepare
 713     if not options.ftp:
 714         logger.write("Find archive products ... ")
 715         d_archives, l_pinfo_vcs = get_archives(config, logger)
 716         logger.write("Done\n")
 717
 718     d_archives_vcs = {}
 719     if not options.with_vcs and len(l_pinfo_vcs) > 0:
 720         # Make archives with the products that are not prepared using an archive
 721         # (git, cvs, svn, etc)
 722         logger.write("Construct archives for vcs products ... ")
 723         d_archives_vcs = get_archives_vcs(l_pinfo_vcs,
 724                                           sat,
 725                                           config,
 726                                           logger,
 727                                           tmp_working_dir)
 728         logger.write("Done\n")
 729
 730     # Create a project
 731     logger.write("Create the project ... ")
 732     d_project = create_project_for_src_package(config,
 733                                                tmp_working_dir,
 734                                                options.with_vcs,
 735                                                options.ftp)
 736     logger.write("Done\n")
 737
 738     # Add salomeTools
 739     tmp_sat = add_salomeTools(config, tmp_working_dir)
 740     d_sat = {"salomeTools" : (tmp_sat, "salomeTools")}
 741
 742     # Add a sat symbolic link if not win
 743     if not src.architecture.is_windows():
 744         tmp_satlink_path = os.path.join(tmp_working_dir, 'sat')
 745         try:
 746             t = os.getcwd()
 747         except:
 748             # In the jobs, os.getcwd() can fail
 749             t = config.LOCAL.workdir
 750         os.chdir(tmp_working_dir)
 751         if os.path.lexists(tmp_satlink_path):
 752             os.remove(tmp_satlink_path)
 753         os.symlink(os.path.join('salomeTools', 'sat'), 'sat')
 754         os.chdir(t)
 755
 756         d_sat["sat link"] = (tmp_satlink_path, "sat")
 757
 758     d_source = src.merge_dicts(d_archives, d_archives_vcs, d_project, d_sat)
 759     return d_source
 760
 761 def get_archives(config, logger):
 762     '''Find all the products that are get using an archive and all the products
 763        that are get using a vcs (git, cvs, svn) repository.
 764
 765     :param config Config: The global configuration.
 766     :param logger Logger: the logging instance
 767     :return: the dictionary {name_product :
 768              (local path of its archive, path in the package of its archive )}
 769              and the list of specific configuration corresponding to the vcs
 770              products
 771     :rtype: (Dict, List)
 772     '''
 773     # Get the list of product informations
 774     l_products_name = config.APPLICATION.products.keys()
 775     l_product_info = src.product.get_products_infos(l_products_name,
 776                                                     config)
 777     d_archives = {}
 778     l_pinfo_vcs = []
 779     for p_name, p_info in l_product_info:
 780         # skip product with property not_in_package set to yes
 781         if src.get_property_in_product_cfg(p_info, "not_in_package") == "yes":
 782             continue
 783         # ignore the native and fixed products
 784         if (src.product.product_is_native(p_info)
 785                 or src.product.product_is_fixed(p_info)):
 786             continue
 787         if p_info.get_source == "archive":
 788             archive_path = p_info.archive_info.archive_name
 789             archive_name = os.path.basename(archive_path)
 790             d_archives[p_name] = (archive_path,
 791                                   os.path.join(ARCHIVE_DIR, archive_name))
 792             if (src.appli_test_property(config,"pip", "yes") and
 793                 src.product.product_test_property(p_info,"pip", "yes")):
 794                 # if pip mode is activated, and product is managed by pip
 795                 pip_wheels_dir=os.path.join(config.LOCAL.archive_dir,"wheels")
 796                 pip_wheel_pattern=os.path.join(pip_wheels_dir,
 797                     "%s-%s*" % (p_info.name, p_info.version))
 798                 pip_wheel_path=glob.glob(pip_wheel_pattern)
 799                 msg_pip_not_found="Error in get_archive, pip wheel for "\
 800                                   "product %s-%s was not found in %s directory"
 801                 msg_pip_two_or_more="Error in get_archive, several pip wheels for "\
 802                                   "product %s-%s were found in %s directory"
 803                 if len(pip_wheel_path)==0:
 804                     raise src.SatException(msg_pip_not_found %\
 805                         (p_info.name, p_info.version, pip_wheels_dir))
 806                 if len(pip_wheel_path)>1:
 807                     raise src.SatException(msg_pip_two_or_more %\
 808                         (p_info.name, p_info.version, pip_wheels_dir))
 809
 810                 pip_wheel_name=os.path.basename(pip_wheel_path[0])
 811                 d_archives[p_name+" (pip wheel)"]=(pip_wheel_path[0],
 812                     os.path.join(ARCHIVE_DIR, "wheels", pip_wheel_name))
 813         else:
 814             # this product is not managed by archive,
 815             # an archive of the vcs directory will be created by get_archive_vcs
 816             l_pinfo_vcs.append((p_name, p_info))
 817
 818     return d_archives, l_pinfo_vcs
 819
 820 def add_salomeTools(config, tmp_working_dir):
 821     '''Prepare a version of salomeTools that has a specific local.pyconf file
 822        configured for a source package.
 823
 824     :param config Config: The global configuration.
 825     :param tmp_working_dir str: The temporary local directory containing some
 826                                 specific directories or files needed in the
 827                                 source package
 828     :return: The path to the local salomeTools directory to add in the package
 829     :rtype: str
 830     '''
 831     # Copy sat in the temporary working directory
 832     sat_tmp_path = src.Path(os.path.join(tmp_working_dir, "salomeTools"))
 833     sat_running_path = src.Path(config.VARS.salometoolsway)
 834     sat_running_path.copy(sat_tmp_path)
 835
 836     # Update the local.pyconf file that contains the path to the project
 837     local_pyconf_name = "local.pyconf"
 838     local_pyconf_dir = os.path.join(tmp_working_dir, "salomeTools", "data")
 839     local_pyconf_file = os.path.join(local_pyconf_dir, local_pyconf_name)
 840     # Remove the .pyconf file in the root directory of salomeTools if there is
 841     # any. (For example when launching jobs, a pyconf file describing the jobs
 842     # can be here and is not useful)
 843     files_or_dir_SAT = os.listdir(os.path.join(tmp_working_dir, "salomeTools"))
 844     for file_or_dir in files_or_dir_SAT:
 845         if file_or_dir.endswith(".pyconf") or file_or_dir.endswith(".txt"):
 846             file_path = os.path.join(tmp_working_dir,
 847                                      "salomeTools",
 848                                      file_or_dir)
 849             os.remove(file_path)
 850
 851     ff = open(local_pyconf_file, "w")
 852     ff.write(LOCAL_TEMPLATE)
 853     ff.close()
 854
 855     return sat_tmp_path.path
 856
 857 def get_archives_vcs(l_pinfo_vcs, sat, config, logger, tmp_working_dir):
 858     '''For sources package that require that all products are get using an
 859        archive, one has to create some archive for the vcs products.
 860        So this method calls the clean and source command of sat and then create
 861        the archives.
 862
 863     :param l_pinfo_vcs List: The list of specific configuration corresponding to
 864                              each vcs product
 865     :param sat Sat: The Sat instance that can be called to clean and source the
 866                     products
 867     :param config Config: The global configuration.
 868     :param logger Logger: the logging instance
 869     :param tmp_working_dir str: The temporary local directory containing some
 870                                 specific directories or files needed in the
 871                                 source package
 872     :return: the dictionary that stores all the archives to add in the source
 873              package. {label : (path_on_local_machine, path_in_archive)}
 874     :rtype: dict
 875     '''
 876     # clean the source directory of all the vcs products, then use the source
 877     # command and thus construct an archive that will not contain the patches
 878     l_prod_names = [pn for pn, __ in l_pinfo_vcs]
 879     if False: # clean is dangerous in user/SOURCES, fixed in tmp_local_working_dir
 880       logger.write(_("\nclean sources\n"))
 881       args_clean = config.VARS.application
 882       args_clean += " --sources --products "
 883       args_clean += ",".join(l_prod_names)
 884       logger.write("WARNING: get_archives_vcs clean\n         '%s'\n" % args_clean, 1)
 885       sat.clean(args_clean, batch=True, verbose=0, logger_add_link = logger)
 886     if True:
 887       # source
 888       logger.write(_("get sources\n"))
 889       args_source = config.VARS.application
 890       args_source += " --products "
 891       args_source += ",".join(l_prod_names)
 892       svgDir = sat.cfg.APPLICATION.workdir
 893       tmp_local_working_dir = os.path.join(sat.cfg.APPLICATION.workdir, "tmp_package")  # to avoid too much big files in /tmp
 894       sat.cfg.APPLICATION.workdir = tmp_local_working_dir
 895       # DBG.write("SSS sat config.APPLICATION.workdir", sat.cfg.APPLICATION, True)
 896       # DBG.write("sat config id", id(sat.cfg), True)
 897       # shit as config is not same id() as for sat.source()
 898       # sat.source(args_source, batch=True, verbose=5, logger_add_link = logger)
 899       import source
 900       source.run(args_source, sat, logger) #use this mode as runner.cfg reference
 901
 902       # make the new archives
 903       d_archives_vcs = {}
 904       for pn, pinfo in l_pinfo_vcs:
 905           path_archive = make_archive(pn, pinfo, tmp_local_working_dir)
 906           logger.write("make archive vcs '%s'\n" % path_archive)
 907           d_archives_vcs[pn] = (path_archive,
 908                                 os.path.join(ARCHIVE_DIR, pn + ".tgz"))
 909       sat.cfg.APPLICATION.workdir = svgDir
 910       # DBG.write("END sat config", sat.cfg.APPLICATION, True)
 911     return d_archives_vcs
 912
 913 def make_archive(prod_name, prod_info, where):
 914     '''Create an archive of a product by searching its source directory.
 915
 916     :param prod_name str: The name of the product.
 917     :param prod_info Config: The specific configuration corresponding to the
 918                              product
 919     :param where str: The path of the repository where to put the resulting
 920                       archive
 921     :return: The path of the resulting archive
 922     :rtype: str
 923     '''
 924     path_targz_prod = os.path.join(where, prod_name + ".tgz")
 925     tar_prod = tarfile.open(path_targz_prod, mode='w:gz')
 926     local_path = prod_info.source_dir
 927     tar_prod.add(local_path,
 928                  arcname=prod_name,
 929                  exclude=exclude_VCS_and_extensions)
 930     tar_prod.close()
 931     return path_targz_prod
 932
 933 def create_project_for_src_package(config, tmp_working_dir, with_vcs, with_ftp):
 934     '''Create a specific project for a source package.
 935
 936     :param config Config: The global configuration.
 937     :param tmp_working_dir str: The temporary local directory containing some
 938                                 specific directories or files needed in the
 939                                 source package
 940     :param with_vcs boolean: True if the package is with vcs products (not
 941                              transformed into archive products)
 942     :param with_ftp boolean: True if the package use ftp servers to get archives
 943     :return: The dictionary
 944              {"project" : (produced project, project path in the archive)}
 945     :rtype: Dict
 946     '''
 947
 948     # Create in the working temporary directory the full project tree
 949     project_tmp_dir = os.path.join(tmp_working_dir, PROJECT_DIR)
 950     products_pyconf_tmp_dir = os.path.join(project_tmp_dir,
 951                                          "products")
 952     compil_scripts_tmp_dir = os.path.join(project_tmp_dir,
 953                                          "products",
 954                                          "compil_scripts")
 955     env_scripts_tmp_dir = os.path.join(project_tmp_dir,
 956                                          "products",
 957                                          "env_scripts")
 958     patches_tmp_dir = os.path.join(project_tmp_dir,
 959                                          "products",
 960                                          "patches")
 961     application_tmp_dir = os.path.join(project_tmp_dir,
 962                                          "applications")
 963     for directory in [project_tmp_dir,
 964                       compil_scripts_tmp_dir,
 965                       env_scripts_tmp_dir,
 966                       patches_tmp_dir,
 967                       application_tmp_dir]:
 968         src.ensure_path_exists(directory)
 969
 970     # Create the pyconf that contains the information of the project
 971     project_pyconf_name = "project.pyconf"
 972     project_pyconf_file = os.path.join(project_tmp_dir, project_pyconf_name)
 973     ff = open(project_pyconf_file, "w")
 974     ff.write(PROJECT_TEMPLATE)
 975     if with_ftp and len(config.PATHS.ARCHIVEFTP) > 0:
 976         ftp_path='ARCHIVEFTP : "'+config.PATHS.ARCHIVEFTP[0]
 977         for ftpserver in config.PATHS.ARCHIVEFTP[1:]:
 978             ftp_path=ftp_path+":"+ftpserver
 979         ftp_path+='"'
 980         ff.write("# ftp servers where to search for prerequisite archives\n")
 981         ff.write(ftp_path)
 982     # add licence paths if any
 983     if len(config.PATHS.LICENCEPATH) > 0:
 984         licence_path='LICENCEPATH : "'+config.PATHS.LICENCEPATH[0]
 985         for path in config.PATHS.LICENCEPATH[1:]:
 986             licence_path=licence_path+":"+path
 987         licence_path+='"'
 988         ff.write("\n# Where to search for licences\n")
 989         ff.write(licence_path)
 990
 991
 992     ff.close()
 993
 994     # Loop over the products to get there pyconf and all the scripts
 995     # (compilation, environment, patches)
 996     # and create the pyconf file to add to the project
 997     lproducts_name = config.APPLICATION.products.keys()
 998     l_products = src.product.get_products_infos(lproducts_name, config)
 999     for p_name, p_info in l_products:
1000         # skip product with property not_in_package set to yes
1001         if src.get_property_in_product_cfg(p_info, "not_in_package") == "yes":
1002             continue
1003         find_product_scripts_and_pyconf(p_name,
1004                                         p_info,
1005                                         config,
1006                                         with_vcs,
1007                                         compil_scripts_tmp_dir,
1008                                         env_scripts_tmp_dir,
1009                                         patches_tmp_dir,
1010                                         products_pyconf_tmp_dir)
1011
1012     find_application_pyconf(config, application_tmp_dir)
1013
1014     d_project = {"project" : (project_tmp_dir, PROJECT_DIR )}
1015     return d_project
1016
1017 def find_product_scripts_and_pyconf(p_name,
1018                                     p_info,
1019                                     config,
1020                                     with_vcs,
1021                                     compil_scripts_tmp_dir,
1022                                     env_scripts_tmp_dir,
1023                                     patches_tmp_dir,
1024                                     products_pyconf_tmp_dir):
1025     '''Create a specific pyconf file for a given product. Get its environment
1026        script, its compilation script and patches and put it in the temporary
1027        working directory. This method is used in the source package in order to
1028        construct the specific project.
1029
1030     :param p_name str: The name of the product.
1031     :param p_info Config: The specific configuration corresponding to the
1032                              product
1033     :param config Config: The global configuration.
1034     :param with_vcs boolean: True if the package is with vcs products (not
1035                              transformed into archive products)
1036     :param compil_scripts_tmp_dir str: The path to the temporary compilation
1037                                        scripts directory of the project.
1038     :param env_scripts_tmp_dir str: The path to the temporary environment script
1039                                     directory of the project.
1040     :param patches_tmp_dir str: The path to the temporary patch scripts
1041                                 directory of the project.
1042     :param products_pyconf_tmp_dir str: The path to the temporary product
1043                                         scripts directory of the project.
1044     '''
1045
1046     # read the pyconf of the product
1047     product_pyconf_cfg = src.pyconf.Config(p_info.from_file)
1048
1049     # find the compilation script if any
1050     if src.product.product_has_script(p_info):
1051         compil_script_path = src.Path(p_info.compil_script)
1052         compil_script_path.copy(compil_scripts_tmp_dir)
1053
1054     # find the environment script if any
1055     if src.product.product_has_env_script(p_info):
1056         env_script_path = src.Path(p_info.environ.env_script)
1057         env_script_path.copy(env_scripts_tmp_dir)
1058
1059     # find the patches if any
1060     if src.product.product_has_patches(p_info):
1061         patches = src.pyconf.Sequence()
1062         for patch_path in p_info.patches:
1063             p_path = src.Path(patch_path)
1064             p_path.copy(patches_tmp_dir)
1065             patches.append(os.path.basename(patch_path), "")
1066
1067     if (not with_vcs) and src.product.product_is_vcs(p_info):
1068         # in non vcs mode, if the product is not archive, then make it become archive.
1069
1070         # depending upon the incremental mode, select impacted sections
1071         if "properties" in p_info and "incremental" in p_info.properties and\
1072             p_info.properties.incremental == "yes":
1073             sections = ["default", "default_win", p_info.section, p_info.section+"_win"]
1074         else:
1075             sections = [p_info.section]
1076         for section in sections:
1077             if section in product_pyconf_cfg and "get_source" in product_pyconf_cfg[section]:
1078                 DBG.write("sat package set archive mode to archive for product %s and section %s" %\
1079                           (p_name,section))
1080                 product_pyconf_cfg[section].get_source = "archive"
1081                 if not "archive_info" in product_pyconf_cfg[section]:
1082                     product_pyconf_cfg[section].addMapping("archive_info",
1083                                         src.pyconf.Mapping(product_pyconf_cfg),
1084                                         "")
1085                     product_pyconf_cfg[section].archive_info.archive_name =\
1086                         p_info.name + ".tgz"
1087
1088     # write the pyconf file to the temporary project location
1089     product_tmp_pyconf_path = os.path.join(products_pyconf_tmp_dir,
1090                                            p_name + ".pyconf")
1091     ff = open(product_tmp_pyconf_path, 'w')
1092     ff.write("#!/usr/bin/env python\n#-*- coding:utf-8 -*-\n\n")
1093     product_pyconf_cfg.__save__(ff, 1)
1094     ff.close()
1095
1096 def find_application_pyconf(config, application_tmp_dir):
1097     '''Find the application pyconf file and put it in the specific temporary
1098        directory containing the specific project of a source package.
1099
1100     :param config Config: The global configuration.
1101     :param application_tmp_dir str: The path to the temporary application
1102                                        scripts directory of the project.
1103     '''
1104     # read the pyconf of the application
1105     application_name = config.VARS.application
1106     application_pyconf_path = src.find_file_in_lpath(
1107                                             application_name + ".pyconf",
1108                                             config.PATHS.APPLICATIONPATH)
1109     application_pyconf_cfg = src.pyconf.Config(application_pyconf_path)
1110
1111     # Change the workdir
1112     application_pyconf_cfg.APPLICATION.workdir = src.pyconf.Reference(
1113                                     application_pyconf_cfg,
1114                                     src.pyconf.DOLLAR,
1115                                     'VARS.salometoolsway + $VARS.sep + ".."')
1116
1117     # Prevent from compilation in base
1118     application_pyconf_cfg.APPLICATION.no_base = "yes"
1119
1120     #remove products that are not in config (which were filtered by --without_properties)
1121     for product_name in application_pyconf_cfg.APPLICATION.products.keys():
1122         if product_name not in config.APPLICATION.products.keys():
1123             application_pyconf_cfg.APPLICATION.products.__delitem__(product_name)
1124
1125     # write the pyconf file to the temporary application location
1126     application_tmp_pyconf_path = os.path.join(application_tmp_dir,
1127                                                application_name + ".pyconf")
1128
1129     ff = open(application_tmp_pyconf_path, 'w')
1130     ff.write("#!/usr/bin/env python\n#-*- coding:utf-8 -*-\n\n")
1131     application_pyconf_cfg.__save__(ff, 1)
1132     ff.close()
1133
1134 def sat_package(config, tmp_working_dir, options, logger):
1135     '''Prepare a dictionary that stores all the needed directories and files to
1136        add in a salomeTool package.
1137
1138     :param tmp_working_dir str: The temporary local working directory
1139     :param options OptResult: the options of the launched command
1140     :return: the dictionary that stores all the needed directories and files to
1141              add in a salomeTool package.
1142              {label : (path_on_local_machine, path_in_archive)}
1143     :rtype: dict
1144     '''
1145     d_project = {}
1146
1147     # we include sat himself
1148     d_project["all_sat"]=(config.VARS.salometoolsway, "")
1149
1150     # and we overwrite local.pyconf with a clean wersion.
1151     local_pyconf_tmp_path = os.path.join(tmp_working_dir, "local.pyconf")
1152     local_file_path = os.path.join(config.VARS.datadir, "local.pyconf")
1153     local_cfg = src.pyconf.Config(local_file_path)
1154     local_cfg.PROJECTS.project_file_paths=src.pyconf.Sequence(local_cfg.PROJECTS)
1155     local_cfg.LOCAL["base"] = "default"
1156     local_cfg.LOCAL["workdir"] = "default"
1157     local_cfg.LOCAL["log_dir"] = "default"
1158     local_cfg.LOCAL["archive_dir"] = "default"
1159     local_cfg.LOCAL["VCS"] = "None"
1160     local_cfg.LOCAL["tag"] = src.get_salometool_version(config)
1161
1162     # if the archive contains a project, we write its relative path in local.pyconf
1163     if options.project:
1164         project_arch_path = os.path.join("projects", options.project,
1165                                          os.path.basename(options.project_file_path))
1166         local_cfg.PROJECTS.project_file_paths.append(project_arch_path, "")
1167
1168     ff = open(local_pyconf_tmp_path, 'w')
1169     local_cfg.__save__(ff, 1)
1170     ff.close()
1171     d_project["local.pyconf"]=(local_pyconf_tmp_path, "data/local.pyconf")
1172     return d_project
1173
1174
1175 def project_package(config, name_project, project_file_path, ftp_mode, tmp_working_dir, embedded_in_sat, logger):
1176     '''Prepare a dictionary that stores all the needed directories and files to
1177        add in a project package.
1178
1179     :param project_file_path str: The path to the local project.
1180     :param ftp_mode boolean: Do not embed archives, the archive will rely on ftp mode to retrieve them.
1181     :param tmp_working_dir str: The temporary local directory containing some
1182                                 specific directories or files needed in the
1183                                 project package
1184     :param embedded_in_sat boolean : the project package is embedded in a sat package
1185     :return: the dictionary that stores all the needed directories and files to
1186              add in a project package.
1187              {label : (path_on_local_machine, path_in_archive)}
1188     :rtype: dict
1189     '''
1190     d_project = {}
1191     # Read the project file and get the directories to add to the package
1192
1193     try:
1194       project_pyconf_cfg = config.PROJECTS.projects.__getattr__(name_project)
1195     except:
1196       logger.write("""
1197 WARNING: inexisting config.PROJECTS.projects.%s, try to read now from:\n%s\n""" % (name_project, project_file_path))
1198       project_pyconf_cfg = src.pyconf.Config(project_file_path)
1199       project_pyconf_cfg.PWD = os.path.dirname(project_file_path)
1200
1201     paths = {"APPLICATIONPATH" : "applications",
1202              "PRODUCTPATH" : "products",
1203              "JOBPATH" : "jobs",
1204              "MACHINEPATH" : "machines"}
1205     if not ftp_mode:
1206         paths["ARCHIVEPATH"] = "archives"
1207
1208     # Loop over the project paths and add it
1209     project_file_name = os.path.basename(project_file_path)
1210     for path in paths:
1211         if path not in project_pyconf_cfg:
1212             continue
1213         if embedded_in_sat:
1214             dest_path = os.path.join("projects", name_project, paths[path])
1215             project_file_dest = os.path.join("projects", name_project, project_file_name)
1216         else:
1217             dest_path = paths[path]
1218             project_file_dest = project_file_name
1219
1220         # Add the directory to the files to add in the package
1221         d_project[path] = (project_pyconf_cfg[path], dest_path)
1222
1223         # Modify the value of the path in the package
1224         project_pyconf_cfg[path] = src.pyconf.Reference(
1225                                     project_pyconf_cfg,
1226                                     src.pyconf.DOLLAR,
1227                                     'project_path + "/' + paths[path] + '"')
1228
1229     # Modify some values
1230     if "project_path" not in project_pyconf_cfg:
1231         project_pyconf_cfg.addMapping("project_path",
1232                                       src.pyconf.Mapping(project_pyconf_cfg),
1233                                       "")
1234     project_pyconf_cfg.project_path = src.pyconf.Reference(project_pyconf_cfg,
1235                                                            src.pyconf.DOLLAR,
1236                                                            'PWD')
1237     # we don't want to export these two fields
1238     project_pyconf_cfg.__delitem__("file_path")
1239     project_pyconf_cfg.__delitem__("PWD")
1240     if ftp_mode:
1241         project_pyconf_cfg.__delitem__("ARCHIVEPATH")
1242
1243     # Write the project pyconf file
1244     project_pyconf_tmp_path = os.path.join(tmp_working_dir, project_file_name)
1245     ff = open(project_pyconf_tmp_path, 'w')
1246     ff.write("#!/usr/bin/env python\n#-*- coding:utf-8 -*-\n\n")
1247     project_pyconf_cfg.__save__(ff, 1)
1248     ff.close()
1249     d_project["Project hat file"] = (project_pyconf_tmp_path, project_file_dest)
1250
1251     return d_project
1252
1253 def add_readme(config, options, where):
1254     readme_path = os.path.join(where, "README")
1255     with codecs.open(readme_path, "w", 'utf-8') as f:
1256
1257     # templates for building the header
1258         readme_header="""
1259 # This package was generated with sat $version
1260 # Date: $date
1261 # User: $user
1262 # Distribution : $dist
1263
1264 In the following, $$ROOT represents the directory where you have installed
1265 SALOME (the directory where this file is located).
1266
1267 """
1268         readme_compilation_with_binaries="""
1269
1270 compilation based on the binaries used as prerequisites
1271 =======================================================
1272
1273 If you fail to compile the complete application (for example because
1274 you are not root on your system and cannot install missing packages), you
1275 may try a partial compilation based on the binaries.
1276 For that it is necessary to copy the binaries from BINARIES to INSTALL,
1277 and do some substitutions on cmake and .la files (replace the build directories
1278 with local paths).
1279 The procedure to do it is:
1280  1) Remove or rename INSTALL directory if it exists
1281  2) Execute the shell script install_bin.sh:
1282  > cd $ROOT
1283  > ./install_bin.sh
1284  3) Use SalomeTool (as explained in Sources section) and compile only the
1285     modules you need to (with -p option)
1286
1287 """
1288         readme_header_tpl=string.Template(readme_header)
1289         readme_template_path_bin = os.path.join(config.VARS.internal_dir,
1290                 "README_BIN.template")
1291         readme_template_path_bin_launcher = os.path.join(config.VARS.internal_dir,
1292                 "README_LAUNCHER.template")
1293         readme_template_path_bin_virtapp = os.path.join(config.VARS.internal_dir,
1294                 "README_BIN_VIRTUAL_APP.template")
1295         readme_template_path_src = os.path.join(config.VARS.internal_dir,
1296                 "README_SRC.template")
1297         readme_template_path_pro = os.path.join(config.VARS.internal_dir,
1298                 "README_PROJECT.template")
1299         readme_template_path_sat = os.path.join(config.VARS.internal_dir,
1300                 "README_SAT.template")
1301
1302         # prepare substitution dictionary
1303         d = dict()
1304         d['user'] = config.VARS.user
1305         d['date'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
1306         d['version'] = src.get_salometool_version(config)
1307         d['dist'] = config.VARS.dist
1308         f.write(readme_header_tpl.substitute(d)) # write the general header (common)
1309
1310         if options.binaries or options.sources:
1311             d['application'] = config.VARS.application
1312             f.write("# Application: " + d['application'] + "\n")
1313             if 'KERNEL' in config.APPLICATION.products:
1314                 VersionSalome = src.get_salome_version(config)
1315                 # Case where SALOME has the launcher that uses the SalomeContext API
1316                 if VersionSalome >= 730:
1317                     d['launcher'] = config.APPLICATION.profile.launcher_name
1318                 else:
1319                     d['virtual_app'] = 'runAppli' # this info is not used now)
1320
1321         # write the specific sections
1322         if options.binaries:
1323             f.write(src.template.substitute(readme_template_path_bin, d))
1324             if "virtual_app" in d:
1325                 f.write(src.template.substitute(readme_template_path_bin_virtapp, d))
1326             if "launcher" in d:
1327                 f.write(src.template.substitute(readme_template_path_bin_launcher, d))
1328
1329         if options.sources:
1330             f.write(src.template.substitute(readme_template_path_src, d))
1331
1332         if options.binaries and options.sources:
1333             f.write(readme_compilation_with_binaries)
1334
1335         if options.project:
1336             f.write(src.template.substitute(readme_template_path_pro, d))
1337
1338         if options.sat:
1339             f.write(src.template.substitute(readme_template_path_sat, d))
1340
1341     return readme_path
1342
1343 def update_config(config, prop, value):
1344     '''Remove from config.APPLICATION.products the products that have the property given as input.
1345
1346     :param config Config: The global config.
1347     :param prop str: The property to filter
1348     :param value str: The value of the property to filter
1349     '''
1350     # if there is no APPLICATION (ex sat package -t) : nothing to do
1351     if "APPLICATION" in config:
1352         l_product_to_remove = []
1353         for product_name in config.APPLICATION.products.keys():
1354             prod_cfg = src.product.get_product_config(config, product_name)
1355             if src.get_property_in_product_cfg(prod_cfg, prop) == value:
1356                 l_product_to_remove.append(product_name)
1357         for product_name in l_product_to_remove:
1358             config.APPLICATION.products.__delitem__(product_name)
1359
1360 def description():
1361     '''method that is called when salomeTools is called with --help option.
1362
1363     :return: The text to display for the package command description.
1364     :rtype: str
1365     '''
1366     return _("""
1367 The package command creates a tar file archive of a product.
1368 There are four kinds of archive, which can be mixed:
1369
1370  1 - The binary archive.
1371      It contains the product installation directories plus a launcher.
1372  2 - The sources archive.
1373      It contains the product archives, a project (the application plus salomeTools).
1374  3 - The project archive.
1375      It contains a project (give the project file path as argument).
1376  4 - The salomeTools archive.
1377      It contains code utility salomeTools.
1378
1379 example:
1380  >> sat package SALOME-master --binaries --sources""")
1381
1382 def run(args, runner, logger):
1383     '''method that is called when salomeTools is called with package parameter.
1384     '''
1385
1386     # Parse the options
1387     (options, args) = parser.parse_args(args)
1388
1389     # Check that a type of package is called, and only one
1390     all_option_types = (options.binaries,
1391                         options.sources,
1392                         options.project not in ["", None],
1393                         options.sat)
1394
1395     # Check if no option for package type
1396     if all_option_types.count(True) == 0:
1397         msg = _("Error: Precise a type for the package\nUse one of the "
1398                 "following options: --binaries, --sources, --project or"
1399                 " --salometools")
1400         logger.write(src.printcolors.printcError(msg), 1)
1401         logger.write("\n", 1)
1402         return 1
1403
1404     # The repository where to put the package if not Binary or Source
1405     package_default_path = runner.cfg.LOCAL.workdir
1406
1407     # if the package contains binaries or sources:
1408     if options.binaries or options.sources:
1409         # Check that the command has been called with an application
1410         src.check_config_has_application(runner.cfg)
1411
1412         # Display information
1413         logger.write(_("Packaging application %s\n") % src.printcolors.printcLabel(
1414                                                     runner.cfg.VARS.application), 1)
1415
1416         # Get the default directory where to put the packages
1417         package_default_path = os.path.join(runner.cfg.APPLICATION.workdir, "PACKAGE")
1418         src.ensure_path_exists(package_default_path)
1419
1420     # if the package contains a project:
1421     if options.project:
1422         # check that the project is visible by SAT
1423         projectNameFile = options.project + ".pyconf"
1424         foundProject = None
1425         for i in runner.cfg.PROJECTS.project_file_paths:
1426             baseName = os.path.basename(i)
1427             if baseName == projectNameFile:
1428                 foundProject = i
1429                 break
1430
1431         if foundProject is None:
1432             local_path = os.path.join(runner.cfg.VARS.salometoolsway, "data", "local.pyconf")
1433             msg = _("""ERROR: the project %(1)s is not visible by salomeTools.
1434 known projects are:
1435 %(2)s
1436
1437 Please add it in file:
1438 %(3)s""" % \
1439                     {"1": options.project, "2": "\n  ".join(runner.cfg.PROJECTS.project_file_paths), "3": local_path})
1440             logger.write(src.printcolors.printcError(msg), 1)
1441             logger.write("\n", 1)
1442             return 1
1443         else:
1444             options.project_file_path = foundProject
1445             src.printcolors.print_value(logger, "Project path", options.project_file_path, 2)
1446
1447     # Remove the products that are filtered by the --without_properties option
1448     if options.without_properties:
1449         app = runner.cfg.APPLICATION
1450         logger.trace("without_properties all products:\n %s\n" % PP.pformat(sorted(app.products.keys())))
1451         prop, value = options.without_properties
1452         update_config(runner.cfg, prop, value)
1453         logger.warning("without_properties selected products:\n %s\n" % PP.pformat(sorted(app.products.keys())))
1454
1455     # Remove from config the products that have the not_in_package property
1456     update_config(runner.cfg, "not_in_package", "yes")
1457
1458     # get the name of the archive or build it
1459     if options.name:
1460         if os.path.basename(options.name) == options.name:
1461             # only a name (not a path)
1462             archive_name = options.name
1463             dir_name = package_default_path
1464         else:
1465             archive_name = os.path.basename(options.name)
1466             dir_name = os.path.dirname(options.name)
1467
1468         # suppress extension
1469         if archive_name[-len(".tgz"):] == ".tgz":
1470             archive_name = archive_name[:-len(".tgz")]
1471         if archive_name[-len(".tar.gz"):] == ".tar.gz":
1472             archive_name = archive_name[:-len(".tar.gz")]
1473
1474     else:
1475         archive_name=""
1476         dir_name = package_default_path
1477         if options.binaries or options.sources:
1478             archive_name = runner.cfg.APPLICATION.name
1479
1480         if options.binaries:
1481             archive_name += "-"+runner.cfg.VARS.dist
1482
1483         if options.sources:
1484             archive_name += "-SRC"
1485             if options.with_vcs:
1486                 archive_name += "-VCS"
1487
1488         if options.sat:
1489             archive_name += ("salomeTools_" + src.get_salometool_version(runner.cfg))
1490
1491         if options.project:
1492             if options.sat:
1493                 archive_name += "_"
1494             project_name = options.project
1495             archive_name += ("satproject_" + project_name)
1496
1497         if len(archive_name)==0: # no option worked
1498             msg = _("Error: Cannot name the archive\n"
1499                     " check if at least one of the following options was "
1500                     "selected : --binaries, --sources, --project or"
1501                     " --salometools")
1502             logger.write(src.printcolors.printcError(msg), 1)
1503             logger.write("\n", 1)
1504             return 1
1505
1506     path_targz = os.path.join(dir_name, archive_name + ".tgz")
1507
1508     src.printcolors.print_value(logger, "Package path", path_targz, 2)
1509
1510     # Create a working directory for all files that are produced during the
1511     # package creation and that will be removed at the end of the command
1512     tmp_working_dir = os.path.join(runner.cfg.VARS.tmp_root, runner.cfg.VARS.datehour)
1513     src.ensure_path_exists(tmp_working_dir)
1514     logger.write("\n", 5)
1515     logger.write(_("The temporary working directory: %s\n" % tmp_working_dir),5)
1516
1517     logger.write("\n", 3)
1518
1519     msg = _("Preparation of files to add to the archive")
1520     logger.write(src.printcolors.printcLabel(msg), 2)
1521     logger.write("\n", 2)
1522
1523     d_files_to_add={}  # content of the archive
1524
1525     # a dict to hold paths that will need to be substitute for users recompilations
1526     d_paths_to_substitute={}
1527
1528     if options.binaries:
1529         d_bin_files_to_add = binary_package(runner.cfg,
1530                                             logger,
1531                                             options,
1532                                             tmp_working_dir)
1533         # for all binaries dir, store the substitution that will be required
1534         # for extra compilations
1535         for key in d_bin_files_to_add:
1536             if key.endswith("(bin)"):
1537                 source_dir = d_bin_files_to_add[key][0]
1538                 path_in_archive = d_bin_files_to_add[key][1].replace(
1539                    runner.cfg.INTERNAL.config.binary_dir + runner.cfg.VARS.dist,
1540                    runner.cfg.INTERNAL.config.install_dir)
1541                 if os.path.basename(source_dir)==os.path.basename(path_in_archive):
1542                     # if basename is the same we will just substitute the dirname
1543                     d_paths_to_substitute[os.path.dirname(source_dir)]=\
1544                         os.path.dirname(path_in_archive)
1545                 else:
1546                     d_paths_to_substitute[source_dir]=path_in_archive
1547
1548         d_files_to_add.update(d_bin_files_to_add)
1549
1550     if options.sources:
1551         d_files_to_add.update(source_package(runner,
1552                                         runner.cfg,
1553                                         logger,
1554                                         options,
1555                                         tmp_working_dir))
1556         if options.binaries:
1557             # for archives with bin and sources we provide a shell script able to
1558             # install binaries for compilation
1559             file_install_bin=produce_install_bin_file(runner.cfg,logger,
1560                                                       tmp_working_dir,
1561                                                       d_paths_to_substitute,
1562                                                       "install_bin.sh")
1563             d_files_to_add.update({"install_bin" : (file_install_bin, "install_bin.sh")})
1564             logger.write("substitutions that need to be done later : \n", 5)
1565             logger.write(str(d_paths_to_substitute), 5)
1566             logger.write("\n", 5)
1567     else:
1568         # --salomeTool option is not considered when --sources is selected, as this option
1569         # already brings salomeTool!
1570         if options.sat:
1571             d_files_to_add.update(sat_package(runner.cfg, tmp_working_dir,
1572                                   options, logger))
1573
1574     if options.project:
1575         DBG.write("config for package %s" % project_name, runner.cfg)
1576         d_files_to_add.update(project_package(runner.cfg, project_name, options.project_file_path, options.ftp, tmp_working_dir, options.sat, logger))
1577
1578     if not(d_files_to_add):
1579         msg = _("Error: Empty dictionnary to build the archive!\n")
1580         logger.write(src.printcolors.printcError(msg), 1)
1581         logger.write("\n", 1)
1582         return 1
1583
1584     # Add the README file in the package
1585     local_readme_tmp_path = add_readme(runner.cfg, options, tmp_working_dir)
1586     d_files_to_add["README"] = (local_readme_tmp_path, "README")
1587
1588     # Add the additional files of option add_files
1589     if options.add_files:
1590         for file_path in options.add_files:
1591             if not os.path.exists(file_path):
1592                 msg = _("WARNING: the file %s is not accessible.\n" % file_path)
1593                 continue
1594             file_name = os.path.basename(file_path)
1595             d_files_to_add[file_name] = (file_path, file_name)
1596
1597     logger.write("\n", 2)
1598     logger.write(src.printcolors.printcLabel(_("Actually do the package")), 2)
1599     logger.write("\n", 2)
1600     logger.write("\nfiles and directories to add:\n%s\n\n" % PP.pformat(d_files_to_add), 5)
1601
1602     res = 0
1603     try:
1604         # Creating the object tarfile
1605         tar = tarfile.open(path_targz, mode='w:gz')
1606
1607         # get the filtering function if needed
1608         filter_function = exclude_VCS_and_extensions
1609
1610         # Add the files to the tarfile object
1611         res = add_files(tar, archive_name, d_files_to_add, logger, f_exclude=filter_function)
1612         tar.close()
1613     except KeyboardInterrupt:
1614         logger.write(src.printcolors.printcError("\nERROR: forced interruption\n"), 1)
1615         logger.write(_("Removing the temporary working directory '%s'... ") % tmp_working_dir, 1)
1616         # remove the working directory
1617         shutil.rmtree(tmp_working_dir)
1618         logger.write(_("OK"), 1)
1619         logger.write(_("\n"), 1)
1620         return 1
1621
1622     # case if no application, only package sat as 'sat package -t'
1623     try:
1624         app = runner.cfg.APPLICATION
1625     except:
1626         app = None
1627
1628     # unconditionaly remove the tmp_local_working_dir
1629     if app is not None:
1630         tmp_local_working_dir = os.path.join(app.workdir, "tmp_package")
1631         if os.path.isdir(tmp_local_working_dir):
1632             shutil.rmtree(tmp_local_working_dir)
1633
1634     # remove the tmp directory, unless user has registered as developer
1635     if os.path.isdir(tmp_working_dir) and (not DBG.isDeveloper()):
1636         shutil.rmtree(tmp_working_dir)
1637
1638     # Print again the path of the package
1639     logger.write("\n", 2)
1640     src.printcolors.print_value(logger, "Package path", path_targz, 2)
1641
1642     return res