X-Git-Url: http://git.salome-platform.org/gitweb/?a=blobdiff_plain;f=commands%2Fpackage.py;h=2d67ac917a56141b0aa0d40911acabbff8703c22;hb=620d5d2e941088d41984043bb7fae889da5a52b1;hp=40914e4dbe2a302794e51212054431dc8ccfe31a;hpb=a73105d0c22ff213d31a6502a0b4cf1b77bb754a;p=tools%2Fsat.git diff --git a/commands/package.py b/commands/package.py index 40914e4..2d67ac9 100644 --- a/commands/package.py +++ b/commands/package.py @@ -23,6 +23,7 @@ import datetime import tarfile import codecs import string +import pprint as PP import src @@ -91,7 +92,12 @@ parser.add_option('s', 'sources', 'boolean', 'sources', _('Optional: Produce a compilable archive of the sources of the ' 'application.'), False) parser.add_option('', 'with_vcs', 'boolean', 'with_vcs', - _('Optional: Only source package: do not make archive of vcs products.'), + _('Optional: Do not make archive for products in VCS mode (git, cvs, svn). ' + 'Sat prepare will use VCS mode instead to retrieve them'), + False) +parser.add_option('', 'ftp', 'boolean', 'ftp', + _('Optional: Do not embed archives for products in archive mode.' + 'Sat prepare will use ftp instead to retrieve them'), False) parser.add_option('p', 'project', 'string', 'project', _('Optional: Produce an archive that contains a project.'), "") @@ -103,9 +109,9 @@ parser.add_option('', 'add_files', 'list2', 'add_files', _('Optional: The list of additional files to add to the archive.'), []) parser.add_option('', 'without_commercial', 'boolean', 'without_commercial', _('Optional: do not add commercial licence.'), False) -parser.add_option('', 'without_property', 'string', 'without_property', +parser.add_option('', 'without_properties', 'properties', 'without_properties', _('Optional: Filter the products by their properties.\n\tSyntax: ' - '--without_property :')) + '--without_properties :')) def add_files(tar, name_archive, d_content, logger, f_exclude=None): @@ -128,9 +134,12 @@ def add_files(tar, name_archive, d_content, logger, f_exclude=None): success = 0 # loop over each directory or file stored in the d_content dictionary - for name in sorted(d_content.keys()): + names = sorted(d_content.keys()) + DBG.write("add tar names", names) + + for name in names: # display information - len_points = max_len - len(name) + len_points = max_len - len(name) + 3 local_path, archive_path = d_content[name] in_archive = os.path.join(name_archive, archive_path) logger.write(name + " " + len_points * "." + " "+ in_archive + " ", 3) @@ -193,14 +202,19 @@ def produce_relative_launcher(config, bin_kernel_install_dir = os.path.join(kernel_root_dir,"bin","salome") # check if the application contains an application module - l_product_info = src.product.get_products_infos(config.APPLICATION.products.keys(), - config) + # check also if the application has a distene product, + # in this case get its licence file name + l_product_info = src.product.get_products_infos(config.APPLICATION.products.keys(), config) salome_application_name="Not defined" + distene_licence_file_name=False for prod_name, prod_info in l_product_info: - # look for a salome application + # look for a "salome application" and a distene product + if src.get_property_in_product_cfg(prod_info, "is_distene") == "yes": + distene_licence_file_name = src.product.product_has_licence(prod_info, + config.PATHS.LICENCEPATH) if src.get_property_in_product_cfg(prod_info, "is_salome_application") == "yes": salome_application_name=prod_info.name - continue + # if the application contains an application module, we set ABSOLUTE_APPLI_PATH to it # if not we set it to KERNEL_INSTALL_DIR, which is sufficient, except for salome test if salome_application_name == "Not defined": @@ -209,7 +223,10 @@ def produce_relative_launcher(config, app_root_dir=os.path.join(binaries_dir_name, salome_application_name) # Get the launcher template and do substitutions - withProfile = src.fileEnviron.withProfile + if "python3" in config.APPLICATION and config.APPLICATION.python3 == "yes": + withProfile = src.fileEnviron.withProfile3 + else: + withProfile = src.fileEnviron.withProfile withProfile = withProfile.replace( "ABSOLUTE_APPLI_PATH'] = 'KERNEL_INSTALL_DIR'", @@ -218,8 +235,7 @@ def produce_relative_launcher(config, " 'BIN_KERNEL_INSTALL_DIR'", " out_dir_Path + '" + config.VARS.sep + bin_kernel_install_dir + "'") - before, after = withProfile.split( - "# here your local standalone environment\n") + before, after = withProfile.split("# here your local standalone environment\n") # create an environment file writer writer = src.environment.FileEnvWriter(config, @@ -243,7 +259,9 @@ def produce_relative_launcher(config, # A hack to put a call to a file for distene licence. # It does nothing to an application that has no distene product - hack_for_distene_licence(filepath) + if distene_licence_file_name: + logger.write("Application has a distene licence file! We use it in package launcher", 5) + hack_for_distene_licence(filepath, distene_licence_file_name) # change the rights in order to make the file executable for everybody os.chmod(filepath, @@ -257,7 +275,7 @@ def produce_relative_launcher(config, return filepath -def hack_for_distene_licence(filepath): +def hack_for_distene_licence(filepath, licence_file): '''Replace the distene licence env variable by a call to a file. :param filepath Str: The path to the launcher to modify. @@ -285,10 +303,10 @@ def hack_for_distene_licence(filepath): del text[num_line +1] text_to_insert =""" import imp try: - distene = imp.load_source('distene_licence', '/data/tmpsalome/salome/prerequis/install/LICENSE/dlim8.var.py') + distene = imp.load_source('distene_licence', '%s') distene.set_distene_variables(context) except: - pass\n""" + pass\n""" % licence_file text.insert(num_line + 1, text_to_insert) for line in text: fout.write(line) @@ -482,7 +500,16 @@ def binary_package(config, logger, options, tmp_working_dir): l_not_installed = [] l_sources_not_present = [] generate_mesa_launcher = False # a flag to know if we generate a mesa launcher + if ("APPLICATION" in config and + "properties" in config.APPLICATION and + "mesa_launcher_in_package" in config.APPLICATION.properties and + config.APPLICATION.properties.mesa_launcher_in_package == "yes") : + generate_mesa_launcher=True + for prod_name, prod_info in l_product_info: + # skip product with property not_in_package set to yes + if src.get_property_in_product_cfg(prod_info, "not_in_package") == "yes": + continue # Add the sources of the products that have the property # sources_in_package : "yes" @@ -493,10 +520,6 @@ def binary_package(config, logger, options, tmp_working_dir): else: l_sources_not_present.append(prod_name) - # if at least one of the application products has the "is_mesa" property - if src.get_property_in_product_cfg(prod_info, "is_mesa") == "yes": - generate_mesa_launcher = True # we will generate a mesa launcher - # ignore the native and fixed products for install directories if (src.product.product_is_native(prod_info) or src.product.product_is_fixed(prod_info) @@ -664,10 +687,15 @@ def source_package(sat, config, logger, options, tmp_working_dir): :rtype: dict ''' + d_archives={} # Get all the products that are prepared using an archive - logger.write("Find archive products ... ") - d_archives, l_pinfo_vcs = get_archives(config, logger) - logger.write("Done\n") + # unless ftp mode is specified (in this case the user of the + # archive will get the sources through the ftp mode of sat prepare + if not options.ftp: + logger.write("Find archive products ... ") + d_archives, l_pinfo_vcs = get_archives(config, logger) + logger.write("Done\n") + d_archives_vcs = {} if not options.with_vcs and len(l_pinfo_vcs) > 0: # Make archives with the products that are not prepared using an archive @@ -683,8 +711,9 @@ def source_package(sat, config, logger, options, tmp_working_dir): # Create a project logger.write("Create the project ... ") d_project = create_project_for_src_package(config, - tmp_working_dir, - options.with_vcs) + tmp_working_dir, + options.with_vcs, + options.ftp) logger.write("Done\n") # Add salomeTools @@ -729,6 +758,9 @@ def get_archives(config, logger): d_archives = {} l_pinfo_vcs = [] for p_name, p_info in l_product_info: + # skip product with property not_in_package set to yes + if src.get_property_in_product_cfg(p_info, "not_in_package") == "yes": + continue # ignore the native and fixed products if (src.product.product_is_native(p_info) or src.product.product_is_fixed(p_info)): @@ -856,7 +888,7 @@ def make_archive(prod_name, prod_info, where): tar_prod.close() return path_targz_prod -def create_project_for_src_package(config, tmp_working_dir, with_vcs): +def create_project_for_src_package(config, tmp_working_dir, with_vcs, with_ftp): '''Create a specific project for a source package. :param config Config: The global configuration. @@ -865,6 +897,7 @@ def create_project_for_src_package(config, tmp_working_dir, with_vcs): source package :param with_vcs boolean: True if the package is with vcs products (not transformed into archive products) + :param with_ftp boolean: True if the package use ftp servers to get archives :return: The dictionary {"project" : (produced project, project path in the archive)} :rtype: Dict @@ -897,6 +930,23 @@ def create_project_for_src_package(config, tmp_working_dir, with_vcs): project_pyconf_file = os.path.join(project_tmp_dir, project_pyconf_name) ff = open(project_pyconf_file, "w") ff.write(PROJECT_TEMPLATE) + if with_ftp and len(config.PATHS.ARCHIVEFTP) > 0: + ftp_path='ARCHIVEFTP : "'+config.PATHS.ARCHIVEFTP[0] + for ftpserver in config.PATHS.ARCHIVEFTP[1:]: + ftp_path=ftp_path+":"+ftpserver + ftp_path+='"' + ff.write("# ftp servers where to search for prerequisite archives\n") + ff.write(ftp_path) + # add licence paths if any + if len(config.PATHS.LICENCEPATH) > 0: + licence_path='LICENCEPATH : "'+config.PATHS.LICENCEPATH[0] + for path in config.PATHS.LICENCEPATH[1:]: + licence_path=licence_path+":"+path + licence_path+='"' + ff.write("\n# Where to search for licences\n") + ff.write(licence_path) + + ff.close() # Loop over the products to get there pyconf and all the scripts @@ -905,6 +955,9 @@ def create_project_for_src_package(config, tmp_working_dir, with_vcs): lproducts_name = config.APPLICATION.products.keys() l_products = src.product.get_products_infos(lproducts_name, config) for p_name, p_info in l_products: + # skip product with property not_in_package set to yes + if src.get_property_in_product_cfg(p_info, "not_in_package") == "yes": + continue find_product_scripts_and_pyconf(p_name, p_info, config, @@ -1025,22 +1078,71 @@ def find_application_pyconf(config, application_tmp_dir): # Prevent from compilation in base application_pyconf_cfg.APPLICATION.no_base = "yes" + #remove products that are not in config (which were filtered by --without_properties) + for product_name in application_pyconf_cfg.APPLICATION.products.keys(): + if product_name not in config.APPLICATION.products.keys(): + application_pyconf_cfg.APPLICATION.products.__delitem__(product_name) + # write the pyconf file to the temporary application location application_tmp_pyconf_path = os.path.join(application_tmp_dir, application_name + ".pyconf") + ff = open(application_tmp_pyconf_path, 'w') ff.write("#!/usr/bin/env python\n#-*- coding:utf-8 -*-\n\n") application_pyconf_cfg.__save__(ff, 1) ff.close() -def project_package(config, name_project, project_file_path, tmp_working_dir, logger): +def sat_package(config, tmp_working_dir, options, logger): + '''Prepare a dictionary that stores all the needed directories and files to + add in a salomeTool package. + + :param tmp_working_dir str: The temporary local working directory + :param options OptResult: the options of the launched command + :return: the dictionary that stores all the needed directories and files to + add in a salomeTool package. + {label : (path_on_local_machine, path_in_archive)} + :rtype: dict + ''' + d_project = {} + + # we include sat himself + d_project["all_sat"]=(config.VARS.salometoolsway, "") + + # and we overwrite local.pyconf with a clean wersion. + local_pyconf_tmp_path = os.path.join(tmp_working_dir, "local.pyconf") + local_file_path = os.path.join(config.VARS.datadir, "local.pyconf") + local_cfg = src.pyconf.Config(local_file_path) + local_cfg.PROJECTS.project_file_paths=src.pyconf.Sequence(local_cfg.PROJECTS) + local_cfg.LOCAL["base"] = "default" + local_cfg.LOCAL["workdir"] = "default" + local_cfg.LOCAL["log_dir"] = "default" + local_cfg.LOCAL["archive_dir"] = "default" + local_cfg.LOCAL["VCS"] = "None" + local_cfg.LOCAL["tag"] = src.get_salometool_version(config) + + # if the archive contains a project, we write its relative path in local.pyconf + if options.project: + project_arch_path = os.path.join("projects", options.project, + os.path.basename(options.project_file_path)) + local_cfg.PROJECTS.project_file_paths.append(project_arch_path, "") + + ff = open(local_pyconf_tmp_path, 'w') + local_cfg.__save__(ff, 1) + ff.close() + d_project["local.pyconf"]=(local_pyconf_tmp_path, "data/local.pyconf") + return d_project + + +def project_package(config, name_project, project_file_path, ftp_mode, tmp_working_dir, embedded_in_sat, logger): '''Prepare a dictionary that stores all the needed directories and files to add in a project package. :param project_file_path str: The path to the local project. + :param ftp_mode boolean: Do not embed archives, the archive will rely on ftp mode to retrieve them. :param tmp_working_dir str: The temporary local directory containing some specific directories or files needed in the project package + :param embedded_in_sat boolean : the project package is embedded in a sat package :return: the dictionary that stores all the needed directories and files to add in a project package. {label : (path_on_local_machine, path_in_archive)} @@ -1057,17 +1159,28 @@ WARNING: inexisting config.PROJECTS.projects.%s, try to read now from:\n%s\n""" project_pyconf_cfg = src.pyconf.Config(project_file_path) project_pyconf_cfg.PWD = os.path.dirname(project_file_path) - paths = {"ARCHIVEPATH" : "archives", - "APPLICATIONPATH" : "applications", + paths = {"APPLICATIONPATH" : "applications", "PRODUCTPATH" : "products", "JOBPATH" : "jobs", "MACHINEPATH" : "machines"} + if not ftp_mode: + paths["ARCHIVEPATH"] = "archives" + # Loop over the project paths and add it + project_file_name = os.path.basename(project_file_path) for path in paths: if path not in project_pyconf_cfg: continue + if embedded_in_sat: + dest_path = os.path.join("projects", name_project, paths[path]) + project_file_dest = os.path.join("projects", name_project, project_file_name) + else: + dest_path = paths[path] + project_file_dest = project_file_name + # Add the directory to the files to add in the package - d_project[path] = (project_pyconf_cfg[path], paths[path]) + d_project[path] = (project_pyconf_cfg[path], dest_path) + # Modify the value of the path in the package project_pyconf_cfg[path] = src.pyconf.Reference( project_pyconf_cfg, @@ -1082,15 +1195,19 @@ WARNING: inexisting config.PROJECTS.projects.%s, try to read now from:\n%s\n""" project_pyconf_cfg.project_path = src.pyconf.Reference(project_pyconf_cfg, src.pyconf.DOLLAR, 'PWD') + # we don't want to export these two fields + project_pyconf_cfg.__delitem__("file_path") + project_pyconf_cfg.__delitem__("PWD") + if ftp_mode: + project_pyconf_cfg.__delitem__("ARCHIVEPATH") # Write the project pyconf file - project_file_name = os.path.basename(project_file_path) project_pyconf_tmp_path = os.path.join(tmp_working_dir, project_file_name) ff = open(project_pyconf_tmp_path, 'w') ff.write("#!/usr/bin/env python\n#-*- coding:utf-8 -*-\n\n") project_pyconf_cfg.__save__(ff, 1) ff.close() - d_project["Project hat file"] = (project_pyconf_tmp_path, project_file_name) + d_project["Project hat file"] = (project_pyconf_tmp_path, project_file_dest) return d_project @@ -1147,7 +1264,7 @@ The procedure to do it is: d = dict() d['user'] = config.VARS.user d['date'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") - d['version'] = config.INTERNAL.sat_version + d['version'] = src.get_salometool_version(config) d['dist'] = config.VARS.dist f.write(readme_header_tpl.substitute(d)) # write the general header (common) @@ -1191,14 +1308,15 @@ def update_config(config, prop, value): :param prop str: The property to filter :param value str: The value of the property to filter ''' - src.check_config_has_application(config) - l_product_to_remove = [] - for product_name in config.APPLICATION.products.keys(): - prod_cfg = src.product.get_product_config(config, product_name) - if src.get_property_in_product_cfg(prod_cfg, prop) == value: - l_product_to_remove.append(product_name) - for product_name in l_product_to_remove: - config.APPLICATION.products.__delitem__(product_name) + # if there is no APPLICATION (ex sat package -t) : nothing to do + if "APPLICATION" in config: + l_product_to_remove = [] + for product_name in config.APPLICATION.products.keys(): + prod_cfg = src.product.get_product_config(config, product_name) + if src.get_property_in_product_cfg(prod_cfg, prop) == value: + l_product_to_remove.append(product_name) + for product_name in l_product_to_remove: + config.APPLICATION.products.__delitem__(product_name) def description(): '''method that is called when salomeTools is called with --help option. @@ -1257,8 +1375,7 @@ def run(args, runner, logger): runner.cfg.VARS.application), 1) # Get the default directory where to put the packages - package_default_path = os.path.join(runner.cfg.APPLICATION.workdir, - "PACKAGE") + package_default_path = os.path.join(runner.cfg.APPLICATION.workdir, "PACKAGE") src.ensure_path_exists(package_default_path) # if the package contains a project: @@ -1273,9 +1390,7 @@ def run(args, runner, logger): break if foundProject is None: - local_path = os.path.join(runner.cfg.VARS.salometoolsway, - "data", - "local.pyconf") + local_path = os.path.join(runner.cfg.VARS.salometoolsway, "data", "local.pyconf") msg = _("""ERROR: the project %(1)s is not visible by salomeTools. known projects are: %(2)s @@ -1290,10 +1405,16 @@ Please add it in file: options.project_file_path = foundProject src.printcolors.print_value(logger, "Project path", options.project_file_path, 2) - # Remove the products that are filtered by the --without_property option - if options.without_property: - [prop, value] = options.without_property.split(":") + # Remove the products that are filtered by the --without_properties option + if options.without_properties: + app = runner.cfg.APPLICATION + logger.trace("without_properties all products:\n %s\n" % PP.pformat(sorted(app.products.keys()))) + prop, value = options.without_properties update_config(runner.cfg, prop, value) + logger.warning("without_properties selected products:\n %s\n" % PP.pformat(sorted(app.products.keys()))) + + # Remove from config the products that have the not_in_package property + update_config(runner.cfg, "not_in_package", "yes") # get the name of the archive or build it if options.name: @@ -1325,12 +1446,15 @@ Please add it in file: if options.with_vcs: archive_name += "-VCS" + if options.sat: + archive_name += ("salomeTools_" + src.get_salometool_version(runner.cfg)) + if options.project: + if options.sat: + archive_name += "_" project_name = options.project - archive_name += ("PROJECT-" + project_name) + archive_name += ("satproject_" + project_name) - if options.sat: - archive_name += ("salomeTools_" + runner.cfg.INTERNAL.sat_version) if len(archive_name)==0: # no option worked msg = _("Error: Cannot name the archive\n" " check if at least one of the following options was " @@ -1403,11 +1527,12 @@ Please add it in file: # --salomeTool option is not considered when --sources is selected, as this option # already brings salomeTool! if options.sat: - d_files_to_add.update({"salomeTools" : (runner.cfg.VARS.salometoolsway, "")}) + d_files_to_add.update(sat_package(runner.cfg, tmp_working_dir, + options, logger)) if options.project: DBG.write("config for package %s" % project_name, runner.cfg) - d_files_to_add.update(project_package(runner.cfg, project_name, options.project_file_path, tmp_working_dir, logger)) + d_files_to_add.update(project_package(runner.cfg, project_name, options.project_file_path, options.ftp, tmp_working_dir, options.sat, logger)) if not(d_files_to_add): msg = _("Error: Empty dictionnary to build the archive!\n") @@ -1429,10 +1554,11 @@ Please add it in file: d_files_to_add[file_name] = (file_path, file_name) logger.write("\n", 2) - logger.write(src.printcolors.printcLabel(_("Actually do the package")), 2) logger.write("\n", 2) - + logger.write("\nfiles and directories to add:\n%s\n\n" % PP.pformat(d_files_to_add), 5) + + res = 0 try: # Creating the object tarfile tar = tarfile.open(path_targz, mode='w:gz') @@ -1452,13 +1578,20 @@ Please add it in file: logger.write(_("\n"), 1) return 1 + # case if no application, only package sat as 'sat package -t' + try: + app = runner.cfg.APPLICATION + except: + app = None + # unconditionaly remove the tmp_local_working_dir - tmp_local_working_dir = os.path.join(runner.cfg.APPLICATION.workdir, "tmp_package") - if os.path.isdir(tmp_local_working_dir): - shutil.rmtree(tmp_local_working_dir) + if app is not None: + tmp_local_working_dir = os.path.join(app.workdir, "tmp_package") + if os.path.isdir(tmp_local_working_dir): + shutil.rmtree(tmp_local_working_dir) - # to decide... - DBG.tofix("make shutil.rmtree('%s') effective" % tmp_working_dir, "", True) + # have to decide some time + DBG.tofix("make shutil.rmtree('%s') effective" % tmp_working_dir, "", DBG.isDeveloper()) # Print again the path of the package logger.write("\n", 2)