From: CROUZET Nicolas Date: Wed, 1 Sep 2021 13:37:22 +0000 (+0200) Subject: Merge branch 'nct/ete21' X-Git-Tag: V9_9_0~26 X-Git-Url: http://git.salome-platform.org/gitweb/?a=commitdiff_plain;h=54d88a65a3735ff7820a68667575aa1fba120d94;hp=2b52151efa4dd6165f35f7160ab2dab017cb3f47;p=tools%2Fsat.git Merge branch 'nct/ete21' --- diff --git a/commands/config.py b/commands/config.py index e855c57..95a28c0 100644 --- a/commands/config.py +++ b/commands/config.py @@ -407,7 +407,9 @@ class ConfigManager: "LICENCEPATH"]: if PATH not in cfg.PROJECTS.projects[project]: continue - cfg.PATHS[PATH].append(cfg.PROJECTS.projects[project][PATH], "") + pathlist=cfg.PROJECTS.projects[project][PATH].split(":") + for path in pathlist: + cfg.PATHS[PATH].append(path, "") # apply overwrite from command line if needed for rule in self.get_command_line_overrides(options, ["PATHS"]): diff --git a/commands/install.py b/commands/install.py new file mode 100644 index 0000000..2811a6a --- /dev/null +++ b/commands/install.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- +# Copyright (C) 2010-2012 CEA/DEN +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os +import shutil +import re +import subprocess + +import src +import prepare +import src.debug as DBG + +PACKAGE_EXT=".tar.gz" # the extension we use for the packages + +# Define all possible option for patch command : sat patch +parser = src.options.Options() +parser.add_option('p', 'products', 'list2', 'products', + _('Optional: products from which to get the sources. This option accepts a comma separated list.')) + + +def get_binary_from_archive(config, product_name, product_info, install_dir, logger): + '''The method get the binary of the product from an archive + + :param config Config: The global configuration + :param product_name : The name of the product + :param product_info Config: The configuration specific to + the product to be prepared + :param install_dir Path: The Path instance corresponding to the + directory where to put the sources + :param logger Logger: The logger instance to use for the display and logging + :return: True if it succeed, else False + :rtype: boolean + ''' + + + # check archive exists + + # the expected name of the bin archive, as produced by sat package --bin_products + archive_name = product_name + '-' + product_info.version + "-" + config.VARS.dist + PACKAGE_EXT + # we search this archive in bin directory + bin_arch_name = os.path.join("bin",archive_name) + # search in the config.PATHS.ARCHIVEPATH + arch_path = src.find_file_in_lpath(archive_name, config.PATHS.ARCHIVEPATH, "bin") + if not arch_path: + # bin archive was not found locally in ARCHIVEPATH + # search on ftp site + logger.write("\n The bin archive is not found on local file system, we try ftp\n", 3) + ret=src.find_file_in_ftppath(archive_name, config.PATHS.ARCHIVEFTP, + config.LOCAL.archive_dir, logger, "bin") + + if ret: + # archive was found on ftp and stored in ret + arch_path = ret + else: + logger.write('%s ' % src.printcolors.printc(src.OK_STATUS), 3, False) + msg = _("Archive not found in ARCHIVEPATH, nor on ARCHIVEFTP: '%s'") % bin_arch_name + logger.write(msg, 3) + return 1 + + logger.write('arc:%s ... ' % + src.printcolors.printcInfo(archive_name), + 3, + False) + logger.flush() + # Call the system function that do the extraction in archive mode + retcode, NameExtractedDirectory = src.system.archive_extract(arch_path, + install_dir.dir(), logger) + + # Rename the source directory if + # it does not match with product_info.source_dir + if (NameExtractedDirectory.replace('/', '') != + os.path.basename(product_info.install_dir)): + shutil.move(os.path.join(os.path.dirname(product_info.install_dir), + NameExtractedDirectory), + product_info.install_dir) + + return retcode + + + +def get_all_product_binaries(config, products, logger): + '''Get all the product sources. + + :param config Config: The global configuration + :param products List: The list of tuples (product name, product informations) + :param logger Logger: The logger instance to be used for the logging + :return: the tuple (number of success, dictionary product_name/success_fail) + :rtype: (int,dict) + ''' + + # Initialize the variables that will count the fails and success + results = dict() + good_result = 0 + + # Get the maximum name length in order to format the terminal display + max_product_name_len = 1 + if len(products) > 0: + max_product_name_len = max(map(lambda l: len(l), products[0])) + 4 + + # The loop on all the products from which to get the binaries + for product_name, product_info in products: + # display and log + logger.write('%s: ' % src.printcolors.printcLabel(product_name), 3) + logger.write(' ' * (max_product_name_len - len(product_name)), 3, False) + logger.write("\n", 4, False) + # + do_install_prod=True + # check if there is something to do! + if src.product.product_is_fixed(product_info): + do_install_prod=False + msg = _("INFO : Not doing anything because the products %s is fixed\n") % product_name + elif src.product.product_is_native(product_info): + do_install_prod=False + msg = _("INFO : Not doing anything because the products %s is native\n") % product_name + elif src.appli_test_property(config,"pip", "yes") and \ + src.product.product_test_property(product_info,"pip", "yes"): + do_install_prod=False + msg = _("INFO : Not doing anything because the products %s is managed by pip\n") % product_name + else: + install_dir=src.Path(product_info.install_dir) + if install_dir.exists(): + do_install_prod=False + msg = _("INFO : Not doing anything because the install directory already exists:\n %s\n") % install_dir + + if not do_install_prod: + logger.write('%s ' % src.printcolors.printc(src.OK_STATUS), 3, False) + logger.write(msg, 3) + good_result = good_result + 1 + # Do not get the binaries and go to next product + continue + + # we neeed to install binaries for the product + retcode = get_binary_from_archive(config, product_name, product_info, install_dir, logger) + + # Check that the sources are correctly get using the files to be tested + # in product information + if retcode: + pass + # CNC TODO check md5sum + #check_OK, wrong_path = check_sources(product_info, logger) + #if not check_OK: + # # Print the missing file path + # msg = _("The required file %s does not exists. " % wrong_path) + # logger.write(src.printcolors.printcError("\nERROR: ") + msg, 3) + # retcode = False +# does post install substitutions +#for f in $(grep -RIl -e /volatile/salome/jenkins/workspace/Salome_master_CO7/SALOME-9.7.0-CO7/INSTALL INSTALL); do +# sed -i " +# s?/volatile/salome/jenkins/workspace/Salome_master_CO7/SALOME-9.7.0-CO7/INSTALL?$(pwd)/INSTALL?g +# " $f +#done + + + # show results + results[product_name] = retcode + if retcode: + # The case where it succeed + res = src.OK_STATUS + good_result = good_result + 1 + else: + # The case where it failed + res = src.KO_STATUS + + # print the result + if do_install_prod: + logger.write('%s\n' % src.printcolors.printc(res), 3, False) + + return good_result, results + +def check_sources(product_info, logger): + '''Check that the sources are correctly get, using the files to be tested + in product information + + :param product_info Config: The configuration specific to + the product to be prepared + :return: True if the files exists (or no files to test is provided). + :rtype: boolean + ''' + # Get the files to test if there is any + if ("present_files" in product_info and + "source" in product_info.present_files): + l_files_to_be_tested = product_info.present_files.source + for file_path in l_files_to_be_tested: + # The path to test is the source directory + # of the product joined the file path provided + path_to_test = os.path.join(product_info.source_dir, file_path) + logger.write(_("\nTesting existence of file: \n"), 5) + logger.write(path_to_test, 5) + if not os.path.exists(path_to_test): + return False, path_to_test + logger.write(src.printcolors.printcSuccess(" OK\n"), 5) + return True, "" + +def description(): + '''method that is called when salomeTools is called with --help option. + + :return: The text to display for the source command description. + :rtype: str + ''' + return _("The install command gets the binaries of the application products " + "from local (ARCHIVEPATH) or ftp server.\n\nexample:" + "\nsat install SALOME-master --products GEOM,SMESH") + +def run(args, runner, logger): + '''method that is called when salomeTools is called with install parameter. + ''' + DBG.write("install.run()", args) + # Parse the options + (options, args) = parser.parse_args(args) + + # check that the command has been called with an application + src.check_config_has_application( runner.cfg ) + + # Print some informations + logger.write(_('Getting binaries of the application %s\n') % + src.printcolors.printcLabel(runner.cfg.VARS.application), 1) + src.printcolors.print_value(logger, 'workdir', + runner.cfg.APPLICATION.workdir, 2) + logger.write("\n", 2, False) + + + # Get the list of all application products, and create its dependency graph + all_products_infos = src.product.get_products_infos(runner.cfg.APPLICATION.products, + runner.cfg) + from compile import get_dependencies_graph,depth_search_graph + all_products_graph=get_dependencies_graph(all_products_infos) + #logger.write("Dependency graph of all application products : %s\n" % all_products_graph, 6) + DBG.write("Dependency graph of all application products : ", all_products_graph) + + products_infos=[] + if options.products is None: + #implicit selection of all products + products_infos = all_products_infos + else: + # a list of products is specified + products_list=options.products + # we evaluate the complete list including dependencies (~ to the --with-fathers of sat compile) + + # Extend the list with all recursive dependencies of the given products + visited=[] + for p_name in products_list: + visited=depth_search_graph(all_products_graph, p_name, visited) + products_list = visited + logger.write("Product we have to compile (as specified by user) : %s\n" % products_list, 5) + + # Create a dict of all products to facilitate products_infos sorting + all_products_dict={} + for (pname,pinfo) in all_products_infos: + all_products_dict[pname]=(pname,pinfo) + + # build products_infos for the products we have to install + for product in products_list: + products_infos.append(all_products_dict[product]) + + + + # Call to the function that gets all the sources + good_result, results = get_all_product_binaries(runner.cfg, + products_infos, + logger) + + # Display the results (how much passed, how much failed, etc...) + status = src.OK_STATUS + details = [] + + logger.write("\n", 2, False) + if good_result == len(products_infos): + res_count = "%d / %d" % (good_result, good_result) + else: + status = src.KO_STATUS + res_count = "%d / %d" % (good_result, len(products_infos)) + + for product in results: + if results[product] == 0 or results[product] is None: + details.append(product) + + result = len(products_infos) - good_result + + # write results + logger.write(_("Getting binaries of the application:"), 1) + logger.write(" " + src.printcolors.printc(status), 1, False) + logger.write(" (%s)\n" % res_count, 1, False) + + if len(details) > 0: + logger.write(_("Following binaries haven't been get:\n"), 2) + logger.write(" ".join(details), 2) + logger.write("\n", 2, False) + + return result diff --git a/commands/launcher.py b/commands/launcher.py index aec0ff8..bfbdc92 100644 --- a/commands/launcher.py +++ b/commands/launcher.py @@ -33,6 +33,8 @@ parser.add_option('n', 'name', 'string', 'name', _('Optional: The name of the' parser.add_option('e', 'exe', 'string', 'path_exe', _('Use this option to generate a launcher which sets' ' the environment and call the executable given as argument' ' (its relative path to application workdir, or an exe name present in appli PATH)')) +parser.add_option('p', 'products', 'list2', 'products', + _("Optional: Includes only the specified products.")) parser.add_option('c', 'catalog', 'string', 'catalog', _('Optional: The resources catalog to use')) parser.add_option('', 'gencat', 'string', 'gencat', @@ -54,6 +56,7 @@ def generate_launch_file(config, launcher_name, pathlauncher, path_exe, + env_info, display=True, additional_env={}, no_path_init=False): @@ -69,6 +72,7 @@ def generate_launch_file(config, :param display boolean: If False, do not print anything in the terminal :param additional_env dict: The dict giving additional environment variables + :param env_info str: The list of products to add in the files. :return: The launcher file path. :rtype: str ''' @@ -151,8 +155,8 @@ def generate_launch_file(config, writer = src.environment.FileEnvWriter(config, logger, pathlauncher, - src_root=None, - env_info=None) + None, + env_info) # Display some information if display: @@ -302,6 +306,14 @@ def run(args, runner, logger): src.check_config_has_application( runner.cfg ) # Determine the launcher name (from option, profile section or by default "salome") + if options.products is None: + environ_info = None + else: + # add products specified by user (only products + # included in the application) + environ_info = filter(lambda l: + l in runner.cfg.APPLICATION.products.keys(), + options.products) if options.name: launcher_name = options.name else: @@ -341,6 +353,7 @@ def run(args, runner, logger): launcher_path, options.path_exe, additional_env = additional_environ, + env_info=environ_info, no_path_init = no_path_initialisation ) return 0 diff --git a/commands/package.py b/commands/package.py index 5cc26e6..f23290d 100644 --- a/commands/package.py +++ b/commands/package.py @@ -96,6 +96,8 @@ parser.add_option('f', 'force_creation', 'boolean', 'force_creation', parser.add_option('s', 'sources', 'boolean', 'sources', _('Optional: Produce a compilable archive of the sources of the ' 'application.'), False) +parser.add_option('', 'bin_products', 'boolean', 'bin_products', + _('Optional: Create binary archives for all products.'), False) parser.add_option('', 'with_vcs', 'boolean', 'with_vcs', _('Optional: Do not make archive for products in VCS mode (git, cvs, svn). ' 'Sat prepare will use VCS mode instead to retrieve them'), @@ -566,6 +568,51 @@ def product_appli_creation_script(config, return tmp_file_path +def bin_products_archives(config, logger): + '''Prepare binary packages for all products + :param config Config: The global configuration. + :return: the error status + :rtype: bool + ''' + + logger.write("Make %s binary archives\n" % config.VARS.dist) + # Get the default directory where to put the packages + binpackage_path = os.path.join(config.APPLICATION.workdir, "PACKAGE", "products") + src.ensure_path_exists(binpackage_path) + # Get the list of product installation to add to the archive + l_products_name = sorted(config.APPLICATION.products.keys()) + l_product_info = src.product.get_products_infos(l_products_name, + config) + # first loop on products : filter products, analyse properties, + # and store the information that will be used to create the archive in the second loop + l_not_installed=[] # store not installed products for warning at the end + for prod_name, prod_info in l_product_info: + # ignore the native and fixed products for install directories + if (src.get_property_in_product_cfg(prod_info, "not_in_package") == "yes" + or src.product.product_is_native(prod_info) + or src.product.product_is_fixed(prod_info) + or not src.product.product_compiles(prod_info)): + continue + if not src.product.check_installation(config, prod_info): + l_not_installed.append(prod_name) + continue # product is not installed, we skip it + # prepare call to make_bin_archive + path_targz_prod = os.path.join(binpackage_path, prod_name + '-' + prod_info.version + "-" + config.VARS.dist + PACKAGE_EXT) + targz_prod = tarfile.open(path_targz_prod, mode='w:gz') + bin_path = prod_info.install_dir + targz_prod.add(bin_path) + targz_prod.close() + # Python program to find MD5 hash value of a file + import hashlib + with open(path_targz_prod,"rb") as f: + bytes = f.read() # read file as bytes + readable_hash = hashlib.md5(bytes).hexdigest(); + with open(path_targz_prod+".md5", "w") as md5sum: + md5sum.write("%s %s" % (readable_hash, os.path.basename(path_targz_prod))) + logger.write(" archive : %s (md5sum = %s)\n" % (path_targz_prod, readable_hash)) + + return 0 + def binary_package(config, logger, options, tmp_working_dir): '''Prepare a dictionary that stores all the needed directories and files to add in a binary package. @@ -1025,6 +1072,24 @@ def get_archives_vcs(l_pinfo_vcs, sat, config, logger, tmp_working_dir): # DBG.write("END sat config", sat.cfg.APPLICATION, True) return d_archives_vcs +def make_bin_archive(prod_name, prod_info, where): + '''Create an archive of a product by searching its source directory. + + :param prod_name str: The name of the product. + :param prod_info Config: The specific configuration corresponding to the + product + :param where str: The path of the repository where to put the resulting + archive + :return: The path of the resulting archive + :rtype: str + ''' + path_targz_prod = os.path.join(where, prod_name + PACKAGE_EXT) + tar_prod = tarfile.open(path_targz_prod, mode='w:gz') + bin_path = prod_info.install_dir + tar_prod.add(bin_path, arcname=path_targz_prod) + tar_prod.close() + return path_targz_prod + def make_archive(prod_name, prod_info, where): '''Create an archive of a product by searching its source directory. @@ -1524,26 +1589,37 @@ def run(args, runner, logger): # Parse the options (options, args) = parser.parse_args(args) + # Check that a type of package is called, and only one all_option_types = (options.binaries, options.sources, options.project not in ["", None], - options.sat) + options.sat, + options.bin_products) # Check if no option for package type if all_option_types.count(True) == 0: msg = _("Error: Precise a type for the package\nUse one of the " "following options: --binaries, --sources, --project or" - " --salometools") + " --salometools, --bin_products") logger.write(src.printcolors.printcError(msg), 1) logger.write("\n", 1) return 1 - + do_create_package = options.binaries or options.sources or options.project or options.sat + + if options.bin_products: + ret = bin_products_archives(runner.cfg, logger) + if ret!=0: + return ret + if not do_create_package: + return 0 + + # continue to create a tar.gz package + # The repository where to put the package if not Binary or Source package_default_path = runner.cfg.LOCAL.workdir - # if the package contains binaries or sources: - if options.binaries or options.sources: + if options.binaries or options.sources or options.bin_products: # Check that the command has been called with an application src.check_config_has_application(runner.cfg) diff --git a/complete_sat.sh b/complete_sat.sh index 7470b1b..d8e623b 100755 --- a/complete_sat.sh +++ b/complete_sat.sh @@ -212,7 +212,7 @@ _salomeTools_complete() return 0 ;; launcher) - opts="--name --exe --catalog --gencat --no_path_init --use_mesa" + opts="--products --name --exe --catalog --gencat --no_path_init --use_mesa" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; diff --git a/doc/src/commands/launcher.rst b/doc/src/commands/launcher.rst index 8ec433f..0a4efef 100644 --- a/doc/src/commands/launcher.rst +++ b/doc/src/commands/launcher.rst @@ -52,6 +52,15 @@ Usage sat launcher --use_mesa +* Generate the environment files only with the given products: + + .. code-block:: bash + + # This will create a launcher file called salome-pre, + # which will contain GEOM,SMESH and their prerequisites. + # It is useful if you want to create a launcher with only a part of Salome + sat launcher --product GEOM,SMESH -n salome-pre + Configuration ============= diff --git a/src/__init__.py b/src/__init__.py index 95bd104..9a4a846 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -456,7 +456,7 @@ def find_file_in_lpath(file_name, lpath, additional_dir = ""): return os.path.join(dir_complete, file_name) return False -def find_file_in_ftppath(file_name, ftppath, installation_dir, logger): +def find_file_in_ftppath(file_name, ftppath, installation_dir, logger, additional_dir = ""): """\ Find in all ftp servers in ftppath the file called file_name If it is found then return the destination path of the file @@ -495,8 +495,20 @@ def find_file_in_ftppath(file_name, ftppath, installation_dir, logger): for directory in ftp_archive_split[1:]: logger.write(" Change directory to %s\n" % directory, 3) ftp.cwd(directory) + if additional_dir: + ftp.cwd(additional_dir) except: logger.error("while connecting to ftp server %s\n" % ftp_server) + continue + + try: # get md5 file if it exists + file_name_md5=file_name + ".md5" + destination_md5=destination + ".md5" + if ftp.size(file_name_md5) > 0: + with open(destination_md5,'wb') as dest_file_md5: + ftp.retrbinary("RETR "+file_name_md5, dest_file_md5.write) + except: + pass try: if ftp.size(file_name) > 0: @@ -507,7 +519,6 @@ def find_file_in_ftppath(file_name, ftppath, installation_dir, logger): return destination except: logger.error("File not found in ftp_archive %s\n" % ftp_server) - pass return False diff --git a/src/environment.py b/src/environment.py index f44c131..0699a3d 100644 --- a/src/environment.py +++ b/src/environment.py @@ -229,11 +229,11 @@ class SalomeEnviron: return "%s(\n%s\n)" % (self.__class__.__name__, PP.pformat(res)) def __set_sorted_products_list(self): - from compile import get_dependencies_graph, depth_first_topo_graph all_products_infos = src.product.get_products_infos( self.cfg.APPLICATION.products, self.cfg) + from compile import get_dependencies_graph,depth_first_topo_graph all_products_graph=get_dependencies_graph(all_products_infos, self.forBuild) visited_nodes=[] sorted_nodes=[] @@ -245,6 +245,7 @@ class SalomeEnviron: visited_nodes, sorted_nodes) self.sorted_product_list=sorted_nodes + self.all_products_graph=all_products_graph def append(self, key, value, sep=os.pathsep): @@ -760,7 +761,7 @@ class SalomeEnviron: def set_full_environ(self, logger, env_info): """\ - Sets the full environment for products + Sets the full environment for products, with their dependencies specified in env_info dictionary. :param logger Logger: The logger instance to display messages @@ -773,9 +774,13 @@ class SalomeEnviron: # use the sorted list of all products to sort the list of products # we have to set + visited=[] + from compile import depth_search_graph # to get the dependencies + for p_name in env_info: + visited=depth_search_graph(self.all_products_graph, p_name, visited) sorted_product_list=[] for n in self.sorted_product_list: - if n in env_info: + if n in visited: sorted_product_list.append(n) if "Python" in sorted_product_list: diff --git a/src/system.py b/src/system.py index b7cc6c2..c54452b 100644 --- a/src/system.py +++ b/src/system.py @@ -98,7 +98,9 @@ def git_extract(from_what, tag, git_options, where, logger, environment=None): cmd = r""" set -x git clone %(git_options)s %(remote)s %(where)s -touch -d "$(git --git-dir=%(where_git)s log -1 --format=date_format)" %(where)s +res=$? +if [ $res -eq 0 ]; then touch -d "$(git --git-dir=%(where_git)s log -1 --format=date_format)" %(where)s;fi +exit $res """ cmd = cmd % {'git_options': git_options, 'remote': from_what, 'tag': tag, 'where': str(where), 'where_git': where_git} else: @@ -115,7 +117,7 @@ git clone %(git_options)s %(remote)s %(where)s && \ git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(tag)s res=$? git --git-dir=%(where_git)s status|grep HEAD -if [ $? -ne 0 ]; then touch -d "$(git --git-dir=%(where_git)s log -1 --format=date_format)" %(where)s;fi +if [ $res -eq 0 -a $? -ne 0 ]; then touch -d "$(git --git-dir=%(where_git)s log -1 --format=date_format)" %(where)s;fi exit $res """ cmd = cmd % {'git_options': git_options,