From 8104e1516931aca9b4df1e1497760686a6580e18 Mon Sep 17 00:00:00 2001 From: Aymeric Sonolet Date: Wed, 31 Jan 2024 13:40:58 +0100 Subject: [PATCH] spns #40791: adding sat update command --- commands/source_update.py | 664 ++++++++++++++++++++++++++++++++++++++ commands/update.py | 296 +++++++++++++++++ complete_sat.sh | 2 +- src/system.py | 72 +++++ 4 files changed, 1033 insertions(+), 1 deletion(-) create mode 100644 commands/source_update.py create mode 100644 commands/update.py diff --git a/commands/source_update.py b/commands/source_update.py new file mode 100644 index 0000000..c934d62 --- /dev/null +++ b/commands/source_update.py @@ -0,0 +1,664 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +# Copyright (C) 2010-2012 CEA/DEN +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os +import shutil +import re +import subprocess + +import src +import prepare +import src.debug as DBG + +# Define all possible option for patch command : sat patch +parser = src.options.Options() +parser.add_option( + "p", + "products", + "list2", + "products", + _( + "Optional: products from which to get the sources. This option accepts a comma separated list." + ), +) + + +def update_source_for_dev(config, product_info, source_dir, logger, pad): + """The method called if the product is in development mode + + :param config Config: The global configuration + :param product_info Config: The configuration specific to + the product to be prepared + :param source_dir Path: The Path instance corresponding to the + directory where to put the sources + :param logger Logger: The logger instance to use for the display and logging + :param pad int: The gap to apply for the terminal display + :return: True if it succeed, else False + :rtype: boolean + """ + + # Call the function corresponding to get the sources with True checkout + retcode = update_product_sources( + config, product_info, True, source_dir, logger, pad, checkout=True + ) + logger.write("\n", 3, False) + # +2 because product name is followed by ': ' + logger.write(" " * (pad + 2), 3, False) + + logger.write( + "dev: %s ... " % src.printcolors.printcInfo(product_info.source_dir), 3, False + ) + logger.flush() + + return retcode + + +def update_source_from_git( + config, product_info, source_dir, logger, pad, is_dev=False, environ=None +): + """The method called if the product is to be get in git mode + + :param product_info Config: The configuration specific to + the product to be prepared + :param source_dir Path: The Path instance corresponding to the + directory where to put the sources + :param logger Logger: The logger instance to use for the display and logging + :param pad int: The gap to apply for the terminal display + :param is_dev boolean: True if the product is in development mode + :param environ src.environment.Environ: The environment to source when + extracting. + :return: True if it succeed, else False + :rtype: boolean + """ + # The str to display + coflag = "git" + + use_repo_dev = False + if ( + "APPLICATION" in config + and "properties" in config.APPLICATION + and "repo_dev" in config.APPLICATION.properties + and config.APPLICATION.properties.repo_dev == "yes" + ): + use_repo_dev = True + + # Get the repository address. + # If the application has the repo_dev property + # Or if the product is in dev mode + # Then we use repo_dev if the key exists + if (is_dev or use_repo_dev) and "repo_dev" in product_info.git_info: + coflag = src.printcolors.printcHighlight(coflag.upper()) + repo_git = product_info.git_info.repo_dev + else: + repo_git = product_info.git_info.repo + + # Display informations + logger.write("%s:%s" % (coflag, src.printcolors.printcInfo(repo_git)), 3, False) + logger.write(" " * (pad + 50 - len(repo_git)), 3, False) + logger.write( + " tag:%s" % src.printcolors.printcInfo(product_info.git_info.tag), 3, False + ) + logger.write(" %s. " % ("." * (10 - len(product_info.git_info.tag))), 3, False) + logger.flush() + logger.write("\n", 5, False) + + git_options = "" + if "git_options" in product_info.git_info: + git_options = product_info.git_info.git_options + + # Call the system function that do the checkout in git mode + retcode = src.system.git_checkout( + product_info.git_info.tag, + git_options, + source_dir, + logger, + environ, + ) + if not retcode: + logger.write( + r""" +Error, %(repo)s could not checkout %(branch)s. Please check you don't +have uncommited work and branch or tag %(branch)s exists.""" + % {"repo": repo_git, "branch": product_info.git_info.tag} + ) + raise Exception() + + # Call the system function that do the checkout in git mode + retcode = src.system.git_pull( + repo_git, + product_info.git_info.tag, + git_options, + source_dir, + logger, + environ, + ) + if not retcode: + logger.write( + r""" +Error, %(repo)s could not pull branch %(branch)s. Please check that your branch +has not diverged from repo.""" + % {"repo": repo_git, "branch": product_info.git_info.tag} + ) + raise Exception() + return retcode + + +def get_source_from_archive(config, product_info, source_dir, logger): + """The method called if the product is to be get in archive mode + + :param config Config: The global configuration + :param product_info Config: The configuration specific to + the product to be prepared + :param source_dir Path: The Path instance corresponding to the + directory where to put the sources + :param logger Logger: The logger instance to use for the display and logging + :return: True if it succeed, else False + :rtype: boolean + """ + + # check if pip should be used : pip mode id activated if the application + # and product have pip property + if src.appli_test_property( + config, "pip", "yes" + ) and src.product.product_test_property(product_info, "pip", "yes"): + pip_msg = "PIP : do nothing, product will be downloaded later at compile time " + logger.write(pip_msg, 3) + return True + + # check archive exists + if not os.path.exists(product_info.archive_info.archive_name): + # The archive is not found on local file system (ARCHIVEPATH) + # We try ftp! + logger.write( + "\n The archive is not found on local file system, we try ftp\n", 3 + ) + ret = src.find_file_in_ftppath( + product_info.archive_info.archive_name, + config.PATHS.ARCHIVEFTP, + config.LOCAL.archive_dir, + logger, + ) + if ret: + # archive was found on ftp and stored in ret + product_info.archive_info.archive_name = ret + else: + raise src.SatException( + _("Archive not found in ARCHIVEPATH, nor on ARCHIVEFTP: '%s'") + % product_info.archive_info.archive_name + ) + + logger.write( + "arc:%s ... " + % src.printcolors.printcInfo(product_info.archive_info.archive_name), + 3, + False, + ) + logger.flush() + # Call the system function that do the extraction in archive mode + retcode, NameExtractedDirectory = src.system.archive_extract( + product_info.archive_info.archive_name, source_dir.dir(), logger + ) + + # Rename the source directory if + # it does not match with product_info.source_dir + if NameExtractedDirectory.replace("/", "") != os.path.basename( + product_info.source_dir + ): + shutil.move( + os.path.join( + os.path.dirname(product_info.source_dir), NameExtractedDirectory + ), + product_info.source_dir, + ) + + return retcode + + +def get_source_from_dir(product_info, source_dir, logger): + if "dir_info" not in product_info: + msg = _( + "Error: you must put a dir_info section" + " in the file %s.pyconf" % product_info.name + ) + logger.write("\n%s\n" % src.printcolors.printcError(msg), 1) + return False + + if "dir" not in product_info.dir_info: + msg = _( + "Error: you must put a dir in the dir_info section" + " in the file %s.pyconf" % product_info.name + ) + logger.write("\n%s\n" % src.printcolors.printcError(msg), 1) + return False + + # check that source exists + if not os.path.exists(product_info.dir_info.dir): + msg = _( + "Error: the dir %s defined in the file" + " %s.pyconf does not exists" + % (product_info.dir_info.dir, product_info.name) + ) + logger.write("\n%s\n" % src.printcolors.printcError(msg), 1) + return False + + logger.write( + "DIR: %s ... " % src.printcolors.printcInfo(product_info.dir_info.dir), 3 + ) + logger.flush() + + retcode = src.Path(product_info.dir_info.dir).copy(source_dir) + + return retcode + + +def get_source_from_cvs( + user, product_info, source_dir, checkout, logger, pad, environ=None +): + """The method called if the product is to be get in cvs mode + + :param user str: The user to use in for the cvs command + :param product_info Config: The configuration specific to + the product to be prepared + :param source_dir Path: The Path instance corresponding to the + directory where to put the sources + :param checkout boolean: If True, get the source in checkout mode + :param logger Logger: The logger instance to use for the display and logging + :param pad int: The gap to apply for the terminal display + :param environ src.environment.Environ: The environment to source when + extracting. + :return: True if it succeed, else False + :rtype: boolean + """ + # Get the protocol to use in the command + if "protocol" in product_info.cvs_info: + protocol = product_info.cvs_info.protocol + else: + protocol = "pserver" + + # Construct the line to display + if "protocol" in product_info.cvs_info: + cvs_line = "%s:%s@%s:%s" % ( + protocol, + user, + product_info.cvs_info.server, + product_info.cvs_info.product_base, + ) + else: + cvs_line = "%s / %s" % ( + product_info.cvs_info.server, + product_info.cvs_info.product_base, + ) + + coflag = "cvs" + if checkout: + coflag = src.printcolors.printcHighlight(coflag.upper()) + + logger.write("%s:%s" % (coflag, src.printcolors.printcInfo(cvs_line)), 3, False) + logger.write(" " * (pad + 50 - len(cvs_line)), 3, False) + logger.write( + " src:%s" % src.printcolors.printcInfo(product_info.cvs_info.source), 3, False + ) + logger.write(" " * (pad + 1 - len(product_info.cvs_info.source)), 3, False) + logger.write( + " tag:%s" % src.printcolors.printcInfo(product_info.cvs_info.tag), 3, False + ) + # at least one '.' is visible + logger.write(" %s. " % ("." * (10 - len(product_info.cvs_info.tag))), 3, False) + logger.flush() + logger.write("\n", 5, False) + + # Call the system function that do the extraction in cvs mode + retcode = src.system.cvs_extract( + protocol, + user, + product_info.cvs_info.server, + product_info.cvs_info.product_base, + product_info.cvs_info.tag, + product_info.cvs_info.source, + source_dir, + logger, + checkout, + environ, + ) + return retcode + + +def get_source_from_svn(user, product_info, source_dir, checkout, logger, environ=None): + """The method called if the product is to be get in svn mode + + :param user str: The user to use in for the svn command + :param product_info Config: The configuration specific to + the product to be prepared + :param source_dir Path: The Path instance corresponding to the + directory where to put the sources + :param checkout boolean: If True, get the source in checkout mode + :param logger Logger: The logger instance to use for the display and logging + :param environ src.environment.Environ: The environment to source when + extracting. + :return: True if it succeed, else False + :rtype: boolean + """ + coflag = "svn" + if checkout: + coflag = src.printcolors.printcHighlight(coflag.upper()) + + logger.write( + "%s:%s ... " % (coflag, src.printcolors.printcInfo(product_info.svn_info.repo)), + 3, + False, + ) + logger.flush() + logger.write("\n", 5, False) + # Call the system function that do the extraction in svn mode + retcode = src.system.svn_extract( + user, + product_info.svn_info.repo, + product_info.svn_info.tag, + source_dir, + logger, + checkout, + environ, + ) + return retcode + + +def update_product_sources( + config, product_info, is_dev, source_dir, logger, pad, checkout=False +): + """Get the product sources. + + :param config Config: The global configuration + :param product_info Config: The configuration specific to + the product to be prepared + :param is_dev boolean: True if the product is in development mode + :param source_dir Path: The Path instance corresponding to the + directory where to put the sources + :param logger Logger: The logger instance to use for the display and logging + :param pad int: The gap to apply for the terminal display + :param checkout boolean: If True, get the source in checkout mode + :return: True if it succeed, else False + :rtype: boolean + """ + + # Get the application environment + logger.write(_("Set the application environment\n"), 5) + env_appli = src.environment.SalomeEnviron( + config, src.environment.Environ(dict(os.environ)) + ) + env_appli.set_application_env(logger) + + # Call the right function to get sources regarding the product settings + if not checkout and is_dev: + return get_source_for_dev(config, product_info, source_dir, logger, pad) + + if product_info.get_source == "git": + return update_source_from_git( + config, product_info, source_dir, logger, pad, is_dev, env_appli + ) + + if product_info.get_source == "archive": + return get_source_from_archive(config, product_info, source_dir, logger) + + if product_info.get_source == "dir": + return get_source_from_dir(product_info, source_dir, logger) + + if product_info.get_source == "cvs": + cvs_user = config.USER.cvs_user + return get_source_from_cvs( + cvs_user, product_info, source_dir, checkout, logger, pad, env_appli + ) + + if product_info.get_source == "svn": + svn_user = config.USER.svn_user + return get_source_from_svn( + svn_user, product_info, source_dir, checkout, logger, env_appli + ) + + if product_info.get_source == "native": + # for native products we check the corresponding system packages are installed + logger.write("Native : Checking system packages are installed\n", 3) + check_cmd = src.system.get_pkg_check_cmd( + config.VARS.dist_name + ) # (either rmp or apt) + run_pkg, build_pkg = src.product.check_system_dep( + config.VARS.dist, check_cmd, product_info + ) + result = True + for pkg in run_pkg: + logger.write(" - " + pkg + " : " + run_pkg[pkg] + "\n", 1) + if "KO" in run_pkg[pkg]: + result = False + for pkg in build_pkg: + logger.write(" - " + pkg + " : " + build_pkg[pkg] + "\n", 1) + if "KO" in build_pkg[pkg]: + result = False + if result == False: + logger.error( + "some system dependencies are missing, please install them with " + + check_cmd[0] + ) + return result + + if product_info.get_source == "fixed": + # skip + logger.write("%s " % src.printcolors.printc(src.OK_STATUS), 3, False) + msg = "FIXED : %s\n" % product_info.install_dir + + if not os.path.isdir(product_info.install_dir): + logger.warning( + "The fixed path do not exixts!! Please check it : %s\n" + % product_info.install_dir + ) + else: + logger.write(msg, 3) + return True + + # if the get_source is not in [git, archive, cvs, svn, fixed, native] + logger.write( + _('Unknown get source method "%(get)s" for product %(product)s') + % {"get": product_info.get_source, "product": product_info.name}, + 3, + False, + ) + logger.write(" ... ", 3, False) + logger.flush() + return False + + +def update_all_product_sources(config, products, logger): + """Update/get all the product sources. + + :param config Config: The global configuration + :param products List: The list of tuples (product name, product informations) + :param logger Logger: The logger instance to be used for the logging + :return: the tuple (number of success, dictionary product_name/success_fail) + :rtype: (int,dict) + """ + + # Initialize the variables that will count the fails and success + results = dict() + good_result = 0 + + # Get the maximum name length in order to format the terminal display + max_product_name_len = 1 + if len(products) > 0: + max_product_name_len = max(map(lambda l: len(l), products[0])) + 4 + + # The loop on all the products from which to get the sources + # DBG.write("source.get_all_product_sources config id", id(config), True) + for product_name, product_info in products: + # get product name, product informations and the directory where to put + # the sources + if not ( + src.product.product_is_fixed(product_info) + or src.product.product_is_native(product_info) + ): + source_dir = src.Path(product_info.source_dir) + else: + source_dir = src.Path("") + + # display and log + logger.write("%s: " % src.printcolors.printcLabel(product_name), 3) + logger.write(" " * (max_product_name_len - len(product_name)), 3, False) + logger.write("\n", 4, False) + + # TODO: check that there is no pb on not .git tracked products + # if source_dir.exists(): + # logger.write("%s " % src.printcolors.printc(src.OK_STATUS), 3, False) + # msg = ( + # _( + # "INFO : Not doing anything because the source directory already exists:\n %s\n" + # ) + # % source_dir + # ) + # logger.write(msg, 3) + # good_result = good_result + 1 + # # Do not get the sources and go to next product + # continue + + is_dev = src.product.product_is_dev(product_info) + # Call to the function that get the sources for one product + retcode = update_product_sources( + config, + product_info, + is_dev, + source_dir, + logger, + max_product_name_len, + checkout=False, + ) + + # Check that the sources are correctly get using the files to be tested + # in product information + if retcode: + check_OK, wrong_path = check_sources(product_info, logger) + if not check_OK: + # Print the missing file path + msg = _("The required file %s does not exists. " % wrong_path) + logger.write(src.printcolors.printcError("\nERROR: ") + msg, 3) + retcode = False + + # show results + results[product_name] = retcode + if retcode: + # The case where it succeed + res = src.OK_STATUS + good_result = good_result + 1 + else: + # The case where it failed + res = src.KO_STATUS + + # print the result + if not ( + src.product.product_is_fixed(product_info) + or src.product.product_is_native(product_info) + ): + logger.write("%s\n" % src.printcolors.printc(res), 3, False) + + return good_result, results + + +def check_sources(product_info, logger): + """Check that the sources are correctly get, using the files to be tested + in product information + + :param product_info Config: The configuration specific to + the product to be prepared + :return: True if the files exists (or no files to test is provided). + :rtype: boolean + """ + # Get the files to test if there is any + if "present_files" in product_info and "source" in product_info.present_files: + l_files_to_be_tested = product_info.present_files.source + for file_path in l_files_to_be_tested: + # The path to test is the source directory + # of the product joined the file path provided + path_to_test = os.path.join(product_info.source_dir, file_path) + logger.write(_("\nTesting existence of file: \n"), 5) + logger.write(path_to_test, 5) + if not os.path.exists(path_to_test): + return False, path_to_test + logger.write(src.printcolors.printcSuccess(" OK\n"), 5) + return True, "" + + +def description(): + """method that is called when salomeTools is called with --help option. + + :return: The text to display for the source command description. + :rtype: str + """ + return _( + "The source command gets the sources of the application products " + "from cvs, git or an archive.\n\nexample:" + "\nsat source SALOME-master --products KERNEL,GUI" + ) + + +def run(args, runner, logger): + """method that is called when salomeTools is called with source parameter.""" + DBG.write("source.run()", args) + # Parse the options + (options, args) = parser.parse_args(args) + + # check that the command has been called with an application + src.check_config_has_application(runner.cfg) + + # Print some informations + logger.write( + _("Getting sources of the application %s\n") + % src.printcolors.printcLabel(runner.cfg.VARS.application), + 1, + ) + src.printcolors.print_value(logger, "workdir", runner.cfg.APPLICATION.workdir, 2) + logger.write("\n", 2, False) + + # Get the products list with products informations regarding the options + products_infos = src.product.get_products_list(options, runner.cfg, logger) + + # Call to the function that gets all the sources + good_result, results = update_all_product_sources(runner.cfg, products_infos, logger) + + # Display the results (how much passed, how much failed, etc...) + status = src.OK_STATUS + details = [] + + logger.write("\n", 2, False) + if good_result == len(products_infos): + res_count = "%d / %d" % (good_result, good_result) + else: + status = src.KO_STATUS + res_count = "%d / %d" % (good_result, len(products_infos)) + + for product in results: + if results[product] == 0 or results[product] is None: + details.append(product) + + result = len(products_infos) - good_result + + # write results + logger.write(_("Getting sources of the application:"), 1) + logger.write(" " + src.printcolors.printc(status), 1, False) + logger.write(" (%s)\n" % res_count, 1, False) + + if len(details) > 0: + logger.write(_("Following sources haven't been get:\n"), 2) + logger.write(" ".join(details), 2) + logger.write("\n", 2, False) + + return result diff --git a/commands/update.py b/commands/update.py new file mode 100644 index 0000000..9952d82 --- /dev/null +++ b/commands/update.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +# Copyright (C) 2010-2012 CEA/DEN +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import re +import os +import pprint as PP + +import src +import src.debug as DBG + + +# Define all possible option for update command : sat update +parser = src.options.Options() +parser.add_option( + "p", + "products", + "list2", + "products", + _("Optional: products to update. This option accepts a comma separated list."), +) +parser.add_option('f', 'force', 'boolean', 'force', + _("Optional: force to update the products in development mode.")) +parser.add_option('', 'force_patch', 'boolean', 'force_patch', + _("Optional: force to apply patch to the products in development mode.")) + +def find_products_already_prepared(l_products): + """function that returns the list of products that have an existing source + directory. + + :param l_products List: The list of products to check + :return: The list of product configurations that have an existing source + directory. + :rtype: List + """ + l_res = [] + for p_name_p_cfg in l_products: + __, prod_cfg = p_name_p_cfg + if "source_dir" in prod_cfg and os.path.exists(prod_cfg.source_dir): + l_res.append(p_name_p_cfg) + return l_res + +def find_git_products(l_products): + """ + function that returns the list of products that have an existing source + directory and a git configuration. Those products will be updated using : + + git checkout TARGET_TAG + git pull origin TARGET_TAG --ff-only + + Not committed dev or conflict with origin during pull will trigger an error. + + :param l_products List: The list of products to check + :return: The list of product configurations that have an existing source + directory and a git history. + :rtype: List + """ + l_res = [] + for p_name_p_cfg in l_products: + __, prod_cfg = p_name_p_cfg + if "source_dir" in prod_cfg and os.path.exists(prod_cfg.source_dir): + if prod_cfg.get_source == "git": + l_res.append(p_name_p_cfg) + return l_res + + +def find_products_with_patchs(l_products): + """function that returns the list of products that have one or more patches. + + :param l_products List: The list of products to check + :return: The list of product configurations that have one or more patches. + :rtype: List + """ + l_res = [] + for p_name_p_cfg in l_products: + __, prod_cfg = p_name_p_cfg + l_patchs = src.get_cfg_param(prod_cfg, "patches", []) + if len(l_patchs) > 0: + l_res.append(p_name_p_cfg) + return l_res + + +def description(): + """method that is called when salomeTools is called with --help option. + + :return: The text to display for the update command description. + :rtype: str + """ + return _( + "The update command updates the sources under git and gets the sources of " + "the application products and apply the patches if there is any." + "\n\nexample:\nsat update SALOME-master --products KERNEL,GUI" + ) + +class UpdateOp: + """ + This is an operation class. It is prepared though the init and launched + with the launch method. + + This operation updates the products, meaning it get the missing ones, and + pull the TARGET_TAG for the already present ones. It prevents from erasing + everything, especially .git/ files. + + In case you have uncommited work, the operation will stop. + + In case the remote tracking branch can't be pulled fast-forward (after a + checkout to the matching local branch), the operation will stop. + """ + + def __init__(self, args, runner, logger): + """ + Initialisation of the UpdateOp. The runner and the plateform are + checked. + + :args: arguments passed to sat + :runner: Sat class instance + :logger: Current logger + """ + # check that the command has been called with an application + src.check_config_has_application(runner.cfg) + + # write warning if platform is not declared as supported + src.check_platform_is_supported(runner.cfg, logger) + + # Parse the options + (options, args) = parser.parse_args(args) + self._list_of_products = options.products + self._force_patch = options.force_patch + self._force = options.force + + self.runner = runner + self.logger = logger + self.products_infos = src.product.get_products_list(options, self.runner.cfg, self.logger) + + # Construct the arguments to pass to the clean, source and patch commands + self.args_appli = runner.cfg.VARS.application + " " # useful whitespace + + @property + def products(self): + if self._list_of_products: + return list(self._list_of_products) + return [name for name, tmp in self.products_infos] + + def getProductsToPrepare(self): + """ + Remove products that are already prepared and under git tracking so + that only new products (and not tracked ones) are prepared. + """ + pi_already_prepared = find_git_products(self.products_infos) + l_already_prepared = [i for i, tmp in pi_already_prepared] + newList, removedList = removeInList(self.products, l_already_prepared) + if len(newList) == 0 and len(removedList) > 0: + msg = "\nAll the products are already installed, do nothing!\n" + self.logger.write(src.printcolors.printcWarning(msg), 1) + return 0 + if len(removedList) > 0: + msg = ( + "\nList of already prepared products that are skipped : %s\n" + % ",".join(removedList) + ) + self.logger.write(msg, 3) + return newList + + def getProductsToUpdate(self): + pi_already_prepared = find_git_products(self.products_infos) + productsToUpdate = [i for i, tmp in pi_already_prepared] + return productsToUpdate + + def getProductsToClean(self, listProdToPrepare): + ldev_products = [p for p in self.products_infos if src.product.product_is_dev(p[1])] + productsToClean = listProdToPrepare # default + if len(ldev_products) > 0: + l_products_not_getted = find_products_already_prepared(ldev_products) + listNot = [i for i, tmp in l_products_not_getted] + productsToClean, removedList = removeInList(listProdToPrepare, listNot) + if len(removedList) > 0: + msg = _( + """ + Do not get the source of the following products in + development mode. + """ + ) + msg += "\n%s\n" % ",".join(removedList) + self.logger.write(src.printcolors.printcWarning(msg), 1) + return productsToClean + + def getProductsToPatch(self, listProdToPrepare): + productsToPatch = listProdToPrepare # default + ldev_products = [p for p in self.products_infos if src.product.product_is_dev(p[1])] + if not self._force_patch and len(ldev_products) > 0: + l_products_with_patchs = find_products_with_patchs(ldev_products) + listNot = [i for i, tmp in l_products_with_patchs] + productsToPatch, removedList = removeInList(listProdToPrepare, listNot) + if len(removedList) > 0: + msg = _( + """ + Do not patch the following products in development mode. + Use the --force_patch option to overwrite it. + """ + ) + msg += "\n%s\n" % ",".join(removedList) + self.logger.write(src.printcolors.printcWarning(msg), 1) + return productsToPatch + + def launch(self): + productsToPrepare = self.getProductsToPrepare() + args_product_to_prepare_opt = "--products " + ",".join(productsToPrepare) + + productsToUpdate = self.getProductsToUpdate() + args_product_to_update_opt = "--products " + ",".join(productsToUpdate) + + productsToClean = self.getProductsToClean(productsToPrepare) + args_product_opt_clean = "--products " + ",".join(productsToClean) + + productsToPatch = self.getProductsToPatch(productsToPrepare) + args_product_opt_patch = "--products " + ",".join(productsToPatch) + + + # Initialize the results to a running status + res_clean = 0 + res_source = 0 + res_patch = 0 + + # Call the commands using the API + if self._force: + if len(productsToClean) > 0: + msg = _("Clean the source directories ...") + self.logger.write(msg, 3) + self.logger.flush() + args_clean = self.args_appli + args_product_opt_clean + " --sources" + res_clean = self.runner.clean(args_clean, batch=True, verbose = 0, logger_add_link = self.logger) + if res_clean == 0: + self.logger.write('%s\n' % src.printcolors.printc(src.OK_STATUS), 3) + else: + self.logger.write('%s\n' % src.printcolors.printc(src.KO_STATUS), 3) + if len(productsToPrepare) > 0: + msg = _("Get the sources of the products ...") + self.logger.write(msg, 5) + args_source = self.args_appli + args_product_to_prepare_opt + res_source = self.runner.source(args_source, logger_add_link=self.logger) + if res_source == 0: + self.logger.write("%s\n" % src.printcolors.printc(src.OK_STATUS), 5) + else: + self.logger.write("%s\n" % src.printcolors.printc(src.KO_STATUS), 5) + if len(productsToPatch) > 0: + msg = _("Patch the product sources (if any) ...") + self.logger.write(msg, 5) + args_patch = self.args_appli + args_product_opt_patch + res_patch = self.runner.patch(args_patch, logger_add_link=self.logger) + if res_patch == 0: + self.logger.write("%s\n" % src.printcolors.printc(src.OK_STATUS), 5) + else: + self.logger.write("%s\n" % src.printcolors.printc(src.KO_STATUS), 5) + if len(productsToUpdate) > 0: + msg = _("Update the sources of the products ...") + self.logger.write(msg, 5) + args_source = self.args_appli + args_product_to_update_opt + res_source = self.runner.source_update(args_source, logger_add_link=self.logger) + if res_source == 0: + self.logger.write("%s\n" % src.printcolors.printc(src.OK_STATUS), 5) + else: + self.logger.write("%s\n" % src.printcolors.printc(src.KO_STATUS), 5) + return res_clean + res_source + res_patch + + +def run(args, runner, logger): + """method that is called when salomeTools is called with update parameter.""" + updateOp = UpdateOp(args, runner, logger) + res = updateOp.launch() + return res + + +def removeInList(aList, removeList): + """Removes elements of removeList list from aList + + :param aList: (list) The list from which to remove elements + :param removeList: (list) The list which contains elements to remove + :return: (list, list) (list with elements removed, list of elements removed) + """ + res1 = [i for i in aList if i not in removeList] + res2 = [i for i in aList if i in removeList] + return (res1, res2) diff --git a/complete_sat.sh b/complete_sat.sh index f3206ad..a42432d 100755 --- a/complete_sat.sh +++ b/complete_sat.sh @@ -107,7 +107,7 @@ _salomeTools_complete() # first argument => show available commands if [[ ${argc} == 1 ]] then - opts="config log source patch prepare environ clean configure make makeinstall compile launcher run jobs job shell test package generate find_duplicates application template base check profile script init --help --overwrite --debug --verbose --batch --all_in_terminal --logs_paths_in_file" + opts="config log source patch prepare environ clean configure make makeinstall compile launcher run jobs job shell update test package generate find_duplicates application template base check profile script init --help --overwrite --debug --verbose --batch --all_in_terminal --logs_paths_in_file" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 fi diff --git a/src/system.py b/src/system.py index acb6fea..8ffcd5a 100644 --- a/src/system.py +++ b/src/system.py @@ -196,6 +196,78 @@ exit $res return rc.isOk() +def git_checkout(tag, git_options, where, logger, environment=None): + '''Checkout sources from a git repository. +87 + :param tag str: The tag. + :param git_options str: git options + :param where str: The path where to extract. + :param logger Logger: The logger instance to use. + :param environment src.environment.Environ: The environment to source when extracting. + :return: True if the extraction is successful + :rtype: boolean + ''' + DBG.write("git_checkout", [tag, str(where)]) + if not where.exists(): + where.make() + where_git = os.path.join(str(where), ".git") + cmd = r""" +git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(git_options)s --guess %(tag)s +""" + cmd = cmd % {'git_options': git_options, 'tag': tag, 'where': str(where), 'where_git': where_git} + + cmd=cmd.replace('date_format', '"%ai"') + cmd=cmd.replace('--no-guess ', '') + isOk = launch_command(cmd, logger, where, environment) + return isOk + +def git_pull(from_what, tag, git_options, where, logger, environment=None): + '''Checkout sources from a git repository. +87 + :param from_what str: The remote git repository. + :param tag str: The tag. + :param git_options str: git options + :param where str: The path where to extract. + :param logger Logger: The logger instance to use. + :param environment src.environment.Environ: The environment to source when extracting. + :return: True if the extraction is successful + :rtype: boolean + ''' + DBG.write("git_pull", [from_what, tag, str(where)]) + if not where.exists(): + where.make() + where_git = os.path.join(str(where), ".git") + cmd = r""" +git --git-dir=%(where_git)s --work-tree=%(where)s pull %(git_options)s --recurse-submodule --ff-only origin %(tag)s +""" + cmd = cmd % {'git_options': git_options, 'tag': tag, 'where': str(where), 'where_git': where_git} + + cmd=cmd.replace('date_format', '"%ai"') + isOk = launch_command(cmd, logger, where, environment) + return isOk + + +def launch_command(cmd, logger, where, environment): + logger.logTxtFile.write("\n" + cmd + "\n") + logger.logTxtFile.flush() + + DBG.write("cmd", cmd) + # git commands may fail sometimes for various raisons + # (big module, network troubles, tuleap maintenance) + # therefore we give several tries + i_try = 0 + max_number_of_tries = 3 + sleep_delay = 30 # seconds + while (True): + i_try += 1 + rc = UTS.Popen(cmd, cwd=str(where.dir()), env=environment.environ.environ, logger=logger) + if rc.isOk() or (i_try>=max_number_of_tries): + break + logger.write('\ngit command failed! Wait %d seconds and give an other try (%d/%d)\n' % \ + (sleep_delay, i_try + 1, max_number_of_tries), 3) + time.sleep(sleep_delay) # wait a little + + return rc.isOk() def git_extract_sub_dir(from_what, tag, git_options, where, sub_dir, logger, environment=None): '''Extracts sources from a subtree sub_dir of a git repository. -- 2.39.2