From f9fd44a4933db899cc33b63ba8f4fb44c3881304 Mon Sep 17 00:00:00 2001 From: Nabil Ghodbane Date: Thu, 9 May 2024 23:30:40 +0200 Subject: [PATCH] spns #40779: improved implementation - servers are now defined in project configuration file like e.g. salome.pyconf --- commands/config.py | 32 ++++++++++++++----- commands/package.py | 1 + commands/prepare.py | 9 +++++- commands/source.py | 7 +++-- doc/src/commands/prepare.rst | 5 +++ doc/src/configuration.rst | 3 +- src/__init__.py | 34 +++++++++++++-------- src/environment.py | 2 +- src/internal_config/git_repo_servers.pyconf | 10 ------ src/product.py | 28 ++++++----------- 10 files changed, 76 insertions(+), 55 deletions(-) delete mode 100644 src/internal_config/git_repo_servers.pyconf diff --git a/commands/config.py b/commands/config.py index eec74a3..4cf6116 100644 --- a/commands/config.py +++ b/commands/config.py @@ -200,13 +200,7 @@ class ConfigManager: # particular win case if src.architecture.is_windows() : var['tmp_root'] = os.path.expanduser('~') + os.sep + 'tmp' - repositories_cfg = src.pyconf.Config( osJoin(var['srcDir'], 'internal_config', 'git_repo_servers.pyconf')) - var['repositories_servers'] = [] - var['opensource_repositories_servers'] =[] - for repo_server in repositories_cfg.REPOSITORIES_SERVERS: - var['repositories_servers']+=[repo_server] - if repositories_cfg.REPOSITORIES_SERVERS[repo_server] == 'opensource': - var['opensource_repositories_servers']+=[repo_server] + return var def get_command_line_overrides(self, options, sections): @@ -421,11 +415,33 @@ class ConfigManager: for rule in self.get_command_line_overrides(options, ["PATHS"]): exec('cfg.' + rule) # this cannot be factorized because of the exec + # add git servers if any + cfg.addMapping("git_info", src.pyconf.Mapping(cfg), "The repositories\n") + cfg.VARS['git_servers'] = [] + cfg.VARS['opensource_git_servers'] =[] + + for project in cfg.PROJECTS.projects: + if 'git_info' not in cfg.PROJECTS.projects[project]: + logger.warning("Project: {} does not have any git_info section! Please define one!") + continue + if 'git_server' in cfg.PROJECTS.projects[project]['git_info']: + git_servers=cfg.PROJECTS.projects[project]['git_info']['git_server'] + for git_server in git_servers: + cfg.VARS['git_servers']+=[git_server] + if git_servers[git_server]['opensource_only'] == 'yes' : + cfg.VARS['opensource_git_servers']+=[git_server] + if 'default_git_server_dev' in cfg.PROJECTS.projects[project]['git_info'].keys(): + cfg.VARS['git_servers']+=['tuleap'] + cfg.VARS['default_git_server_dev'] = cfg.PROJECTS.projects[project]['git_info']['default_git_server_dev'] + if 'default_git_server' in cfg.PROJECTS.projects[project]['git_info'].keys(): + cfg.VARS['git_servers']+=['gitpub'] + cfg.VARS['opensource_git_servers']+=['gitpub'] + cfg.VARS['default_git_server'] = cfg.PROJECTS.projects[project]['git_info']['default_git_server'] + # AT END append APPLI_TEST directory in APPLICATIONPATH, for unittest appli_test_dir = osJoin(satdir, "test", "APPLI_TEST") if appli_test_dir not in cfg.PATHS.APPLICATIONPATH: cfg.PATHS.APPLICATIONPATH.append(appli_test_dir, "unittest APPLI_TEST path") - # ===================================================================== # Load APPLICATION config file if application is not None: diff --git a/commands/package.py b/commands/package.py index 61b518d..52f8c96 100644 --- a/commands/package.py +++ b/commands/package.py @@ -1003,6 +1003,7 @@ def get_archives(config, logger): has_git_server_non_opensource = src.check_git_server_has_non_opensource( config, git_server) if has_git_server_non_opensource and is_not_prod_opensource: logger.warning("%s is a closed-source software and is not available on %s" % (product, git_server)) + logger.flush() continue if p_info.get_source == "archive": diff --git a/commands/prepare.py b/commands/prepare.py index c86d5d6..061c5e8 100644 --- a/commands/prepare.py +++ b/commands/prepare.py @@ -87,6 +87,9 @@ def run(args, runner, logger): # check that the command has been called with an application src.check_config_has_application( runner.cfg ) + # check if application configuration file was migrated to newer repository approach + src.check_application_syntax_deprecated(runner.cfg, logger) + # write warning if platform is not declared as supported src.check_platform_is_supported( runner.cfg, logger ) @@ -103,7 +106,11 @@ def run(args, runner, logger): if src.check_git_server_has_non_opensource( runner.cfg, git_server): not_opensource_products = [p for p in products_infos if src.product.product_is_not_opensource(p[1])] listProd = [p for p in listProd if p not in [name for name, tmp in not_opensource_products]] - + logger.flush() + if len(not_opensource_products) > 0: + lp = ','.join([ name for name, tmp in not_opensource_products]) + msg = "WARNING: Following products are not available, since these are closed-source products: %s !" % lp + logger.write("\n%s\n\n" % src.printcolors.printcWarning(msg), 1) if options.complete: # remove products that are already prepared 'completion mode) pi_already_prepared=find_products_already_prepared(products_infos) diff --git a/commands/source.py b/commands/source.py index a99a5b3..7f0d8a0 100644 --- a/commands/source.py +++ b/commands/source.py @@ -94,18 +94,19 @@ def get_source_from_git(config, git_server = src.get_git_server(config,logger) product_file = product_info.from_file.split('/').pop() if 'git_info' in product_info and 'repositories' in product_info.git_info: - if git_server in product_info.git_info.repositories.keys(): + if git_server in product_info.git_info.repositories.keys(): # keys are git servers repo_git = product_info.git_info.repositories[git_server] elif 'properties' in product_info and 'is_opensource' in product_info.properties and product_info.properties.is_opensource == 'yes' : for git_server in product_info.git_info.repositories.keys(): - if git_server in config.VARS.opensource_repositories_servers: + if git_server in config.VARS.opensource_git_servers: repo_git = product_info.git_info.repositories[git_server] break elif 'properties' in product_info and not 'is_opensource' in product_info.properties: for git_server in product_info.git_info.repositories.keys(): - if git_server in config.VARS.opensource_repositories_servers: + if git_server in config.VARS.opensource_git_servers: repo_git = product_info.git_info.repositories[git_server] logger.warning("Using opensource repository ({}) for product {}".format(git_server, product_info.name)) + logger.flush() break else: logger.error("Error in configuration file: define git repository for product: {} in file {}".format(product_info.name, product_file)) diff --git a/doc/src/commands/prepare.rst b/doc/src/commands/prepare.rst index 5ea8eba..9df51c2 100644 --- a/doc/src/commands/prepare.rst +++ b/doc/src/commands/prepare.rst @@ -90,6 +90,11 @@ Usage sat prepare --complete +* Use GitHub repositories. + To prepare SALOME sources which are available on GitHub, run:: + + sat -o 'APPLICATION.properties.git_server="github"' prepare + Some useful configuration paths ================================= diff --git a/doc/src/configuration.rst b/doc/src/configuration.rst index c9f7381..ab66ab1 100644 --- a/doc/src/configuration.rst +++ b/doc/src/configuration.rst @@ -73,7 +73,7 @@ At the beginning of the APPLICATION sections, global variables and flags are def * **name** : the name of the application (mandatory) * **workdir** : the directory in which the application is produced (mandatory) * **tag** : the default tag to use for the git bases - * **dev** : activate the dev mode. In dev mode git bases are checked out only one time, to avoid risks of removing developments. +. * **verbose** : activate verbosity in the compilation * **debug** : activate debug mode in the compilation, i.e -g option * **python3** : 'yes/no' tell sat that the application uses python3 @@ -537,7 +537,6 @@ Command line overwriting is triggered by sat **-o** option, followed in double q In the following example, we suppose that the application SALOME-9.4.0 has set both flags debug and verbose to "no", and that we want to recompile MEDCOUPLING in debug mode, with cmake verbosity activated. The command to use is: .. code-block:: bash - # recompile MEDCOUPLING in debug mode (-g) and with verbosity ./sat -t -o "APPLICATION.verbose='yes'" -o "APPLICATION.debug='yes'" compile\ SALOME-9.4.0 -p MEDCOUPLING --clean_all diff --git a/src/__init__.py b/src/__init__.py index bbad2a7..a935525 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -53,7 +53,6 @@ KO_STATUS = "KO" NA_STATUS = "NA" KNOWNFAILURE_STATUS = "KF" TIMEOUT_STATUS = "TIMEOUT" - class SatException(Exception): """sat exception class""" def message(self, arg): @@ -107,6 +106,20 @@ def check_config_has_profile( config, details = None ): details.append(message) raise SatException( message ) +def check_application_syntax_deprecated(config, logger): + """\ + check that the application has the key git_server. + else, raise a warning + + :param config class 'common.pyconf.Config': The config. + :param logger Logger: The logging instance to use for the prints. + """ + if 'APPLICATION' in config and 'properties' in config.APPLICATION and not 'git_server' in config.APPLICATION.properties : + msg = 'WARNING: Your application is using repo_dev key which is deprecated and will be removed from future SAT releases!\n' + msg+= ' Please upgrade your application configuration file and add a valid git_server key.\n' + msg+= ' git_server key values need to be defined in the project file (e.g. salome.pyconf)!' + logger.write("\n%s\n\n" % printcolors.printcWarning(msg), 1) + def appli_test_property(config,property_name, property_value): """Generic function to test if an application has a property set to a value :param config class 'common.pyconf.Config': The config. @@ -127,7 +140,6 @@ def appli_test_property(config,property_name, property_value): result = eval(eval_expression) return result - def config_has_application( config ): return 'APPLICATION' in config @@ -624,28 +636,26 @@ def activate_mesa_property(config): config.APPLICATION.properties.use_mesa="yes" def check_git_server_has_non_opensource( config, the_git_server): - """check that the git repositories server contains non public repositories + """check that the git server contains non public repositories :param config class 'common.pyconf.Config': The config. :param logger Logger: The logging instance to use for the prints. """ - if 'opensource_repositories_servers' in config.VARS: - for git_server in config.VARS.opensource_repositories_servers: + if 'opensource_git_servers' in config.VARS: + for git_server in config.VARS['opensource_git_servers']: if git_server == the_git_server: return True return False return def get_git_server(config, logger): - git_server= None + the_git_server= None has_properties = 'properties' in config.APPLICATION if has_properties and "git_server" in config.APPLICATION.properties: - git_server = config.APPLICATION.properties.git_server + the_git_server = config.APPLICATION.properties.git_server elif has_properties and "repo_dev" in config.APPLICATION.properties: # Fall back to deprecated approach but issue a warning that this approach is deprecated - logger.warning("repo_dev is deprecated and will be removed from future SAT releases!") - logger.warning("Please upgrade your application configuration file and the product description file!") if config.APPLICATION.properties.repo_dev == 'yes': - git_server = [ repositories_server for repositories_server in config.VARS.repositories_servers if repositories_server not in config.VARS.opensource_repositories_servers][0] + the_git_server = [ git_server for git_server in config.VARS.git_servers if git_server not in config.VARS.opensource_git_servers][0] else: - git_server = [ repositories_server for repositories_server in config.VARS.repositories_servers if repositories_server in config.VARS.opensource_repositories_servers][0] - return git_server + the_git_server = [ git_server for git_server in config.VARS.git_servers if git_server in config.VARS.opensource_git_servers][0] + return the_git_server diff --git a/src/environment.py b/src/environment.py index f8df300..33623c2 100644 --- a/src/environment.py +++ b/src/environment.py @@ -606,7 +606,7 @@ class SalomeEnviron: # skip product if git server misses non opensource products is_not_prod_opensource = src.product.product_is_not_opensource(pi) - git_server= get_git_server(self.cfg, logger) + git_server= src.get_git_server(self.cfg, logger) has_git_server_non_opensource = src.check_git_server_has_non_opensource( self.cfg, git_server) if has_git_server_non_opensource and is_not_prod_opensource: logger.warning("%s is a closed-source software and is not available on %s" % (pi.name, git_server)) diff --git a/src/internal_config/git_repo_servers.pyconf b/src/internal_config/git_repo_servers.pyconf deleted file mode 100644 index 3d7213e..0000000 --- a/src/internal_config/git_repo_servers.pyconf +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python -#-*- coding:utf-8 -*- - -REPOSITORIES_SERVERS : -{ - "tuleap" : "all" - "github" : "opensource" - "gitpub" : "opensource" -} - diff --git a/src/product.py b/src/product.py index 1ef5094..75e10d4 100644 --- a/src/product.py +++ b/src/product.py @@ -805,23 +805,15 @@ def get_products_list(options, cfg, logger): continue if 'get_source' in prod_info and prod_info.get_source == 'git': is_prod_opensource = not src.product.product_is_not_opensource(prod_info) - git_server= None - if not 'git_server' in cfg.APPLICATION.properties: - if 'repo_dev' in cfg.APPLICATION.properties: - # Fall back to deprecated approach but issue a warning that this approach is deprecated - warning_msg ="repo_dev is deprecated and will be removed from future SAT releases!\n" - warning_msg+="Please upgrade your application configuration file and the product description file!" - if cfg.APPLICATION.properties.repo_dev == 'yes': - git_server = [ repositories_server for repositories_server in cfg.VARS.repositories_servers if repositories_server not in cfg.VARS.opensource_repositories_servers][0] - else: - git_server = [ repositories_server for repositories_server in cfg.VARS.repositories_servers if repositories_server in cfg.VARS.opensource_repositories_servers][0] - else: - git_server = cfg.APPLICATION.properties.git_server - - has_git_server_non_opensource = src.check_git_server_has_non_opensource( cfg, git_server) - if has_git_server_non_opensource and not is_prod_opensource: - logger.warning("%s is a closed-source software and is not available on %s" % (product, git_server)) - continue + git_server = src.get_git_server(cfg,logger) + else: + git_server = cfg.VARS['default_git_server_dev'] + + has_git_server_non_opensource = src.check_git_server_has_non_opensource( cfg, git_server) + if has_git_server_non_opensource and not is_prod_opensource: + logger.warning("%s is a closed-source software and is not available on %s" % (product, git_server)) + logger.flush() + continue products+=[product] products = src.getProductNames(cfg, products, logger) else: @@ -1126,7 +1118,7 @@ def product_is_not_opensource(product_info): :param product_info Config: The configuration specific to the product - :return: True if the product is an closed-source, False otherwise + :return: True if the product is a closed-source, False otherwise :rtype: boolean """ return ("properties" in product_info and -- 2.39.2