From 0a5bd2bdb9e1dbd08eeb423728e0c364a85cfc00 Mon Sep 17 00:00:00 2001 From: Nabil Ghodbane Date: Sat, 4 May 2024 14:49:05 +0200 Subject: [PATCH] spns #40779: support multi repositories --- commands/compile.py | 4 +- commands/config.py | 9 ++++- commands/package.py | 12 ++++-- commands/prepare.py | 5 ++- commands/source.py | 42 ++++++++++++++------- data/local.pyconf | 3 +- src/__init__.py | 27 +++++++++++++ src/environment.py | 8 +++- src/internal_config/git_repo_servers.pyconf | 10 +++++ src/product.py | 35 ++++++++++++++--- 10 files changed, 125 insertions(+), 30 deletions(-) create mode 100644 src/internal_config/git_repo_servers.pyconf diff --git a/commands/compile.py b/commands/compile.py index afdad9b..545a502 100644 --- a/commands/compile.py +++ b/commands/compile.py @@ -168,7 +168,7 @@ def compile_all_products(sat, config, options, products_infos, all_products_dict if src.product.product_is_salome(p_info): check_salome_configuration=True - if config.APPLICATION.properties.github == 'yes' and src.product.product_test_property(p_info,"is_opensource", "no"): + if src.product.product_test_property(p_info,"is_opensource", "no") and not src.check_git_repository_has_non_opensource(config, config.APPLICATION.properties.git_server): continue # nothing to clean for native or fixed products if (not src.product.product_compiles(p_info)) or\ @@ -277,7 +277,7 @@ def compile_all_products(sat, config, options, products_infos, all_products_dict log_step(logger, header, "ignored") logger.write("\n", 3, False) continue - if config.APPLICATION.properties.github == 'yes' and src.product.product_test_property(p_info,"is_opensource", "no"): + if src.product.product_test_property(p_info,"is_opensource", "no") and not src.check_git_repository_has_non_opensource(config, config.APPLICATION.properties.git_server): log_step(logger, header, "ignored") logger.write("\n", 3, False) continue diff --git a/commands/config.py b/commands/config.py index a5997bb..fa6a809 100644 --- a/commands/config.py +++ b/commands/config.py @@ -201,7 +201,13 @@ class ConfigManager: # particular win case if src.architecture.is_windows() : var['tmp_root'] = os.path.expanduser('~') + os.sep + 'tmp' - + repositories_cfg = src.pyconf.Config( osJoin(var['srcDir'], 'internal_config', 'git_repo_servers.pyconf')) + var['repositories_servers'] = [] + var['opensource_repositories_servers'] =[] + for repo_server in repositories_cfg.REPOSITORIES_SERVERS: + var['repositories_servers']+=[repo_server] + if repositories_cfg.REPOSITORIES_SERVERS[repo_server] == 'opensource': + var['opensource_repositories_servers']+=[repo_server] return var def get_command_line_overrides(self, options, sections): @@ -381,7 +387,6 @@ class ConfigManager: cfg.addMapping("PATHS", src.pyconf.Mapping(cfg), "The paths\n") cfg.PATHS["APPLICATIONPATH"] = src.pyconf.Sequence(cfg.PATHS) cfg.PATHS.APPLICATIONPATH.append(cfg.VARS.personal_applications_dir, "") - cfg.PATHS["PRODUCTPATH"] = src.pyconf.Sequence(cfg.PATHS) cfg.PATHS.PRODUCTPATH.append(cfg.VARS.personal_products_dir, "") diff --git a/commands/package.py b/commands/package.py index 9ee7033..1eb702b 100644 --- a/commands/package.py +++ b/commands/package.py @@ -678,6 +678,7 @@ def binary_package(config, logger, options, tmp_working_dir): config.APPLICATION.properties.mesa_launcher_in_package == "yes") : generate_mesa_launcher=True + has_properties = "APPLICATION" in config and "properties" in config.APPLICATION # first loop on products : filter products, analyse properties, # and store the information that will be used to create the archive in the second loop for prod_name, prod_info in l_product_info: @@ -685,8 +686,7 @@ def binary_package(config, logger, options, tmp_working_dir): if src.get_property_in_product_cfg(prod_info, "not_in_package") == "yes": continue - # skip product if github and product is not opensource - if config.APPLICATION.properties.github == "yes" and src.get_property_in_product_cfg(prod_info, "is_opensource") == "no": + if src.product.product_is_not_opensource(prod_info) and src.check_git_repository_has_non_opensource( cfg, git_server): continue # Add the sources of the products that have the property @@ -997,8 +997,12 @@ def get_archives(config, logger): or src.product.product_is_fixed(p_info)): continue - # skip product if github and product is not opensource - if config.APPLICATION.properties.github == "yes" and src.get_property_in_product_cfg(p_info, "is_opensource") == "no": + # skip product if git server misses non opensource products + is_not_prod_opensource = src.product.product_is_not_opensource(p_info) + git_server = src.get_git_server(config,logger) + has_git_server_non_opensource = src.check_git_repository_has_non_opensource( config, git_server) + if has_git_server_non_opensource and is_not_prod_opensource: + logger.warning("%s is a closed-source software and is not available on %s" % (product, git_server)) continue if p_info.get_source == "archive": diff --git a/commands/prepare.py b/commands/prepare.py index d147699..c86f769 100644 --- a/commands/prepare.py +++ b/commands/prepare.py @@ -97,7 +97,10 @@ def run(args, runner, logger): listProd = list(options.products) else: # no product interpeted as all products listProd = [name for name, tmp in products_infos] - if runner.cfg.APPLICATION.properties.github == 'yes': + + git_server = src.get_git_server(runner.cfg,logger) + + if src.check_git_repository_has_non_opensource( runner.cfg, git_server): not_opensource_products = [p for p in products_infos if src.product.product_is_not_opensource(p[1])] listProd = [p for p in listProd if p not in [name for name, tmp in not_opensource_products]] diff --git a/commands/source.py b/commands/source.py index 9ed87ed..b8b8da5 100644 --- a/commands/source.py +++ b/commands/source.py @@ -85,25 +85,41 @@ def get_source_from_git(config, ''' # The str to display coflag = 'git' - - use_repo_dev = "APPLICATION" in config and "properties" in config.APPLICATION and "github" in config.APPLICATION.properties and not config.APPLICATION.properties.github == "yes" - if ("APPLICATION" in config and - "properties" in config.APPLICATION and - "repo_dev" in config.APPLICATION.properties and - config.APPLICATION.properties.repo_dev == "yes" and - use_repo_dev == True ) : - use_repo_dev=True - # Get the repository address. # If the application has the repo_dev property # Or if the product is in dev mode # Then we use repo_dev if the key exists - if (is_dev or use_repo_dev) and 'repo_dev' in product_info.git_info: - coflag = src.printcolors.printcHighlight(coflag.upper()) - repo_git = product_info.git_info.repo_dev + coflag = src.printcolors.printcHighlight(coflag.upper()) + repo_git = None + git_server = src.get_git_server(config,logger) + product_file = product_info.from_file.split('/').pop() + if 'git_info' in product_info and 'repo_name' in product_info.git_info: + if git_server in product_info.git_info.repo_name.keys(): + repo_git = product_info.git_info.repo_name[git_server] + elif 'properties' in product_info and 'is_opensource' in product_info.properties and product_info.properties.is_opensource == 'yes' : + for git_server in product_info.git_info.repo_name.keys(): + if git_server in config.VARS.opensource_repositories_servers: + repo_git = product_info.git_info.repo_name[git_server] + break + elif 'properties' in product_info and not 'is_opensource' in product_info.properties: + for git_server in product_info.git_info.repo_name.keys(): + if git_server in config.VARS.opensource_repositories_servers: + repo_git = product_info.git_info.repo_name[git_server] + logger.warning("Using opensource repository ({}) for product {}".format(git_server, product_info.name)) + break + else: + logger.error("Error in configuration file: define git repository for product: {} in file {}".format(product_info.name, product_file)) + return False + + elif 'repo_dev' in product_info.git_info: + repo_git = product_info.git_info.repo_dev else: - repo_git = product_info.git_info.repo + logger.error("Error in configuration file: define git repository for product: {}".format(product_info.name)) + return False + if repo_git is None: + logger.error("Error in configuration file: define git repository for product: {} in file {}.".format(product_info.name, product_file)) + return False # Display informations logger.write('%s:%s' % (coflag, src.printcolors.printcInfo(repo_git)), 3, diff --git a/data/local.pyconf b/data/local.pyconf index 24ee9eb..8096362 100644 --- a/data/local.pyconf +++ b/data/local.pyconf @@ -1,4 +1,3 @@ - LOCAL : { base : 'default' @@ -12,5 +11,7 @@ { project_file_paths : [ + '../SAT_SALOME/salome.pyconf' + '../SAT_SALOMEAPP/salomeapp.pyconf' ] } diff --git a/src/__init__.py b/src/__init__.py index 9f3e8d1..af2ef89 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -622,3 +622,30 @@ def activate_mesa_property(config): if not 'properties' in config.APPLICATION: config.APPLICATION.addMapping( 'properties', pyconf.Mapping(), None ) config.APPLICATION.properties.use_mesa="yes" + +def check_git_repository_has_non_opensource( config, the_git_server): + """check that the git repository contains non public repositories + :param config class 'common.pyconf.Config': The config. + :param logger Logger: The logging instance to use for the prints. + """ + if 'opensource_repositories_servers' in config.VARS: + for git_server in config.VARS.opensource_repositories_servers: + if git_server == the_git_server: + return True + return False + return + +def get_git_server(config, logger): + git_server= None + has_properties = 'properties' in config.APPLICATION + if has_properties and "git_server" in config.APPLICATION.properties: + git_server = config.APPLICATION.properties.git_server + elif has_properties and "repo_dev" in config.APPLICATION.properties: + # Fall back to deprecated approach but issue a warning that this approach is deprecated + logger.warning("repo_dev is deprecated and will be removed from future SAT releases!") + logger.warning("Please upgrade your application configuration file and the product description file!") + if config.APPLICATION.properties.repo_dev == 'yes': + git_server = [ repositories_server for repositories_server in config.VARS.repositories_servers if repositories_server not in config.VARS.opensource_repositories_servers][0] + else: + git_server = [ repositories_server for repositories_server in config.VARS.repositories_servers if repositories_server in config.VARS.opensource_repositories_servers][0] + return git_server diff --git a/src/environment.py b/src/environment.py index b928f93..525c4c5 100644 --- a/src/environment.py +++ b/src/environment.py @@ -604,8 +604,12 @@ class SalomeEnviron: # src.appli_test_property(self.cfg,"pip_install_dir", "python") ): # return - # skip product if github and product is not opensource - if self.cfg.APPLICATION.properties.github == "yes" and src.get_property_in_product_cfg(pi, "is_opensource") == "no": + # skip product if git server misses non opensource products + is_not_prod_opensource = src.product.product_is_not_opensource(pi) + git_server= get_fit_server(self.cfg, logger) + has_git_server_non_opensource = src.check_git_repository_has_non_opensource( self.cfg, git_server) + if has_git_server_non_opensource and is_not_prod_opensource: + logger.warning("%s is a closed-source software and is not available on %s" % (pi.name, git_server)) return # skip mesa products (if any) at run time, diff --git a/src/internal_config/git_repo_servers.pyconf b/src/internal_config/git_repo_servers.pyconf new file mode 100644 index 0000000..3d7213e --- /dev/null +++ b/src/internal_config/git_repo_servers.pyconf @@ -0,0 +1,10 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- + +REPOSITORIES_SERVERS : +{ + "tuleap" : "all" + "github" : "opensource" + "gitpub" : "opensource" +} + diff --git a/src/product.py b/src/product.py index 8bce492..a2f80f7 100644 --- a/src/product.py +++ b/src/product.py @@ -393,7 +393,6 @@ Please provide a 'compil_script' key in its definition.""") % product_name # Set the install_dir key prod_info.install_dir,prod_info.install_mode = get_install_dir(config, version, prod_info) - return prod_info def get_product_section(config, product_name, version, section=None): @@ -798,7 +797,34 @@ def get_products_list(options, cfg, logger): # Get the products to be prepared, regarding the options if options.products is None: # No options, get all products sources - products = cfg.APPLICATION.products + products=[] + for product in cfg.APPLICATION.products.keys(): + prod_info = get_product_config(cfg, product) + print(prod_info.name) + if prod_info is None: + logger.error("%s does not have associated information" % (product)) + continue + if 'get_source' in prod_info and prod_info.get_source == 'git': + is_prod_opensource = not src.product.product_is_not_opensource(prod_info) + git_server= None + if not 'git_server' in cfg.APPLICATION.properties: + if 'repo_dev' in cfg.APPLICATION.properties: + # Fall back to deprecated approach but issue a warning that this approach is deprecated + warning_msg ="repo_dev is deprecated and will be removed from future SAT releases!\n" + warning_msg+="Please upgrade your application configuration file and the product description file!" + if cfg.APPLICATION.properties.repo_dev == 'yes': + git_server = [ repositories_server for repositories_server in cfg.VARS.repositories_servers if repositories_server not in cfg.VARS.opensource_repositories_servers][0] + else: + git_server = [ repositories_server for repositories_server in cfg.VARS.repositories_servers if repositories_server in cfg.VARS.opensource_repositories_servers][0] + else: + git_server = cfg.APPLICATION.properties.git_server + + has_git_server_non_opensource = src.check_git_repository_has_non_opensource( cfg, git_server) + if has_git_server_non_opensource and not is_prod_opensource: + logger.warning("%s is a closed-source software and is not available on %s" % (product, git_server)) + continue + products+=[product] + products = src.getProductNames(cfg, products, logger) else: # if option --products, check that all products of the command line # are present in the application. @@ -810,7 +836,6 @@ def get_products_list(options, cfg, logger): { 'product': p, 'application': cfg.VARS.application} )""" products = src.getProductNames(cfg, options.products, logger) - # Construct the list of tuple containing # the products name and their definition resAll = src.product.get_products_infos(products, cfg) @@ -1098,11 +1123,11 @@ def product_is_cpp(product_info): product_info.properties.cpp == "yes") def product_is_not_opensource(product_info): - """Check if a given product is not open + """Check if a given product is closed-source :param product_info Config: The configuration specific to the product - :return: True if the product is an opensource, False otherwise + :return: True if the product is an closed-source, False otherwise :rtype: boolean """ return ("properties" in product_info and -- 2.39.2