From: Nabil Ghodbane Date: Mon, 18 Mar 2024 15:00:16 +0000 (+0100) Subject: spns #40779: implement git multiserver approach: github, gitpub, tuleap X-Git-Tag: V9_13_0~10 X-Git-Url: http://git.salome-platform.org/gitweb/?a=commitdiff_plain;h=refs%2Fheads%2Fgb262689%2Fpython3-12-compatibility-import-imp-removed-issue;p=tools%2Fsat.git spns #40779: implement git multiserver approach: github, gitpub, tuleap --- diff --git a/commands/compile.py b/commands/compile.py index d084b2d..1a6cee8 100644 --- a/commands/compile.py +++ b/commands/compile.py @@ -167,7 +167,9 @@ def compile_all_products(sat, config, options, products_infos, all_products_dict p_name, p_info = p_name_info if src.product.product_is_salome(p_info): check_salome_configuration=True - + # if product is closed source and git server is public skip the current product + if src.product.product_is_not_opensource(p_info) and not src.git_server_has_all_repositories(config, config.APPLICATION.properties.git_server): + continue # nothing to clean for native or fixed products if (not src.product.product_compiles(p_info)) or\ src.product.product_is_native(p_info) or\ @@ -276,6 +278,13 @@ def compile_all_products(sat, config, options, products_infos, all_products_dict logger.write("\n", 3, False) continue + # skip product if git server does not host all git repositories + # product is not opensource and git server does not have all repositories (closed and open sources) + if src.product.product_is_not_opensource(p_info) and not src.git_server_has_all_repositories(config, config.APPLICATION.properties.git_server): + log_step(logger, header, "ignored") + logger.write("\n", 3, False) + continue + # Do nothing if the product is native if src.product.product_is_native(p_info): log_step(logger, header, "native") @@ -298,12 +307,11 @@ def compile_all_products(sat, config, options, products_infos, all_products_dict is_pip= (src.appli_test_property(config,"pip", "yes") and src.product.product_test_property(p_info,"pip", "yes")) # don't check sources with option --show # or for products managed by pip (there sources are in wheels stored in LOCAL.ARCHIVE - if not (options.no_compile or is_pip): + if not (options.no_compile or is_pip): if not check_source: logger.write(_("Sources of product not found (try 'sat -h prepare') \n")) res += 1 # one more error continue - # if we don't force compilation, check if the was already successfully installed. # we don't compile in this case. if (not options.force) and src.product.check_installation(config, p_info): @@ -311,12 +319,12 @@ def compile_all_products(sat, config, options, products_infos, all_products_dict logger.write(_(" in %s" % p_info.install_dir), 4) logger.write(_("\n")) continue - + # If the show option was called, do not launch the compilation if options.no_compile: logger.write(_("Not installed in %s\n" % p_info.install_dir)) continue - + # Check if the dependencies are installed l_depends_not_installed = check_dependencies(config, p_name_info, all_products_dict) if len(l_depends_not_installed) > 0: @@ -327,7 +335,7 @@ def compile_all_products(sat, config, options, products_infos, all_products_dict logger.write(src.printcolors.printcError(prod_name + " ")) logger.write("\n") continue - + # Call the function to compile the product res_prod, len_end_line, error_step = compile_product( sat, p_name_info, config, options, logger, header, len_end_line) @@ -744,7 +752,7 @@ def run(args, runner, logger): 'directories of the products of ' 'the application %s\n') % src.printcolors.printcLabel(runner.cfg.VARS.application), 1) - + info = [ (_("SOURCE directory"), os.path.join(runner.cfg.APPLICATION.workdir, 'SOURCES')), diff --git a/commands/config.py b/commands/config.py index 24a95a8..3b9fb9c 100644 --- a/commands/config.py +++ b/commands/config.py @@ -148,7 +148,6 @@ class ConfigManager: var['datadir'] = osJoin(var['salometoolsway'], 'data') if datadir is not None: var['datadir'] = datadir - var['personalDir'] = osJoin(os.path.expanduser('~'), '.salomeTools') src.ensure_path_exists(var['personalDir']) @@ -201,7 +200,7 @@ class ConfigManager: # particular win case if src.architecture.is_windows() : var['tmp_root'] = os.path.expanduser('~') + os.sep + 'tmp' - + return var def get_command_line_overrides(self, options, sections): @@ -381,7 +380,6 @@ class ConfigManager: cfg.addMapping("PATHS", src.pyconf.Mapping(cfg), "The paths\n") cfg.PATHS["APPLICATIONPATH"] = src.pyconf.Sequence(cfg.PATHS) cfg.PATHS.APPLICATIONPATH.append(cfg.VARS.personal_applications_dir, "") - cfg.PATHS["PRODUCTPATH"] = src.pyconf.Sequence(cfg.PATHS) cfg.PATHS.PRODUCTPATH.append(cfg.VARS.personal_products_dir, "") @@ -417,11 +415,33 @@ class ConfigManager: for rule in self.get_command_line_overrides(options, ["PATHS"]): exec('cfg.' + rule) # this cannot be factorized because of the exec + # add git servers if any + cfg.addMapping("git_info", src.pyconf.Mapping(cfg), "The repositories\n") + cfg.VARS['git_servers'] = [] + cfg.VARS['opensource_git_servers'] =[] + + for project in cfg.PROJECTS.projects: + if 'git_info' not in cfg.PROJECTS.projects[project]: + logger.warning("Project: {} does not have any git_info section! Please define one!") + continue + if 'git_server' in cfg.PROJECTS.projects[project]['git_info']: + git_servers=cfg.PROJECTS.projects[project]['git_info']['git_server'] + for git_server in git_servers: + cfg.VARS['git_servers']+=[git_server] + if git_servers[git_server]['opensource_only'] == 'yes' : + cfg.VARS['opensource_git_servers']+=[git_server] + if 'default_git_server_dev' in cfg.PROJECTS.projects[project]['git_info'].keys(): + cfg.VARS['git_servers']+=['tuleap'] + cfg.VARS['default_git_server_dev'] = cfg.PROJECTS.projects[project]['git_info']['default_git_server_dev'] + if 'default_git_server' in cfg.PROJECTS.projects[project]['git_info'].keys(): + cfg.VARS['git_servers']+=['gitpub'] + cfg.VARS['opensource_git_servers']+=['gitpub'] + cfg.VARS['default_git_server'] = cfg.PROJECTS.projects[project]['git_info']['default_git_server'] + # AT END append APPLI_TEST directory in APPLICATIONPATH, for unittest appli_test_dir = osJoin(satdir, "test", "APPLI_TEST") if appli_test_dir not in cfg.PATHS.APPLICATIONPATH: cfg.PATHS.APPLICATIONPATH.append(appli_test_dir, "unittest APPLI_TEST path") - # ===================================================================== # Load APPLICATION config file if application is not None: diff --git a/commands/package.py b/commands/package.py index c5ffbff..050df51 100644 --- a/commands/package.py +++ b/commands/package.py @@ -678,6 +678,7 @@ def binary_package(config, logger, options, tmp_working_dir): config.APPLICATION.properties.mesa_launcher_in_package == "yes") : generate_mesa_launcher=True + has_properties = "APPLICATION" in config and "properties" in config.APPLICATION # first loop on products : filter products, analyse properties, # and store the information that will be used to create the archive in the second loop for prod_name, prod_info in l_product_info: @@ -685,6 +686,9 @@ def binary_package(config, logger, options, tmp_working_dir): if src.get_property_in_product_cfg(prod_info, "not_in_package") == "yes": continue + if src.product.product_is_not_opensource(prod_info) and not src.git_server_has_all_repositories( cfg, git_server): + continue + # Add the sources of the products that have the property # sources_in_package : "yes" if src.get_property_in_product_cfg(prod_info, @@ -992,6 +996,15 @@ def get_archives(config, logger): if (src.product.product_is_native(p_info) or src.product.product_is_fixed(p_info)): continue + + # skip product if git server misses non opensource products + is_not_prod_opensource = src.product.product_is_not_opensource(p_info) + git_server = src.get_git_server(config,logger) + if src.product.product_is_not_opensource(p_info) and not src.git_server_has_all_repositories(config, git_server): + logger.warning("%s is a closed-source software and is not available on %s" % (product, git_server)) + logger.flush() + continue + if p_info.get_source == "archive": archive_path = p_info.archive_info.archive_name archive_name = os.path.basename(archive_path) diff --git a/commands/prepare.py b/commands/prepare.py index 71a44a8..5b65d5d 100644 --- a/commands/prepare.py +++ b/commands/prepare.py @@ -36,7 +36,6 @@ parser.add_option('c', 'complete', 'boolean', 'complete', _("Optional: completion mode, only prepare products not present in SOURCES dir."), False) - def find_products_already_prepared(l_products): '''function that returns the list of products that have an existing source directory. @@ -88,11 +87,13 @@ def run(args, runner, logger): # check that the command has been called with an application src.check_config_has_application( runner.cfg ) + # check if application configuration file was migrated to newer repository approach + src.check_application_syntax_deprecated(runner.cfg, logger) + # write warning if platform is not declared as supported src.check_platform_is_supported( runner.cfg, logger ) products_infos = src.product.get_products_list(options, runner.cfg, logger) - # Construct the arguments to pass to the clean, source and patch commands args_appli = runner.cfg.VARS.application + " " # useful whitespace if options.products: @@ -100,6 +101,17 @@ def run(args, runner, logger): else: # no product interpeted as all products listProd = [name for name, tmp in products_infos] + git_server = src.get_git_server(runner.cfg,logger) + + # current git server hosts only opensource repositories - then remove products which are not hosted + if not src.git_server_has_all_repositories(runner.cfg, git_server): + not_opensource_products = [p for p in products_infos if src.product.product_is_not_opensource(p[1])] + listProd = [p for p in listProd if p not in [name for name, tmp in not_opensource_products]] + logger.flush() + if len(not_opensource_products) > 0: + lp = ','.join([ name for name, tmp in not_opensource_products]) + msg = "WARNING: Following products are not available, since these are closed-source products: %s !" % lp + logger.write("\n%s\n\n" % src.printcolors.printcWarning(msg), 1) if options.complete: # remove products that are already prepared 'completion mode) pi_already_prepared=find_products_already_prepared(products_infos) @@ -156,7 +168,6 @@ Use the --force_patch option to overwrite it. args_clean = args_appli + args_product_opt_clean + " --sources" args_source = args_appli + args_product_opt args_patch = args_appli + args_product_opt_patch - # Initialize the results to a running status res_clean = 0 res_source = 0 diff --git a/commands/source.py b/commands/source.py index 832e484..7f0d8a0 100644 --- a/commands/source.py +++ b/commands/source.py @@ -85,24 +85,42 @@ def get_source_from_git(config, ''' # The str to display coflag = 'git' - - use_repo_dev=False - if ("APPLICATION" in config and - "properties" in config.APPLICATION and - "repo_dev" in config.APPLICATION.properties and - config.APPLICATION.properties.repo_dev == "yes") : - use_repo_dev=True - # Get the repository address. # If the application has the repo_dev property # Or if the product is in dev mode # Then we use repo_dev if the key exists - if (is_dev or use_repo_dev) and 'repo_dev' in product_info.git_info: - coflag = src.printcolors.printcHighlight(coflag.upper()) - repo_git = product_info.git_info.repo_dev + coflag = src.printcolors.printcHighlight(coflag.upper()) + repo_git = None + git_server = src.get_git_server(config,logger) + product_file = product_info.from_file.split('/').pop() + if 'git_info' in product_info and 'repositories' in product_info.git_info: + if git_server in product_info.git_info.repositories.keys(): # keys are git servers + repo_git = product_info.git_info.repositories[git_server] + elif 'properties' in product_info and 'is_opensource' in product_info.properties and product_info.properties.is_opensource == 'yes' : + for git_server in product_info.git_info.repositories.keys(): + if git_server in config.VARS.opensource_git_servers: + repo_git = product_info.git_info.repositories[git_server] + break + elif 'properties' in product_info and not 'is_opensource' in product_info.properties: + for git_server in product_info.git_info.repositories.keys(): + if git_server in config.VARS.opensource_git_servers: + repo_git = product_info.git_info.repositories[git_server] + logger.warning("Using opensource repository ({}) for product {}".format(git_server, product_info.name)) + logger.flush() + break + else: + logger.error("Error in configuration file: define git repository for product: {} in file {}".format(product_info.name, product_file)) + return False + + elif 'repo_dev' in product_info.git_info: + repo_git = product_info.git_info.repo_dev else: - repo_git = product_info.git_info.repo + logger.error("Error in configuration file: define git repository for product: {}".format(product_info.name)) + return False + if repo_git is None: + logger.error("Error in configuration file: define git repository for product: {} in file {}.".format(product_info.name, product_file)) + return False # Display informations logger.write('%s:%s' % (coflag, src.printcolors.printcInfo(repo_git)), 3, diff --git a/data/local.pyconf b/data/local.pyconf index eb81d5a..24ee9eb 100644 --- a/data/local.pyconf +++ b/data/local.pyconf @@ -1,4 +1,4 @@ - + LOCAL : { base : 'default' diff --git a/doc/src/commands/prepare.rst b/doc/src/commands/prepare.rst index 5ea8eba..9df51c2 100644 --- a/doc/src/commands/prepare.rst +++ b/doc/src/commands/prepare.rst @@ -90,6 +90,11 @@ Usage sat prepare --complete +* Use GitHub repositories. + To prepare SALOME sources which are available on GitHub, run:: + + sat -o 'APPLICATION.properties.git_server="github"' prepare + Some useful configuration paths ================================= diff --git a/doc/src/configuration.rst b/doc/src/configuration.rst index c9f7381..ab66ab1 100644 --- a/doc/src/configuration.rst +++ b/doc/src/configuration.rst @@ -73,7 +73,7 @@ At the beginning of the APPLICATION sections, global variables and flags are def * **name** : the name of the application (mandatory) * **workdir** : the directory in which the application is produced (mandatory) * **tag** : the default tag to use for the git bases - * **dev** : activate the dev mode. In dev mode git bases are checked out only one time, to avoid risks of removing developments. +. * **verbose** : activate verbosity in the compilation * **debug** : activate debug mode in the compilation, i.e -g option * **python3** : 'yes/no' tell sat that the application uses python3 @@ -537,7 +537,6 @@ Command line overwriting is triggered by sat **-o** option, followed in double q In the following example, we suppose that the application SALOME-9.4.0 has set both flags debug and verbose to "no", and that we want to recompile MEDCOUPLING in debug mode, with cmake verbosity activated. The command to use is: .. code-block:: bash - # recompile MEDCOUPLING in debug mode (-g) and with verbosity ./sat -t -o "APPLICATION.verbose='yes'" -o "APPLICATION.debug='yes'" compile\ SALOME-9.4.0 -p MEDCOUPLING --clean_all diff --git a/src/__init__.py b/src/__init__.py index 34aecde..7cc6486 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -53,7 +53,6 @@ KO_STATUS = "KO" NA_STATUS = "NA" KNOWNFAILURE_STATUS = "KF" TIMEOUT_STATUS = "TIMEOUT" - class SatException(Exception): """sat exception class""" def message(self, arg): @@ -107,6 +106,20 @@ def check_config_has_profile( config, details = None ): details.append(message) raise SatException( message ) +def check_application_syntax_deprecated(config, logger): + """\ + check that the application has the key git_server. + else, raise a warning + + :param config class 'common.pyconf.Config': The config. + :param logger Logger: The logging instance to use for the prints. + """ + if 'APPLICATION' in config and 'properties' in config.APPLICATION and not 'git_server' in config.APPLICATION.properties : + msg = 'WARNING: Your application is using repo_dev key which is deprecated and will be removed from future SAT releases!\n' + msg+= ' Please upgrade your application configuration file and add a valid git_server key.\n' + msg+= ' git_server key values need to be defined in the project file (e.g. salome.pyconf)!' + logger.write("\n%s\n\n" % printcolors.printcWarning(msg), 1) + def appli_test_property(config,property_name, property_value): """Generic function to test if an application has a property set to a value :param config class 'common.pyconf.Config': The config. @@ -127,7 +140,6 @@ def appli_test_property(config,property_name, property_value): result = eval(eval_expression) return result - def config_has_application( config ): return 'APPLICATION' in config @@ -622,3 +634,27 @@ def activate_mesa_property(config): if not 'properties' in config.APPLICATION: config.APPLICATION.addMapping( 'properties', pyconf.Mapping(), None ) config.APPLICATION.properties.use_mesa="yes" + +def git_server_has_all_repositories( config, the_git_server): + """check that the git server contains all repositories (closed and open) + :param config class 'common.pyconf.Config': The config. + :param logger Logger: The logging instance to use for the prints. + """ + if 'opensource_git_servers' in config.VARS: + for git_server in config.VARS['opensource_git_servers']: + if git_server == the_git_server: + return False + return True + +def get_git_server(config, logger): + the_git_server= None + has_properties = 'properties' in config.APPLICATION + if has_properties and "git_server" in config.APPLICATION.properties: + the_git_server = config.APPLICATION.properties.git_server + elif has_properties and "repo_dev" in config.APPLICATION.properties: + # Fall back to deprecated approach but issue a warning that this approach is deprecated + if config.APPLICATION.properties.repo_dev == 'yes': + the_git_server = [ git_server for git_server in config.VARS.git_servers if git_server not in config.VARS.opensource_git_servers][0] + else: + the_git_server = [ git_server for git_server in config.VARS.git_servers if git_server in config.VARS.opensource_git_servers][0] + return the_git_server diff --git a/src/environment.py b/src/environment.py index 74e6bbd..7bf1ec6 100644 --- a/src/environment.py +++ b/src/environment.py @@ -604,6 +604,12 @@ class SalomeEnviron: # src.appli_test_property(self.cfg,"pip_install_dir", "python") ): # return + # skip product if git server does not host all git repositories + git_server= src.get_git_server(self.cfg, logger) + if src.product.product_is_not_opensource(pi) and not src.git_server_has_all_repositories( self.cfg, git_server): + logger.warning("%s is a closed-source software and is not available on %s" % (pi.name, git_server)) + return + # skip mesa products (if any) at run time, # unless use_mesa property was activated if not self.forBuild: diff --git a/src/product.py b/src/product.py index 091ffe2..b644681 100644 --- a/src/product.py +++ b/src/product.py @@ -393,7 +393,6 @@ Please provide a 'compil_script' key in its definition.""") % product_name # Set the install_dir key prod_info.install_dir,prod_info.install_mode = get_install_dir(config, version, prod_info) - return prod_info def get_product_section(config, product_name, version, section=None): @@ -798,7 +797,23 @@ def get_products_list(options, cfg, logger): # Get the products to be prepared, regarding the options if options.products is None: # No options, get all products sources - products = cfg.APPLICATION.products + products=[] + for product in cfg.APPLICATION.products.keys(): + prod_info = get_product_config(cfg, product) + if prod_info is None: + logger.error("%s does not have associated information" % (product)) + continue + if 'get_source' in prod_info and prod_info.get_source == 'git': + git_server = src.get_git_server(cfg,logger) + else: + git_server = cfg.VARS['default_git_server_dev'] + + if src.product.product_is_not_opensource(prod_info) and not src.git_server_has_all_repositories(cfg, git_server): + logger.warning("%s is a closed-source software and is not available on %s" % (product, git_server)) + logger.flush() + continue + products+=[product] + products = src.getProductNames(cfg, products, logger) else: # if option --products, check that all products of the command line # are present in the application. @@ -810,7 +825,6 @@ def get_products_list(options, cfg, logger): { 'product': p, 'application': cfg.VARS.application} )""" products = src.getProductNames(cfg, options.products, logger) - # Construct the list of tuple containing # the products name and their definition resAll = src.product.get_products_infos(products, cfg) @@ -1097,6 +1111,18 @@ def product_is_cpp(product_info): "cpp" in product_info.properties and product_info.properties.cpp == "yes") +def product_is_not_opensource(product_info): + """Check if a given product is closed-source + + :param product_info Config: The configuration specific to + the product + :return: True if the product is a closed-source, False otherwise + :rtype: boolean + """ + return ("properties" in product_info and + "is_opensource" in product_info.properties and + product_info.properties.is_opensource == "no") + def product_compiles(product_info): """\ Know if a product compiles or not