From 056ce23991faa4c0b52cf272441ac544e99f265f Mon Sep 17 00:00:00 2001 From: crouzet Date: Tue, 8 Oct 2019 15:35:37 +0200 Subject: [PATCH 1/1] sat #17833 : nouveau mode pour les bases compatible avec environment module --- commands/environ.py | 27 +++--- doc/src/commands/compile.rst | 1 + doc/src/commands/environ.rst | 24 +++++- src/environment.py | 157 +++++++++++++++++++++++++---------- src/fileEnviron.py | 79 ++++++++++++++++++ src/product.py | 37 +++++++-- 6 files changed, 263 insertions(+), 62 deletions(-) diff --git a/commands/environ.py b/commands/environ.py index b34aed7..7b26763 100644 --- a/commands/environ.py +++ b/commands/environ.py @@ -34,8 +34,8 @@ parser.add_option('t', 'target', 'string', 'out_dir', None) # list of available shells with extensions -C_SHELLS = { "bash": "sh", "bat": "bat", "cfg" : "cfg" } -C_ALL_SHELL = [ "bash", "bat", "cfg" ] +C_SHELLS = { "bash": "sh", "bat": "bat", "cfg" : "cfg", "tcl" : ""} +C_ALL_SHELL = [ "bash", "bat", "cfg", "tcl" ] ## @@ -100,15 +100,22 @@ def write_all_source_files(config, for_build = True for_launch = False for shell in shells_list: - files.append(writer.write_env_file("%s_launch.%s" % - (prefix, shell.extension), - for_launch, - shell.name)) - files.append(writer.write_env_file("%s_build.%s" % - (prefix, shell.extension), - for_build, - shell.name)) + if shell.name=="tcl": + files.append(writer.write_tcl_files(for_launch, + shell.name)) + else: + files.append(writer.write_env_file("%s_launch.%s" % + (prefix, shell.extension), + for_launch, + shell.name)) + files.append(writer.write_env_file("%s_build.%s" % + (prefix, shell.extension), + for_build, + shell.name)) + for f in files: + if f: + logger.write(" "+f+"\n", 3) return files ################################################## diff --git a/doc/src/commands/compile.rst b/doc/src/commands/compile.rst index 0926acd..1b50caa 100644 --- a/doc/src/commands/compile.rst +++ b/doc/src/commands/compile.rst @@ -78,3 +78,4 @@ The main options are: * **cmake_options** : additional options for cmake. * **nb_proc** : number of jobs to use with make for this product. * **check_install** : allow to specify a list of path (relative to install directory), that sat will check after installation. This flag allow to check an installation is complete. + * **install_dir** : allow to change the default install dir. If the value is set to *'base'*, the product will by default be installed in salomeTool base. Unless no_base was set to 'yes' or base was set to 'no' in application pyconf. diff --git a/doc/src/commands/environ.rst b/doc/src/commands/environ.rst index d487e4c..ad52dc7 100644 --- a/doc/src/commands/environ.rst +++ b/doc/src/commands/environ.rst @@ -28,9 +28,9 @@ Usage sat environ * Create the environment files of the application for a given shell. - Options are bash, bat (for windows) and cfg (the configuration format used by SALOME_): :: + Options are bash, bat (for windows), tcl, cfg (the configuration format used by SALOME_): :: - sat environ --shell [bash|cfg|all] + sat environ --shell [bash|bat|cfg|tcl|all] * Use a different prefix for the files (default is 'env'): @@ -57,6 +57,26 @@ Usage # sat to compile a given product. sat environ --product ,, ... +* Generate tcl modules for use with *environment-modules* package. + + .. code-block:: bash + + sat environ --shell tcl + +Use this command to generate tcl modules associated to a module base. +The production of a module base is triggered when the flag *base* in the application pyconf is set to a value not equal to *yes*. + + .. code-block:: bash + + APPLICATION : + { + ... + # trigger the production of a environment module base which name is salome9 + base : 'salome9' + } + +In this example, the module base will be produced in *BASE/apps/salome9*, and the tcl modules associated in the directory tcl *BASE/apps/modulefiles/salome9*. +Later, it will be possible to enable these modules with the shell command *module use --append .../SAT/BASE/modulefiles*. Configuration ============= diff --git a/src/environment.py b/src/environment.py index 81741d2..c094300 100644 --- a/src/environment.py +++ b/src/environment.py @@ -61,8 +61,9 @@ class Environ: try: value = zt.substitute(self.environ) except KeyError as exc: - raise src.SatException(_("Missing definition " - "in environment: %s") % str(exc)) + pass + #raise src.SatException(_("Missing definition " + # "in environment: %s") % str(exc)) return value def append_value(self, key, value, sep=os.pathsep): @@ -346,30 +347,6 @@ class SalomeEnviron: self.python_lib = self.get('PYTHON_LIBDIR') self.has_python = True - def get_names(self, lProducts): - """\ - Get the products name to add in SALOME_MODULES environment variable - It is the name of the product, except in the case where the is a - component name. And it has to be in SALOME_MODULES variable only - if the product has the property has_salome_hui = "yes" - - :param lProducts list: List of products to potentially add - """ - lProdHasGui = [p for p in lProducts if 'properties' in - src.product.get_product_config(self.cfg, p) and - 'has_salome_gui' in - src.product.get_product_config(self.cfg, p).properties and - src.product.get_product_config(self.cfg, - p).properties.has_salome_gui=='yes'] - lProdName = [] - for ProdName in lProdHasGui: - pi = src.product.get_product_config(self.cfg, ProdName) - if 'component_name' in pi: - lProdName.append(pi.component_name) - else: - lProdName.append(ProdName) - return lProdName - def set_application_env(self, logger): """\ Sets the environment defined in the APPLICATION file. @@ -414,27 +391,33 @@ class SalomeEnviron: # set root dir DBG.write("set_salome_minimal_product_env", product_info) root_dir = product_info.name + "_ROOT_DIR" - if not self.is_defined(root_dir): - if 'install_dir' in product_info and product_info.install_dir: - self.set(root_dir, product_info.install_dir) - elif not self.silent: - logger.write(" " + _("No install_dir for product %s\n") % - product_info.name, 5) - + if 'install_dir' in product_info and product_info.install_dir: + self.set(root_dir, product_info.install_dir) + elif not self.silent: + logger.write(" " + _("No install_dir for product %s\n") % + product_info.name, 5) + source_in_package = src.get_property_in_product_cfg(product_info, "sources_in_package") if not self.for_package or source_in_package == "yes": # set source dir, unless no source dir if not src.product.product_is_fixed(product_info): src_dir = product_info.name + "_SRC_DIR" - if not self.is_defined(src_dir): - if not self.for_package: - self.set(src_dir, product_info.source_dir) - else: - self.set(src_dir, os.path.join("out_dir_Path", - "SOURCES", - os.path.basename(product_info.source_dir))) + if not self.for_package: + self.set(src_dir, product_info.source_dir) + else: + self.set(src_dir, os.path.join("out_dir_Path", + "SOURCES", + os.path.basename(product_info.source_dir))) + def expand_salome_modules(self, pi): + if 'component_name' in pi: + compo_name = pi.component_name + else: + compo_name = pi.name + self.prepend('SALOME_MODULES', compo_name, ',') + + def set_salome_generic_product_env(self, pi): """\ Sets the generic environment for a SALOME product. @@ -619,7 +602,12 @@ class SalomeEnviron: # set environment using definition of the product self.set_salome_minimal_product_env(pi, logger) self.set_salome_generic_product_env(pi) + + # Expand SALOME_MODULES variable for products which have a salome gui + if src.product.product_has_salome_gui(pi): + self.expand_salome_modules(pi) + # use variable LICENCE_FILE to communicate the licence file name to the environment script licence_file_name = src.product.product_has_licence(pi, self.cfg.PATHS.LICENCEPATH) if licence_file_name: @@ -786,6 +774,91 @@ class FileEnvWriter: self.silent = True self.env_info = env_info + def write_tcl_files(self, + forBuild, + shell, + for_package = None, + no_path_init=False, + additional_env = {}): + """\ + Create tcl environment files for environment module. + + :param forBuild bool: if true, the build environment + :param shell str: the type of file wanted (.sh, .bat) + :param for_package bool: if true do specific stuff for required for packages + :param no_path_init bool: if true generate a environ file that do not reinitialise paths + :param additional_env dict: contains sat_ prefixed variables to help the génération, + and also variables to add in the environment. + :return: The path to the generated file + :rtype: str + """ + + # get the products informations + all_products=self.config.APPLICATION.products + products_infos = src.product.get_products_infos(all_products, self.config) + + # set a global environment (we need it to resolve variable references + # between dependent products + global_environ = src.environment.SalomeEnviron(self.config, + src.environment.Environ(additional_env), + False) + global_environ.set_products(self.logger) + + # The loop on the products + for product in all_products: + # create one file per product + pi = src.product.get_product_config(self.config, product) + if "base" not in pi: # we write tcl files only for products in base + continue + + # get the global environment, and complete it with sat_ prefixed + # prefixed variables which are used to transfer info to + # TclFileEnviron class + product_env = copy.deepcopy(global_environ.environ) + product_env.environ["sat_product_name"] = pi.name + product_env.environ["sat_product_version"] = pi.version + product_env.environ["sat_product_base_path"] = src.get_base_path(self.config) + product_env.environ["sat_product_base_name"] = pi.base + + # store infos in sat_product_load_depend to set dependencies in tcl file + sat_product_load_depend="" + for p_name,p_info in products_infos: + if p_name in pi.depend: + sat_product_load_depend+="module load %s/%s/%s;" % (pi.base, + p_info.name, + p_info.version) + if len(sat_product_load_depend)>0: + # if there are dependencies, store the module to load (get rid of trailing ;) + product_env.environ["sat_product_load_depend"]=sat_product_load_depend[0:-1] + + + env_file_name = os.path.join(product_env.environ["sat_product_base_path"], + "modulefiles", + product_env.environ["sat_product_base_name"], + product_env.environ["sat_product_name"], + product_env.environ["sat_product_version"]) + prod_dir_name=os.path.dirname(env_file_name) + if not os.path.isdir(prod_dir_name): + os.makedirs(prod_dir_name) + + env_file = open(env_file_name, "w") + file_environ = src.fileEnviron.get_file_environ(env_file, + "tcl", product_env) + env = SalomeEnviron(self.config, + file_environ, + False, + for_package=for_package) + if "Python" in pi.depend: + # short cut, env.python_lib is required by set_a_product for salome modules + env.has_python="True" + env.python_lib=global_environ.get("PYTHON_LIBDIR") + env.set_a_product(product, self.logger) + env_file.close() + if not self.silent: + self.logger.write(_(" Create tcl module environment file %s\n") % + src.printcolors.printcLabel(env_file_name), 3) + + def write_env_file(self, filename, forBuild, @@ -838,10 +911,6 @@ class FileEnvWriter: # set env from the APPLICATION env.set_application_env(self.logger) - # The list of products to launch - lProductsName = env.get_names(self.config.APPLICATION.products.keys()) - env.set( "SALOME_MODULES", ','.join(lProductsName)) - # set the products env.set_products(self.logger, src_root=self.src_root) diff --git a/src/fileEnviron.py b/src/fileEnviron.py index 98691c6..e43e0e5 100644 --- a/src/fileEnviron.py +++ b/src/fileEnviron.py @@ -33,6 +33,8 @@ def get_file_environ(output, shell, environ=None): environ=src.environment.Environ({}) if shell == "bash": return BashFileEnviron(output, environ) + if shell == "tcl": + return TclFileEnviron(output, environ) if shell == "bat": return BatFileEnviron(output, environ) if shell == "cfgForPy": @@ -241,6 +243,77 @@ class FileEnviron(object): return res +class TclFileEnviron(FileEnviron): + """\ + Class for tcl shell. + """ + def __init__(self, output, environ=None): + """Initialization + + :param output file: the output file stream. + :param environ dict: a potential additional environment. + """ + self._do_init(output, environ) + self.output.write(tcl_header.replace("", + self.environ.get("sat_product_name"))) + self.output.write("\nset software %s\n" % self.environ.get("sat_product_name") ) + self.output.write("set version %s\n" % self.environ.get("sat_product_version") ) + root=os.path.join(self.environ.get("sat_product_base_path"), + "apps", + self.environ.get("sat_product_base_name"), + "$software", + "$version") + self.output.write("set root %s\n" % root) + modules_to_load=self.environ.get("sat_product_load_depend") + if len(modules_to_load)>0: + # write module load commands for product dependencies + self.output.write("\n") + for module_to_load in modules_to_load.split(";"): + self.output.write(module_to_load+"\n") + + def set(self, key, value): + """Set the environment variable "key" to value "value" + + :param key str: the environment variable to set + :param value str: the value + """ + #print "CNC TclFileEnviron set ", key, " to ", value + self.output.write('setenv %s "%s"\n' % (key, value)) + self.environ.set(key, value) + + def get(self, key): + """\ + Get the value of the environment variable "key" + + :param key str: the environment variable + """ + return self.environ.get(key) + + def append_value(self, key, value, sep=os.pathsep): + """append value to key using sep + + :param key str: the environment variable to append + :param value str: the value to append to key + :param sep str: the separator string + """ + if sep==os.pathsep: + self.output.write('append-path %s %s\n' % (key, value)) + else: + self.output.write('append-path --delim=\%c %s %s\n' % (sep, key, value)) + + def prepend_value(self, key, value, sep=os.pathsep): + """prepend value to key using sep + + :param key str: the environment variable to prepend + :param value str: the value to prepend to key + :param sep str: the separator string + """ + if sep==os.pathsep: + self.output.write('prepend-path %s %s\n' % (key, value)) + else: + self.output.write('prepend-path --delim=\%c %s %s\n' % (sep, key, value)) + + class BashFileEnviron(FileEnviron): """\ Class for bash shell. @@ -617,6 +690,12 @@ rem The following variables are used only in case of a sat package set out_dir_Path=%~dp0 """ +tcl_header="""\ +#%Module -*- tcl -*- +# +# module for use with 'environment-modules' package +# +""" bash_header="""\ #!/bin/bash diff --git a/src/product.py b/src/product.py index 5a5a7cf..e063970 100644 --- a/src/product.py +++ b/src/product.py @@ -113,6 +113,8 @@ def get_product_config(config, product_name, with_install_dir=True): # Get the base if any if 'base' in dic_version: base = dic_version.base + elif 'base' in config.APPLICATION: + base = config.APPLICATION.base # Get the section if any if 'section' in dic_version: @@ -254,6 +256,8 @@ Please add a section in it.""") % {"1" : vv, "2" : prod_pyconf_path} prod_info.dev = dev prod_info.hpc = hpc prod_info.version = version + if base != 'maybe': + prod_info.base = base # Set the archive_info if the product is get in archive mode if prod_info.get_source == "archive": @@ -363,7 +367,7 @@ Please provide a 'compil_script' key in its definition.""") % product_name prod_info.install_dir = prod_info.install_dir_save # Set the install_dir key - prod_info.install_dir = get_install_dir(config, base, version, prod_info) + prod_info.install_dir = get_install_dir(config, version, prod_info) return prod_info @@ -477,7 +481,7 @@ def get_product_section(config, product_name, version, section=None): # (product_name, version, section), prod_info) return prod_info -def get_install_dir(config, base, version, prod_info): +def get_install_dir(config, version, prod_info): """Compute the installation directory of a given product :param config Config: The global configuration @@ -494,14 +498,16 @@ def get_install_dir(config, base, version, prod_info): """ install_dir = "" in_base = False - # base : corresponds to what is specified in application pyconf (either from the global key, or from a product dict) + # prod_info.base : corresponds to what is specified in application pyconf (either from the global key, or from a product dict) # prod_info.install_dir : corresponds to what is specified in product pyconf (usually "base" for prerequisites) - if (("install_dir" in prod_info and prod_info.install_dir == "base") - or base == "yes"): + if ( ("install_dir" in prod_info and prod_info.install_dir == "base") + or ("base" in prod_info and prod_info.base != "no") ): + # a product goes in base if install_dir is set to base, or if product was declared based in application pyconf in_base = True + # what was declared in application has precedence over what was said in product pyconf # no_base="yes" has precedence over base == "yes" - if (base == "no" or ("no_base" in config.APPLICATION + if ( ("base" in prod_info and prod_info.base == "no") or ("no_base" in config.APPLICATION and config.APPLICATION.no_base == "yes")): in_base = False @@ -543,10 +549,29 @@ def get_base_install_dir(config, prod_info, version): :param product_info Config: The configuration specific to the product :param version str: The version of the product + :param base str: This corresponds to the value given by user in its + application.pyconf for the specific product. If "yes", the + user wants the product to be in base. If "no", he wants the + product to be in the application workdir. + if it contains something else, is is interpreted as the name + of a base we build for module load. :return: The path of the product installation :rtype: str """ base_path = src.get_base_path(config) + if "base" in prod_info and prod_info.base != "no" and prod_info.base != "yes": + # we are in the case of a named base + if ( src.appli_test_property(config,"pip", "yes") and + src.product.product_test_property(prod_info,"pip", "yes") and + src.appli_test_property(config,"pip_install_dir", "python") ): + # when pip mode is activated in the application + # and product is pip, and pip_install_dir is set to python + python_info=get_product_config(config, "Python") + prod_dir = os.path.join(base_path, "apps", prod_info.base, "Python", python_info.version) + else: + prod_dir = os.path.join(base_path, "apps", prod_info.base, prod_info.name, version) + return prod_dir + prod_dir = os.path.join(base_path, prod_info.name + "-" + version) if not os.path.exists(prod_dir): return os.path.join(prod_dir, "config-1") -- 2.30.2