From: Serge Rehbinder Date: Thu, 29 Dec 2016 13:47:26 +0000 (+0100) Subject: Add the script command, change the compile command in order to take the script comman... X-Git-Tag: 5.0.0a1~38^2~36 X-Git-Url: http://git.salome-platform.org/gitweb/?a=commitdiff_plain;h=b8acb176c4fe8849acc7aec3ad7b9c03eb8ae857;p=tools%2Fsat.git Add the script command, change the compile command in order to take the script command into account, and add the steps logs into the /LOGS/ directory --- diff --git a/commands/compile.py b/commands/compile.py index 9a2d095..c1493e8 100644 --- a/commands/compile.py +++ b/commands/compile.py @@ -401,12 +401,76 @@ def compile_product(sat, p_name_info, config, options, logger, header, len_end): :param config Config: The global configuration :param logger Logger: The logger instance to use for the display and logging + :param header Str: the header to display when logging + :param len_end Int: the lenght of the the end of line (used in display) :return: 1 if it fails, else 0. :rtype: int ''' + __, p_info = p_name_info + + # Get the build procedure from the product configuration. + # It can be : + # build_sources : autotools -> build_configure, configure, make, make install + # build_sources : cmake -> cmake, make, make install + # build_sources : script -> script executions + res = 0 + if (src.product.product_is_autotools(p_info) or + src.product.product_is_cmake(p_info)): + res, len_end_line, error_step = compile_product_cmake_autotools(sat, + p_name_info, + config, + options, + logger, + header, + len_end) + if src.product.product_has_script(p_info): + res, len_end_line, error_step = compile_product_script(sat, + p_name_info, + config, + options, + logger, + header, + len_end) + + # Check that the install directory exists + if res==0 and not(os.path.exists(p_info.install_dir)): + res = 1 + error_step = "NO INSTALL DIR" + msg = _("Error: despite the fact that all the steps ended successfully," + " no install directory was found !") + logger.write(src.printcolors.printcError(msg), 4) + logger.write("\n", 4) + + # Add the config file corresponding to the dependencies/versions of the + # product that have been successfully compiled + if res==0: + logger.write(_("Add the config file in installation directory\n"), 5) + add_compile_config_file(p_info, config) + + return res, len_end_line, error_step + +def compile_product_cmake_autotools(sat, + p_name_info, + config, + options, + logger, + header, + len_end): + '''Execute the proper build procedure for autotools or cmake + in the product build directory. + + :param p_name_info tuple: (str, Config) => (product_name, product_info) + :param config Config: The global configuration + :param logger Logger: The logger instance to use for the display + and logging + :param header Str: the header to display when logging + :param len_end Int: the lenght of the the end of line (used in display) + :return: 1 if it fails, else 0. + :rtype: int + ''' p_name, p_info = p_name_info - + # Execute "sat configure", "sat make" and "sat install" res = 0 error_step = "" @@ -422,7 +486,7 @@ def compile_product(sat, p_name_info, config, options, logger, header, len_end): if res_c > 0: error_step = "CONFIGURE" - else: + else: # Logging and sat command call for make step # Logging take account of the fact that the product has a compilation # script or not @@ -461,23 +525,42 @@ def compile_product(sat, p_name_info, config, options, logger, header, len_end): if res_mi > 0: error_step = "MAKE INSTALL" + + return res, len_end_line, error_step - # Check that the install directory exists - if res==0 and not(os.path.exists(p_info.install_dir)): - res = 1 - error_step = "NO INSTALL DIR" - msg = _("Error: despite the fact that all the steps ended successfully," - " no install directory was found !") - logger.write(src.printcolors.printcError(msg), 4) - logger.write("\n", 4) +def compile_product_script(sat, + p_name_info, + config, + options, + logger, + header, + len_end): + '''Execute the script build procedure in the product build directory. - # Add the config file corresponding to the dependencies/versions of the - # product that have been successfully compiled - if res==0: - logger.write(_("Add the config file in installation directory\n"), 5) - add_compile_config_file(p_info, config) + :param p_name_info tuple: (str, Config) => (product_name, product_info) + :param config Config: The global configuration + :param logger Logger: The logger instance to use for the display + and logging + :param header Str: the header to display when logging + :param len_end Int: the lenght of the the end of line (used in display) + :return: 1 if it fails, else 0. + :rtype: int + ''' + p_name, p_info = p_name_info - return res, len_end_line, error_step + # Execute "sat configure", "sat make" and "sat install" + error_step = "" + + # Logging and sat command call for the script step + scrit_path_display = src.printcolors.printcLabel(p_info.compil_script) + log_step(logger, header, "SCRIPT " + scrit_path_display) + len_end_line = len_end + len(scrit_path_display) + res = sat.script(config.VARS.application + " --products " + p_name, + verbose = 0, + logger_add_link = logger) + log_res_step(logger, res) + + return res, len_end_line, error_step def add_compile_config_file(p_info, config): '''Execute the proper configuration command(s) diff --git a/commands/make.py b/commands/make.py index 6cd7bac..4b15ee1 100644 --- a/commands/make.py +++ b/commands/make.py @@ -142,19 +142,11 @@ def make_product(p_name_info, make_option, config, logger): # Execute cmake if the product is cmake len_end_line = 20 res = 0 - if not src.product.product_has_script(p_info): - nb_proc, make_opt_without_j = get_nb_proc(p_info, config, make_option) - log_step(logger, header, "MAKE -j" + str(nb_proc)) - res_m = builder.make(nb_proc, make_opt_without_j) - log_res_step(logger, res_m) - res += res_m - else: - scrit_path_display = src.printcolors.printcLabel(p_info.compil_script) - log_step(logger, header, "SCRIPT " + scrit_path_display) - len_end_line += len(scrit_path_display) - res_s = builder.do_script_build(p_info.compil_script) - log_res_step(logger, res_s) - res += res_s + nb_proc, make_opt_without_j = get_nb_proc(p_info, config, make_option) + log_step(logger, header, "MAKE -j" + str(nb_proc)) + res_m = builder.make(nb_proc, make_opt_without_j) + log_res_step(logger, res_m) + res += res_m # Log the result if res > 0: diff --git a/commands/script.py b/commands/script.py new file mode 100644 index 0000000..bb7fd8d --- /dev/null +++ b/commands/script.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- +# Copyright (C) 2010-2012 CEA/DEN +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os +import re + +import src + +# Define all possible option for the script command : sat script +parser = src.options.Options() +parser.add_option('p', 'products', 'list2', 'products', + _('Optional: products to configure. This option can be' + ' passed several time to configure several products.')) +parser.add_option('', 'nb_proc', 'int', 'nb_proc', + _('Optional: The number of processors to use in the script if the make ' + 'command is used in it.\n\tWarning: the script has to be correctly written ' + 'if you want this option to work.\n\tThe $MAKE_OPTIONS has to be ' + 'used.'), "") + +def get_products_list(options, cfg, logger): + '''method that gives the product list with their informations from + configuration regarding the passed options. + + :param options Options: The Options instance that stores the commands + arguments + :param cfg Config: The global configuration + :param logger Logger: The logger instance to use for the display and + logging + :return: The list of (product name, product_informations). + :rtype: List + ''' + # Get the products to be prepared, regarding the options + if options.products is None: + # No options, get all products sources + products = cfg.APPLICATION.products + else: + # if option --products, check that all products of the command line + # are present in the application. + products = options.products + for p in products: + if p not in cfg.APPLICATION.products: + raise src.SatException(_("Product %(product)s " + "not defined in application %(application)s") % + { 'product': p, 'application': cfg.VARS.application} ) + + # Construct the list of tuple containing + # the products name and their definition + products_infos = src.product.get_products_infos(products, cfg) + + products_infos = [pi for pi in products_infos if not( + src.product.product_is_native(pi[1]) or + src.product.product_is_fixed(pi[1]))] + + return products_infos + +def log_step(logger, header, step): + logger.write("\r%s%s" % (header, " " * 20), 3) + logger.write("\r%s%s" % (header, step), 3) + logger.write("\n==== %s \n" % src.printcolors.printcInfo(step), 4) + logger.flush() + +def log_res_step(logger, res): + if res == 0: + logger.write("%s \n" % src.printcolors.printcSuccess("OK"), 4) + logger.flush() + else: + logger.write("%s \n" % src.printcolors.printcError("KO"), 4) + logger.flush() + +def run_script_all_products(config, products_infos, nb_proc, logger): + '''Execute the script in each product build directory. + + :param config Config: The global configuration + :param products_info list: List of + (str, Config) => (product_name, product_info) + :param nb_proc int: The number of processors to use + :param logger Logger: The logger instance to use for the display and logging + :return: the number of failing commands. + :rtype: int + ''' + res = 0 + for p_name_info in products_infos: + res_prod = run_script_of_product(p_name_info, + nb_proc, + config, + logger) + if res_prod != 0: + res += 1 + return res + +def run_script_of_product(p_name_info, nb_proc, config, logger): + '''Execute the proper configuration command(s) + in the product build directory. + + :param p_name_info tuple: (str, Config) => (product_name, product_info) + :param nb_proc int: The number of processors to use + :param config Config: The global configuration + :param logger Logger: The logger instance to use for the display + and logging + :return: 1 if it fails, else 0. + :rtype: int + ''' + + p_name, p_info = p_name_info + + # Logging + logger.write("\n", 4, False) + logger.write("################ ", 4) + header = _("Running script of %s") % src.printcolors.printcLabel(p_name) + header += " %s " % ("." * (20 - len(p_name))) + logger.write(header, 3) + logger.write("\n", 4, False) + logger.flush() + + # Do nothing if he product is not compilable or has no compilation script + if (("properties" in p_info and "compilation" in p_info.properties and + p_info.properties.compilation == "no") or + (not src.product.product_has_script(p_info))): + log_step(logger, header, "ignored") + logger.write("\n", 3, False) + return 0 + + # Instantiate the class that manages all the construction commands + # like cmake, make, make install, make test, environment management, etc... + builder = src.compilation.Builder(config, logger, p_info) + + # Prepare the environment + log_step(logger, header, "PREPARE ENV") + res_prepare = builder.prepare() + log_res_step(logger, res_prepare) + + # Execute the script + len_end_line = 20 + script_path_display = src.printcolors.printcLabel(p_info.compil_script) + log_step(logger, header, "SCRIPT " + script_path_display) + len_end_line += len(script_path_display) + res = builder.do_script_build(p_info.compil_script, number_of_proc=nb_proc) + log_res_step(logger, res) + + # Log the result + if res > 0: + logger.write("\r%s%s" % (header, " " * len_end_line), 3) + logger.write("\r" + header + src.printcolors.printcError("KO")) + logger.write("==== %(KO)s in script execution of %(name)s \n" % + { "name" : p_name , "KO" : src.printcolors.printcInfo("ERROR")}, 4) + logger.flush() + else: + logger.write("\r%s%s" % (header, " " * len_end_line), 3) + logger.write("\r" + header + src.printcolors.printcSuccess("OK")) + logger.write("==== %s \n" % src.printcolors.printcInfo("OK"), 4) + logger.write("==== Script execution of %(name)s %(OK)s \n" % + { "name" : p_name , "OK" : src.printcolors.printcInfo("OK")}, 4) + logger.flush() + logger.write("\n", 3, False) + + return res + +def description(): + '''method that is called when salomeTools is called with --help option. + + :return: The text to display for the script command description. + :rtype: str + ''' + return _("The script command executes the script(s) of the the given " + "products in the build directory.\nThis is done only for the " + "products that are constructed using a script (build_source " + ": \"script\").\nOtherwise, nothing is done." + "\n\nexample:\nsat script SALOME-master --products Python,numpy") + +def run(args, runner, logger): + '''method that is called when salomeTools is called with make parameter. + ''' + + # Parse the options + (options, args) = parser.parse_args(args) + + # check that the command has been called with an application + src.check_config_has_application( runner.cfg ) + + # Get the list of products to treat + products_infos = get_products_list(options, runner.cfg, logger) + + # Print some informations + logger.write(_('Executing the script in the build ' + 'directories of the application %s\n') % + src.printcolors.printcLabel(runner.cfg.VARS.application), 1) + + info = [(_("BUILD directory"), + os.path.join(runner.cfg.APPLICATION.workdir, 'BUILD'))] + src.print_info(logger, info) + + # Call the function that will loop over all the products and execute + # the right command(s) + if options.nb_proc is None: + options.nb_proc = 0 + res = run_script_all_products(runner.cfg, + products_infos, + options.nb_proc, + logger) + + # Print the final state + nb_products = len(products_infos) + if res == 0: + final_status = "OK" + else: + final_status = "KO" + + logger.write(_("\nScript: %(status)s " + "(%(valid_result)d/%(nb_products)d)\n") % \ + { 'status': src.printcolors.printc(final_status), + 'valid_result': nb_products - res, + 'nb_products': nb_products }, 1) + + return res \ No newline at end of file diff --git a/complete_sat.sh b/complete_sat.sh index a95ac24..00d37a6 100755 --- a/complete_sat.sh +++ b/complete_sat.sh @@ -104,7 +104,7 @@ _salomeTools_complete() # first argument => show available commands if [[ ${argc} == 1 ]] then - opts="config log testcommand source patch prepare environ clean configure make makeinstall compile launcher run jobs job shell test package generate find_duplicates application template base profile --help" + opts="config log testcommand source patch prepare environ clean configure make makeinstall compile launcher run jobs job shell test package generate find_duplicates application template base profile script --help" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 fi @@ -268,6 +268,11 @@ _salomeTools_complete() COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; + script) + opts="--products --nb_proc" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; *) return 0 ;; esac diff --git a/src/compilation.py b/src/compilation.py index 0cf9b16..bdd789f 100644 --- a/src/compilation.py +++ b/src/compilation.py @@ -82,12 +82,16 @@ class Builder: #environ_info.append(self.product_info.name) # create build environment - self.build_environ = src.environment.SalomeEnviron(self.config, src.environment.Environ(dict(os.environ)), True) + self.build_environ = src.environment.SalomeEnviron(self.config, + src.environment.Environ(dict(os.environ)), + True) self.build_environ.silent = (self.config.USER.output_verbose_level < 5) self.build_environ.set_full_environ(self.logger, environ_info) # create runtime environment - self.launch_environ = src.environment.SalomeEnviron(self.config, src.environment.Environ(dict(os.environ)), False) + self.launch_environ = src.environment.SalomeEnviron(self.config, + src.environment.Environ(dict(os.environ)), + False) self.launch_environ.silent = True # no need to show here self.launch_environ.set_full_environ(self.logger, environ_info) @@ -125,6 +129,7 @@ class Builder: stderr=subprocess.STDOUT) if res == 0: + self.put_txt_log_in_appli_log_dir("cmake") return res else: return 1 @@ -148,6 +153,7 @@ class Builder: stderr=subprocess.STDOUT) if res == 0: + self.put_txt_log_in_appli_log_dir("build_configure") return res else: return 1 @@ -172,6 +178,7 @@ class Builder: stderr=subprocess.STDOUT) if res == 0: + self.put_txt_log_in_appli_log_dir("configure") return res else: return 1 @@ -223,6 +230,7 @@ CC=\\"hack_libtool\\"%g" libtool''' stderr=subprocess.STDOUT) if res == 0: + self.put_txt_log_in_appli_log_dir("make") return res else: return 1 @@ -257,6 +265,7 @@ CC=\\"hack_libtool\\"%g" libtool''' stderr=subprocess.STDOUT) if res == 0: + self.put_txt_log_in_appli_log_dir("make") return res else: return 1 @@ -283,6 +292,7 @@ CC=\\"hack_libtool\\"%g" libtool''' stderr=subprocess.STDOUT) if res == 0: + self.put_txt_log_in_appli_log_dir("makeinstall") return res else: return 1 @@ -382,6 +392,7 @@ CC=\\"hack_libtool\\"%g" libtool''' pymodule = imp.load_source(product + "_compile_script", script) self.nb_proc = nb_proc retcode = pymodule.compil(self.config, self, self.logger) + self.put_txt_log_in_appli_log_dir("script") except: __, exceptionValue, exceptionTraceback = sys.exc_info() self.logger.write(str(exceptionValue), 1) @@ -423,15 +434,19 @@ CC=\\"hack_libtool\\"%g" libtool''' env=self.build_environ.environ.environ) if res == 0: + self.put_txt_log_in_appli_log_dir("script") return res else: return 1 - def do_script_build(self, script): + def do_script_build(self, script, number_of_proc=0): # define make options (may not be used by the script) - nb_proc = src.get_cfg_param(self.product_info,"nb_proc", 0) - if nb_proc == 0: - nb_proc = self.config.VARS.nb_proc + if number_of_proc==0: + nb_proc = src.get_cfg_param(self.product_info,"nb_proc", 0) + if nb_proc == 0: + nb_proc = self.config.VARS.nb_proc + else: + nb_proc = min(number_of_proc, self.config.VARS.nb_proc) extension = script.split('.')[-1] if extension in ["bat","sh"]: @@ -440,4 +455,25 @@ CC=\\"hack_libtool\\"%g" libtool''' return self.do_python_script_build(script, nb_proc) msg = _("The script %s must have .sh, .bat or .py extension." % script) - raise src.SatException(msg) \ No newline at end of file + raise src.SatException(msg) + + def put_txt_log_in_appli_log_dir(self, file_name): + '''Put the txt log (that contain the system logs, like make command + output) in the directory /LOGS// + + :param file_name Str: the name of the file to write + ''' + if self.logger.logTxtFile == sys.__stdout__: + return + dir_where_to_put = os.path.join(self.config.APPLICATION.workdir, + "LOGS", + self.product_info.name) + file_path = os.path.join(dir_where_to_put, file_name) + src.ensure_path_exists(dir_where_to_put) + # write the logTxtFile copy it to the destination, and then recreate + # it as it was + self.logger.logTxtFile.close() + os.rename(self.logger.txtFilePath, file_path) + self.logger.logTxtFile = open(str(self.logger.txtFilePath), 'w') + self.logger.logTxtFile.write(open(file_path, "r").read()) + \ No newline at end of file