From aaf1f0cac95c1e3baf0228751db882316eae35c7 Mon Sep 17 00:00:00 2001 From: SRE Date: Mon, 29 May 2017 14:13:32 +0200 Subject: [PATCH] New command sait init. USER.workdir is replaced by LOCAL.workdir. Idem for USER.base and USER.log_dir --- commands/config.py | 51 +++++++-------- commands/init.py | 154 ++++++++++++++++++++++++++++++++++++++++++++ commands/jobs.py | 16 ++++- commands/log.py | 4 +- commands/package.py | 39 +++++------ commands/test.py | 6 +- complete_sat.sh | 10 ++- data/local.pyconf | 16 +++++ data/site.pyconf | 19 ------ src/__init__.py | 34 ++++++++-- src/logger.py | 7 +- 11 files changed, 274 insertions(+), 82 deletions(-) create mode 100644 commands/init.py create mode 100644 data/local.pyconf delete mode 100644 data/site.pyconf diff --git a/commands/config.py b/commands/config.py index deef661..084df54 100644 --- a/commands/config.py +++ b/commands/config.py @@ -247,30 +247,40 @@ class ConfigManager: exec('cfg.' + rule) # this cannot be factorized because of the exec # ===================================================================== - # Load SITE config file + # Load LOCAL config file # search only in the data directory src.pyconf.streamOpener = ConfigOpener([cfg.VARS.datadir]) try: - site_cfg = src.pyconf.Config(open(os.path.join(cfg.VARS.datadir, - 'site.pyconf')), - PWD = ('SITE', cfg.VARS.datadir) ) + local_cfg = src.pyconf.Config(open(os.path.join(cfg.VARS.datadir, + 'local.pyconf')), + PWD = ('LOCAL', cfg.VARS.datadir) ) except src.pyconf.ConfigError as e: raise src.SatException(_("Error in configuration file: " - "site.pyconf\n %(error)s") % \ + "local.pyconf\n %(error)s") % \ {'error': str(e) }) except IOError as error: e = str(error) - if "site.pyconf" in e : - e += ("\nYou can copy data" - + cfg.VARS.sep - + "site.template.pyconf to data" - + cfg.VARS.sep - + "site.pyconf and edit the file") raise src.SatException( e ); - merger.merge(cfg, site_cfg) + merger.merge(cfg, local_cfg) + + # When the key is "unknown", put the default value + if cfg.LOCAL.base == "unknown": + cfg.LOCAL.base = os.path.abspath( + os.path.join(cfg.VARS.salometoolsway, + "..", + "BASE")) + if cfg.LOCAL.workdir == "unknown": + cfg.LOCAL.workdir = os.path.abspath( + os.path.join(cfg.VARS.salometoolsway, + "..")) + if cfg.LOCAL.log_dir == "unknown": + cfg.LOCAL.log_dir = os.path.abspath( + os.path.join(cfg.VARS.salometoolsway, + "..", + "LOGS")) # apply overwrite from command line if needed - for rule in self.get_command_line_overrides(options, ["SITE"]): + for rule in self.get_command_line_overrides(options, ["LOCAL"]): exec('cfg.' + rule) # this cannot be factorized because of the exec # ===================================================================== @@ -482,10 +492,6 @@ class ConfigManager: # user_cfg.addMapping('USER', src.pyconf.Mapping(user_cfg), "") - # - user_cfg.USER.addMapping('workdir', os.path.expanduser('~'), - "This is where salomeTools will work. " - "You may (and probably do) change it.\n") user_cfg.USER.addMapping('cvs_user', config.VARS.user, "This is the user name used to access salome cvs base.\n") user_cfg.USER.addMapping('svn_user', config.VARS.user, @@ -516,16 +522,9 @@ class ConfigManager: src.pyconf.DOLLAR, 'workdir + $VARS.sep + "BASE"'), "The products installation base (could be " - "ignored if this key exists in the site.pyconf" + "ignored if this key exists in the local.pyconf" " file of salomTools).\n") - - user_cfg.USER.addMapping("log_dir", - src.pyconf.Reference( - user_cfg, - src.pyconf.DOLLAR, - 'workdir + $VARS.sep + "LOGS"'), - "The log repository\n") - + # src.ensure_path_exists(config.VARS.personalDir) src.ensure_path_exists(os.path.join(config.VARS.personalDir, diff --git a/commands/init.py b/commands/init.py new file mode 100644 index 0000000..0a0103a --- /dev/null +++ b/commands/init.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- +# Copyright (C) 2010-2012 CEA/DEN +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os + +import src + +# Define all possible option for the init command : sat init +parser = src.options.Options() +parser.add_option('b', 'base', 'string', 'base', + _('Optional: The path to the products base')) +parser.add_option('w', 'workdir', 'string', 'workdir', + _('Optional: The path to the working directory ' + '(where to install the applications')) +parser.add_option('v', 'VCS', 'string', 'VCS', + _('Optional: The address of the repository of SAT ' + '(only informative)')) +parser.add_option('t', 'tag', 'string', 'tag', + _('Optional: The tag of SAT (only informative)')) +parser.add_option('l', 'log_dir', 'string', 'log_dir', + _('Optional: The directory where to put all the logs of SAT')) + +def set_local_value(config, key, value, logger): + """ Edit the site.pyconf file and change a value. + + :param config Config: The global configuration. + :param key Str: The key from which to change the value. + :param value Str: The path to change. + :param logger Logger: The logger instance. + :return: 0 if all is OK, else 1 + :rtype: int + """ + local_file_path = os.path.join(config.VARS.datadir, "local.pyconf") + # Update the local.pyconf file + try: + local_cfg = src.pyconf.Config(local_file_path) + local_cfg.LOCAL[key] = value + ff = open(local_file_path, 'w') + local_cfg.__save__(ff, 1) + ff.close() + if key != "log_dir": + config.LOCAL[key] = value + except Exception as e: + err = str(e) + msg = _("Unable to update the local.pyconf file: %s\n" % err) + logger.write(msg, 1) + return 1 + + return 0 + +def display_local_values(config, logger): + """ Display the base path + + :param config Config: The global configuration. + :param key Str: The key from which to change the value. + :param logger Logger: The logger instance. + """ + info = [("base", config.LOCAL.base), + ("workdir", config.LOCAL.workdir), + ("log_dir", config.LOCAL.log_dir), + ("VCS", config.LOCAL.VCS), + ("tag", config.LOCAL.tag)] + src.print_info(logger, info) + + return 0 + +def check_path(path_to_check, logger): + """ Verify that the given path is not a file and can be created. + + :param path_to_check Str: The path to check. + :param logger Logger: The logger instance. + """ + if path_to_check == "unknown": + return 0 + + # Get the path + path = src.Path(path_to_check) + + # If it is a file, do nothing and return error + if path.isfile(): + msg = _("Error: The given path is a file. Please provide a path to " + "a directory") + logger.write(src.printcolors.printcError(msg), 1) + return 1 + + # Try to create the given path + try: + src.ensure_path_exists(str(path)) + except Exception as e: + err = src.printcolors.printcError(str(e)) + msg = _("Unable to create the directory %s: %s\n" % (str(path), + err)) + logger.write(msg, 1) + return 1 + + return 0 + +def description(): + '''method that is called when salomeTools is called with --help option. + + :return: The text to display for the init command description. + :rtype: str + ''' + return _("Changes the local settings of SAT.") + +def run(args, runner, logger): + '''method that is called when salomeTools is called with init parameter. + ''' + + # Parse the options + (options, args) = parser.parse_args(args) + + # Print some informations + logger.write(_('Local Settings of SAT %s\n\n') % + src.printcolors.printcLabel(runner.cfg.VARS.salometoolsway), 1) + + res = 0 + + # Set the options corresponding to a directory + for opt in [("base" , options.base), + ("workdir", options.workdir), + ("log_dir", options.log_dir)]: + key, value = opt + if value: + res_check = check_path(value, logger) + res += res_check + if res_check == 0: + res_set = set_local_value(runner.cfg, key, value, logger) + res += res_set + + # Set the options corresponding to an informative value + for opt in [("VCS", options.VCS), ("tag", options.tag)]: + key, value = opt + res_set = set_local_value(runner.cfg, key, value, logger) + res += res_set + + display_local_values(runner.cfg, logger) + + return res \ No newline at end of file diff --git a/commands/jobs.py b/commands/jobs.py index 3c0e17a..64073bb 100644 --- a/commands/jobs.py +++ b/commands/jobs.py @@ -839,6 +839,15 @@ class Jobs(object): self.logger) machine.distribution = out_dist.read().decode().replace("\n", "") + + # set the local settings of sat on the remote machine using + # the init command + (__, __, __) = machine.exec_command( + os.path.join(machine.sat_path, + "sat init --base unknown --workdir unknown" + " --log_dir unknown"), + self.logger) + # Print the status of the copy if res_copy == 0: self.logger.write('\r%s' % @@ -1731,7 +1740,8 @@ def run(args, runner, logger): config_jobs.__save__(f) # log the paramiko problems - paramiko_log_dir_path = os.path.join(runner.cfg.USER.log_dir, "JOBS") + log_dir = src.get_log_path(runner.cfg) + paramiko_log_dir_path = os.path.join(log_dir, "JOBS") src.ensure_path_exists(paramiko_log_dir_path) paramiko.util.log_to_file(os.path.join(paramiko_log_dir_path, logger.txtFileName)) @@ -1753,7 +1763,7 @@ def run(args, runner, logger): logger.flush() # Copy the stylesheets in the log directory - log_dir = runner.cfg.USER.log_dir + log_dir = log_dir xsl_dir = os.path.join(runner.cfg.VARS.srcDir, 'xsl') files_to_copy = [] files_to_copy.append(os.path.join(xsl_dir, STYLESHEET_GLOBAL)) @@ -1764,7 +1774,7 @@ def run(args, runner, logger): # Instanciate the Gui in order to produce the xml files that contain all # the boards - gui = Gui(runner.cfg.USER.log_dir, + gui = Gui(log_dir, today_jobs.ljobs, today_jobs.ljobs_not_today, runner.cfg.VARS.datehour, diff --git a/commands/log.py b/commands/log.py index ed7439e..08e5f23 100644 --- a/commands/log.py +++ b/commands/log.py @@ -203,7 +203,7 @@ def run(args, runner, logger): (options, args) = parser.parse_args(args) # get the log directory. - logDir = runner.cfg.USER.log_dir + logDir = src.get_log_path(runner.cfg) # Print a header nb_files_log_dir = len(glob.glob(os.path.join(logDir, "*"))) @@ -322,7 +322,7 @@ def run(args, runner, logger): # Create or update the hat xml that gives access to all the commands log files logger.write(_("Generating the hat log file (can be long) ... "), 3) xmlHatFilePath = os.path.join(logDir, 'hat.xml') - src.logger.update_hat_xml(runner.cfg.USER.log_dir, + src.logger.update_hat_xml(logDir, application = runner.cfg.VARS.application, notShownCommands = notShownCommands) logger.write(src.printcolors.printc("OK"), 3) diff --git a/commands/package.py b/commands/package.py index aa49bb5..593834b 100644 --- a/commands/package.py +++ b/commands/package.py @@ -59,16 +59,17 @@ JOBPATH : $project_path + "jobs/" MACHINEPATH : $project_path + "machines/" """ -SITE_TEMPLATE = ("""#!/usr/bin/env python +LOCAL_TEMPLATE = ("""#!/usr/bin/env python #-*- coding:utf-8 -*- -SITE : -{ - log : - { - log_dir : $USER.workdir + "/LOGS" - } -} + LOCAL : + { + base : 'unknown' + workdir : 'unknown' + log_dir : 'unknown' + VCS : None + tag : None + } PROJECTS : { @@ -621,7 +622,7 @@ def get_archives(config, logger): return d_archives, l_pinfo_vcs def add_salomeTools(config, tmp_working_dir): - '''Prepare a version of salomeTools that has a specific site.pyconf file + '''Prepare a version of salomeTools that has a specific local.pyconf file configured for a source package. :param config Config: The global configuration. @@ -636,12 +637,12 @@ def add_salomeTools(config, tmp_working_dir): sat_running_path = src.Path(config.VARS.salometoolsway) sat_running_path.copy(sat_tmp_path) - # Update the site.pyconf file that contains the path to the project - site_pyconf_name = "site.pyconf" - site_pyconf_dir = os.path.join(tmp_working_dir, "salomeTools", "data") - site_pyconf_file = os.path.join(site_pyconf_dir, site_pyconf_name) - ff = open(site_pyconf_file, "w") - ff.write(SITE_TEMPLATE) + # Update the local.pyconf file that contains the path to the project + local_pyconf_name = "local.pyconf" + local_pyconf_dir = os.path.join(tmp_working_dir, "salomeTools", "data") + local_pyconf_file = os.path.join(local_pyconf_dir, local_pyconf_name) + ff = open(local_pyconf_file, "w") + ff.write(LOCAL_TEMPLATE) ff.close() return sat_tmp_path.path @@ -1096,12 +1097,12 @@ def run(args, runner, logger): if options.project: # check that the project is visible by SAT if options.project not in runner.cfg.PROJECTS.project_file_paths: - site_path = os.path.join(runner.cfg.VARS.salometoolsway, + local_path = os.path.join(runner.cfg.VARS.salometoolsway, "data", - "site.pyconf") + "local.pyconf") msg = _("ERROR: the project %(proj)s is not visible by salomeTools." - "\nPlease add it in the %(site)s file." % { - "proj" : options.project, "site" : site_path}) + "\nPlease add it in the %(local)s file." % { + "proj" : options.project, "local" : local_path}) logger.write(src.printcolors.printcError(msg), 1) logger.write("\n", 1) return 1 diff --git a/commands/test.py b/commands/test.py index 393d7ba..8feae64 100644 --- a/commands/test.py +++ b/commands/test.py @@ -536,7 +536,8 @@ def generate_history_xml_path(config, test_base): test_base_name = os.path.basename(test_base) history_xml_name += test_base_name history_xml_name += ".xml" - return os.path.join(config.USER.log_dir, "TEST", history_xml_name) + log_dir = src.get_log_path(config) + return os.path.join(log_dir, "TEST", history_xml_name) def run(args, runner, logger): '''method that is called when salomeTools is called with test parameter. @@ -682,7 +683,8 @@ def run(args, runner, logger): logger.write("\n", 2, False) logger.write(_("\nGenerate the specific test log\n"), 5) - out_dir = os.path.join(runner.cfg.USER.log_dir, "TEST") + log_dir = src.get_log_path(runner.cfg) + out_dir = os.path.join(log_dir, "TEST") src.ensure_path_exists(out_dir) name_xml_board = logger.logFileName.split(".")[0] + "board" + ".xml" historic_xml_path = generate_history_xml_path(runner.cfg, test_base) diff --git a/complete_sat.sh b/complete_sat.sh index ec85d50..6873137 100755 --- a/complete_sat.sh +++ b/complete_sat.sh @@ -58,6 +58,9 @@ _show_applications() base) opts2=$(echo --set $opts2) ;; + init) + opts2=$(echo --base --workdir --VCS --tag --log_dir $opts2) + ;; esac COMPREPLY=( $(compgen -W "${opts2}" -- ${cur}) ) @@ -104,7 +107,7 @@ _salomeTools_complete() # first argument => show available commands if [[ ${argc} == 1 ]] then - opts="config log source patch prepare environ clean configure make makeinstall compile launcher run jobs job shell test package generate find_duplicates application template base check profile script --help --overwrite --debug --verbose --batch --all_in_terminal --logs_paths_in_file" + opts="config log source patch prepare environ clean configure make makeinstall compile launcher run jobs job shell test package generate find_duplicates application template base check profile script init --help --overwrite --debug --verbose --batch --all_in_terminal --logs_paths_in_file" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 fi @@ -278,6 +281,11 @@ _salomeTools_complete() COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; + init) + opts="--base --workdir --VCS --tag --log_dir" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; *) return 0 ;; esac diff --git a/data/local.pyconf b/data/local.pyconf new file mode 100644 index 0000000..baf2b8c --- /dev/null +++ b/data/local.pyconf @@ -0,0 +1,16 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- + LOCAL : + { + base : 'unknown' + workdir : 'unknown' + log_dir : 'unknown' + VCS : None + tag : None + } + PROJECTS : + { + project_file_paths : + [ + ] + } diff --git a/data/site.pyconf b/data/site.pyconf deleted file mode 100644 index 33a9050..0000000 --- a/data/site.pyconf +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -#-*- coding:utf-8 -*- - -SITE : -{ - base : $USER.workdir + $VARS.sep + "BASE" - test :{ - tmp_dir_with_application : '/tmp' + $VARS.sep + $VARS.user + $VARS.sep + $APPLICATION.name + $VARS.sep + 'test' - tmp_dir : '/tmp' + $VARS.sep + $VARS.user + $VARS.sep + 'test' - timeout : 150 - } -} - -PROJECTS : -{ -project_file_paths : -[ -] -} diff --git a/src/__init__.py b/src/__init__.py index abf59b0..fd75079 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -119,20 +119,40 @@ def print_info(logger, info): logger.write("\n", 2) def get_base_path(config): - '''Returns the path of the product base. + '''Returns the path of the products base. :param config Config: The global Config instance. - :return: The path of the product base. + :return: The path of the products base. :rtype: str ''' - if "base" not in config.SITE: - site_file_path = os.path.join(config.VARS.salometoolsway, + if "base" not in config.LOCAL: + local_file_path = os.path.join(config.VARS.salometoolsway, "data", - "site.pyconf") - msg = _("Please define a base path in the file %s" % site_file_path) + "local.pyconf") + msg = _("Please define a base path in the file %s" % local_file_path) raise SatException(msg) + + base_path = os.path.abspath(config.LOCAL.base) + + return base_path - return config.SITE.base +def get_log_path(config): + '''Returns the path of the logs. + + :param config Config: The global Config instance. + :return: The path of the logs. + :rtype: str + ''' + if "log_dir" not in config.LOCAL: + local_file_path = os.path.join(config.VARS.salometoolsway, + "data", + "local.pyconf") + msg = _("Please define a log_dir in the file %s" % local_file_path) + raise SatException(msg) + + log_dir_path = os.path.abspath(config.LOCAL.log_dir) + + return log_dir_path def get_salome_version(config): if hasattr(config.APPLICATION, 'version_salome'): diff --git a/src/logger.py b/src/logger.py index eb2ee58..843f7b7 100644 --- a/src/logger.py +++ b/src/logger.py @@ -56,11 +56,12 @@ class Logger(object): config.VARS.command + "_" + config.VARS.hostname) logFileName = prefix + hour_command_host + ".xml" - logFilePath = os.path.join(config.USER.log_dir, logFileName) + log_dir = src.get_log_path(config) + logFilePath = os.path.join(log_dir, logFileName) # Construct txt file location in order to log # the external commands calls (cmake, make, git clone, etc...) txtFileName = prefix + hour_command_host + ".txt" - txtFilePath = os.path.join(config.USER.log_dir, "OUT", txtFileName) + txtFilePath = os.path.join(log_dir, "OUT", txtFileName) src.ensure_path_exists(os.path.dirname(logFilePath)) src.ensure_path_exists(os.path.dirname(txtFilePath)) @@ -230,7 +231,7 @@ class Logger(object): self.xmlFile.write_tree(stylesheet = "command.xsl") # Dump the config in a pyconf file in the log directory - logDir = self.config.USER.log_dir + logDir = src.get_log_path(self.config) dumpedPyconfFileName = (self.config.VARS.datehour + "_" + self.config.VARS.command -- 2.39.2