From ca9eed199ad15ac77ee7c9a62d0d7d5891df42dd Mon Sep 17 00:00:00 2001 From: Serge Rehbinder Date: Wed, 24 Feb 2016 14:41:59 +0100 Subject: [PATCH] Rename software into modules. Add first version of source command, it handle git modules --- commands/config.py | 18 +-- commands/source.py | 231 ++++++++++++++++++++++++++++++ data/softwares/softA.pyconf | 36 ----- data/softwares/softB.pyconf | 36 ----- src/__init__.py | 128 ++++++++++++++++- src/module.py | 57 ++++++++ src/system.py | 35 ++++- test/config/create_user_pyconf.py | 8 +- 8 files changed, 462 insertions(+), 87 deletions(-) create mode 100644 commands/source.py delete mode 100644 data/softwares/softA.pyconf delete mode 100644 data/softwares/softB.pyconf create mode 100644 src/module.py diff --git a/commands/config.py b/commands/config.py index a53dd18..3008884 100644 --- a/commands/config.py +++ b/commands/config.py @@ -276,19 +276,19 @@ class ConfigManager: exec('cfg.' + rule) # ===================================================================== - # Load softwares config files in SOFTWARE section + # Load modules config files in MODULES section # The directory containing the softwares definition - softsDir = os.path.join(cfg.VARS.dataDir, 'softwares') + modules_dir = os.path.join(cfg.VARS.dataDir, 'modules') - # Loop on all files that are in softsDir directory + # Loop on all files that are in softsDir directory # and read their config - for fName in os.listdir(softsDir): + for fName in os.listdir(modules_dir): if fName.endswith(".pyconf"): - src.pyconf.streamOpener = ConfigOpener([softsDir]) + src.pyconf.streamOpener = ConfigOpener([modules_dir]) try: - soft_cfg = src.pyconf.Config(open( - os.path.join(softsDir, fName))) + mod_cfg = src.pyconf.Config(open( + os.path.join(modules_dir, fName))) except src.pyconf.ConfigError as e: raise src.SatException(_( "Error in configuration file: %(soft)s\n %(error)s") % \ @@ -297,10 +297,10 @@ class ConfigManager: e = str(error) raise src.SatException( e ); - merger.merge(cfg, soft_cfg) + merger.merge(cfg, mod_cfg) # apply overwrite from command line if needed - for rule in self.get_command_line_overrides(options, ["SOFTWARE"]): + for rule in self.get_command_line_overrides(options, ["MODULES"]): exec('cfg.' + rule) # this cannot be factorized because of the exec diff --git a/commands/source.py b/commands/source.py new file mode 100644 index 0000000..c3e4dbf --- /dev/null +++ b/commands/source.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- +# Copyright (C) 2010-2012 CEA/DEN +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os +import shutil + +import src + +# Define all possible option for log command : sat log +parser = src.options.Options() +parser.add_option('m', 'module', 'list2', 'modules', + _('modules to get the sources. This option can be' + ' passed several time to get the sources of several modules.')) +parser.add_option('', 'no_sample', 'boolean', 'no_sample', + _("do not prepare sample modules.")) + +def prepare_for_dev(config, module_info, source_dir, logger, pad): + + retcode = 'N\A' + # if module sources dir does not exists in dev, + # get it in checkout mode + if not os.path.exists(module_info.source_dir): + retcode = get_module_sources(config, module_info, True, source_dir, logger, pad, checkout=True) + logger.write("\n", 3, False) + logger.write(" " * (pad+2), 3, False) # +2 because module name is followed by ': ' + + logger.write('dev: %s ... ' % src.printcolors.printcInfo(module_info.source_dir), 3, False) + logger.flush() + + return retcode + +def prepare_from_git(module_info, source_dir, logger, pad, is_dev=False): + '''Prepares a module from git + ''' + coflag = 'git' + + if is_dev and 'repo_dev' in module_info.git_info: + coflag = src.printcolors.printcHighlight(coflag.upper()) + repo_git = module_info.git_info.repo_dev + else: + repo_git = module_info.git_info.repo + + + logger.write('%s:%s' % (coflag, src.printcolors.printcInfo(repo_git)), 3, False) + logger.write(' ' * (pad + 50 - len(repo_git)), 3, False) + logger.write(' tag:%s' % src.printcolors.printcInfo(module_info.git_info.tag), 3, False) + logger.write(' %s. ' % ('.' * (10 - len(module_info.git_info.tag))), 3, False) + logger.flush() + logger.write('\n', 5, False) + retcode = src.system.git_extract(repo_git, + module_info.git_info.tag, + source_dir, logger) + return retcode + +def prepare_from_archive(module_info, source_dir, logger): + # check archive exists + if not os.path.exists(module_info.archive_info.archive_name): + raise src.SatException(_("Archive not found: '%s'") % module_info.archive_info.archive_name) + + logger.write('arc:%s ... ' % src.printcolors.printcInfo(module_info.archive_info.archive_name), 3, False) + logger.flush() + retcode, NameExtractedDirectory = src.system.archive_extract(module_info.archive_info.archive_name, + source_dir.dir(), logger) + + # Rename the source directory if it does not match with module_info.source_dir + if NameExtractedDirectory.replace('/', '') != os.path.basename(module_info.source_dir): + shutil.move(os.path.join(os.path.dirname(module_info.source_dir), NameExtractedDirectory), module_info.source_dir) + + return retcode + +def get_module_sources(config, module_info, is_dev, source_dir, logger, pad, checkout=False): + '''Get the module sources. + + ''' + if not checkout and is_dev: + return prepare_for_dev(config, module_info, source_dir, logger, pad) + + if module_info.get_method == "git": + return prepare_from_git(module_info, source_dir, logger, pad, is_dev) + + if module_info.get_method == "archive": + return prepare_from_archive(module_info, source_dir, logger) + ''' + if module_info.get_method == "cvs": + cvs_user = common.get_cfg_param(module_info.cvs_info, "cvs_user", config.USER.cvs_user) + return prepare_from_cvs(cvs_user, module_info, source_dir, checkout, logger, pad) + + if module_info.get_method == "svn": + svn_user = common.get_cfg_param(module_info.svn_info, "svn_user", config.USER.svn_user) + return prepare_from_svn(svn_user, module_info, source_dir, checkout, logger) + + if module_info.get_method == "dir": + return prepare_from_dir(module_info, source_dir, logger) + ''' + + if len(module_info.get_method) == 0: + # skip + logger.write('%s ...' % _("ignored"), 3, False) + return True + + # + logger.write(_("Unknown get_mehtod %(get)s for module %(module)s") % \ + { 'get': module_info.get_method, 'module': module_info.name }, 3, False) + logger.write(" ... ", 3, False) + logger.flush() + return False + +def get_all_module_sources(config, modules, logger): + '''Get all the module sources. + + ''' + + results = dict() + good_result = 0 + + max_module_name_len = 1 + if len(modules) > 0: + max_module_name_len = max(map(lambda l: len(l), modules[0])) + 4 + for module in modules: + module_name = module[0] + module_info = module[1] + source_dir = src.Path(module_info.source_dir) + + logger.write('%s: ' % src.printcolors.printcLabel(module_name), 3) + logger.write(' ' * (max_module_name_len - len(module_name)), 3, False) + logger.write("\n", 4, False) + + is_dev = "dev_modules" in config.APPLICATION and module_name in config.APPLICATION.dev_modules + if source_dir.exists() and not is_dev: + logger.write(" " + _('remove %s') % source_dir, 4) + logger.write("\n ", 4, False) + source_dir.rm() + + retcode = get_module_sources(config, module_info, is_dev, source_dir, logger, max_module_name_len, checkout=False) + + ''' + if 'no_rpath' in module_info.keys(): + if module_info.no_rpath: + hack_no_rpath(config, module_info, logger) + ''' + + # show results + results[module_name] = retcode + if retcode == 'N\A': + res =src.printcolors.printc(src.OK_STATUS) + src.printcolors.printcWarning(_(' source directory already exists')) + good_result = good_result + 1 + elif retcode: + res = src.OK_STATUS + good_result = good_result + 1 + else: + res = src.KO_STATUS + + logger.write('%s\n' % src.printcolors.printc(res), 3, False) + + return good_result, results + +def run(args, runner, logger): + '''method that is called when salomeTools is called with source parameter. + ''' + # Parse the options + (options, args) = parser.parse_args(args) + + # check that the command has been called with an application + src.check_config_has_application( runner.cfg ) + + logger.write(_('Preparing sources of product %s\n') % + src.printcolors.printcLabel(runner.cfg.VARS.application), 1) + src.printcolors.print_value(logger, 'out_dir', + runner.cfg.APPLICATION.out_dir, 2) + logger.write("\n", 2, False) + + if options.modules is None: + modules = runner.cfg.APPLICATION.modules + else: + modules = options.modules + for m in modules: + if m not in runner.cfg.APPLICATION.modules: + raise src.SatException(_("Module %(module)s not defined in product %(product)s") % + { 'module': m, 'product': runner.cfg.VARS.product} ) + + modules_infos = src.module.get_modules_infos(modules, runner.cfg) + + + if options.no_sample: + modules_infos = filter(lambda l: not src.module.module_is_sample(l[1]), + modules_infos) + + good_result, results = get_all_module_sources(runner.cfg, modules_infos, logger) + + status = src.OK_STATUS + details = [] + + logger.write("\n", 2, False) + if good_result == len(modules): + res_count = "%d / %d" % (good_result, good_result) + else: + status = src.KO_STATUS + res_count = "%d / %d" % (good_result, len(modules)) + + for module in results: + if results[module] == 0 or results[module] is None: + details.append(module) + + result = len(modules) - good_result + + # write results + logger.write(_("Preparing of product's sources:"), 1) + logger.write(" " + src.printcolors.printc(status), 1, False) + logger.write(" (%s)\n" % res_count, 1, False) + + if len(details) > 0: + logger.write(_("Following sources haven't been prepared:\n"), 2) + logger.write(" ".join(details), 2) + logger.write("\n", 2, False) + + return result diff --git a/data/softwares/softA.pyconf b/data/softwares/softA.pyconf deleted file mode 100644 index 0a1f036..0000000 --- a/data/softwares/softA.pyconf +++ /dev/null @@ -1,36 +0,0 @@ -SOFTWARE : -{ - softA : - { - name : "softA" - has_gui : "no" - compile_method : "cmake" # ou autotools, ou script - get_method : "git" # "archive", embedded", "native" "fixed" - cvs_info: - { - server : $SITE.prepare.default_cvs_server - module_base : $SITE.prepare.cvs_dir + $name - source : 'softA_SRC' - tag : '' - } - git_info: - { - repo : $SITE.prepare.default_git_server + $VARS.sep + $name - repo_dev : $SITE.prepare.default_git_server_dev + $VARS.sep + $name - tag : $APPLICATION.default_version_to_download - } - archive_info: - { - archive_name : $SITE.prepare.archive_dir + $VARS.sep + $name + '.tar.gz' - } - environ : - { - "_LD_LIBRARY_PATH" : "${SOFT_ROOT_DIR}" + $VARS.sep + "lib" - "_PYTHONPATH" : ["${SOFT_ROOT_DIR}" + $VARS.sep + "lib" - "${SOFT_ROOT_DIR}" + $VARS.sep + "${PYTHON_LIBDIR0}" - "${SOFT_ROOT_DIR}" + $VARS.sep + "${PYTHON_LIBDIR1}"] - } - depend : [] - opt_depend : [] - } -} \ No newline at end of file diff --git a/data/softwares/softB.pyconf b/data/softwares/softB.pyconf deleted file mode 100644 index 3a34586..0000000 --- a/data/softwares/softB.pyconf +++ /dev/null @@ -1,36 +0,0 @@ -SOFTWARE : -{ - softB : - { - name : "softB" - has_gui : "no" - compile_method : "cmake" # ou autotools, ou script - get_method : "git" # "archive", embedded", "native" "fixed" - cvs_info: - { - server : $SITE.prepare.default_cvs_server - module_base : $SITE.prepare.cvs_dir + $name - source : 'softB_SRC' - tag : '' - } - git_info: - { - repo : $SITE.prepare.default_git_server + $VARS.sep + $name - repo_dev : $SITE.prepare.default_git_server_dev + $VARS.sep + $name - tag : $APPLICATION.default_version_to_download - } - archive_info: - { - archive_name : $SITE.prepare.archive_dir + $VARS.sep + $name + '.tar.gz' - } - environ : - { - "_LD_LIBRARY_PATH" : "${SOFT_ROOT_DIR}" + $VARS.sep + "lib" - "_PYTHONPATH" : ["${SOFT_ROOT_DIR}" + $VARS.sep + "lib" - "${SOFT_ROOT_DIR}" + $VARS.sep + "${PYTHON_LIBDIR0}" - "${SOFT_ROOT_DIR}" + $VARS.sep + "${PYTHON_LIBDIR1}"] - } - depend : ['softA'] - opt_depend : [] - } -} \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py index 242c570..5c4980e 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -17,6 +17,9 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import os +import shutil +import errno +import stat from . import pyconf from . import architecture @@ -25,6 +28,11 @@ from . import options from . import system from . import ElementTree from . import logger +from . import module + +OK_STATUS = "OK" +KO_STATUS = "KO" +NA_STATUS = "NA" class SatException(Exception): '''rename Exception Class @@ -66,4 +74,122 @@ def print_info(logger, info): for i in info: sp = " " * (smax - len(i[0])) printcolors.print_value(logger, sp + i[0], i[1], 2) - logger.write("\n", 2) \ No newline at end of file + logger.write("\n", 2) + +## +# Utils class to simplify path manipulations. +class Path: + def __init__(self, path): + self.path = str(path) + + def __add__(self, other): + return Path(os.path.join(self.path, str(other))) + + def __abs__(self): + return Path(os.path.abspath(self.path)) + + def __str__(self): + return self.path + + def __eq__(self, other): + return self.path == other.path + + def exists(self): + return self.islink() or os.path.exists(self.path) + + def islink(self): + return os.path.islink(self.path) + + def isdir(self): + return os.path.isdir(self.path) + + def list(self): + return [Path(p) for p in os.listdir(self.path)] + + def dir(self): + return Path(os.path.dirname(self.path)) + + def base(self): + return Path(os.path.basename(self.path)) + + def make(self, mode=None): + os.makedirs(self.path) + if mode: + os.chmod(self.path, mode) + + def chmod(self, mode): + os.chmod(self.path, mode) + + def rm(self): + if self.islink(): + os.remove(self.path) + else: + shutil.rmtree( self.path, onerror = handleRemoveReadonly ) + + def copy(self, path, smart=False): + if not isinstance(path, Path): + path = Path(path) + + if os.path.islink(self.path): + return self.copylink(path) + elif os.path.isdir(self.path): + return self.copydir(path, smart) + else: + return self.copyfile(path) + + def smartcopy(self, path): + return self.copy(path, True) + + def readlink(self): + if self.islink(): + return os.readlink(self.path) + else: + return False + + def symlink(self, path): + try: + os.symlink(str(path), self.path) + return True + except: + return False + + def copylink(self, path): + try: + os.symlink(os.readlink(self.path), str(path)) + return True + except: + return False + + def copydir(self, dst, smart=False): + try: + names = self.list() + + if not dst.exists(): + dst.make() + + for name in names: + if name == dst: + continue + if smart and (str(name) in [".git", "CVS", ".svn"]): + continue + srcname = self + name + dstname = dst + name + srcname.copy(dstname, smart) + return True + except: + return False + + def copyfile(self, path): + try: + shutil.copy2(self.path, str(path)) + return True + except: + return False + +def handleRemoveReadonly(func, path, exc): + excvalue = exc[1] + if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES: + os.chmod(path, stat.S_IRWXU| stat.S_IRWXG| stat.S_IRWXO) # 0777 + func(path) + else: + raise \ No newline at end of file diff --git a/src/module.py b/src/module.py new file mode 100644 index 0000000..da07bdd --- /dev/null +++ b/src/module.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- +# Copyright (C) 2010-2012 CEA/DEN +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import src + +def get_module_config(config, module_name, version): + vv = version + for c in ".-": vv = vv.replace(c, "_") # substitute some character with _ + full_module_name = module_name + '_' + vv + + mod_info = None + if full_module_name in config.MODULES: + # returns specific information for the given version + mod_info = config.MODULES[full_module_name] + + elif module_name in config.MODULES: + # returns the generic information (given version not found) + mod_info = config.MODULES[module_name] + + # merge opt_depend in depend + if mod_info is not None and 'opt_depend' in mod_info: + for depend in mod_info.opt_depend: + if depend in config.MODULES: + mod_info.depend.append(depend,'') + return mod_info + +def get_modules_infos(lmodules, config): + modules_infos = [] + for mod in lmodules: + version_mod = config.APPLICATION.modules[mod][0] + mod_info = get_module_config(config, mod, version_mod) + if mod_info is not None: + modules_infos.append((mod, mod_info)) + else: + msg = _("The %s module has no definition in the configuration.") % mod + raise src.SatException(msg) + return modules_infos + + +def module_is_sample(module_info): + mtype = module_info.module_type + return mtype.lower() == 'sample' \ No newline at end of file diff --git a/src/system.py b/src/system.py index 0107cdb..24c0f69 100644 --- a/src/system.py +++ b/src/system.py @@ -22,6 +22,8 @@ like open a browser or an editor, or call a git command ''' import subprocess +import os +import tarfile from . import printcolors @@ -47,4 +49,35 @@ def show_in_editor(editor, filePath, logger): except: logger.write(printcolors.printcError(_("Unable to edit file %s\n") % filePath), 1) - \ No newline at end of file + +## +# Extracts sources from a git repository. +def git_extract(from_what, tag, where, logger): + if not where.exists(): + where.make() + if tag == "master" or tag == "HEAD": + command = "git clone %(remote)s %(where)s" % \ + { 'remote': from_what, 'tag': tag, 'where': str(where) } + else: + # NOTICE: this command only works with recent version of git + # because --work-tree does not work with an absolute path + where_git = os.path.join( str(where), ".git" ) + command = "rmdir %(where)s && git clone %(remote)s %(where)s && " + \ + "git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(tag)s" + command = command % { 'remote': from_what, 'tag': tag, 'where': str(where), 'where_git': where_git } + + logger.write(command + "\n", 5) + + res = subprocess.call(command, cwd=str(where.dir()), shell=True, + stdout=logger.logTxtFile, stderr=subprocess.STDOUT) + return (res == 0) + +def archive_extract(from_what, where, logger): + try: + archive = tarfile.open(from_what) + for i in archive.getmembers(): + archive.extract(i, path=str(where)) + return True, os.path.commonprefix(archive.getnames()) + except Exception as exc: + logger.write("archive_extract: %s\n" % exc) + return False, None \ No newline at end of file diff --git a/test/config/create_user_pyconf.py b/test/config/create_user_pyconf.py index 0bc6e96..f59f450 100644 --- a/test/config/create_user_pyconf.py +++ b/test/config/create_user_pyconf.py @@ -119,16 +119,16 @@ class TestConfig(unittest.TestCase): # pyunit method to compare 2 str self.assertEqual(OK, "OK") - def test_override_SOFTWARE(self): - '''override SOFTWARE + def test_override_MODULES(self): + '''override MODULES ''' OK = "KO" # The command to test - sat = Sat("-oSOFTWARE.softA.compile_method='test'") + sat = Sat("-oMODULES.softA.compile_method='test'") sat.config('') - if sat.cfg.SOFTWARE.softA.compile_method == 'test': + if sat.cfg.MODULES.softA.compile_method == 'test': OK = "OK" # pyunit method to compare 2 str -- 2.39.2