From f44ef88e646f05a4b911fb42f9af7986af64d3d4 Mon Sep 17 00:00:00 2001 From: Christian Van Wambeke Date: Tue, 17 Jul 2018 11:07:39 +0200 Subject: [PATCH] sat #12016 for sat prepare git clone only sub_dir a.gommlich@hzdr.de --- commands/source.py | 22 +++++++-- src/logger.py | 63 ++++++++++++++++++----- src/system.py | 121 +++++++++++++++++++++++++++++++-------------- src/utilsSat.py | 4 +- 4 files changed, 155 insertions(+), 55 deletions(-) diff --git a/commands/source.py b/commands/source.py index 9580ef2..83c037a 100644 --- a/commands/source.py +++ b/commands/source.py @@ -108,10 +108,24 @@ def get_source_from_git(product_info, False) logger.flush() logger.write('\n', 5, False) - # Call the system function that do the extraction in git mode - retcode = src.system.git_extract(repo_git, - product_info.git_info.tag, - source_dir, logger, environ) + + sub_dir = None + if not is_dev and "sub_dir" in product_info.git_info: + sub_dir = product_info.git_info.sub_dir + + if sub_dir is None: + # Call the system function that do the extraction in git mode + retcode = src.system.git_extract(repo_git, + product_info.git_info.tag, + source_dir, logger, environ) + else: + # Call the system function that do the extraction of a sub_dir in git mode + logger.write("sub_dir:%s " % sub_dir, 3) + retcode = src.system.git_extract_sub_dir(repo_git, + product_info.git_info.tag, + source_dir, sub_dir, logger, environ) + + return retcode def get_source_from_archive(product_info, source_dir, logger): diff --git a/src/logger.py b/src/logger.py index 4400540..94f81d6 100755 --- a/src/logger.py +++ b/src/logger.py @@ -201,20 +201,57 @@ class Logger(object): sys.stdout.write(message) self.flush() - def error(self, message): - """Print an error. - - :param message str: The message to print. - """ - # Print in the log file - self.xmlFile.append_node_text("traces", _('ERROR:') + message) + def error(self, message, prefix="ERROR: "): + """Print an error. + + :param message str: The message to print. + """ + # Print in the log file + self.xmlFile.append_node_text("traces", prefix + message) + + # Print in the terminal and clean colors if the terminal + # is redirected by user + if not ('isatty' in dir(sys.stderr) and sys.stderr.isatty()): + sys.stderr.write(printcolors.printcError(prefix + message + "\n")) + else: + sys.stderr.write(prefix + message + "\n") + + def step(self, message): + """Print an step message. + + :param message str: The message to print. + """ + self.write('STEP: ' + message, level=4) + + def trace(self, message): + """Print an trace message. + + :param message str: The message to print. + """ + self.write('TRACE: ' + message, level=5) + + def debug(self, message): + """Print an debug message. + + :param message str: The message to print. + """ + self.write('DEBUG: ' + message, level=6) + + def warning(self, message): + """Print an warning message. + + :param message str: The message to print. + """ + self.error(message, prefix="WARNING: ") + + def critical(self, message): + """Print an critical message. + + :param message str: The message to print. + """ + self.error(message, prefix="CRITICAL: ") + - # Print in the terminal and clean colors if the terminal - # is redirected by user - if not ('isatty' in dir(sys.stderr) and sys.stderr.isatty()): - sys.stderr.write(printcolors.printcError(_('ERROR:') + message)) - else: - sys.stderr.write(_('ERROR:') + message) def flush(self): """Flush terminal""" diff --git a/src/system.py b/src/system.py index 78e9946..4c2838a 100644 --- a/src/system.py +++ b/src/system.py @@ -25,6 +25,9 @@ import subprocess import os import tarfile +import debug as DBG +import utilsSat as UTS + from . import printcolors def show_in_editor(editor, filePath, logger): @@ -52,44 +55,90 @@ def show_in_editor(editor, filePath, logger): def git_extract(from_what, tag, where, logger, environment=None): - '''Extracts sources from a git repository. - - :param from_what str: The remote git repository. - :param tag str: The tag. - :param where str: The path where to extract. - :param logger Logger: The logger instance to use. - :param environment src.environment.Environ: The environment to source when - extracting. - :return: True if the extraction is successful - :rtype: boolean - ''' - if not where.exists(): - where.make() - if tag == "master" or tag == "HEAD": - command = "git clone %(remote)s %(where)s" % \ - { 'remote': from_what, 'tag': tag, 'where': str(where) } - else: - # NOTICE: this command only works with recent version of git - # because --work-tree does not work with an absolute path - where_git = os.path.join( str(where), ".git" ) - command = "rmdir %(where)s && git clone %(remote)s %(where)s && " + \ - "git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(tag)s" - command = command % {'remote': from_what, - 'tag': tag, - 'where': str(where), - 'where_git': where_git } + '''Extracts sources from a git repository. - logger.write(command + "\n", 5) + :param from_what str: The remote git repository. + :param tag str: The tag. + :param where str: The path where to extract. + :param logger Logger: The logger instance to use. + :param environment src.environment.Environ: The environment to source when extracting. + :return: True if the extraction is successful + :rtype: boolean + ''' + DBG.write("git_extract", [from_what, tag, str(where)]) + if not where.exists(): + where.make() + if tag == "master" or tag == "HEAD": + command = "git clone %(remote)s %(where)s" % \ + {'remote': from_what, 'tag': tag, 'where': str(where)} + else: + # NOTICE: this command only works with recent version of git + # because --work-tree does not work with an absolute path + where_git = os.path.join(str(where), ".git") + command = "rmdir %(where)s && git clone %(remote)s %(where)s && " + \ + "git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(tag)s" + command = command % {'remote': from_what, + 'tag': tag, + 'where': str(where), + 'where_git': where_git} + + logger.write(command + "\n", 5) + + logger.logTxtFile.write("\n" + command + "\n") + logger.logTxtFile.flush() + res = subprocess.call(command, + cwd=str(where.dir()), + env=environment.environ.environ, + shell=True, + stdout=logger.logTxtFile, + stderr=subprocess.STDOUT) + return (res == 0) + + +def git_extract_sub_dir(from_what, tag, where, sub_dir, logger, environment=None): + '''Extracts sources from a subtree sub_dir of a git repository. + + :param from_what str: The remote git repository. + :param tag str: The tag. + :param where str: The path where to extract. + :param sub_dir str: The relative path of subtree to extract. + :param logger Logger: The logger instance to use. + :param environment src.environment.Environ: The environment to source when extracting. + :return: True if the extraction is successful + :rtype: boolean + ''' + strWhere = str(where) + tmpWhere = strWhere + '_tmp' + parentWhere = os.path.dirname(strWhere) + if not os.path.exists(parentWhere): + logger.error("not existing directory: %s" % parentWhere) + return False + if os.path.isdir(strWhere): + logger.error("do not override existing directory: %s" % strWhere) + return False + aDict = {'remote': from_what, + 'tag': tag, + 'sub_dir': sub_dir, + 'where': strWhere, + 'parentWhere': parentWhere, + 'tmpWhere': tmpWhere, + } + DBG.write("git_extract_sub_dir", aDict) + + cmd = r""" +export tmpDir=%(tmpWhere)s && \ +rm -rf $tmpDir && \ +git clone %(remote)s $tmpDir && \ +cd $tmpDir && \ +git checkout %(tag)s && \ +mv %(sub_dir)s %(where)s && \ +git log -1 > %(where)s/README_git_log.txt && \ +rm -rf $tmpDir +""" % aDict + DBG.write("cmd", cmd) + rc = UTS.Popen(cmd, cwd=parentWhere, env=environment.environ.environ, logger=logger) + return rc.isOk() - logger.logTxtFile.write("\n" + command + "\n") - logger.logTxtFile.flush() - res = subprocess.call(command, - cwd=str(where.dir()), - env=environment.environ.environ, - shell=True, - stdout=logger.logTxtFile, - stderr=subprocess.STDOUT) - return (res == 0) def archive_extract(from_what, where, logger): '''Extracts sources from an archive. diff --git a/src/utilsSat.py b/src/utilsSat.py index e60eec9..ea42232 100644 --- a/src/utilsSat.py +++ b/src/utilsSat.py @@ -59,12 +59,12 @@ def Popen(command, shell=True, cwd=None, env=None, stdout=SP.PIPE, stderr=SP.PIP if rc == 0: if logger is not None: - logger.trace(" launch command rc=%s cwd=%s:\n%s" % (rc, cwd, command)) + logger.trace(" launch command rc=%s cwd=%s:\n%s" % (rc, cwd, command)) logger.trace(" result command stdout&stderr:\n%s" % res_out) return RCO.ReturnCode("OK", "Popen command done", value=res_out) else: if logger is not None: - logger.warning(" launch command rc=%s cwd=%s:\n%s" % (rc, cwd, command)) + logger.warning(" launch command rc=%s cwd=%s:\n%s" % (rc, cwd, command)) logger.warning(" result command stdout&stderr:\n%s" % res_out) return RCO.ReturnCode("KO", "Popen command problem", value=res_out) else: #except Exception as e: -- 2.39.2