From ecd252023f5d4fedb34223b979d0fd462ab1e9ca Mon Sep 17 00:00:00 2001 From: Nabil Ghodbane Date: Thu, 26 Sep 2024 20:06:09 +0200 Subject: [PATCH] spns #43062: add new git_commands key --- commands/source.py | 7 +- sat | 3 + src/system.py | 208 +++++++++++++++++++++++++++++++-------------- 3 files changed, 150 insertions(+), 68 deletions(-) diff --git a/commands/source.py b/commands/source.py index 79c11df..fc5166a 100644 --- a/commands/source.py +++ b/commands/source.py @@ -140,6 +140,9 @@ def get_source_from_git(config, git_options= '' if "git_options" in product_info.git_info: git_options = product_info.git_info.git_options + git_commands= [] + if "git_commands" in product_info.git_info: + git_commands = product_info.git_info.git_commands sub_dir = None # what do we do with git tree structure and history @@ -153,13 +156,13 @@ def get_source_from_git(config, if sub_dir is None: # Call the system function that do the extraction in git mode retcode = src.system.git_extract(repo_git, - product_info.git_info.tag, git_options, + product_info.git_info.tag, git_options, git_commands, source_dir, logger, environ) else: # Call the system function that do the extraction of a sub_dir in git mode logger.write("sub_dir:%s " % sub_dir, 3) retcode = src.system.git_extract_sub_dir(repo_git, - product_info.git_info.tag,git_options, + product_info.git_info.tag,git_options, git_commands, source_dir, sub_dir, logger, environ) diff --git a/sat b/sat index 04bdd16..9a05f38 100755 --- a/sat +++ b/sat @@ -26,6 +26,9 @@ import os import sys import platform +if sys.version_info[:2] >= (3,12): + import warnings + warnings.filterwarnings('ignore') # exit OKSYS and KOSYS seems equal on linux or windows OKSYS = 0 # OK KOSYS = 1 # KO diff --git a/src/system.py b/src/system.py index 7b8d149..ade5368 100644 --- a/src/system.py +++ b/src/system.py @@ -118,12 +118,13 @@ def git_describe(repo_path): return tag_description -def git_extract(from_what, tag, git_options, where, logger, environment=None): +def git_extract(from_what, tag, git_options, git_commands, where, logger, environment=None): '''Extracts sources from a git repository. 87 :param from_what str: The remote git repository. :param tag str: The tag. :param git_options str: git options + :param git_commands array: git command lines :param where str: The path where to extract. :param logger Logger: The logger instance to use. :param environment src.environment.Environ: The environment to source when extracting. @@ -137,47 +138,88 @@ def git_extract(from_what, tag, git_options, where, logger, environment=None): if tag == "master" or tag == "HEAD": if src.architecture.is_windows(): cmd = "git clone %(git_options)s %(remote)s %(where)s" + cmd+= "\n" + "if NOT %ERRORLEVEL% == 0 (" + cmd+= "\n" + " exit 1" + cmd+= "\n" + ")" + if len(git_commands) > 0: + cmd+= "\n" + "cd %(where)s" + for git_command in git_commands: + cmd+= "\n" + git_command + cmd+= "\n" + "if NOT %ERRORLEVEL% == 0 (" + cmd+= "\n" + " exit 1" + cmd+= "\n" + ")" else: - cmd = r""" -set -x -git clone %(git_options)s %(remote)s %(where)s -res=$? -if [ $res -eq 0 ]; then - touch -d "$(git --git-dir=%(where_git)s log -1 --format=date_format)" %(where)s -fi -exit $res -""" - cmd = cmd % {'git_options': git_options, 'remote': from_what, 'tag': tag, 'where': str(where), 'where_git': where_git} + cmd = "\n" + "set -x" + cmd+= "\n" + "git clone %(git_options)s %(remote)s %(where)s" + cmd+= "\n" + "if [ $? -ne 0 ]; then" + cmd+= "\n" + " exit 1" + cmd+= "\n" + "fi" + cmd+= "\n" + "touch -d \"$(git --git-dir=%(where_git)s log -1 --format=date_format)\" %(where)s" + cmd+= "\n" + "if [ $? -ne 0 ]; then" + cmd+= "\n" + " exit 1" + cmd+= "\n" + "fi" + if len(git_commands) > 0: + cmd+= "\n" + "cd %(where)s" + for git_command in git_commands: + cmd+= "\n" + git_command + cmd+= "\n" + "if [ $? -ne 0 ]; then" + cmd+= "\n" + " exit 1" + cmd+= "\n" + "fi" else: - # NOTICE: this command only works with recent version of git - # because --work-tree does not work with an absolute path if src.architecture.is_windows(): - cmd = "rmdir /S /Q %(where)s && git clone %(git_options)s %(remote)s %(where)s && git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(tag)s" + cmd = "rmdir /S /Q %(where)s" + cmd+= "\n" + "git clone %(git_options)s %(remote)s %(where)s" + cmd+= "\n" + "if NOT %ERRORLEVEL% == 0 (" + cmd+= "\n" + " exit 1" + cmd+= "\n" + ")" + cmd+= "\n" + "git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(tag)s" + cmd+= "\n" + "if NOT %ERRORLEVEL% == 0 (" + cmd+= "\n" + " exit 1" + cmd+= "\n" + ")" + if len(git_commands) > 0: + cmd+= "\n" + "cd %(where)s" + for git_command in git_commands: + cmd+= "\n" + git_command + cmd+= "\n" + "if NOT %ERRORLEVEL% == 0 (" + cmd+= "\n" + " exit 1" + cmd+= "\n" + ")" else: -# for sat compile --update : changes the date of directory, only for branches, not tag - cmd = r""" -set -x -rm -rf %(where)s -git clone %(git_options)s %(remote)s %(where)s && \ -git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(tag)s -res=$? -git --git-dir=%(where_git)s status | grep HEAD -if [ $res -eq 0 -a $? -ne 0 ]; then - touch -d "$(git --git-dir=%(where_git)s log -1 --format=date_format)" %(where)s -fi -exit $res -""" - cmd = cmd % {'git_options': git_options, - 'remote': from_what, - 'tag': tag, - 'where': str(where), - 'where_git': where_git} - - + cmd = "\n" + "set -x" + cmd+= "\n" + "git clone %(git_options)s %(remote)s %(where)s" + cmd+= "\n" + "if [ $? -ne 0 ]; then" + cmd+= "\n" + " exit 1" + cmd+= "\n" + "fi" + cmd+= "\n" + "git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(tag)s" + cmd+= "\n" + "if [ $? -ne 0 ]; then" + cmd+= "\n" + " exit 1" + cmd+= "\n" + "fi" + if len(git_commands) > 0: + cmd+= "\n" + "cd %(where)s" + for git_command in git_commands: + cmd+= "\n" + git_command + cmd+= "\n" + "if [ $? -ne 0 ]; then" + cmd+= "\n" + " exit 1" + cmd+= "\n" + "fi" + cmd+= "\n" + "git --git-dir=%(where_git)s status | grep HEAD" + cmd+= "\n" + "if [ $? -ne 0 ]; then" + cmd+= "\n" + " exit 1" + cmd+= "\n" + "fi" + cmd+= "\n" + "touch -d \"$(git --git-dir=%(where_git)s log -1 --format=date_format)\" %(where)s" + cmd+= "\n" + "if [ $? -ne 0 ]; then" + cmd+= "\n" + " exit 1" + cmd+= "\n" + "fi" + cmd+= "\n" + "exit 0" + aDict = {'%(git_options)s': git_options, + '%(remote)s' : from_what, + '%(where)s' : str(where), + '%(tag)s' : tag, + '%(where_git)s': where_git + } + for k, v in aDict.items(): + cmd= cmd.replace(k,v) cmd=cmd.replace('date_format', '"%ai"') logger.logTxtFile.write("\n" + cmd + "\n") logger.logTxtFile.flush() - DBG.write("cmd", cmd) # git commands may fail sometimes for various raisons # (big module, network troubles, tuleap maintenance) @@ -197,12 +239,13 @@ exit $res return rc.isOk() -def git_extract_sub_dir(from_what, tag, git_options, where, sub_dir, logger, environment=None): +def git_extract_sub_dir(from_what, tag, git_options, git_commands, where, sub_dir, logger, environment=None): '''Extracts sources from a subtree sub_dir of a git repository. :param from_what str: The remote git repository. :param tag str: The tag. :param git_options str: git options + :param git_commands array: git command lines :param where str: The path where to extract. :param sub_dir str: The relative path of subtree to extract. :param logger Logger: The logger instance to use. @@ -219,40 +262,73 @@ def git_extract_sub_dir(from_what, tag, git_options, where, sub_dir, logger, env if os.path.isdir(strWhere): logger.error("do not override existing directory: %s" % strWhere) return False - aDict = {'git_options': git_options, - 'remote': from_what, - 'tag': tag, - 'sub_dir': sub_dir, - 'where': strWhere, - 'parentWhere': parentWhere, - 'tmpWhere': tmpWhere, + aDict = {'%(git_options)s': git_options, + '%(remote)s' : from_what, + '%(tag)s' : tag, + '%(sub_dir)s' : sub_dir, + '%(where)s' : strWhere, + '%(parentWhere)s': parentWhere, + '%(tmpWhere)s' : tmpWhere } DBG.write("git_extract_sub_dir", aDict) if not src.architecture.is_windows(): - cmd = r""" -set -x -export tmpDir=%(tmpWhere)s && \ -rm -rf $tmpDir -git clone %(git_options)s %(remote)s $tmpDir && \ -cd $tmpDir && \ -git checkout %(tag)s && \ -mv %(sub_dir)s %(where)s && \ -git log -1 > %(where)s/README_git_log.txt && \ -rm -rf $tmpDir -""" % aDict + cmd = "\n" + "set -x" + cmd+= "\n" + "export tmpDir=%(tmpWhere)s" + cmd+= "\n" + "rm -rf $tmpDir" + cmd+= "\n" + "git clone %(git_options)s %(remote)s $tmpDir" + cmd+= "\n" + "if [ $? -ne 0 ]; then" + cmd+= "\n" + " exit 1" + cmd+= "\n" + "fi" + cmd+= "\n" + "cd $tmpDir" + cmd+= "\n" + "git checkout %(tag)s" + cmd+= "\n" + "if [ $? -ne 0 ]; then" + cmd+= "\n" + " exit 1" + cmd+= "\n" + "fi" + if len(git_commands) > 0: + for git_command in git_commands: + cmd+= "\n" + git_command + cmd+= "\n" + "if [ $? -ne 0 ]; then" + cmd+= "\n" + " exit 1" + cmd+= "\n" + "fi" + cmd+= "\n" + "mv %(sub_dir)s %(where)s" + cmd+= "\n" + "git log -1 > %(where)s/README_git_log.txt" + cmd+= "\n" + "if [ $? -ne 0 ]; then" + cmd+= "\n" + " exit 1" + cmd+= "\n" + "fi" + cmd+= "\n" + "rm -rf ${tmpDir}" else: - cmd = r""" - -set tmpDir=%(tmpWhere)s && \ -rmdir /S /Q %tmpDir% -git clone %(git_options)s %(remote)s %tmpDir% && \ -cd %tmpDir% && \ -git checkout %(tag)s && \ -mv %(sub_dir)s %(where)s && \ -git log -1 > %(where)s\\README_git_log.txt && \ -rmdir /S /Q %tmpDir% -""" % aDict - + cmd+= "\n" + "set tmpDir=%(tmpWhere)s" + cmd+= "\n" + "rmdir /S /Q %tmpDir%" + cmd+= "\n" + "git clone %(git_options)s %(remote)s %tmpDir%" + cmd+= "\n" + "if NOT %ERRORLEVEL% == 0 (" + cmd+= "\n" + " exit 1" + cmd+= "\n" + ")" + cmd+= "\n" + "cd %tmpDir%" + cmd+= "\n" + "git checkout %(tag)s" + cmd+= "\n" + "if NOT %ERRORLEVEL% == 0 (" + cmd+= "\n" + " exit 1" + cmd+= "\n" + ")" + if len(git_commands) > 0: + for git_command in git_commands: + cmd+= "\n" + git_command + cmd+= "\n" + "if NOT %ERRORLEVEL% == 0 (" + cmd+= "\n" + " exit 1" + cmd+= "\n" + ")" + cmd+= "\n" + "mv %(sub_dir)s %(where)s" + cmd+= "\n" + "if NOT %ERRORLEVEL% == 0 (" + cmd+= "\n" + " exit 1" + cmd+= "\n" + ")" + cmd+= "\n" + "git log -1 > %(where)s\\README_git_log.txt" + cmd+= "\n" + "if NOT %ERRORLEVEL% == 0 (" + cmd+= "\n" + " exit 1" + cmd+= "\n" + ")" + cmd+= "\n" + "rmdir /S /Q %tmpDir%" + cmd+= "\n" + "if NOT %ERRORLEVEL% == 0 (" + cmd+= "\n" + " exit 1" + cmd+= "\n" + ")" + + for k, v in aDict.items(): + cmd= cmd.replace(k,v) DBG.write("cmd", cmd) for nbtry in range(0,3): # retries case of network problem -- 2.39.2