From abdba0a000f0e0436aa604ecae09224b4a63d178 Mon Sep 17 00:00:00 2001 From: Christian Van Wambeke Date: Fri, 10 Dec 2021 14:41:46 +0100 Subject: [PATCH] add sat log on http.server 8765 --- commands/log.py | 2 +- commands/source.py | 33 ++++------- src/ElementTree.py | 4 +- src/debug.py | 17 +++--- src/system.py | 104 +++++++++++++++++++++++---------- src/xmlManager.py | 7 ++- src/xsl/command.xsl | 4 +- src/xsl/hat.xsl | 15 +++-- src/xsl/jobs_board_report.xsl | 2 +- src/xsl/jobs_global_report.xsl | 2 +- src/xsl/test.xsl | 2 +- src/xsl/test_history.xsl | 2 +- 12 files changed, 118 insertions(+), 76 deletions(-) diff --git a/commands/log.py b/commands/log.py index e2c1902..22a7885 100644 --- a/commands/log.py +++ b/commands/log.py @@ -347,7 +347,7 @@ def run(args, runner, logger): # open the hat xml in the user editor if not options.no_browser: logger.write(_("\nOpening the hat log file %s\n" % xmlHatFilePath), 3) - src.system.show_in_editor(runner.cfg.USER.browser, xmlHatFilePath, logger) + src.system.show_in_webbrowser(runner.cfg.USER.browser, xmlHatFilePath, logger) else: logger.write("\nHat log File is %s\n" % xmlHatFilePath, 3) return 0 diff --git a/commands/source.py b/commands/source.py index 2b078cd..0740ba1 100644 --- a/commands/source.py +++ b/commands/source.py @@ -120,8 +120,8 @@ def get_source_from_git(config, git_options= '' if "git_options" in product_info.git_info: git_options = product_info.git_info.git_options - sub_dir = None + sub_dir = None # what do we do with git tree structure and history if is_dev and "sub_dir" in product_info.git_info: logger.error("dev mode for product is incompatible with 'sub_dir' option") @@ -130,10 +130,10 @@ def get_source_from_git(config, if not is_dev and "sub_dir" in product_info.git_info: sub_dir = product_info.git_info.sub_dir - if sub_dir is None: + if sub_dir is None: # Call the system function that do the extraction in git mode retcode = src.system.git_extract(repo_git, - product_info.git_info.tag,git_options, + product_info.git_info.tag, git_options, source_dir, logger, environ) else: # Call the system function that do the extraction of a sub_dir in git mode @@ -376,7 +376,7 @@ def get_product_sources(config, if product_info.get_source == "git": return get_source_from_git(config, product_info, source_dir, logger, pad, - is_dev,env_appli) + is_dev, env_appli) if product_info.get_source == "archive": return get_source_from_archive(config, product_info, source_dir, logger) @@ -386,20 +386,13 @@ def get_product_sources(config, if product_info.get_source == "cvs": cvs_user = config.USER.cvs_user - return get_source_from_cvs(cvs_user, - product_info, - source_dir, - checkout, - logger, - pad, - env_appli) + return get_source_from_cvs(cvs_user, product_info, source_dir, + checkout, logger, pad, env_appli) if product_info.get_source == "svn": svn_user = config.USER.svn_user return get_source_from_svn(svn_user, product_info, source_dir, - checkout, - logger, - env_appli) + checkout, logger, env_appli) if product_info.get_source == "native": # for native products we check the corresponding system packages are installed @@ -487,12 +480,8 @@ def get_all_product_sources(config, products, logger): continue # Call to the function that get the sources for one product - retcode = get_product_sources(config, - product_info, - is_dev, - source_dir, - logger, - max_product_name_len, + retcode = get_product_sources(config, product_info, is_dev, + source_dir, logger, max_product_name_len, checkout=False) ''' @@ -583,9 +572,7 @@ def run(args, runner, logger): products_infos = src.product.get_products_list(options, runner.cfg, logger) # Call to the function that gets all the sources - good_result, results = get_all_product_sources(runner.cfg, - products_infos, - logger) + good_result, results = get_all_product_sources(runner.cfg, products_infos, logger) # Display the results (how much passed, how much failed, etc...) status = src.OK_STATUS diff --git a/src/ElementTree.py b/src/ElementTree.py index 8b1b92b..b871200 100644 --- a/src/ElementTree.py +++ b/src/ElementTree.py @@ -13,14 +13,14 @@ _versionPython = sys.version_info[0] if _versionPython < 3: # python2 previous historic mode import src.ElementTreePython2 as etree - DBG.write("ElementTree Python2", etree.__file__, DBG.isDeveloper()) + DBG.write("ElementTree Python2 %s" % etree.VERSION, etree.__file__, DBG.isDeveloper()) tostring = etree.tostring else: # python3 mode # import xml.etree.ElementTree as etree # native version import src.ElementTreePython3 as etree # VERSION 1.3.0 plus _serialize 'pretty_xml' - DBG.write("ElementTree Python3 VERSION 1.3.0", etree.__file__, DBG.isDeveloper()) + DBG.write("ElementTree Python3 %s" % etree.VERSION, etree.__file__, DBG.isDeveloper()) def tostring(node, encoding='utf-8'): """ diff --git a/src/debug.py b/src/debug.py index 1fbdcd4..330fb6d 100755 --- a/src/debug.py +++ b/src/debug.py @@ -72,14 +72,17 @@ except ImportError: _debug = [False] #support push/pop for temporary activate debug outputs -# wambeke is christian at home -_developers = ["xchristian", "xwambeke"] - def isDeveloper(): - """if you are a developer, sometimes you want verbose traces etc.""" - res = src.architecture.get_user() in _developers - return res + """ + if you are a developer, sometimes you want verbose traces unconditionally + export SAT_DEVELOPER_MODE=1 before launch sat command to do that + """ + res = os.getenv("SAT_DEVELOPER_MODE", "0") + if res in "YES yes 1 Y y".split(): + return True + else: + return False def indent(text, amount=2, ch=' '): """indent multi lines message""" @@ -148,7 +151,7 @@ def format_exception(msg, limit=None, trace=None): """ Format a stack trace and the exception information. as traceback.format_exception(), without color - with traceback only if (_debug) or (DBG._user in DBG._developers) + with traceback only if (_debug) or (DBG.isDeveloper()) """ etype, value, tb = sys.exc_info() res = msg diff --git a/src/system.py b/src/system.py index c54452b..553aab9 100644 --- a/src/system.py +++ b/src/system.py @@ -22,9 +22,12 @@ like open a browser or an editor, or call a git command ''' import os -import subprocess +import subprocess as SP import time import tarfile +import psutil +import time + import debug as DBG import utilsSat as UTS @@ -49,21 +52,62 @@ def show_in_editor(editor, filePath, logger): # launch cmd using subprocess.Popen cmd = editor % filePath logger.write('Launched command:\n' + cmd + '\n', 5) - p = subprocess.Popen(cmd, shell=True) + p = SP.Popen(cmd, shell=True) p.communicate() except: logger.write(printcolors.printcError(_("Unable to edit file %s\n") % filePath), 1) +def show_in_webbrowser(editor, filePath, logger): + '''open filePath using web browser firefox, chromium etc... + if file is xml, previous http sever is done before to fix new security problems + + :param editor str: The web browser to use. + :param filePath str: The path to the file to open. + ''' + # default editor is firefox + if editor is None or len(editor) == 0: + editor = 'firefox' + + path, namefile = os.path.split(filePath) + basefile, ext = os.path.splitext(namefile) + + # previouly http.server 8765 kill ... or not ? TODO wait and see REX + for proc in psutil.process_iter(): + # help(proc) + cmdline = " ".join(proc.cmdline()) + if "python3 -m http.server 8765" in cmdline: + print("kill previous process '%s'" % cmdline) + proc.kill() # TODO may be not owner ? -> change 8765+n + + # The argument -u will make it so that stdout and stderr are not buffered + # and will be written in real time. + cmd = """ +set -x +cd %(path)s +python3 -m http.server 8765 &> /dev/null & +%(editor)s http://localhost:8765/%(namefile)s +""" % {"path": path, "editor": editor, "namefile": namefile} + + # print("show_in_webbrowser:\n%s" % cmd) + + try: + # launch cmd using subprocess.Popen + logger.write('Launched command:\n%s\n' % cmd, 5) + p = SP.Popen(cmd, shell=True, stdout=SP.PIPE, stderr=SP.STDOUT) + res_out, res_err = p.communicate() # res_err = None as stderr=SP.STDOUT + # print("Launched command stdout:\n%s" % res_out) + except Exception as e: + logger.write(printcolors.printcError(_("Unable to display file %s\n%s\n") + % (filePath, e)), 1) + + def git_describe(repo_path): '''Use git describe --tags command to return tag description of the git repository" :param repo_path str: The git repository to describe ''' git_cmd="cd %s;git describe --tags" % repo_path - p = subprocess.Popen(git_cmd, shell=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = SP.Popen(git_cmd, shell=True, stdin=SP.PIPE, stdout=SP.PIPE, stderr=SP.PIPE) p.wait() if p.returncode != 0: return False @@ -99,7 +143,9 @@ def git_extract(from_what, tag, git_options, where, logger, environment=None): set -x git clone %(git_options)s %(remote)s %(where)s res=$? -if [ $res -eq 0 ]; then touch -d "$(git --git-dir=%(where_git)s log -1 --format=date_format)" %(where)s;fi +if [ $res -eq 0 ]; then + touch -d "$(git --git-dir=%(where_git)s log -1 --format=date_format)" %(where)s +fi exit $res """ cmd = cmd % {'git_options': git_options, 'remote': from_what, 'tag': tag, 'where': str(where), 'where_git': where_git} @@ -116,8 +162,10 @@ rm -rf %(where)s git clone %(git_options)s %(remote)s %(where)s && \ git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(tag)s res=$? -git --git-dir=%(where_git)s status|grep HEAD -if [ $res -eq 0 -a $? -ne 0 ]; then touch -d "$(git --git-dir=%(where_git)s log -1 --format=date_format)" %(where)s;fi +git --git-dir=%(where_git)s status | grep HEAD +if [ $res -eq 0 -a $? -ne 0 ]; then + touch -d "$(git --git-dir=%(where_git)s log -1 --format=date_format)" %(where)s +fi exit $res """ cmd = cmd % {'git_options': git_options, @@ -127,7 +175,7 @@ exit $res 'where_git': where_git} - cmd=cmd.replace('date_format','"%ai"') + cmd=cmd.replace('date_format', '"%ai"') logger.logTxtFile.write("\n" + cmd + "\n") logger.logTxtFile.flush() @@ -279,12 +327,11 @@ def cvs_extract(protocol, user, server, base, tag, product, where, logger.logTxtFile.write("\n" + command + "\n") logger.logTxtFile.flush() - res = subprocess.call(command, - cwd=str(where.dir()), - env=environment.environ.environ, - shell=True, - stdout=logger.logTxtFile, - stderr=subprocess.STDOUT) + res = SP.call(command, cwd=str(where.dir()), + env=environment.environ.environ, + shell=True, + stdout=logger.logTxtFile, + stderr=SP.STDOUT) return (res == 0) def svn_extract(user, @@ -331,12 +378,11 @@ def svn_extract(user, logger.write(command + "\n", 5) logger.logTxtFile.write("\n" + command + "\n") logger.logTxtFile.flush() - res = subprocess.call(command, - cwd=str(where.dir()), - env=environment.environ.environ, - shell=True, - stdout=logger.logTxtFile, - stderr=subprocess.STDOUT) + res = SP.call(command, cwd=str(where.dir()), + env=environment.environ.environ, + shell=True, + stdout=logger.logTxtFile, + stderr=SP.STDOUT) return (res == 0) def get_pkg_check_cmd(dist_name): @@ -354,12 +400,12 @@ def get_pkg_check_cmd(dist_name): cmd_which_apt=["which", "apt"] with open(os.devnull, 'w') as devnull: # 1) we search for apt (debian based systems) - completed=subprocess.call(cmd_which_apt,stdout=devnull, stderr=subprocess.STDOUT) + completed=SP.call(cmd_which_apt,stdout=devnull, stderr=SP.STDOUT) if completed==0 and linux=="DB": cmd_is_package_installed=["apt", "list", "--installed"] else: # 2) if apt not found search for rpm (redhat) - completed=subprocess.call(cmd_which_rpm,stdout=devnull, stderr=subprocess.STDOUT) # only 3.8! ,capture_output=True) + completed=SP.call(cmd_which_rpm,stdout=devnull, stderr=SP.STDOUT) # only 3.8! ,capture_output=True) if completed==0 and linux=="RH": cmd_is_package_installed=["rpm", "-q"] else: @@ -389,19 +435,15 @@ def check_system_pkg(check_cmd,pkg): # also apt do not return status, we need to use grep # and apt output is too messy for being used cmd_is_package_installed[-1]+="*" # we don't specify in pyconf the exact name because of version numbers - p=subprocess.Popen(cmd_is_package_installed, - stdout=subprocess.PIPE, - stderr=FNULL) + p=SP.Popen(cmd_is_package_installed, stdout=SP.PIPE, stderr=FNULL) try: - output = subprocess.check_output(['grep', pkg], stdin=p.stdout) + output = SP.check_output(['grep', pkg], stdin=p.stdout) msg_status=src.printcolors.printcSuccess("OK") except: msg_status=src.printcolors.printcError("KO") msg_status+=" (package is not installed!)\n" else: - p=subprocess.Popen(cmd_is_package_installed, - stdout=subprocess.PIPE, - stderr=FNULL) + p=SP.Popen(cmd_is_package_installed, stdout=SP.PIPE, stderr=FNULL) output, err = p.communicate() rc = p.returncode if rc==0: diff --git a/src/xmlManager.py b/src/xmlManager.py index 4b4fa61..6a4bb66 100644 --- a/src/xmlManager.py +++ b/src/xmlManager.py @@ -61,7 +61,12 @@ class XmlLogFile(object): with open(log_file_path, 'w') as f: f.write("\n") if stylesheet: - f.write("\n" % stylesheet) + # example as href='./hat.xsl' + # as local file xml with local file xsl + # with 'python3 -m http.server 8765 &' and + # 'chromium-browser http://localhost:8765/hat.xml' or + # 'firefox http://localhost:8765/hat.xml' + f.write("\n" % stylesheet) pass res= etree.tostring(self.xmlroot, encoding='utf-8') f.write(res) diff --git a/src/xsl/command.xsl b/src/xsl/command.xsl index aab7b64..1f5e1c0 100644 --- a/src/xsl/command.xsl +++ b/src/xsl/command.xsl @@ -12,12 +12,12 @@ border-collapse:collapse; empty-cells : show; } - td { vertical-align : center; padding: 15px; } + td { vertical-align : center; padding: 10px; } h1 { text-align : center; font-style: italic; font-size: 20px; } .legend { font-weight : bold; text-align : center; } - .def { font-family: Arial, Verdana, "Times New Roman", Times, serif;} + .def { font-family: monospace;} hr.note { color: #BFBFBF; } .note { text-align : right; font-style: italic; font-size: small; } div.release { -moz-column-count: 2; diff --git a/src/xsl/hat.xsl b/src/xsl/hat.xsl index 87931fc..0683a38 100644 --- a/src/xsl/hat.xsl +++ b/src/xsl/hat.xsl @@ -7,17 +7,18 @@ SAlomeTools log diff --git a/src/xsl/jobs_global_report.xsl b/src/xsl/jobs_global_report.xsl index 9b7e800..79873e6 100644 --- a/src/xsl/jobs_global_report.xsl +++ b/src/xsl/jobs_global_report.xsl @@ -74,7 +74,7 @@ h4 { text-align : left; font-size: small; font-weight: bold; display: inline; } h_err { text-align : left; font-size: small; font-weight: normal; display: inline; color: red; } .legend { text-align : center; } - .def { font-family: Arial, Verdana, "Times New Roman", Times, serif; background-color:#FFFFCC;} + .def { font-family: monospace, Arial, Verdana, "Times New Roman", Times, serif; background-color:#FFFFCC;} diff --git a/src/xsl/test.xsl b/src/xsl/test.xsl index 24dbfc5..96bad11 100644 --- a/src/xsl/test.xsl +++ b/src/xsl/test.xsl @@ -7,7 +7,7 @@ Tests of <xsl:value-of select="//product/@name" />