From eeb032521b26710f0ad1a36a106800feb0365be7 Mon Sep 17 00:00:00 2001 From: Christian Van Wambeke Date: Wed, 23 May 2018 16:46:09 +0200 Subject: [PATCH] fix linksXml.py etc --- commands/clean.py | 2 +- commands/prepare.py | 99 +++++++++++--------------- src/coloringSat.py | 20 ++++-- src/configManager.py | 4 ++ src/internal_config/salomeTools.pyconf | 2 +- src/linksXml.py | 47 ++++++++++-- src/loggingSat.py | 15 ++-- src/returnCode.py | 7 ++ src/salomeTools.py | 32 ++++++++- src/xmlManager.py | 27 ++++--- 10 files changed, 165 insertions(+), 90 deletions(-) diff --git a/commands/clean.py b/commands/clean.py index c8979f4..b029a54 100644 --- a/commands/clean.py +++ b/commands/clean.py @@ -227,5 +227,5 @@ def suppress_directories(l_paths, logger): else: logger.info(_("Removing %s ...") % strpath ) path.rm() - logger.info('') + logger.info('OK') diff --git a/commands/prepare.py b/commands/prepare.py index b484a7f..6d7adde 100644 --- a/commands/prepare.py +++ b/commands/prepare.py @@ -58,7 +58,7 @@ class Command(_BaseCommand): def run(self, cmd_arguments): """method called for command 'sat prepare '""" argList = self.assumeAsList(cmd_arguments) - + # print general help and returns if len(argList) == 0: self.print_help() @@ -69,7 +69,7 @@ class Command(_BaseCommand): if self._options.help: self.print_help() return RCO.ReturnCode("OK", "Done 'sat %s --help'" % self.name) - + # shortcuts runner = self.getRunner() config = self.getConfig() @@ -83,105 +83,88 @@ class Command(_BaseCommand): # Construct the arguments to pass to the clean, source and patch commands args_appli = config.VARS.application - args_product_opt = '--products ' if options.products: - for p_name in options.products: - args_product_opt += ',' + p_name + listProd = list(options.products) else: - for p_name, __ in products_infos: - args_product_opt += ',' + p_name - + listProd = [name for name, tmp in products_infos] + args_product_opt = '--products ' + ",".join(listProd) + do_source = (len(listProd) > 0) + ldev_products = [p for p in products_infos if PROD.product_is_dev(p[1])] - args_product_opt_clean = args_product_opt + newList = listProd if not options.force and len(ldev_products) > 0: l_products_not_getted = find_products_already_getted(ldev_products) - if len(l_products_not_getted) > 0: + listNot = [i for i, tmp in l_products_not_getted] + newList, removedList = removeInList(listProd, listNot) + if len(removedList) > 0: msg = _("""\ Do not get the source of the following products in development mode. Use the --force option to overwrite it. """) - logger.error(UTS.red(msg)) - args_product_opt_clean = remove_products(args_product_opt_clean, - l_products_not_getted, - logger) - + logger.error(msg + "\n%s" % ",".join(removedList)) - args_product_opt_patch = args_product_opt + args_product_opt_clean = '--products ' + ",".join(newList) + do_clean = (len(newList) > 0) + + newList = listProd if not options.force_patch and len(ldev_products) > 0: l_products_with_patchs = find_products_with_patchs(ldev_products) - if len(l_products_with_patchs) > 0: + listNot = [i for i, tmp in l_products_with_patchs] + newList, removedList = removeInList(listProd, listNot) + if len(removedList) > 0: msg = _(""" Do not patch the following products in development mode. Use the --force_patch option to overwrite it. """) - logger.error(UTS.red(msg)) - args_product_opt_patch = remove_products(args_product_opt_patch, - l_products_with_patchs, - logger) - + logger.error(msg + "\n%s" % ",".join(removedList)) + + args_product_opt_patch = '--products ' + ",".join(newList) + do_patch = (len(newList) > 0) + # Construct the final commands arguments - args_clean = "%s --sources" % (args_product_opt_clean) - args_source = "%s" % (args_product_opt) - args_patch = "%s" % (args_product_opt_patch) - - # If there is no more any product in the command arguments, - # do not call the concerned command - oExpr = re.compile("^--products *$") - do_clean = not(oExpr.search(args_product_opt_clean)) - do_source = not(oExpr.search(args_product_opt)) - do_patch = not(oExpr.search(args_product_opt_patch)) - + args_clean = "%s --sources" % args_product_opt_clean + args_source = "%s" % args_product_opt + args_patch = "%s" % args_product_opt_patch + # Initialize the results to Ok but nothing done status res_clean = RCO.ReturnCode("OK", "nothing done") res_source = RCO.ReturnCode("OK", "nothing done") res_patch = RCO.ReturnCode("OK", "nothing done") # Call the commands using the API + # If do_etc there is no more any product in the command arguments if do_clean: msg = _("Clean the source directories ...") logger.info(msg + "(%s)" % args_clean) mCmd = self.getMicroCommand("clean", args_appli) - res_clean = mCmd.run(args_clean) - logger.step(str(res_clean)) - logger.closeFileHandlerForCommand(mCmd) + res_clean = self.runMicroCommand(mCmd, args_clean) if do_source: msg = _("Get the sources of the products ...") logger.info(msg + "(%s)" % args_source) mCmd = self.getMicroCommand("source", args_appli) - res_source = mCmd.run(args_source) - logger.step(str(res_source)) - logger.closeFileHandlerForCommand(mCmd) + res_source = self.runMicroCommand(mCmd, args_source) if do_patch: msg = _("Patch the product sources (if any) ...") logger.info(msg + "(%s)" % args_patch) mCmd = self.getMicroCommand("patch", args_appli) - res_patch = mCmd.run(args_patch) - logger.step(str(res_patch)) - logger.closeFileHandlerForCommand(mCmd) + res_patch = self.runMicroCommand(mCmd, args_patch) return res_clean + res_source + res_patch -def remove_products(arguments, l_products_info, logger): - """Removes the products in l_products_info from arguments list. +def removeInList(aList, removeList): + """Removes elements of removeList list from aList - :param arguments: (str) The arguments from which to remove products - :param l_products_info: (list) - List of (str, Config) => (product_name, product_info) - :param logger: (Logger) - The logger instance to use for the display and logging - :return: (str) The updated arguments. + :param aList: (list) The list from which to remove elements + :param removeList: (list) The list which contains elements to remove + :return: (list, list) (list with elements removed, list of elements removed) """ - args = arguments - for i, (product_name, __) in enumerate(l_products_info): - args = args.replace(',' + product_name, '') - end_text = ', ' - if i+1 == len(l_products_info): - end_text = '\n' - logger.info(product_name + end_text) - return args + res1 = [i for i in aList if i not in removeList] + res2 = [i for i in aList if i in removeList] + return (res1, res2) + def find_products_already_getted(l_products): """Returns the list of products that have an existing source directory. diff --git a/src/coloringSat.py b/src/coloringSat.py index 5bc87c6..046b9bc 100755 --- a/src/coloringSat.py +++ b/src/coloringSat.py @@ -72,7 +72,7 @@ note: DIM is not assumed in win32 # dir(ST) # order matters for items replaces forward to color -_tags = ( +_tags = [ ("", FG.BLACK), ("", FG.RED), ("", FG.GREEN), @@ -91,13 +91,23 @@ _tags = ( ("", FG.RED), ("", FG.RED + ST.BRIGHT), ("", FG.RED + ST.BRIGHT), - ("", FG.GREEN + ST.BRIGHT + "OK" + ST.RESET_ALL), - ("", FG.RED + ST.BRIGHT + "KO" + ST.RESET_ALL), -) +] # _tagsNone = ( (i, "") for i,j in _tags ) # to clean tags when log not tty # reversed order matters for item replaces backward to no color -_tagsNone = tuple( reversed( [(i, "") for i, j in _tags] ) ) +_tagsNone = list( reversed( [(i, "") for i, j in _tags] ) ) + +# more non empty colored smart tags +_tags = _tags + [ + ("", FG.GREEN + ST.BRIGHT + "OK" + ST.RESET_ALL), + ("", FG.RED + ST.BRIGHT + "KO" + ST.RESET_ALL), +] + +# more non empty colored smart tags reversed order +_tagsNone = [ + (FG.GREEN + ST.BRIGHT + "OK" + ST.RESET_ALL, "OK"), + (FG.RED + ST.BRIGHT + "KO" + ST.RESET_ALL, "KO"), +] + _tagsNone def indent(msg, nb, car=" "): diff --git a/src/configManager.py b/src/configManager.py index 0f3967d..1a5c42c 100644 --- a/src/configManager.py +++ b/src/configManager.py @@ -22,6 +22,7 @@ import shutil import sys import StringIO as SIO +import src # for __version__ import src.debug as DBG import src.loggingSat as LOG import src.returnCode as RCO @@ -229,6 +230,9 @@ class ConfigManager: except Exception as e: raise Exception(msgPb % (afile, str(e))) + if internal_cfg.INTERNAL.sat_version == "auto": + internal_cfg.INTERNAL.sat_version = src.__version__ + merger.merge(cfg, internal_cfg) # apply overwrite from command line if needed diff --git a/src/internal_config/salomeTools.pyconf b/src/internal_config/salomeTools.pyconf index 831216b..baec965 100644 --- a/src/internal_config/salomeTools.pyconf +++ b/src/internal_config/salomeTools.pyconf @@ -3,7 +3,7 @@ INTERNAL : { - sat_version : "5.0.0dev" + sat_version : "auto" # "auto" as from src.__init__.__version__ config : { copy_prefix : "LOCAL_" diff --git a/src/linksXml.py b/src/linksXml.py index f567208..7854da3 100755 --- a/src/linksXml.py +++ b/src/linksXml.py @@ -23,9 +23,13 @@ main command calls sequence of microcommand(s) which calls other sequence of microcommand(s) etc. command(s) are identified (and their logger handler(s)) by '_idCommandHandlers' attribute + +Usage: +>> import src.linksXml as LIXML """ import pprint as PP +import src.debug as DBG class LinkXml(object): """ @@ -60,8 +64,9 @@ class LinkXml(object): aDict = { "idName": self.idName, "log_file_name": self.log_file_name, - "full_launched_cmd": self.full_launched_cmd, + "cmd_name": self.cmd_name, "self.cmd_res": self.cmd_res, + "full_launched_cmd": self.full_launched_cmd, "links": self._linksXml, } tmp = PP.pformat(aDict) @@ -89,7 +94,7 @@ class LinkXml(object): return app def getAllIdNames(self): - """recursive trip for sequence xml""" + """recursive trip in tree to get list sequence xml""" res = [self.idName] for i in self._linksXml: res.extend(i.getAllIdNames()) @@ -101,6 +106,24 @@ class LinkXml(object): else: msg = "setAuthAttr %s attribute not authorized" % nameAttrib raise Exception(msg) + + def toLinkXml(self): + """returns easy to use data for method put_links_fields""" + aDict = { + "command": self.cmd_name, + "launchedCommand": self.full_launched_cmd, + "passed": self.cmd_res, + } + return (self.log_file_name, aDict) + + def toDict(self): + aDict = { + "log_file_name": self.log_file_name, + "cmd_name": self.cmd_name, + "self.cmd_res": self.cmd_res, + "full_launched_cmd": self.full_launched_cmd, + } + return aDict ##################################################### @@ -128,14 +151,26 @@ def appendLinkForCommand(cmdParent, cmdNew): raise Exception(msg) import src.debug as DBG kNew = kParent.appendLink(idNew) - DBG.write("appendLinkForCommand %i for parent %i" % (idNew, idParent), k0, True) + DBG.write("appendLinkForCommand %i for parent" % idNew, idParent, True) return kNew def setAttribLinkForCommand(cmd, nameAttrib, value): """init an attribute value in link of a command in singleton tree""" k0 = getLinksXml() # get singleton - idCmd = cmd.getId() - kCmd = k0.findLink(idCmd) - k0.setAuthAttr(nameAttrib, value) + kCmd = k0.findLink(cmd.getId()) + kCmd.setAuthAttr(nameAttrib, value) + # DBG.write("setAttribLinkForCommand", (nameAttrib, value), True) +def getLinksForXml(idCommand): + """return list of links of one command from its id""" + k0 = getLinksXml() # get singleton + kCommand = k0.findLink(idCommand) + kLinks = kCommand.getAllIdNames()[1:] #avoid first idCommand + res = [kCommand.findLink(k) for k in kLinks] + DBG.write("getLinksForXml", [k.toDict() for k in res], True) + return res +def getLinksForCmd(idCommand): + k0 = getLinksXml() # get singleton + return k0.findLink(idCommand) + \ No newline at end of file diff --git a/src/loggingSat.py b/src/loggingSat.py index 360ccd9..c339db2 100755 --- a/src/loggingSat.py +++ b/src/loggingSat.py @@ -70,6 +70,7 @@ LOGI.TRACE = _TRACE # only for coherency, ################################################################# # utilities methods ################################################################# + def indent(msg, nb, car=" "): """indent nb car (spaces) multi lines message except first one""" s = msg.split("\n") @@ -320,7 +321,7 @@ class LoggerSat(LOGI.Logger): handler.set_config(config) handler.idCommandHandlers = self.idCommandHandlers - fmt = '%(asctime)s :: %(levelname)s :: %(message)s' + fmt = '%(asctime)s :: %(levelname)-8s :: %(message)s' formatter = FileXmlFormatter(fmt, "%y-%m-%d %H:%M:%S") handler.setFormatter(formatter) @@ -333,7 +334,7 @@ class LoggerSat(LOGI.Logger): handler.set_name(nameFileTxt) handler.idCommandHandlers = self.idCommandHandlers - fmt = '%(asctime)s :: %(levelname)s :: %(message)s' + fmt = '%(asctime)s :: %(levelname)-8s :: %(message)s' formatter = FileTxtFormatter(fmt, "%y-%m-%d %H:%M:%S") handler.setFormatter(formatter) @@ -352,7 +353,7 @@ class LoggerSat(LOGI.Logger): handler.set_config(config) handler.idCommandHandlers = self.idCommandHandlers - fmt = '%(asctime)s :: %(levelname)s :: %(message)s' + fmt = '%(asctime)s :: %(levelname)-8s :: %(message)s' formatter = FileXmlFormatter(fmt, "%y-%m-%d %H:%M:%S") handler.setFormatter(formatter) @@ -365,7 +366,7 @@ class LoggerSat(LOGI.Logger): handler.set_name(nameFileTxt) handler.idCommandHandlers = self.idCommandHandlers - fmt = '%(asctime)s :: %(levelname)s :: %(message)s' + fmt = '%(asctime)s :: %(levelname)-8s :: %(message)s' formatter = FileTxtFormatter(fmt, "%y-%m-%d %H:%M:%S") handler.setFormatter(formatter) @@ -613,9 +614,9 @@ class XmlHandler(BufferingHandler): self._target_file = None self._config = None self._log_field = "Uninitialized log" - self._links_fields = [] # list of (log_file_name, cmd_name, cmd_res, full_launched_cmd) self._final_fields = {} # node attributes self.isClosed = False # precaution as write file done yet + self.idCommandHandlers = None # have to be set later to know links def set_target_file(self, filename): """ @@ -664,14 +665,14 @@ class XmlHandler(BufferingHandler): """ # TODO for debug - log("XmlHandler to xml file\n%s" % PP.pformat(getListOfStrLogRecord(self.buffer)), True) + log("XmlHandler to xml file\n%s" % PP.pformat(getListOfStrLogRecord(self.buffer))) self._log_field = self.createLogField() xmlFile = XMLMGR.XmlLogFile(targetFile, "SATcommand") xmlFile.put_initial_fields(config) xmlFile.put_log_field(self._log_field) - xmlFile.put_links_fields(self._links_fields) + xmlFile.put_links_fields(self.idCommandHandlers) xmlFile.put_final_fields(self._final_fields) xmlFile.write_tree(stylesheet = "command.xsl") # xml complete closed file xmlFile.dump_config(config) # create pyconf file in the log directory diff --git a/src/returnCode.py b/src/returnCode.py index 8d6f2f5..de6fe23 100644 --- a/src/returnCode.py +++ b/src/returnCode.py @@ -150,6 +150,13 @@ class ReturnCode(object): return self._TOSYS[self._status] except: return self._TOSYS[self.NA_STATUS] + + def toXmlPassed(self): + """return xml return code as '0' (passed) or '1' (not passed)""" + if self.isOk(): + return "0" + else: + return "1" def getWhy(self): """return why as str or list if sum or some ReturnCode""" diff --git a/src/salomeTools.py b/src/salomeTools.py index 3f7897c..61db9db 100755 --- a/src/salomeTools.py +++ b/src/salomeTools.py @@ -167,6 +167,7 @@ class _BaseCommand(object): self._options = None self._fullName = [] # example '[prepare','clean'] when micro command 'clean' of 'prepare' self._idCommandHandlers = None # as logger.idCommandHandlers for logger handlers of current command + # useless self._lastRunArgs = ["No run arguments yet"] # store last command argument of last run method calls def initFullName(self, parentFullName=[]): """ @@ -216,6 +217,12 @@ class _BaseCommand(object): res = "%s(\n %s)\n" % (self.getClassName(), tmp[1:-1]) return res + '''useless + def getLastRunArgs(self): + """return last command arguments of last run method called""" + return _lastRunArgs[-1] + ''' + def getMicroCommand(self, nameCommandToLoad, nameAppliToLoad): """ get micro command instance from current command instance @@ -241,6 +248,27 @@ class _BaseCommand(object): logger.setFileHandlerForCommand(self, cmdInstance) return cmdInstance + + def runMicroCommand(self, cmdInstance, cmd_arguments): + """create instance of a micro command and launch on arguments""" + logger = self.getLogger() + commandArguments = self.assumeAsList(cmd_arguments) + # Run the micro command using the remainders command arguments + strArgs = " ".join(commandArguments) + msg = "BEGIN launch micro command %s on (%s)" % (cmdInstance.name, strArgs) + logger.step(msg) + returnCode = cmdInstance.run(commandArguments) + msg = "END launch micro command %s on (%s)\n%s" % (cmdInstance.name, strArgs, str(returnCode)) + logger.step(msg) + + import src.linksXml as LKXML + LKXML.setAttribLinkForCommand(cmdInstance, "full_launched_cmd", strArgs) + LKXML.setAttribLinkForCommand(cmdInstance, "cmd_res", returnCode.toXmlPassed()) + + logger.closeFileHandlerForCommand(cmdInstance) + + return returnCode + def run(self, cmd_arguments): """ @@ -572,8 +600,8 @@ class Sat(object): logger.step(msg) import src.linksXml as LKXML - LKXML.setAttribLinkForCommand(cmdInstance, "full_launched_cmd", cmdInstance.getLastRunArgs()) - LKXML.setAttribLinkForCommand(cmdInstance, "cmd_res", returnCode.toXml()) + LKXML.setAttribLinkForCommand(cmdInstance, "full_launched_cmd", strArgs) + LKXML.setAttribLinkForCommand(cmdInstance, "cmd_res", returnCode.toXmlPassed()) logger.closeFileHandlerForCommand(cmdInstance) diff --git a/src/xmlManager.py b/src/xmlManager.py index f3d869d..5a51d07 100644 --- a/src/xmlManager.py +++ b/src/xmlManager.py @@ -177,6 +177,7 @@ class XmlLogFile(object): atts = { "command": cfg.VARS.command, # command name "satversion": cfg.INTERNAL.sat_version, # version of salomeTools + "launchedCommand": "Unknown", "hostname": cfg.VARS.hostname, # machine name "OS": cfg.VARS.dist, # Distribution of the machine "user" : cfg.VARS.user, # The user that have launched the command @@ -203,23 +204,29 @@ class XmlLogFile(object): """ self.set_node_text("Log", text) - def put_links_fields(self, links): + def put_links_fields(self, idCommand): """ Put all fields corresponding to the links context (micro commands) - :param links: (list) The links as list of dict - {fileName, command, passed, launchedCommand} - :param fileName: (str) The file name of the link. - :param command: (str) The name of the command linked. - :param passed: (str) The result of the command linked. "0" or "1" - :param launchedCommand: (str) The full launch command ("sat command ...") + :param idCommand): (int) The id to get command informations + + | as: + | fileName: (str) The file name of the link. + | command: (str) The name of the command linked. + | passed: (str) The result of the command linked. "0" or "1" + | launchedCommand: (str) The full launch command ("sat command ...") """ + import src.linksXml as LIXML + links = LIXML.getLinksForXml(idCommand) xmlLinks = self.xmlroot.find("Links") if len(links) != 0: xmlLinks.text = "" # erase No links - for atts in links: # order matters as time - # DBG.write("put_links_fields", atts) - add_simple_node(xmlLinks, "link", text=atts["fileName"], attrib=atts) + for k in links: # order matters as time + filename, attrib = k.toLinkXml() + add_simple_node(xmlLinks, "link", text=filename, attrib=attrib) + + kCmd = LIXML.getLinksForCmd(idCommand) + self.append_node_attrib("Site", attrib={"launchedCommand": kCmd.full_launched_cmd}) def put_final_fields(self, attribute): """ -- 2.39.2