From: Christian Van Wambeke Date: Wed, 16 May 2018 14:43:10 +0000 (+0200) Subject: fix xmlManager.py X-Git-Url: http://git.salome-platform.org/gitweb/?a=commitdiff_plain;h=238af95e45cc77d4147716d86fc104b96dac76a7;p=tools%2Fsat.git fix xmlManager.py --- diff --git a/src/dateTime.py b/src/dateTime.py index a4d7287..1ff3660 100644 --- a/src/dateTime.py +++ b/src/dateTime.py @@ -32,7 +32,7 @@ import datetime as DT import time as TI # global module variable -verbose = True +verbose = False ##################################################### class DateTime(object): @@ -52,11 +52,14 @@ class DateTime(object): FORMAT_DATE_CONFIG = '%Y%m%d' # for config pyconf FORMAT_DATEHOUR_CONFIG = '%Y%m%d_%H%M%S' # for config pyconf as FORMAT_FILE - FORMAT_PACKAGE = '%Y-%m-%d %H:%M' # for sat package + FORMAT_PACKAGE = '%Y-%m-%d %H:%M' # for sat package' + FORMAT_XML = '%Y/%m/%d %Hh%Mm%Ss' # for sat log file xml MSG_UNDEFINED = "UndefinedTime" def __init__(self, when=None): + self._time = None # set "UndefinedTime", else is a float + if verbose: print "when", when if type(when) == str: if when == "now": self._time = TI.time() # is a float @@ -64,8 +67,22 @@ class DateTime(object): raise Exception("DateTime: unknown when '%s'" % when) elif type(when) == self.__class__: self._time = when.getValue() + elif type(when) == DT.datetime: + # convert from datetime to time + self._time = TI.mktime(when.timetuple()) + elif (type(when) == float) and (when > 1e9): # 1526469510 is may 2018 + self._time = when else: - self._time = None + # UndefinedTime + if verbose:# for debug + msg = "DateTime: unknown when %s '%s' implies 'UndefinedTime'" % (type(when), when) + #raise Exception(msg) + print msg + pass + + def __add__(self, seconds): + """add seconds""" + return DateTime(self._time + seconds) def __repr__(self): """complete with type class as 'DateTime(2018-05-07 12:30:55)'""" @@ -144,6 +161,13 @@ class DateTime(object): res = self.MSG_UNDEFINED return res + def toStrXml(self): + if self.isOk(): + res = TI.strftime(self.FORMAT_XML, self.localTime()) + else: + res = self.MSG_UNDEFINED + return res + def getValue(self): return self._time @@ -245,10 +269,15 @@ class DeltaTime(object): """automatic best unity, hours or minutes or seconds""" if self.isOk(): res = self._t2.toSeconds() - self._t1.toSeconds() - if res < 0: return "%.3fs" % res - if res < 10: return "%.3fs" % res - if res < 60: return "%is" % int(res) - if res < 3600: return "%im%is" % (int(res/60), int(res%60)) + if res < 0: + sign = "-" + res = abs(res) + else: + sign = "" + if res < 0: return sign + "%.3fs" % res + if res < 10: return sign + "%.3fs" % res + if res < 60: return sign + "%is" % int(res) + if res < 3600: return sign + "%im%is" % (int(res/60), int(res%60)) return self.toStrHms() else: res = self.MSG_UNDEFINED @@ -258,10 +287,15 @@ class DeltaTime(object): """all unities, hours and minutes and seconds as '2h34m56s'""" if self.isOk(): res = self._t2.toSeconds() - self._t1.toSeconds() + if res < 0: + sign = "-" + res = abs(res) + else: + sign = "" hh = int(res/3600) mm = int(res%3600)/60 ss = int(res%60) - return "%ih%im%is" % (hh, mm, ss) + return sign + "%ih%im%is" % (hh, mm, ss) else: res = self.MSG_UNDEFINED return res @@ -312,15 +346,26 @@ def sleep(seconds): TI.sleep(seconds) def getWeekDayNow(): - """monday is 0, tuesday is 1 etc.""" + """Returns monday as 0, tuesday as 1 etc.""" return DT.date.weekday(DT.date.today()) def fromTimeStamp(val): + """Returns datetime.datetime""" return DT.datetime.fromtimestamp(val) +def fromDateHourConfig(datehour): + """ + datehour as pyconf config.VARS.datehour 'YYYYMMDD_HHMMSS'. + Returns datetime.datetime + """ + Y, m, dd, H, M, S = date_to_datetime(datehour) + t0 = DT.datetime(int(Y), int(m), int(dd), int(H), int(M), int(S)) + return t0 + def parse_date(date): """ - Transform YYYYMMDD_hhmmss into YYYY-MM-DD hh:mm:ss. + Transform as pyconf config.VARS.datehour 'YYYYMMDD_HHMMSS' + to 'YYYY-MM-DD hh:mm:ss'. :param date: (str) The date to transform :return: (str) The date in the new format @@ -337,8 +382,8 @@ def parse_date(date): def date_to_datetime(date): """ - From a string date in format YYYYMMDD_HHMMSS - returns list year, mon, day, hour, minutes, seconds + From a string date as pyconf config.VARS.datehour 'YYYYMMDD_HHMMSS' + returns [year, month, day, hour, minutes, seconds] :param date: (str) The date in format YYYYMMDD_HHMMSS :return: (tuple) as (str,str,str,str,str,str) @@ -350,6 +395,7 @@ def date_to_datetime(date): H = date[9:11] M = date[11:13] S = date[13:15] + # print "date_to_datetime", date, [Y, m, dd, H, M, S] return Y, m, dd, H, M, S def timedelta_total_seconds(timedelta): diff --git a/src/debug.py b/src/debug.py index 647a630..5e7924c 100644 --- a/src/debug.py +++ b/src/debug.py @@ -108,11 +108,11 @@ def format_color_exception(msg, limit=None, trace=None): if (_debug[-1]) or (_user in _developpers): res = "" + msg if tb: - res += "\nTraceback (most recent call last):\n" + res += "\nTraceback (most recent call last):\n" res += "".join(traceback.format_tb(tb, limit)) #[:-1]) res += "\n" res += "\n".join(traceback.format_exception_only(etype, value)) - return res+ "" + return res + "" else: res = "" + msg # + "" res += "".join(traceback.format_exception_only(etype, value)) diff --git a/src/loggingSat.py b/src/loggingSat.py index 9286bad..25220af 100755 --- a/src/loggingSat.py +++ b/src/loggingSat.py @@ -24,18 +24,23 @@ salomeTools logger. using logging package """ """ -to launch example: - ->> export TRG=SALOME-8.4.0 ->> cd .../sat5.1 ->> src/loggingSat.py ->> AllTestLauncherSat.py -p 'test_???_logging*.py' ->> export TRG=SALOME-8.4.0 ->> sat config $TRG -i KERNEL ->> sat config -v LOCAL.log_dir ->> rm -rf /volatile/wambeke/SAT5/SAT5_S840_MATIX24/LOGS ->> sat prepare $TRG -p KERNEL ->> sat log +# to launch examples: + +export TRG=SALOME-8.4.0 +cd .../sat5.1 +src/loggingSat.py +AllTestLauncherSat.py -p 'test_???_logging*.py' +export TRG=SALOME-8.4.0 +sat config $TRG -i KERNEL +sat config -v LOCAL.log_dir +sat config $TRG -n -v LOCAL.log_dir +rm -rf /volatile/wambeke/SAT5/SAT5_S840_MATIX24/LOGS +sat prepare $TRG -p KERNEL +sat log + +rm -rf TMP +sat prepare $TRG -p KERNEL +more TMP/*xml TMP/OUT/*.txt """ import os @@ -248,6 +253,16 @@ class UnittestFormatter(LOGI.Formatter): return COLS.toColor(res) +################################################################# +class FileTxtFormatter(LOGI.Formatter): + def format(self, record): + # print "", record.levelname #type(record), dir(record) + # nb = len("2018-03-17 12:15:41 :: INFO :: ") + res = super(FileTxtFormatter, self).format(record) + res = indentUnittest(res) + return COLS.cleanColors(res) + + ################################################################# class UnittestStream(object): """ @@ -292,6 +307,8 @@ class XmlHandler(BufferingHandler): super(XmlHandler, self).__init__(capacity) self._target_file = None self._config = None + self._links_fields = [] # list of (log_file_name, cmd_name, cmd_res, full_launched_cmd) + self._final_fields = {} # node attributes def set_target_file(self, filename): """ @@ -313,7 +330,11 @@ class XmlHandler(BufferingHandler): self._config = config def close(self): - """prepare ElementTree from existing logs and write xml file""" + """ + prepare ElementTree from existing logs and write xml file + + warning: avoid sat logging message in logger close phase + """ import src.xmlManager as XMLMGR # avoid import cross utilsSat targetFile = self._target_file config = self._config @@ -323,11 +344,17 @@ class XmlHandler(BufferingHandler): if os.path.exists(targetFile): msg = "XmlHandler target file %s existing yet" % targetFile + log(msg, True) #avoid sat logging message in logger close phase - xmlFile = XMLMGR.XmlLogFile(targetFile, "SATcommand", attrib = {"application" : config.VARS.application}) - xmlFile.write_tree() + xmlFile = XMLMGR.XmlLogFile(targetFile, "SATcommand") + xmlFile.put_initial_fields(config) + xmlFile.put_links_fields(self._links_fields) + xmlFile.put_final_fields(self._final_fields) + xmlFile.write_tree(stylesheet = "command.xsl") + xmlFile.dump_config(config) # create pyconf file in the log directory - super(XmlHandler, self).close() # zaps the buffer to empty + # zaps the buffer to empty as parent class + super(XmlHandler, self).close() ################################################################# # methods to define two LoggerSat instances in salomeTools, @@ -430,7 +457,7 @@ def setFileHandler(logger, config): handler.set_name(nameFileTxt) fmt = '%(asctime)s :: %(levelname)s :: %(message)s' - formatter = UnittestFormatter(fmt, "%Y-%m-%d %H:%M:%S") + formatter = FileTxtFormatter(fmt, "%Y-%m-%d %H:%M:%S") handler.setFormatter(formatter) logger.addHandler(handler) diff --git a/src/utilsSat.py b/src/utilsSat.py index 04434db..91b2519 100644 --- a/src/utilsSat.py +++ b/src/utilsSat.py @@ -55,6 +55,24 @@ def ensure_path_exists(path): if not os.path.exists(path): os.makedirs(path) +def ensure_file_exists(aFile, aDefaultFile): + """ + Create a file if not existing, + copying from default file + + :param aFilepath: (str) The file to ensure existence + :param aDefaultFile: (str) The default file to copy if not existing + """ + isfile = os.path.isfile(aFile) + if isfile: return True + try: + DBG.write("ensure_file_exists %s" % isfile, aDefaultFile + " -->\n" + aFile) + shutil.copy2(aDefaultFile, aFile) + return True + except: + return False + + def replace_in_file(file_in, str_in, str_out): """ Replace by in file . diff --git a/src/xmlManager.py b/src/xmlManager.py index 8e7a0e5..830a43d 100644 --- a/src/xmlManager.py +++ b/src/xmlManager.py @@ -33,6 +33,7 @@ except: import src.ElementTree as ETREE import src.utilsSat as UTS +import dateTime as DATT ############################################################################## @@ -42,7 +43,7 @@ class XmlLogFile(object): """ Class to manage writing in salomeTools xml log file """ - def __init__(self, filePath, rootname, attrib = {}): + def __init__(self, filePath, rootname): """Initialization :param filePath: (str) The path to the file where to write the log file @@ -50,30 +51,59 @@ class XmlLogFile(object): :param attrib: (dict) The dictionary that contains the attributes and value of the root node """ + self._config = None + # Initialize the filePath and ensure that the directory # that contain the file exists (make it if necessary) - self.logFile = filePath - UTS.ensure_path_exists(os.path.dirname(filePath)) + self.xmlFile = filePath + + self.dirXmlFile, baseName = os.path.split(filePath) + prefix, tmp = os.path.splitext(baseName) + self.txtFile = os.path.join(self.dirXmlFile, "OUT", prefix + ".txt") + self.pyconfFile = os.path.join(self.dirXmlFile, "OUT", prefix + ".pyconf") + + UTS.ensure_path_exists(self.dirXmlFile) + UTS.ensure_path_exists(os.path.join(self.dirXmlFile, "OUT")) + # Initialize the field that contain the xml in memory - self.xmlroot = ETREE.Element(rootname, attrib = attrib) + self.xmlroot = ETREE.Element(rootname) + + def set_config(self, config): + """needs do be called at least once""" + self._config = config + + def get_config(self): + return self._config def write_tree(self, stylesheet=None, file_path = None): """Write the xml tree in the log file path. Add the stylesheet if asked. - :param stylesheet: (str) The stylesheet to apply to the xml file + :param stylesheet: (str) The basename stylesheet to apply to the xml file """ - log_file_path = self.logFile + cfg = self._config # shortcut + log_file_path = self.xmlFile if file_path: - log_file_path = file_path + log_file_path = file_path + try: - with open(log_file_path, 'w') as f: - f.write("\n") - if stylesheet: - f.write("\n" % - stylesheet) - f.write(ETREE.tostring(self.xmlroot, encoding='utf-8')) + if stylesheet: + fDef = os.path.join(cfg.VARS.srcDir, "xsl", stylesheet) # original default + fCur = os.path.join(self.dirXmlFile, stylesheet) # local need + UTS.ensure_file_exists(fCur, fDef) + fDef = os.path.join(cfg.VARS.srcDir, "xsl", "LOGO-SAT.png") # original default + fCur = os.path.join(self.dirXmlFile, "LOGO-SAT.png") # local need + UTS.ensure_file_exists(fCur, fDef) except Exception: - raise Exception("problem writing Xml log file: %s" % log_file_path) + raise Exception("problem writing stylesheet file: %s" % styCur) + + try: + with open(log_file_path, 'w') as f: + f.write("\n") + if stylesheet: + f.write("\n" % stylesheet) + f.write(ETREE.tostring(self.xmlroot, encoding='utf-8')) + except Exception: + raise Exception("problem writing Xml log file: %s" % log_file_path) def add_simple_node(self, node_name, text=None, attrib={}): """Add a node with some attibutes and text to the root node. @@ -108,76 +138,114 @@ class XmlLogFile(object): """ self.xmlroot.find(node_name).attrib.update(attrib) + def datehourToXml(self, datehour): + """ + format for attrib xml from config VARS.datehour + from '20180516_090830' to '16/05/2018 09h08m30s' + """ + Y, m, dd, H, M, S = DATT.date_to_datetime(datehour) + res = "%2s/%2s/%4s %2sh%2sm%2ss" % (dd, m, Y, H, M, S) + return res + + def relPath(self, aFile): + """get relative path of aFile from self.dirXmlFile""" + return os.path.relpath(aFile, self.dirXmlFile) - def put_initial_xml_fields(self): + def put_initial_fields(self, config): """ - Called at class initialization: Put all fields - corresponding to the command context (user, time, ...) + Put all fields corresponding to the command context (user, time, ...) """ - # command name - self.xmlFile.add_simple_node("Site", attrib={"command" : - self.config.VARS.command}) - # version of salomeTools - self.xmlFile.append_node_attrib("Site", attrib={"satversion" : - self.config.INTERNAL.sat_version}) - # machine name on which the command has been launched - self.xmlFile.append_node_attrib("Site", attrib={"hostname" : - self.config.VARS.hostname}) - # Distribution of the machine - self.xmlFile.append_node_attrib("Site", attrib={"OS" : - self.config.VARS.dist}) - # The user that have launched the command - self.xmlFile.append_node_attrib("Site", attrib={"user" : - self.config.VARS.user}) - # The time when command was launched - Y, m, dd, H, M, S = date_to_datetime(self.config.VARS.datehour) - date_hour = "%2s/%2s/%4s %2sh%2sm%2ss" % (dd, m, Y, H, M, S) - self.xmlFile.append_node_attrib("Site", attrib={"beginTime" : - date_hour}) - # The application if any - if "APPLICATION" in self.config: - self.xmlFile.append_node_attrib("Site", - attrib={"application" : self.config.VARS.application}) - # The initialization of the trace node - self.xmlFile.add_simple_node("Log",text="") + self.set_config(config) + cfg = self._config # shortcut + + # append attrib application to root node + self.xmlroot.attrib.update({"application" : cfg.VARS.application}) + + # add node Site + atts = { + "command": cfg.VARS.command, # command name + "satversion": cfg.INTERNAL.sat_version, # version of salomeTools + "hostname": cfg.VARS.hostname, # machine name + "OS": cfg.VARS.dist, # Distribution of the machine + "user" : cfg.VARS.user, # The user that have launched the command + "beginTime" : self.datehourToXml(cfg.VARS.datehour), #when command was launched + "application" : cfg.VARS.application, # The application if any + } + self.add_simple_node("Site", attrib=atts) + + # The initialization of the node Log + self.add_simple_node("Log", text="") + # The system commands logs - self.xmlFile.add_simple_node("OutLog", - text=os.path.join("OUT", self.txtFileName)) - # The initialization of the node where - # to put the links to the other sat commands that can be called by any - # command - self.xmlFile.add_simple_node("Links") + self.add_simple_node("OutLog", text=self.relPath(self.txtFile)) + + # The initialization of the node Links + # where to put the links to the other sat commands (micro commands) + # called by any first main command + self.add_simple_node("Links") + + def put_links_fields(self, links): + """ + Put all fields corresponding to the links context (micro commands) + + :param log_file_name: (str) The file name of the link. + :param command_name: (str) The name of the command linked. + :param command_res: (str) The result of the command linked. "0" or "1" + :param full_launched_command: (str) The full lanch command ("sat command ...") + """ + xmlLinks = self.xmlroot.find("Links") + for li in links: + log_file_name, cmd_name, cmd_res, full_launched_cmd = li + atts = { + "command": cmd_name, + "passed": cmd_res, + "launchedCommand" : full_launched_cmd, + } + self.add_simple_node(xmlLinks, "link", text=log_file_name, attrib=atts) - def add_link(self, - log_file_name, - command_name, - command_res, - full_launched_command): - """Add a link to another log file. - - :param log_file_name str: The file name of the link. - :param command_name str: The name of the command linked. - :param command_res str: The result of the command linked. "0" or "1" - :parma full_launched_command str: The full lanch command - ("sat command ...") + def put_final_fields(self, attribute): """ - xmlLinks = self.xmlFile.xmlroot.find("Links") - src.xmlManager.add_simple_node(xmlLinks, - "link", - text = log_file_name, - attrib = {"command" : command_name, - "passed" : command_res, - "launchedCommand" : full_launched_command}) + formerly method end_write. + Called just after ending command. + Put all fields corresponding to the command end context + (as current time). + + :param attribute: (dict) some attribute to set/append to the node "Site". + """ + cfg = self._config # shortcut + t1 = DATT.DateTime(DATT.fromDateHourConfig(cfg.VARS.datehour)) # begin command time + t2 = DATT.DateTime("now") # current time as end command time + # print "t1=%s t2=%s" % (t1, t2) + dt = DATT.DeltaTime(t1, t2) + + # Add the fields end and total time of command + atts = { + "endTime": t2.toStrXml(), + "TotalTime": dt.toStrHms(), + } + self.append_node_attrib("Site", attrib=atts) + + # set/append the attribute passed to the method + self.append_node_attrib("Site", attrib=attribute) + + + def dump_config(self, config): + """Dump the config in a pyconf file in the log directory""" + # no time for logger as closing phase, + # if problem raise error... maybe TOFIX + with open(self.pyconfFile, 'w') as f: + config.__save__(f) + def write(self, message, level=None, screenOnly=False): """ function used in the commands to print in the terminal and the log file. - :param message str: The message to print. - :param level int: The output level corresponding - to the message 0 < level < 6. - :param screenOnly boolean: if True, do not write in log file. + :param message: (str) The message to print. + :param level: (int) + The output level corresponding to the message 0 < level < 6. + :param screenOnly: (bool) if True, do not write in log file. """ # do not write message starting with \r to log file if not message.startswith("\r") and not screenOnly: @@ -203,7 +271,7 @@ class XmlLogFile(object): def error(self, message): """Print an error. - :param message str: The message to print. + :param message: (str:) The message to print. """ # Print in the log file self.xmlFile.append_node_text("traces", _('ERROR:') + message) @@ -216,52 +284,6 @@ class XmlLogFile(object): sys.stderr.write(_('ERROR:') + message) - def end_write(self, attribute): - """ - Called just after command end: Put all fields - corresponding to the command end context (time). - Write the log xml file on the hard drive. - And display the command to launch to get the log - - :param attribute dict: the attribute to add to the node "Site". - """ - # Get current time (end of command) and format it - dt = datetime.datetime.now() - Y, m, dd, H, M, S = date_to_datetime(self.config.VARS.datehour) - t0 = datetime.datetime(int(Y), int(m), int(dd), int(H), int(M), int(S)) - tf = dt - delta = tf - t0 - total_time = timedelta_total_seconds(delta) - hours = int(total_time / 3600) - minutes = int((total_time - hours*3600) / 60) - seconds = total_time - hours*3600 - minutes*60 - # Add the fields corresponding to the end time - # and the total time of command - endtime = dt.strftime('%Y/%m/%d %Hh%Mm%Ss') - self.xmlFile.append_node_attrib("Site", attrib={"endTime" : endtime}) - self.xmlFile.append_node_attrib("Site", - attrib={"TotalTime" : "%ih%im%is" % (hours, minutes, seconds)}) - - # Add the attribute passed to the method - self.xmlFile.append_node_attrib("Site", attrib=attribute) - - # Call the method to write the xml file on the hard drive - self.xmlFile.write_tree(stylesheet = "command.xsl") - - # Dump the config in a pyconf file in the log directory - logDir = src.get_log_path(self.config) - dumpedPyconfFileName = (self.config.VARS.datehour - + "_" - + self.config.VARS.command - + ".pyconf") - dumpedPyconfFilePath = os.path.join(logDir, 'OUT', dumpedPyconfFileName) - try: - f = open(dumpedPyconfFilePath, 'w') - self.config.__save__(f) - f.close() - except IOError: - pass - ############################################################################## class ReadXmlFile(object): diff --git a/test/test_023_dateTime.py b/test/test_023_dateTime.py index 04b7130..852e229 100755 --- a/test/test_023_dateTime.py +++ b/test/test_023_dateTime.py @@ -25,6 +25,7 @@ import pprint as PP import src.debug as DBG import src.dateTime as DATT +import time as TI verbose = False #True @@ -110,15 +111,37 @@ class TestCase(unittest.TestCase): DBG.write("test_042 repr", repr(delta)) delta2 = delta.raiseIfKo() self.assertEqual(delta2.toSeconds(), delta.toSeconds()) - + def test_044(self): + t1 = DATT.DateTime("now") + t2 = DATT.DateTime(t1) + 3.1 # add 3 seconds + delta = DATT.DeltaTime(t1, t2) + self.assertGreater(delta.toSeconds(), 3) + + ti = TI.time() + t4 = DATT.DateTime(ti) + t5 = t4 + 10.1 + DBG.write("test_044 ti", [type(ti), ti, t4, t5]) + delta = DATT.DeltaTime(t4, t5) + self.assertGreater(delta.toSeconds(), 10) + DBG.write("test_044 delta", [delta.toStrHuman(), delta.toStrHms()]) + self.assertNotIn("-", delta.toStrHuman()) + self.assertNotIn("-", delta.toStrHms()) + + delta = DATT.DeltaTime(t5, t4) # negative delta + self.assertLess(delta.toSeconds(), -10) + DBG.write("test_044 delta", [delta.toStrHuman(), delta.toStrHms()]) + self.assertIn("-10s", delta.toStrHuman()) + self.assertIn("-0h0m10s", delta.toStrHms()) + + def test_046(self): for more in [0, 0.56789, 5.6789, 56.789, 61, 3661, 36061]: t1 = DATT.DateTime("now") t2 = DATT.DateTime(t1) t2.addSeconds(more) delta = DATT.DeltaTime(t1, t2) r = delta.toStrHuman() - DBG.write("test_044 str", r) + DBG.write("test_046 str", r) if more < 60: self.assertIn("s", r) self.assertNotIn("m", r)