# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import os
+import sys
import platform
import datetime
import shutil
import gettext
-import sys
+import pprint as PP
import src
+import src.logger as LOG
import src.debug as DBG
+import src.callerName as CALN
+
+logger = LOG.getDefaultLogger()
+
+verbose = False # True for debug
# internationalization
satdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
_("Optional: synthetic list of all properties used in the application"))
parser.add_option('c', 'copy', 'boolean', 'copy',
_("""Optional: copy a config file to the personal config files directory.
-\tWARNING the included files are not copied.
-\tIf a name is given the new config file takes the given name."""))
+WARNING: the included files are not copied.
+If a name is given the new config file takes the given name."""))
parser.add_option('n', 'no_label', 'boolean', 'no_label',
_("Internal use: do not print labels, Works only with --value and --list."))
parser.add_option('', 'completion', 'boolean', 'completion',
parser.add_option('s', 'schema', 'boolean', 'schema',
_("Internal use."))
+def osJoin(*args):
+ """
+ shortcut wrapper to os.path.join
+ plus optionaly print for debug
+ """
+ res = os.path.realpath(os.path.join(*args))
+ if verbose:
+ if True: # ".pyconf" in res:
+ logger.info("osJoin %-80s in %s" % (res, CALN.caller_name()))
+ return res
+
class ConfigOpener:
'''Class that helps to find an application pyconf
in all the possible directories (pathList)
:param pathList list: The list of paths where to search a pyconf.
'''
self.pathList = pathList
+ if verbose:
+ for path in pathList:
+ if not os.path.isdir(path):
+ logger.warning("ConfigOpener inexisting directory: %s" % path)
def __call__(self, name):
if os.path.isabs(name):
return src.pyconf.ConfigInputStream(open(name, 'rb'))
else:
- return src.pyconf.ConfigInputStream(
- open(os.path.join( self.get_path(name), name ), 'rb') )
+ return src.pyconf.ConfigInputStream(open(osJoin(self.get_path(name), name), 'rb'))
raise IOError(_("Configuration file '%s' not found") % name)
def get_path( self, name ):
'''The method that returns the entire path of the pyconf searched
+ returns first found in self.pathList directories
+
:param name str: The name of the searched pyconf.
'''
for path in self.pathList:
- if os.path.exists(os.path.join(path, name)):
+ if os.path.exists(osJoin(path, name)):
return path
raise IOError(_("Configuration file '%s' not found") % name)
'''
var = {}
var['user'] = src.architecture.get_user()
- var['salometoolsway'] = os.path.dirname(
- os.path.dirname(os.path.abspath(__file__)))
- var['srcDir'] = os.path.join(var['salometoolsway'], 'src')
- var['internal_dir'] = os.path.join(var['srcDir'], 'internal_config')
+ var['salometoolsway'] = os.path.dirname( os.path.dirname(os.path.abspath(__file__)))
+ var['srcDir'] = osJoin(var['salometoolsway'], 'src')
+ var['internal_dir'] = osJoin(var['srcDir'], 'internal_config')
var['sep']= os.path.sep
# datadir has a default location
- var['datadir'] = os.path.join(var['salometoolsway'], 'data')
+ var['datadir'] = osJoin(var['salometoolsway'], 'data')
if datadir is not None:
var['datadir'] = datadir
- var['personalDir'] = os.path.join(os.path.expanduser('~'),
- '.salomeTools')
+ var['personalDir'] = osJoin(os.path.expanduser('~'), '.salomeTools')
src.ensure_path_exists(var['personalDir'])
- var['personal_applications_dir'] = os.path.join(var['personalDir'],
- "Applications")
+ var['personal_applications_dir'] = osJoin(var['personalDir'], "Applications")
src.ensure_path_exists(var['personal_applications_dir'])
- var['personal_products_dir'] = os.path.join(var['personalDir'],
- "products")
+ var['personal_products_dir'] = osJoin(var['personalDir'], "products")
src.ensure_path_exists(var['personal_products_dir'])
- var['personal_archives_dir'] = os.path.join(var['personalDir'],
- "Archives")
+ var['personal_archives_dir'] = osJoin(var['personalDir'], "Archives")
src.ensure_path_exists(var['personal_archives_dir'])
- var['personal_jobs_dir'] = os.path.join(var['personalDir'],
- "Jobs")
+ var['personal_jobs_dir'] = osJoin(var['personalDir'], "Jobs")
src.ensure_path_exists(var['personal_jobs_dir'])
- var['personal_machines_dir'] = os.path.join(var['personalDir'],
- "Machines")
+ var['personal_machines_dir'] = osJoin(var['personalDir'], "Machines")
src.ensure_path_exists(var['personal_machines_dir'])
# read linux distributions dictionary
- distrib_cfg = src.pyconf.Config(os.path.join(var['srcDir'],
- 'internal_config',
- 'distrib.pyconf'))
+ distrib_cfg = src.pyconf.Config( osJoin(var['srcDir'], 'internal_config', 'distrib.pyconf'))
# set platform parameters
- dist_name = src.architecture.get_distribution(
- codes=distrib_cfg.DISTRIBUTIONS)
- dist_version = src.architecture.get_distrib_version(dist_name,
- codes=distrib_cfg.VERSIONS)
+ dist_name = src.architecture.get_distribution(codes=distrib_cfg.DISTRIBUTIONS)
+ dist_version = src.architecture.get_distrib_version(dist_name, codes=distrib_cfg.VERSIONS)
+ dist_version_full = src.architecture.get_infosys()
dist = dist_name + dist_version
var['dist_name'] = dist_name
var['dist_version'] = dist_version
var['dist'] = dist
+ var['dist_ref'] = dist_name + dist_version_full
var['python'] = src.architecture.get_python_version()
var['nb_proc'] = src.architecture.get_nb_proc()
# =====================================================================
# create VARS section
- var = self._create_vars(application=application, command=command,
- datadir=datadir)
+ var = self._create_vars(application=application, command=command, datadir=datadir)
+ # DBG.write("create_vars", var, DBG.isDeveloper())
+
# add VARS to config
cfg.VARS = src.pyconf.Mapping(cfg)
for variable in var:
# Load INTERNAL config
# read src/internal_config/salomeTools.pyconf
src.pyconf.streamOpener = ConfigOpener([
- os.path.join(cfg.VARS.srcDir, 'internal_config')])
+ osJoin(cfg.VARS.srcDir, 'internal_config')])
try:
- internal_cfg = src.pyconf.Config(open(os.path.join(cfg.VARS.srcDir,
+ internal_cfg = src.pyconf.Config(open( osJoin(cfg.VARS.srcDir,
'internal_config', 'salomeTools.pyconf')))
except src.pyconf.ConfigError as e:
raise src.SatException(_("Error in configuration file:"
# search only in the data directory
src.pyconf.streamOpener = ConfigOpener([cfg.VARS.datadir])
try:
- local_cfg = src.pyconf.Config(open(os.path.join(cfg.VARS.datadir,
+ local_cfg = src.pyconf.Config(open( osJoin(cfg.VARS.datadir,
'local.pyconf')),
PWD = ('LOCAL', cfg.VARS.datadir) )
except src.pyconf.ConfigError as e:
# When the key is "default", put the default value
if cfg.LOCAL.base == "default":
- cfg.LOCAL.base = os.path.abspath(
- os.path.join(cfg.VARS.salometoolsway,
- "..",
- "BASE"))
+ cfg.LOCAL.base = os.path.abspath(osJoin(cfg.VARS.salometoolsway, "..", "BASE"))
if cfg.LOCAL.workdir == "default":
- cfg.LOCAL.workdir = os.path.abspath(
- os.path.join(cfg.VARS.salometoolsway,
- ".."))
+ cfg.LOCAL.workdir = os.path.abspath(osJoin(cfg.VARS.salometoolsway, ".."))
if cfg.LOCAL.log_dir == "default":
- cfg.LOCAL.log_dir = os.path.abspath(
- os.path.join(cfg.VARS.salometoolsway,
- "..",
- "LOGS"))
+ cfg.LOCAL.log_dir = os.path.abspath(osJoin(cfg.VARS.salometoolsway, "..", "LOGS"))
if cfg.LOCAL.archive_dir == "default":
- cfg.LOCAL.archive_dir = os.path.abspath(
- os.path.join(cfg.VARS.salometoolsway,
- "..",
- "ARCHIVES"))
+ cfg.LOCAL.archive_dir = os.path.abspath( osJoin(cfg.VARS.salometoolsway, "..", "ARCHIVES"))
# apply overwrite from command line if needed
for rule in self.get_command_line_overrides(options, ["LOCAL"]):
exec('cfg.' + rule) # this cannot be factorized because of the exec
# AT END append APPLI_TEST directory in APPLICATIONPATH, for unittest
- appli_test_dir = os.path.join(satdir, "test", "APPLI_TEST")
+ appli_test_dir = osJoin(satdir, "test", "APPLI_TEST")
if appli_test_dir not in cfg.PATHS.APPLICATIONPATH:
cfg.PATHS.APPLICATIONPATH.append(appli_test_dir, "unittest APPLI_TEST path")
'''
# get the expected name and path of the file
self.config_file_name = 'SAT.pyconf'
- self.user_config_file_path = os.path.join(config.VARS.personalDir,
- self.config_file_name)
+ self.user_config_file_path = osJoin(config.VARS.personalDir, self.config_file_name)
# if pyconf does not exist, create it from scratch
if not os.path.isfile(self.user_config_file_path):
"This is the default output_verbose_level you want."
" 0=>no output, 5=>debug.\n")
user_cfg.USER.addMapping('publish_dir',
- os.path.join(os.path.expanduser('~'),
+ osJoin(os.path.expanduser('~'),
'websupport',
'satreport'),
"")
# "The products installation base (could be "
# "ignored if this key exists in the local.pyconf"
# " file of salomTools).\n")
-
- #
+
src.ensure_path_exists(config.VARS.personalDir)
- src.ensure_path_exists(os.path.join(config.VARS.personalDir,
+ src.ensure_path_exists( osJoin(config.VARS.personalDir,
'Applications'))
f = open(cfg_name, 'w')
pinfo = src.product.get_product_config(config, name)
if "depend" in pinfo:
- src.printcolors.print_value(logger,
- "depends on",
- ', '.join(pinfo.depend), 2)
+ src.printcolors.print_value(logger, "depends on", sorted(pinfo.depend), 2)
if "opt_depend" in pinfo:
- src.printcolors.print_value(logger,
- "optional",
- ', '.join(pinfo.opt_depend), 2)
+ src.printcolors.print_value(logger, "optional", sorted(pinfo.opt_depend), 2)
# information on pyconf
logger.write("\n", 2)
logger.write(src.printcolors.printcLabel("configuration:") + "\n", 2)
if "from_file" in pinfo:
- src.printcolors.print_value(logger,
- "pyconf file path",
- pinfo.from_file,
+ src.printcolors.print_value(logger,
+ "pyconf file path",
+ pinfo.from_file,
2)
if "section" in pinfo:
- src.printcolors.print_value(logger,
- "section",
- pinfo.section,
+ src.printcolors.print_value(logger,
+ "section",
+ pinfo.section,
2)
# information on prepare
src.printcolors.print_value(logger, "tag", pinfo.git_info.tag, 2)
elif method == 'archive':
- src.printcolors.print_value(logger,
- "get from",
- check_path(pinfo.archive_info.archive_name),
+ src.printcolors.print_value(logger,
+ "get from",
+ check_path(pinfo.archive_info.archive_name),
2)
if 'patches' in pinfo:
src.printcolors.print_value(logger, "patch", check_path(patch), 2)
if src.product.product_is_fixed(pinfo):
- src.printcolors.print_value(logger, "install_dir",
+ src.printcolors.print_value(logger, "install_dir",
check_path(pinfo.install_dir), 2)
if src.product.product_is_native(pinfo) or src.product.product_is_fixed(pinfo):
if src.product.product_compiles(pinfo):
logger.write("\n", 2)
logger.write(src.printcolors.printcLabel("compile:") + "\n", 2)
- src.printcolors.print_value(logger,
- "compilation method",
- pinfo.build_source,
+ src.printcolors.print_value(logger,
+ "compilation method",
+ pinfo.build_source,
2)
if pinfo.build_source == "script" and "compil_script" in pinfo:
check_path(pinfo.environ.env_script),
2)
- zz = src.environment.SalomeEnviron(config,
+ zz = src.environment.SalomeEnviron(config,
src.fileEnviron.ScreenEnviron(logger),
False)
zz.set_python_libdirs()
od = options.debug[1:]
else:
od = options.debug
- exec("a = runner.cfg.%s" % od)
- res = DBG.indent(DBG.getStrConfigDbg(a))
- logger.write("\nConfig.%s of application %s:\n\n%s\n" % (od, runner.cfg.VARS.application, res))
+ try:
+ aCode = "a = runner.cfg.%s" % od
+ # https://stackoverflow.com/questions/15086040/behavior-of-exec-function-in-python-2-and-python-3
+ aDict = {"runner": runner}
+ exec(aCode, globals(), aDict)
+ # DBG.write("globals()", globals(), True)
+ # DBG.write("aDict", aDict, True)
+ res = DBG.indent(DBG.getStrConfigDbg(aDict["a"]))
+ logger.write("\nConfig.%s of application %s:\n\n%s\n" % (od, runner.cfg.VARS.application, res))
+ except Exception as e:
+ msg = "\nConfig.%s of application %s: Unknown pyconf key\n" % (od, runner.cfg.VARS.application)
+ logger.write(src.printcolors.printcError(msg), 1)
# case : edit user pyconf file or application file
editor = runner.cfg.USER.editor
if ('APPLICATION' not in runner.cfg and
'open_application' not in runner.cfg): # edit user pyconf
- usercfg = os.path.join(runner.cfg.VARS.personalDir,
+ usercfg = osJoin(runner.cfg.VARS.personalDir,
'SAT.pyconf')
logger.write(_("Opening %s\n" % usercfg), 3)
src.system.show_in_editor(editor, usercfg, logger)
else:
# search for file <application>.pyconf and open it
for path in runner.cfg.PATHS.APPLICATIONPATH:
- pyconf_path = os.path.join(path,
+ pyconf_path = osJoin(path,
runner.cfg.VARS.application + ".pyconf")
if os.path.exists(pyconf_path):
logger.write(_("Opening %s\n" % pyconf_path), 3)
if path == runner.cfg.VARS.personalDir:
continue
# loop on all directories that can have pyconf applications
- zz = os.path.join(path, source)
+ zz = osJoin(path, source)
if os.path.exists(zz):
source_full_path = zz
break
dest = runner.cfg.VARS.application
# the full path
- dest_file = os.path.join(runner.cfg.VARS.personalDir,
+ dest_file = osJoin(runner.cfg.VARS.personalDir,
'Applications', dest + '.pyconf')
if os.path.exists(dest_file):
raise src.SatException(_("A personal application"
yacsgen_info = src.product.get_product_config(config, 'YACSGEN')
yacsgen_dir = yacsgen_info.install_dir
yacs_src = _("Using YACSGEN from application")
- elif os.environ.has_key("YACSGEN_ROOT_DIR"):
+ elif "YACSGEN_ROOT_DIR" in os.environ:
yacsgen_dir = os.getenv("YACSGEN_ROOT_DIR")
yacs_src = _("Using YACSGEN from environment")
pass
import src
+
+
import src.ElementTree as etree
STYLESHEET_GLOBAL = "jobs_global_report.xsl"
# Define all possible option for log command : sat log <options>
parser = src.options.Options()
-parser.add_option('t', 'terminal', 'boolean', 'terminal', "Optional: "
- "Terminal log.")
-parser.add_option('l', 'last', 'boolean', 'last', "Show the log of the last "
- "Optional: launched command.")
-parser.add_option('', 'last_terminal', 'boolean', 'last_terminal', "Show the "
- "log of the last compilations"
- "Optional: launched command.")
-parser.add_option('f', 'full', 'boolean', 'full', "Optional: Show the logs of "
- "ALL the launched commands.")
-parser.add_option('c', 'clean', 'int', 'clean', "Optional: Erase the n most "
- "ancient log files.")
-parser.add_option('n', 'no_browser', 'boolean', 'no_browser', "Optional: Do not"
- " launch the browser at the end of the command. Only update "
- "the hat file.")
+parser.add_option('t', 'terminal', 'boolean', 'terminal',
+ "Optional: Show the log (in terminal) of a command, with user choice.")
+parser.add_option('l', 'last', 'boolean', 'last',
+ "Optional: Show the log (in browser) of the last launched command.")
+parser.add_option('', 'last_terminal', 'boolean', 'last_terminal',
+ "Optional: Show the log (in terminal) of the last launched command.")
+parser.add_option('f', 'full', 'boolean', 'full',
+ "Optional: Show the logs of ALL the launched commands.")
+parser.add_option('c', 'clean', 'int', 'clean',
+ "Erase the n most ancient log files.")
+parser.add_option('n', 'no_browser', 'boolean', 'no_browser',
+ "Optional: Do not launch the browser at the end of the command. Only update the hat file.")
def get_last_log_file(logDir, notShownCommands):
'''Used in case of last option. Get the last log command file path.
for name in names:
# display information
- len_points = max_len - len(name)
+ len_points = max_len - len(name) + 3
local_path, archive_path = d_content[name]
in_archive = os.path.join(name_archive, archive_path)
logger.write(name + " " + len_points * "." + " "+ in_archive + " ", 3)
bin_kernel_install_dir = os.path.join(kernel_root_dir,"bin","salome")
# check if the application contains an application module
- l_product_info = src.product.get_products_infos(config.APPLICATION.products.keys(),
- config)
+ l_product_info = src.product.get_products_infos(config.APPLICATION.products.keys(), config)
salome_application_name="Not defined"
for prod_name, prod_info in l_product_info:
# look for a salome application
runner.cfg.VARS.application), 1)
# Get the default directory where to put the packages
- package_default_path = os.path.join(runner.cfg.APPLICATION.workdir,
- "PACKAGE")
+ package_default_path = os.path.join(runner.cfg.APPLICATION.workdir, "PACKAGE")
src.ensure_path_exists(package_default_path)
# if the package contains a project:
break
if foundProject is None:
- local_path = os.path.join(runner.cfg.VARS.salometoolsway,
- "data",
- "local.pyconf")
+ local_path = os.path.join(runner.cfg.VARS.salometoolsway, "data", "local.pyconf")
msg = _("""ERROR: the project %(1)s is not visible by salomeTools.
known projects are:
%(2)s
d_files_to_add[file_name] = (file_path, file_name)
logger.write("\n", 2)
-
logger.write(src.printcolors.printcLabel(_("Actually do the package")), 2)
logger.write("\n", 2)
+ logger.write("\nfiles and directories to add:\n%s\n\n" % PP.pformat(d_files_to_add), 5)
res = 0
try:
logger.write(_("\n"), 1)
return 1
+ # case if no application, only package sat as 'sat package -t'
+ try:
+ app = runner.cfg.APPLICATION
+ except:
+ app = None
+
# unconditionaly remove the tmp_local_working_dir
- tmp_local_working_dir = os.path.join(runner.cfg.APPLICATION.workdir, "tmp_package")
- if os.path.isdir(tmp_local_working_dir):
- shutil.rmtree(tmp_local_working_dir)
+ if app is not None:
+ tmp_local_working_dir = os.path.join(app.workdir, "tmp_package")
+ if os.path.isdir(tmp_local_working_dir):
+ shutil.rmtree(tmp_local_working_dir)
# have to decide some time
DBG.tofix("make shutil.rmtree('%s') effective" % tmp_working_dir, "", DBG.isDeveloper())
except NameError:
pass
+# Python 2/3 compatibility for execfile function
+try:
+ execfile
+except:
+ def execfile(somefile, global_vars, local_vars):
+ with open(somefile) as f:
+ code = compile(f.read(), somefile, 'exec')
+ exec(code, global_vars, local_vars)
+
parser = src.options.Options()
parser.add_option('n', 'name', 'string', 'name',
_("""REQUIRED: the name of the module to create.
return len(val) > 0 and self.check_method(val)
def get_dico_param(dico, key, default):
- if dico.has_key(key):
+ if key in dico:
return dico[key]
return default
# check required parameters in template.info
missing = []
for pp in ["file_subst", "parameters"]:
- if not ldic.has_key(pp): missing.append("'%s'" % pp)
+ if not (pp in ldic): missing.append("'%s'" % pp)
if len(missing) > 0:
raise src.SatException(_(
"Bad format in settings file! %s not defined.") % ", ".join(
# ask user for values
for p in self.parameters:
tp = TParam(p, self.compo_name, dico)
- if dico.has_key(tp.name):
+ if tp.name in dico:
continue
val = ""
while not tp.check_value(val):
- val = raw_input(tp.prompt)
+ val = input(tp.prompt)
if len(val) == 0 and len(tp.default) > 0:
val = tp.default
dico[tp.name] = val
# ask for missing value for pyconf
pyconfparam = self.get_pyconf_parameters()
- for p in filter(lambda l: not dico.has_key(l), pyconfparam):
+ for p in filter(lambda l: not (l in dico), pyconfparam):
rep = ""
while len(rep) == 0:
- rep = raw_input("%s? " % p)
+ rep = input("%s? " % p)
dico[p] = rep
self.dico = dico
import datetime
import gzip
+# Compatibility python 2/3 for input function
+# input stays input for python 3 and input = raw_input for python 2
+try:
+ input = raw_input
+except NameError:
+ pass
+
+verbose = False
+
try:
from hashlib import sha1
except ImportError:
# Define all possible option for the test command : sat test <options>
parser = src.options.Options()
parser.add_option('b', 'base', 'string', 'base',
- _("Optional: Indicate the name of the test base to use.\n\tThis name has to"
- " be registered in your application and in a project.\n\tA path to a "
- "test base can also be used."))
+ _("""Optional: The name of the test base to use."
+ This name has to be registered in your application and in a project.
+ A path to a test base can also be used."""))
parser.add_option('l', 'launcher', 'string', 'launcher',
- _("Optional: Use this option to specify the path to a SALOME launcher to "
- "use to launch the test scripts of the test base."))
+ _("""Optional: Specify the path to a SALOME launcher
+ used to launch the test scripts of the test base."""))
parser.add_option('g', 'grid', 'list', 'grids',
- _('Optional: Indicate which grid(s) to test (subdirectory of the test '
- 'base).'))
-parser.add_option('s', 'session', 'list', 'sessions',
- _('Optional: indicate which session(s) to test (subdirectory of the '
- 'grid).'))
+ _('Optional: Which grid(s) to test (subdirectory of the test base).'))
+parser.add_option('s', 'session', 'list2', 'sessions',
+ _('Optional: Which session(s) to test (subdirectory of the grid).'))
parser.add_option('', 'display', 'string', 'display',
- _("Optional: set the display where to launch SALOME.\n"
-"\tIf value is NO then option --show-desktop=0 will be used to launch SALOME."))
-
+ _("""Optional: Set the display where to launch SALOME.
+ If value is NO then option --show-desktop=0 will be used to launch SALOME."""))
+parser.add_option('', 'keep', 'boolean', 'keeptempdir',
+ _('Optional: keep temporary big tests directories.'))
def description():
'''method that is called when salomeTools is called with --help option.
return _("The test command runs a test base on a SALOME installation.\n\n"
"example:\nsat test SALOME-master --grid GEOM --session light")
-def parse_option(args, config):
+def parse_option_old(args, config):
""" Parse the options and do some verifications about it
:param args List: The list of arguments of the command
options.launcher = ""
elif not os.path.isabs(options.launcher):
if not src.config_has_application(config):
- raise src.SatException(_("An application is required to use a "
- "relative path with option --appli"))
+ msg = _("An application is required to use a relative path with option --appli")
+ raise src.SatException(msg)
options.launcher = os.path.join(config.APPLICATION.workdir,
options.launcher)
return (options, args)
+
+def parse_option(args, config):
+ """ Parse the options and do some verifications about it
+
+ :param args List: The list of arguments of the command
+ :param config Config: The global configuration
+ :return: the options of the current command launch and the full arguments
+ :rtype: Tuple (options, args)
+ """
+ (options, args) = parser.parse_args(args)
+
+ if not options.launcher:
+ options.launcher = ""
+ return (options, args)
+
+ if not os.path.isabs(options.launcher):
+ if not src.config_has_application(config):
+ msg = _("An application is required to use a relative path with option --appli")
+ raise src.SatException(msg)
+ else:
+ options.launcher = os.path.join(config.APPLICATION.workdir, options.launcher)
+ if not os.path.exists(options.launcher):
+ raise src.SatException(_("Launcher not found: %s") % options.launcher)
+
+ # absolute path
+ launcher = os.path.realpath(os.path.expandvars(options.launcher))
+ if os.path.exists(launcher):
+ options.launcher = launcher
+ return (options, args)
+
+ raise src.SatException(_("Launcher not found: %s") % options.launcher)
+
+
def ask_a_path():
"""
"""
- path = raw_input("enter a path where to save the result: ")
+ path = input("enter a path where to save the result: ")
if path == "":
- result = raw_input("the result will be not save. Are you sure to "
+ result = input("the result will be not save. Are you sure to "
"continue ? [y/n] ")
if result == "y":
return path
return ask_a_path()
elif os.path.exists(path):
- result = raw_input("Warning, the content of %s will be deleted. Are you"
+ result = input("Warning, the content of %s will be deleted. Are you"
" sure to continue ? [y/n] " % path)
if result == "y":
return path
os.makedirs(outtestbase)
#logger.write(" copy testbase %s\n" % testbase, 5)
- for grid_ in [m for m in os.listdir(intestbase) if os.path.isdir(
- os.path.join(intestbase, m))]:
+ for grid_ in [m for m in os.listdir(intestbase) \
+ if os.path.isdir(os.path.join(intestbase, m))]:
# ignore source configuration directories
if grid_[:4] == '.git' or grid_ == 'CVS':
continue
else:
logger.write(src.printcolors.printcSuccess(src.OK_STATUS) + "\n\n", 4)
+def findOrCreateNode(parentNode, nameNodeToFind):
+ found = parentNode.find(nameNodeToFind)
+ if found is None:
+ created = add_simple_node(parentNode, nameNodeToFind)
+ return created
+ else:
+ return found
+
+def purgeEmptyNodes(root):
+ """
+ recursive remove node.text and node.tail if empty node
+ as nothing else than whitespace(s) and RCLF(s)
+
+ | this is comes from
+ | 1) pretty print file xml -> creates indentation(s) in text and tail
+ | 2) and reload parse file xml
+ """
+ # print("root", root.tag, root.text)
+ text = root.text
+ tail = root.tail
+ if text is not None:
+ if text.replace(" ", "").replace("\n", "") == "":
+ # print("purgeEmptyNodes text %s" % root.tag)
+ root.text = None
+ if tail is not None:
+ if tail.replace(" ", "").replace("\n", "") == "":
+ # print("purgeEmptyNodes tail %s" % root.tag)
+ root.tail = None
+ for node in root:
+ purgeEmptyNodes(node)
+ return
+
##
# Creates the XML report for a product.
def create_test_report(config,
first_time = False
if not os.path.exists(xml_history_path):
+ print("Log file creation %s" % xml_history_path)
first_time = True
root = etree.Element("salome")
prod_node = etree.Element("product", name=application_name, build=xmlname)
root.append(prod_node)
else:
+ print("Log file modification %s" % xml_history_path)
root = etree.parse(xml_history_path).getroot()
+ purgeEmptyNodes(root)
prod_node = root.find("product")
-
+
+
prod_node.attrib["history_file"] = os.path.basename(xml_history_path)
- prod_node.attrib["global_res"] = retcode
+ prod_node.attrib["global_res"] = str(retcode)
- # OP 14/11/2017 Ajout de traces pour essayer de decouvrir le pb
- # de remontee de log des tests
- #print "TRACES OP - test.py/create_test_report() : xml_history_path = '#%s#'" %xml_history_path
-
if withappli:
if not first_time:
for node in (prod_node.findall("version_to_download") +
prod_node.findall("out_dir")):
prod_node.remove(node)
- add_simple_node(prod_node, "version_to_download",
- config.APPLICATION.name)
-
+ add_simple_node(prod_node, "version_to_download", config.APPLICATION.name)
add_simple_node(prod_node, "out_dir", config.APPLICATION.workdir)
# add environment
if not first_time:
for node in prod_node.findall("exec"):
- prod_node.remove(node)
+ prod_node.remove(node)
exec_node = add_simple_node(prod_node, "exec")
exec_node.append(etree.Element("env", name="Host", value=config.VARS.node))
- exec_node.append(etree.Element("env", name="Architecture",
- value=config.VARS.dist))
- exec_node.append(etree.Element("env", name="Number of processors",
- value=str(config.VARS.nb_proc)))
- exec_node.append(etree.Element("env", name="Begin date",
- value=src.parse_date(date_hour)))
- exec_node.append(etree.Element("env", name="Command",
- value=config.VARS.command))
- exec_node.append(etree.Element("env", name="sat version",
- value=config.INTERNAL.sat_version))
+ exec_node.append(etree.Element("env", name="Architecture", value=config.VARS.dist))
+ exec_node.append(etree.Element("env", name="Number of processors", value=str(config.VARS.nb_proc)))
+ exec_node.append(etree.Element("env", name="Begin date", value=src.parse_date(date_hour)))
+ exec_node.append(etree.Element("env", name="Command", value=config.VARS.command))
+ exec_node.append(etree.Element("env", name="sat version", value=config.INTERNAL.sat_version))
if 'TESTS' in config:
- if first_time:
- tests = add_simple_node(prod_node, "tests")
- known_errors = add_simple_node(prod_node, "known_errors")
- new_errors = add_simple_node(prod_node, "new_errors")
- amend = add_simple_node(prod_node, "amend")
- else:
- tests = prod_node.find("tests")
- known_errors = prod_node.find("known_errors")
- new_errors = prod_node.find("new_errors")
- amend = prod_node.find("amend")
+ tests = findOrCreateNode(prod_node, "tests")
+ known_errors = findOrCreateNode(prod_node, "known_errors")
+ new_errors = findOrCreateNode(prod_node, "new_errors")
+ amend = findOrCreateNode(prod_node, "amend")
tt = {}
for test in config.TESTS:
- if not tt.has_key(test.testbase):
+ if not test.testbase in tt:
tt[test.testbase] = [test]
else:
tt[test.testbase].append(test)
for testbase in tt.keys():
- if first_time:
- gn = add_simple_node(tests, "testbase")
- else:
- gn = tests.find("testbase")
- # initialize all grids and session to "not executed"
- for mn in gn.findall("grid"):
- mn.attrib["executed_last_time"] = "no"
- for tyn in mn.findall("session"):
- tyn.attrib["executed_last_time"] = "no"
- for test_node in tyn.findall('test'):
- for node in test_node.getchildren():
- if node.tag != "history":
- test_node.remove(node)
-
- attribs_to_pop = []
- for attribute in test_node.attrib:
- if (attribute != "script" and
- attribute != "res"):
- attribs_to_pop.append(attribute)
- for attribute in attribs_to_pop:
- test_node.attrib.pop(attribute)
+ if verbose: print("---- create_test_report %s %s" % (testbase, first_time))
+ gn = findOrCreateNode(tests, "testbase")
+
+ # initialize all grids and session to "not executed"
+ for mn in gn.findall("grid"):
+ mn.attrib["executed_last_time"] = "no"
+ for tyn in mn.findall("session"):
+ tyn.attrib["executed_last_time"] = "no"
+ for test_node in tyn.findall('test'):
+ for node in test_node.getchildren():
+ if node.tag != "history":
+ test_node.remove(node)
+
+ attribs_to_pop = []
+ for attribute in test_node.attrib:
+ if (attribute != "script" and
+ attribute != "res"):
+ attribs_to_pop.append(attribute)
+ for attribute in attribs_to_pop:
+ test_node.attrib.pop(attribute)
gn.attrib['name'] = testbase
nb, nb_pass, nb_failed, nb_timeout, nb_not_run = 0, 0, 0, 0, 0
grids = {}
sessions = {}
for test in tt[testbase]:
- if not grids.has_key(test.grid):
+ if not (test.grid in grids):
if first_time:
mn = add_simple_node(gn, "grid")
mn.attrib['name'] = test.grid
mn.attrib["executed_last_time"] = "yes"
- if not sessions.has_key("%s/%s" % (test.grid, test.session)):
+ if not "%s/%s" % (test.grid, test.session) in sessions:
if first_time:
tyn = add_simple_node(mn, "session")
tyn.attrib['name'] = test.session
if not xmlname.endswith(".xml"):
xmlname += ".xml"
- src.xmlManager.write_report(os.path.join(dest_path, xmlname),
- root,
- "test.xsl")
- src.xmlManager.write_report(xml_history_path,
- root,
- "test_history.xsl")
+ src.xmlManager.write_report(os.path.join(dest_path, xmlname), root, "test.xsl")
+ src.xmlManager.write_report(xml_history_path, root, "test_history.xsl")
return src.OK_STATUS
def generate_history_xml_path(config, test_base):
content = "\n".join(lines)
# create hash from context information
- dirname = sha1(content.encode()).hexdigest()
+ # CVW TODO or not dirname = datetime.datetime.now().strftime("%y%m%d_%H%M%S_") + sha1(content.encode()).hexdigest()[0:8]
+ dirname = sha1(content.encode()).hexdigest()[0:8] # only 8 firsts probably good
base_dir = os.path.join(tmp_dir, dirname)
os.makedirs(base_dir)
os.environ['TT_TMP_RESULT'] = base_dir
# create env_info file
- f = open(os.path.join(base_dir, 'env_info.py'), "w")
- f.write(content)
- f.close()
+ with open(os.path.join(base_dir, 'env_info.py'), "w") as f:
+ f.write(content)
# create working dir and bases dir
working_dir = os.path.join(base_dir, 'WORK')
log_dir = src.get_log_path(runner.cfg)
out_dir = os.path.join(log_dir, "TEST")
src.ensure_path_exists(out_dir)
- name_xml_board = logger.logFileName.split(".")[0] + "board" + ".xml"
+ name_xml_board = logger.logFileName.split(".")[0] + "_board.xml"
historic_xml_path = generate_history_xml_path(runner.cfg, test_base)
create_test_report(runner.cfg,
xmlname = name_xml_board)
xml_board_path = os.path.join(out_dir, name_xml_board)
- # OP 14/11/2017 Ajout de traces pour essayer de decouvrir le pb
- # de remontee de log des tests
- #print "TRACES OP - test.py/run() : historic_xml_path = '#%s#'" %historic_xml_path
- #print "TRACES OP - test.py/run() : log_dir = '#%s#'" %log_dir
- #print "TRACES OP - test.py/run() : name_xml_board = '#%s#'" %name_xml_board
-
logger.l_logFiles.append(xml_board_path)
logger.add_link(os.path.join("TEST", name_xml_board),
"board",
retcode,
"Click on the link to get the detailed test results")
-
+ logger.write("\nTests board file %s\n" % xml_board_path, 1)
+
# Add the historic files into the log files list of the command
logger.l_logFiles.append(historic_xml_path)
-
- logger.write(_("Removing the temporary directory: "
- "rm -rf %s\n" % test_runner.tmp_working_dir), 5)
- if os.path.exists(test_runner.tmp_working_dir):
+
+ if not options.keeptempdir:
+ logger.write("Removing the temporary directory: rm -rf %s\n" % test_runner.tmp_working_dir, 5)
+ if os.path.exists(test_runner.tmp_working_dir):
shutil.rmtree(test_runner.tmp_working_dir)
+ else:
+ logger.write("NOT Removing the temporary directory: rm -rf %s\n" % test_runner.tmp_working_dir, 5)
return retcode
global moduleDesktop, currentDesktop, objectsManager
- if not moduleDesktop.has_key( studyID ):
+ if not studyID in moduleDesktop:
moduleDesktop[studyID] = :sat:{PYCMP}Desktop( sgPyQt, sg )
objectsManager = Controller( moduleDesktop[studyID] )
moduleDesktop[studyID].setController( objectsManager )
def OnGUIEvent( commandID ):
"""This method is called when a GUI action is activated"""
- if dict_command.has_key( commandID ):
+ if commandID in dict_command:
dict_command[commandID]()
pass
pass
logger = LOG.getDefaultLogger()
+DBG.write("Python version", sys.version, DBG.isDeveloper())
+
#################################
# MAIN
#################################
--- /dev/null
+#
+# ElementTree
+# $Id: ElementPath.py 3375 2008-02-13 08:05:08Z fredrik $
+#
+# limited xpath support for element trees
+#
+# history:
+# 2003-05-23 fl created
+# 2003-05-28 fl added support for // etc
+# 2003-08-27 fl fixed parsing of periods in element names
+# 2007-09-10 fl new selection engine
+# 2007-09-12 fl fixed parent selector
+# 2007-09-13 fl added iterfind; changed findall to return a list
+# 2007-11-30 fl added namespaces support
+# 2009-10-30 fl added child element value filter
+#
+# Copyright (c) 2003-2009 by Fredrik Lundh. All rights reserved.
+#
+# fredrik@pythonware.com
+# http://www.pythonware.com
+#
+# --------------------------------------------------------------------
+# The ElementTree toolkit is
+#
+# Copyright (c) 1999-2009 by Fredrik Lundh
+#
+# By obtaining, using, and/or copying this software and/or its
+# associated documentation, you agree that you have read, understood,
+# and will comply with the following terms and conditions:
+#
+# Permission to use, copy, modify, and distribute this software and
+# its associated documentation for any purpose and without fee is
+# hereby granted, provided that the above copyright notice appears in
+# all copies, and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# Secret Labs AB or the author not be used in advertising or publicity
+# pertaining to distribution of the software without specific, written
+# prior permission.
+#
+# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
+# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
+# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
+# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
+# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+# OF THIS SOFTWARE.
+# --------------------------------------------------------------------
+
+# Licensed to PSF under a Contributor Agreement.
+# See http://www.python.org/psf/license for licensing details.
+
+##
+# Implementation module for XPath support. There's usually no reason
+# to import this module directly; the <b>ElementTree</b> does this for
+# you, if needed.
+##
+
+import re
+
+xpath_tokenizer_re = re.compile(
+ "("
+ "'[^']*'|\"[^\"]*\"|"
+ "::|"
+ "//?|"
+ "\.\.|"
+ "\(\)|"
+ "[/.*:\[\]\(\)@=])|"
+ "((?:\{[^}]+\})?[^/\[\]\(\)@=\s]+)|"
+ "\s+"
+ )
+
+def xpath_tokenizer(pattern, namespaces=None):
+ for token in xpath_tokenizer_re.findall(pattern):
+ tag = token[1]
+ if tag and tag[0] != "{" and ":" in tag:
+ try:
+ prefix, uri = tag.split(":", 1)
+ if not namespaces:
+ raise KeyError
+ yield token[0], "{%s}%s" % (namespaces[prefix], uri)
+ except KeyError:
+ raise SyntaxError("prefix %r not found in prefix map" % prefix)
+ else:
+ yield token
+
+def get_parent_map(context):
+ parent_map = context.parent_map
+ if parent_map is None:
+ context.parent_map = parent_map = {}
+ for p in context.root.iter():
+ for e in p:
+ parent_map[e] = p
+ return parent_map
+
+def prepare_child(next, token):
+ tag = token[1]
+ def select(context, result):
+ for elem in result:
+ for e in elem:
+ if e.tag == tag:
+ yield e
+ return select
+
+def prepare_star(next, token):
+ def select(context, result):
+ for elem in result:
+ yield from elem
+ return select
+
+def prepare_self(next, token):
+ def select(context, result):
+ yield from result
+ return select
+
+def prepare_descendant(next, token):
+ token = next()
+ if token[0] == "*":
+ tag = "*"
+ elif not token[0]:
+ tag = token[1]
+ else:
+ raise SyntaxError("invalid descendant")
+ def select(context, result):
+ for elem in result:
+ for e in elem.iter(tag):
+ if e is not elem:
+ yield e
+ return select
+
+def prepare_parent(next, token):
+ def select(context, result):
+ # FIXME: raise error if .. is applied at toplevel?
+ parent_map = get_parent_map(context)
+ result_map = {}
+ for elem in result:
+ if elem in parent_map:
+ parent = parent_map[elem]
+ if parent not in result_map:
+ result_map[parent] = None
+ yield parent
+ return select
+
+def prepare_predicate(next, token):
+ # FIXME: replace with real parser!!! refs:
+ # http://effbot.org/zone/simple-iterator-parser.htm
+ # http://javascript.crockford.com/tdop/tdop.html
+ signature = []
+ predicate = []
+ while 1:
+ token = next()
+ if token[0] == "]":
+ break
+ if token[0] and token[0][:1] in "'\"":
+ token = "'", token[0][1:-1]
+ signature.append(token[0] or "-")
+ predicate.append(token[1])
+ signature = "".join(signature)
+ # use signature to determine predicate type
+ if signature == "@-":
+ # [@attribute] predicate
+ key = predicate[1]
+ def select(context, result):
+ for elem in result:
+ if elem.get(key) is not None:
+ yield elem
+ return select
+ if signature == "@-='":
+ # [@attribute='value']
+ key = predicate[1]
+ value = predicate[-1]
+ def select(context, result):
+ for elem in result:
+ if elem.get(key) == value:
+ yield elem
+ return select
+ if signature == "-" and not re.match("\-?\d+$", predicate[0]):
+ # [tag]
+ tag = predicate[0]
+ def select(context, result):
+ for elem in result:
+ if elem.find(tag) is not None:
+ yield elem
+ return select
+ if signature == "-='" and not re.match("\-?\d+$", predicate[0]):
+ # [tag='value']
+ tag = predicate[0]
+ value = predicate[-1]
+ def select(context, result):
+ for elem in result:
+ for e in elem.findall(tag):
+ if "".join(e.itertext()) == value:
+ yield elem
+ break
+ return select
+ if signature == "-" or signature == "-()" or signature == "-()-":
+ # [index] or [last()] or [last()-index]
+ if signature == "-":
+ # [index]
+ index = int(predicate[0]) - 1
+ if index < 0:
+ raise SyntaxError("XPath position >= 1 expected")
+ else:
+ if predicate[0] != "last":
+ raise SyntaxError("unsupported function")
+ if signature == "-()-":
+ try:
+ index = int(predicate[2]) - 1
+ except ValueError:
+ raise SyntaxError("unsupported expression")
+ if index > -2:
+ raise SyntaxError("XPath offset from last() must be negative")
+ else:
+ index = -1
+ def select(context, result):
+ parent_map = get_parent_map(context)
+ for elem in result:
+ try:
+ parent = parent_map[elem]
+ # FIXME: what if the selector is "*" ?
+ elems = list(parent.findall(elem.tag))
+ if elems[index] is elem:
+ yield elem
+ except (IndexError, KeyError):
+ pass
+ return select
+ raise SyntaxError("invalid predicate")
+
+ops = {
+ "": prepare_child,
+ "*": prepare_star,
+ ".": prepare_self,
+ "..": prepare_parent,
+ "//": prepare_descendant,
+ "[": prepare_predicate,
+ }
+
+_cache = {}
+
+class _SelectorContext:
+ parent_map = None
+ def __init__(self, root):
+ self.root = root
+
+# --------------------------------------------------------------------
+
+##
+# Generate all matching objects.
+
+def iterfind(elem, path, namespaces=None):
+ # compile selector pattern
+ cache_key = (path, None if namespaces is None
+ else tuple(sorted(namespaces.items())))
+ if path[-1:] == "/":
+ path = path + "*" # implicit all (FIXME: keep this?)
+ try:
+ selector = _cache[cache_key]
+ except KeyError:
+ if len(_cache) > 100:
+ _cache.clear()
+ if path[:1] == "/":
+ raise SyntaxError("cannot use absolute path on element")
+ next = iter(xpath_tokenizer(path, namespaces)).__next__
+ token = next()
+ selector = []
+ while 1:
+ try:
+ selector.append(ops[token[0]](next, token))
+ except StopIteration:
+ raise SyntaxError("invalid path")
+ try:
+ token = next()
+ if token[0] == "/":
+ token = next()
+ except StopIteration:
+ break
+ _cache[cache_key] = selector
+ # execute selector pattern
+ result = [elem]
+ context = _SelectorContext(elem)
+ for select in selector:
+ result = select(context, result)
+ return result
+
+##
+# Find first matching object.
+
+def find(elem, path, namespaces=None):
+ try:
+ return next(iterfind(elem, path, namespaces))
+ except StopIteration:
+ return None
+
+##
+# Find all matching objects.
+
+def findall(elem, path, namespaces=None):
+ return list(iterfind(elem, path, namespaces))
+
+##
+# Find text for first matching object.
+
+def findtext(elem, path, default=None, namespaces=None):
+ try:
+ elem = next(iterfind(elem, path, namespaces))
+ return elem.text or ""
+ except StopIteration:
+ return default
\ No newline at end of file
-#
-# ElementTree
-# $Id: ElementTree.py 2326 2005-03-17 07:45:21Z fredrik $
-#
-# light-weight XML support for Python 1.5.2 and later.
-#
-# history:
-# 2001-10-20 fl created (from various sources)
-# 2001-11-01 fl return root from parse method
-# 2002-02-16 fl sort attributes in lexical order
-# 2002-04-06 fl TreeBuilder refactoring, added PythonDoc markup
-# 2002-05-01 fl finished TreeBuilder refactoring
-# 2002-07-14 fl added basic namespace support to ElementTree.write
-# 2002-07-25 fl added QName attribute support
-# 2002-10-20 fl fixed encoding in write
-# 2002-11-24 fl changed default encoding to ascii; fixed attribute encoding
-# 2002-11-27 fl accept file objects or file names for parse/write
-# 2002-12-04 fl moved XMLTreeBuilder back to this module
-# 2003-01-11 fl fixed entity encoding glitch for us-ascii
-# 2003-02-13 fl added XML literal factory
-# 2003-02-21 fl added ProcessingInstruction/PI factory
-# 2003-05-11 fl added tostring/fromstring helpers
-# 2003-05-26 fl added ElementPath support
-# 2003-07-05 fl added makeelement factory method
-# 2003-07-28 fl added more well-known namespace prefixes
-# 2003-08-15 fl fixed typo in ElementTree.findtext (Thomas Dartsch)
-# 2003-09-04 fl fall back on emulator if ElementPath is not installed
-# 2003-10-31 fl markup updates
-# 2003-11-15 fl fixed nested namespace bug
-# 2004-03-28 fl added XMLID helper
-# 2004-06-02 fl added default support to findtext
-# 2004-06-08 fl fixed encoding of non-ascii element/attribute names
-# 2004-08-23 fl take advantage of post-2.1 expat features
-# 2005-02-01 fl added iterparse implementation
-# 2005-03-02 fl fixed iterparse support for pre-2.2 versions
-#
-# Copyright (c) 1999-2005 by Fredrik Lundh. All rights reserved.
-#
-# fredrik@pythonware.com
-# http://www.pythonware.com
-#
-# --------------------------------------------------------------------
-# The ElementTree toolkit is
-#
-# Copyright (c) 1999-2005 by Fredrik Lundh
-#
-# By obtaining, using, and/or copying this software and/or its
-# associated documentation, you agree that you have read, understood,
-# and will comply with the following terms and conditions:
-#
-# Permission to use, copy, modify, and distribute this software and
-# its associated documentation for any purpose and without fee is
-# hereby granted, provided that the above copyright notice appears in
-# all copies, and that both that copyright notice and this permission
-# notice appear in supporting documentation, and that the name of
-# Secret Labs AB or the author not be used in advertising or publicity
-# pertaining to distribution of the software without specific, written
-# prior permission.
-#
-# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
-# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
-# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
-# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
-# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
-# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
-# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
-# OF THIS SOFTWARE.
-# --------------------------------------------------------------------
-__all__ = [
- # public symbols
- "Comment",
- "dump",
- "Element", "ElementTree",
- "fromstring",
- "iselement", "iterparse",
- "parse",
- "PI", "ProcessingInstruction",
- "QName",
- "SubElement",
- "tostring",
- "TreeBuilder",
- "VERSION", "XML",
- "XMLTreeBuilder",
- ]
-##
-# The <b>Element</b> type is a flexible container object, designed to
-# store hierarchical data structures in memory. The type can be
-# described as a cross between a list and a dictionary.
-# <p>
-# Each element has a number of properties associated with it:
-# <ul>
-# <li>a <i>tag</i>. This is a string identifying what kind of data
-# this element represents (the element type, in other words).</li>
-# <li>a number of <i>attributes</i>, stored in a Python dictionary.</li>
-# <li>a <i>text</i> string.</li>
-# <li>an optional <i>tail</i> string.</li>
-# <li>a number of <i>child elements</i>, stored in a Python sequence</li>
-# </ul>
-#
-# To create an element instance, use the {@link #Element} or {@link
-# #SubElement} factory functions.
-# <p>
-# The {@link #ElementTree} class can be used to wrap an element
-# structure, and convert it from and to XML.
-##
+"""
+using VERSION 1.3.0 native xml.etree.ElementTree for python3
+appending method tostring serialize 'pretty_xml'
+"""
-import string, sys, re, platform
+import sys
+import debug as DBG
-class _SimpleElementPath:
- # emulate pre-1.2 find/findtext/findall behaviour
- def find(self, element, tag):
- for elem in element:
- if elem.tag == tag:
- return elem
- return None
- def findtext(self, element, tag, default=None):
- for elem in element:
- if elem.tag == tag:
- return elem.text or ""
- return default
- def findall(self, element, tag):
- if tag[:3] == ".//":
- return element.getiterator(tag[3:])
- result = []
- for elem in element:
- if elem.tag == tag:
- result.append(elem)
- return result
+_versionPython = sys.version_info[0]
-try:
- import ElementPath
-except ImportError:
- # FIXME: issue warning in this case?
- ElementPath = _SimpleElementPath()
+if _versionPython < 3:
+ # python2 previous historic mode
+ import src.ElementTreePython2 as etree
+ DBG.write("ElementTree Python2", etree.__file__, DBG.isDeveloper())
+ tostring = etree.tostring
-# TODO: add support for custom namespace resolvers/default namespaces
-# TODO: add improved support for incremental parsing
-
-VERSION = "1.2.6"
-
-##
-# Internal element class. This class defines the Element interface,
-# and provides a reference implementation of this interface.
-# <p>
-# You should not create instances of this class directly. Use the
-# appropriate factory functions instead, such as {@link #Element}
-# and {@link #SubElement}.
-#
-# @see Element
-# @see SubElement
-# @see Comment
-# @see ProcessingInstruction
-
-class _ElementInterface:
- # <tag attrib>text<child/>...</tag>tail
-
- ##
- # (Attribute) Element tag.
-
- tag = None
-
- ##
- # (Attribute) Element attribute dictionary. Where possible, use
- # {@link #_ElementInterface.get},
- # {@link #_ElementInterface.set},
- # {@link #_ElementInterface.keys}, and
- # {@link #_ElementInterface.items} to access
- # element attributes.
-
- attrib = None
-
- ##
- # (Attribute) Text before first subelement. This is either a
- # string or the value None, if there was no text.
-
- text = None
-
- ##
- # (Attribute) Text after this element's end tag, but before the
- # next sibling element's start tag. This is either a string or
- # the value None, if there was no text.
-
- tail = None # text after end tag, if any
-
- def __init__(self, tag, attrib):
- self.tag = tag
- self.attrib = attrib
- self._children = []
-
- def __repr__(self):
- return "<Element %s at %x>" % (self.tag, id(self))
-
- ##
- # Creates a new element object of the same type as this element.
- #
- # @param tag Element tag.
- # @param attrib Element attributes, given as a dictionary.
- # @return A new element instance.
-
- def makeelement(self, tag, attrib):
- return Element(tag, attrib)
-
- ##
- # Returns the number of subelements.
- #
- # @return The number of subelements.
-
- def __len__(self):
- return len(self._children)
-
- ##
- # Returns the given subelement.
- #
- # @param index What subelement to return.
- # @return The given subelement.
- # @exception IndexError If the given element does not exist.
-
- def __getitem__(self, index):
- return self._children[index]
-
- ##
- # Replaces the given subelement.
- #
- # @param index What subelement to replace.
- # @param element The new element value.
- # @exception IndexError If the given element does not exist.
- # @exception AssertionError If element is not a valid object.
-
- def __setitem__(self, index, element):
- assert iselement(element)
- self._children[index] = element
-
- ##
- # Deletes the given subelement.
- #
- # @param index What subelement to delete.
- # @exception IndexError If the given element does not exist.
-
- def __delitem__(self, index):
- del self._children[index]
-
- ##
- # Returns a list containing subelements in the given range.
- #
- # @param start The first subelement to return.
- # @param stop The first subelement that shouldn't be returned.
- # @return A sequence object containing subelements.
-
- def __getslice__(self, start, stop):
- return self._children[start:stop]
-
- ##
- # Replaces a number of subelements with elements from a sequence.
- #
- # @param start The first subelement to replace.
- # @param stop The first subelement that shouldn't be replaced.
- # @param elements A sequence object with zero or more elements.
- # @exception AssertionError If a sequence member is not a valid object.
-
- def __setslice__(self, start, stop, elements):
- for element in elements:
- assert iselement(element)
- self._children[start:stop] = list(elements)
-
- ##
- # Deletes a number of subelements.
- #
- # @param start The first subelement to delete.
- # @param stop The first subelement to leave in there.
-
- def __delslice__(self, start, stop):
- del self._children[start:stop]
-
- ##
- # Adds a subelement to the end of this element.
- #
- # @param element The element to add.
- # @exception AssertionError If a sequence member is not a valid object.
-
- def append(self, element):
- assert iselement(element)
- self._children.append(element)
-
- ##
- # Inserts a subelement at the given position in this element.
- #
- # @param index Where to insert the new subelement.
- # @exception AssertionError If the element is not a valid object.
-
- def insert(self, index, element):
- assert iselement(element)
- self._children.insert(index, element)
-
- ##
- # Removes a matching subelement. Unlike the <b>find</b> methods,
- # this method compares elements based on identity, not on tag
- # value or contents.
- #
- # @param element What element to remove.
- # @exception ValueError If a matching element could not be found.
- # @exception AssertionError If the element is not a valid object.
-
- def remove(self, element):
- assert iselement(element)
- self._children.remove(element)
-
- ##
- # Returns all subelements. The elements are returned in document
- # order.
- #
- # @return A list of subelements.
- # @defreturn list of Element instances
-
- def getchildren(self):
- return self._children
-
- ##
- # Finds the first matching subelement, by tag name or path.
- #
- # @param path What element to look for.
- # @return The first matching element, or None if no element was found.
- # @defreturn Element or None
-
- def find(self, path):
- if ElementPath.find(self, path) == None:
- return ElementPath.find(self, path.encode())
- return ElementPath.find(self, path)
-
- ##
- # Finds text for the first matching subelement, by tag name or path.
- #
- # @param path What element to look for.
- # @param default What to return if the element was not found.
- # @return The text content of the first matching element, or the
- # default value no element was found. Note that if the element
- # has is found, but has no text content, this method returns an
- # empty string.
- # @defreturn string
-
- def findtext(self, path, default=None):
- return ElementPath.findtext(self, path, default)
-
- ##
- # Finds all matching subelements, by tag name or path.
- #
- # @param path What element to look for.
- # @return A list or iterator containing all matching elements,
- # in document order.
- # @defreturn list of Element instances
-
- def findall(self, path):
- return ElementPath.findall(self, path)
-
- ##
- # Resets an element. This function removes all subelements, clears
- # all attributes, and sets the text and tail attributes to None.
-
- def clear(self):
- self.attrib.clear()
- self._children = []
- self.text = self.tail = None
-
- ##
- # Gets an element attribute.
- #
- # @param key What attribute to look for.
- # @param default What to return if the attribute was not found.
- # @return The attribute value, or the default value, if the
- # attribute was not found.
- # @defreturn string or None
-
- def get(self, key, default=None):
- res = self.attrib.get(key, default)
- if not res:
- res = self.attrib.get(key.encode(), default)
- if isinstance(res, bytes):
- return res.decode()
- else:
- return res
-
- ##
- # Sets an element attribute.
- #
- # @param key What attribute to set.
- # @param value The attribute value.
-
- def set(self, key, value):
- self.attrib[key] = value
-
- ##
- # Gets a list of attribute names. The names are returned in an
- # arbitrary order (just like for an ordinary Python dictionary).
- #
- # @return A list of element attribute names.
- # @defreturn list of strings
-
- def keys(self):
- res = []
- for key in self.attrib.keys():
- if isinstance(key, bytes):
- res.append(key.decode())
- else:
- res.append(key)
- return res
-
- ##
- # Gets element attributes, as a sequence. The attributes are
- # returned in an arbitrary order.
- #
- # @return A list of (name, value) tuples for all attributes.
- # @defreturn list of (string, string) tuples
-
- def items(self):
- return self.attrib.items()
-
- ##
- # Creates a tree iterator. The iterator loops over this element
- # and all subelements, in document order, and returns all elements
- # with a matching tag.
- # <p>
- # If the tree structure is modified during iteration, the result
- # is undefined.
- #
- # @param tag What tags to look for (default is to return all elements).
- # @return A list or iterator containing all the matching elements.
- # @defreturn list or iterator
-
- def getiterator(self, tag=None):
- nodes = []
- if tag == "*":
- tag = None
- if tag is None or self.tag == tag:
- nodes.append(self)
- for node in self._children:
- nodes.extend(node.getiterator(tag))
- return nodes
-
-# compatibility
-_Element = _ElementInterface
-
-##
-# Element factory. This function returns an object implementing the
-# standard Element interface. The exact class or type of that object
-# is implementation dependent, but it will always be compatible with
-# the {@link #_ElementInterface} class in this module.
-# <p>
-# The element name, attribute names, and attribute values can be
-# either 8-bit ASCII strings or Unicode strings.
-#
-# @param tag The element name.
-# @param attrib An optional dictionary, containing element attributes.
-# @param **extra Additional attributes, given as keyword arguments.
-# @return An element instance.
-# @defreturn Element
-
-def Element(tag, attrib={}, **extra):
- attrib = attrib.copy()
- attrib.update(extra)
- return _ElementInterface(tag, attrib)
-
-##
-# Subelement factory. This function creates an element instance, and
-# appends it to an existing element.
-# <p>
-# The element name, attribute names, and attribute values can be
-# either 8-bit ASCII strings or Unicode strings.
-#
-# @param parent The parent element.
-# @param tag The subelement name.
-# @param attrib An optional dictionary, containing element attributes.
-# @param **extra Additional attributes, given as keyword arguments.
-# @return An element instance.
-# @defreturn Element
-
-def SubElement(parent, tag, attrib={}, **extra):
- attrib = attrib.copy()
- attrib.update(extra)
- element = parent.makeelement(tag, attrib)
- parent.append(element)
- return element
-
-##
-# Comment element factory. This factory function creates a special
-# element that will be serialized as an XML comment.
-# <p>
-# The comment string can be either an 8-bit ASCII string or a Unicode
-# string.
-#
-# @param text A string containing the comment string.
-# @return An element instance, representing a comment.
-# @defreturn Element
-
-def Comment(text=None):
- element = Element(Comment)
- element.text = text
- return element
-
-##
-# PI element factory. This factory function creates a special element
-# that will be serialized as an XML processing instruction.
-#
-# @param target A string containing the PI target.
-# @param text A string containing the PI contents, if any.
-# @return An element instance, representing a PI.
-# @defreturn Element
-
-def ProcessingInstruction(target, text=None):
- element = Element(ProcessingInstruction)
- element.text = target
- if text:
- element.text = element.text + " " + text
- return element
-
-PI = ProcessingInstruction
-
-##
-# QName wrapper. This can be used to wrap a QName attribute value, in
-# order to get proper namespace handling on output.
-#
-# @param text A string containing the QName value, in the form {uri}local,
-# or, if the tag argument is given, the URI part of a QName.
-# @param tag Optional tag. If given, the first argument is interpreted as
-# an URI, and this argument is interpreted as a local name.
-# @return An opaque object, representing the QName.
-
-class QName:
- def __init__(self, text_or_uri, tag=None):
- if tag:
- text_or_uri = "{%s}%s" % (text_or_uri, tag)
- self.text = text_or_uri
- def __str__(self):
- return self.text
- def __hash__(self):
- return hash(self.text)
- def __cmp__(self, other):
- if isinstance(other, QName):
- return cmp(self.text, other.text)
- return cmp(self.text, other)
-
-##
-# ElementTree wrapper class. This class represents an entire element
-# hierarchy, and adds some extra support for serialization to and from
-# standard XML.
-#
-# @param element Optional root element.
-# @keyparam file Optional file handle or name. If given, the
-# tree is initialized with the contents of this XML file.
-
-class ElementTree:
-
- def __init__(self, element=None, file=None):
- assert element is None or iselement(element)
- self._root = element # first node
- if file:
- self.parse(file)
-
- ##
- # Gets the root element for this tree.
- #
- # @return An element instance.
- # @defreturn Element
-
- def getroot(self):
- return self._root
-
- ##
- # Replaces the root element for this tree. This discards the
- # current contents of the tree, and replaces it with the given
- # element. Use with care.
- #
- # @param element An element instance.
-
- def _setroot(self, element):
- assert iselement(element)
- self._root = element
-
- ##
- # Loads an external XML document into this element tree.
- #
- # @param source A file name or file object.
- # @param parser An optional parser instance. If not given, the
- # standard {@link XMLTreeBuilder} parser is used.
- # @return The document root element.
- # @defreturn Element
-
- def parse(self, source, parser=None):
- if not hasattr(source, "read"):
- source = open(source, "rb")
- if not parser:
- parser = XMLTreeBuilder()
- while 1:
- data = source.read(32768)
- if not data:
- break
- parser.feed(data)
- self._root = parser.close()
- return self._root
-
- ##
- # Creates a tree iterator for the root element. The iterator loops
- # over all elements in this tree, in document order.
- #
- # @param tag What tags to look for (default is to return all elements)
- # @return An iterator.
- # @defreturn iterator
-
- def getiterator(self, tag=None):
- assert self._root is not None
- return self._root.getiterator(tag)
-
- ##
- # Finds the first toplevel element with given tag.
- # Same as getroot().find(path).
- #
- # @param path What element to look for.
- # @return The first matching element, or None if no element was found.
- # @defreturn Element or None
-
- def find(self, path):
- assert self._root is not None
- if path[:1] == "/":
- path = "." + path
- return self._root.find(path)
-
- ##
- # Finds the element text for the first toplevel element with given
- # tag. Same as getroot().findtext(path).
- #
- # @param path What toplevel element to look for.
- # @param default What to return if the element was not found.
- # @return The text content of the first matching element, or the
- # default value no element was found. Note that if the element
- # has is found, but has no text content, this method returns an
- # empty string.
- # @defreturn string
-
- def findtext(self, path, default=None):
- assert self._root is not None
- if path[:1] == "/":
- path = "." + path
- return self._root.findtext(path, default)
-
- ##
- # Finds all toplevel elements with the given tag.
- # Same as getroot().findall(path).
- #
- # @param path What element to look for.
- # @return A list or iterator containing all matching elements,
- # in document order.
- # @defreturn list of Element instances
-
- def findall(self, path):
- assert self._root is not None
- if path[:1] == "/":
- path = "." + path
- return self._root.findall(path)
-
- ##
- # Writes the element tree to a file, as XML.
- #
- # @param file A file name, or a file object opened for writing.
- # @param encoding Optional output encoding (default is US-ASCII).
-
- def write(self, file, encoding="us-ascii"):
- assert self._root is not None
- if not hasattr(file, "write"):
- file = open(file, "wb")
- if not encoding:
- encoding = "us-ascii"
- elif encoding != "utf-8" and encoding != "us-ascii":
- file.write("<?xml version='1.0' encoding='%s'?>\n" % encoding)
- self._write(file, self._root, encoding, {})
-
- def _write(self, file, node, encoding, namespaces, margin=0):
- # write XML to file
- tag = node.tag
- if tag is Comment:
- file.write("<!-- %s -->\n" % _escape_cdata(node.text, encoding))
- elif tag is ProcessingInstruction:
- file.write("<?%s?>\n" % _escape_cdata(node.text, encoding))
- else:
- items = node.items()
- xmlns_items = [] # new namespaces in this scope
- try:
- if isinstance(tag, QName) or tag[:1] == "{":
- tag, xmlns = fixtag(tag, namespaces)
- if xmlns: xmlns_items.append(xmlns)
- except TypeError:
- _raise_serialization_error(tag)
- file.write(' ' * margin)
- file.write(_encode("<", encoding) + _encode(tag, encoding))
- if items or xmlns_items:
- items = sorted(items) # lexical order
- for k, v in items:
- try:
- if isinstance(k, QName) or k[:1] == "{":
- k, xmlns = fixtag(k, namespaces)
- if xmlns: xmlns_items.append(xmlns)
- except TypeError:
- _raise_serialization_error(k)
- try:
- if isinstance(v, QName):
- v, xmlns = fixtag(v, namespaces)
- if xmlns: xmlns_items.append(xmlns)
- except TypeError:
- _raise_serialization_error(v)
- file.write(" %s=\"%s\"" % (k,v))
- for k, v in xmlns_items:
- file.write(" %s=\"%s\"" % (k,v))
- if node.text or len(node):
- file.write(">")
- if node.text:
- file.write(_escape_cdata(node.text, encoding))
- if len(node) > 0: file.write("\n")
- for n in node:
- self._write(file, n, encoding, namespaces, margin + 2)
- if len(node) > 0: file.write(' ' * margin)
- file.write(_encode("</", encoding) + _encode(tag, encoding) + _encode(">\n", encoding))
- else:
- file.write("/>\n")
- for k, v in xmlns_items:
- del namespaces[v]
- if node.tail:
- file.write(_escape_cdata(node.tail, encoding))
-
-# --------------------------------------------------------------------
-# helpers
-
-##
-# Checks if an object appears to be a valid element object.
-#
-# @param An element instance.
-# @return A true value if this is an element object.
-# @defreturn flag
-
-def iselement(element):
- # FIXME: not sure about this; might be a better idea to look
- # for tag/attrib/text attributes
- return isinstance(element, _ElementInterface) or hasattr(element, "tag")
-
-##
-# Writes an element tree or element structure to sys.stdout. This
-# function should be used for debugging only.
-# <p>
-# The exact output format is implementation dependent. In this
-# version, it's written as an ordinary XML file.
-#
-# @param elem An element tree or an individual element.
-
-def dump(elem):
- # debugging
- if not isinstance(elem, ElementTree):
- elem = ElementTree(elem)
- elem.write(sys.stdout)
- tail = elem.getroot().tail
- if not tail or tail[-1] != "\n":
- sys.stdout.write("\n")
-
-def _encode(s, encoding):
- try:
- return s.encode(encoding)
- except AttributeError:
- return s # 1.5.2: assume the string uses the right encoding
-
-if sys.version[:3] == "1.5":
- _escape = re.compile(r"[&<>\"\x80-\xff]+") # 1.5.2
else:
- _escape = re.compile(eval(r'u"[&<>\"\u0080-\uffff]+"'))
-
-_escape_map = {
- "&": "&",
- "<": "<",
- ">": ">",
- '"': """,
-}
-
-_namespace_map = {
- # "well-known" namespace prefixes
- "http://www.w3.org/XML/1998/namespace": "xml",
- "http://www.w3.org/1999/xhtml": "html",
- "http://www.w3.org/1999/02/22-rdf-syntax-ns#": "rdf",
- "http://schemas.xmlsoap.org/wsdl/": "wsdl",
-}
-
-def _raise_serialization_error(text):
- raise TypeError(
- "cannot serialize %r (type %s)" % (text, type(text).__name__)
- )
-
-def _encode_entity(text, pattern=_escape):
- # map reserved and non-ascii characters to numerical entities
- def escape_entities(m, map=_escape_map):
- out = []
- append = out.append
- for char in m.group():
- text = map.get(char)
- if text is None:
- text = "&#%d;" % ord(char)
- append(text)
- return string.join(out, "")
- try:
- return _encode(pattern.sub(escape_entities, text), "ascii")
- except TypeError:
- _raise_serialization_error(text)
-
-#
-# the following functions assume an ascii-compatible encoding
-# (or "utf-16")
-
-def _escape_cdata(text, encoding=None, replace=str.replace):
- # escape character data
- try:
- if platform.python_version()[0] == '2': # python 2.x.y
- if encoding:
- try:
- text = _encode(text, encoding)
- except UnicodeError:
- return _encode_entity(text)
-
- text = replace(text, "&", "&")
- text = replace(text, "<", "<")
- text = replace(text, ">", ">")
- text = replace(text, "####newLine####", "<br \>")
- if encoding:
- try:
- text = _encode(text, encoding)
- except UnicodeError:
- return _encode_entity(text)
- return text
- except (TypeError, AttributeError):
- _raise_serialization_error(text)
-
-def _escape_attrib(text, encoding=None, replace=str.replace):
- # escape attribute value
+ # python3 mode
+ # import xml.etree.ElementTree as etree # native version
+ import src.ElementTreePython3 as etree # VERSION 1.3.0 plus _serialize 'pretty_xml'
+ DBG.write("ElementTree Python3 VERSION 1.3.0", etree.__file__, DBG.isDeveloper())
+
+ def tostring(node, encoding='utf-8'):
+ """
+ fix output as str with encoding='unicode' because python3
+ If encoding is "unicode", a string is returned.
+ Otherwise a bytestring is returned
+ """
try:
- text = replace(text, "&", "&")
- text = replace(text, "'", "'") # FIXME: overkill
- text = replace(text, "\"", """)
- text = replace(text, "<", "<")
- text = replace(text, ">", ">")
- if encoding:
- try:
- text = _encode(text, encoding)
- except UnicodeError:
- return _encode_entity(text)
- return text
- except (TypeError, AttributeError):
- _raise_serialization_error(text)
-
-def fixtag(tag, namespaces):
- # given a decorated tag (of the form {uri}tag), return prefixed
- # tag and namespace declaration, if any
- if isinstance(tag, QName):
- tag = tag.text
- namespace_uri, tag = string.split(tag[1:], "}", 1)
- prefix = namespaces.get(namespace_uri)
- if prefix is None:
- prefix = _namespace_map.get(namespace_uri)
- if prefix is None:
- prefix = "ns%d" % len(namespaces)
- namespaces[namespace_uri] = prefix
- if prefix == "xml":
- xmlns = None
- else:
- xmlns = ("xmlns:%s" % prefix, namespace_uri)
- else:
- xmlns = None
- return "%s:%s" % (prefix, tag), xmlns
-
-##
-# Parses an XML document into an element tree.
-#
-# @param source A filename or file object containing XML data.
-# @param parser An optional parser instance. If not given, the
-# standard {@link XMLTreeBuilder} parser is used.
-# @return An ElementTree instance
-
-def parse(source, parser=None):
- tree = ElementTree()
- tree.parse(source, parser)
- return tree
-
-##
-# Parses an XML document into an element tree incrementally, and reports
-# what's going on to the user.
-#
-# @param source A filename or file object containing XML data.
-# @param events A list of events to report back. If omitted, only "end"
-# events are reported.
-# @return A (event, elem) iterator.
-
-class iterparse:
-
- def __init__(self, source, events=None):
- if not hasattr(source, "read"):
- # OP TEST
- print("iterparse.__init__ source = %s" % source)
- source = open(source, "rb")
- self._file = source
- self._events = []
- self._index = 0
- self.root = self._root = None
- self._parser = XMLTreeBuilder()
- # wire up the parser for event reporting
- parser = self._parser._parser
- append = self._events.append
- if events is None:
- events = ["end"]
- for event in events:
- if event == "start":
- try:
- parser.ordered_attributes = 1
- parser.specified_attributes = 1
- def handler(tag, attrib_in, event=event, append=append,
- start=self._parser._start_list):
- append((event, start(tag, attrib_in)))
- parser.StartElementHandler = handler
- except AttributeError:
- def handler(tag, attrib_in, event=event, append=append,
- start=self._parser._start):
- append((event, start(tag, attrib_in)))
- parser.StartElementHandler = handler
- elif event == "end":
- def handler(tag, event=event, append=append,
- end=self._parser._end):
- append((event, end(tag)))
- parser.EndElementHandler = handler
- elif event == "start-ns":
- def handler(prefix, uri, event=event, append=append):
- try:
- uri = _encode(uri, "ascii")
- except UnicodeError:
- pass
- append((event, (prefix or "", uri)))
- parser.StartNamespaceDeclHandler = handler
- elif event == "end-ns":
- def handler(prefix, event=event, append=append):
- append((event, None))
- parser.EndNamespaceDeclHandler = handler
-
- def next(self):
- while 1:
- try:
- item = self._events[self._index]
- except IndexError:
- if self._parser is None:
- self.root = self._root
- try:
- raise StopIteration
- except NameError:
- raise IndexError
- # load event buffer
- del self._events[:]
- self._index = 0
- data = self._file.read(16384)
- if data:
- self._parser.feed(data)
- else:
- self._root = self._parser.close()
- self._parser = None
- else:
- self._index = self._index + 1
- return item
-
- try:
- iter
- def __iter__(self):
- return self
- except NameError:
- def __getitem__(self, index):
- return self.next()
-
-##
-# Parses an XML document from a string constant. This function can
-# be used to embed "XML literals" in Python code.
-#
-# @param source A string containing XML data.
-# @return An Element instance.
-# @defreturn Element
-
-def XML(text):
- parser = XMLTreeBuilder()
- parser.feed(text)
- return parser.close()
-
-##
-# Parses an XML document from a string constant, and also returns
-# a dictionary which maps from element id:s to elements.
-#
-# @param source A string containing XML data.
-# @return A tuple containing an Element instance and a dictionary.
-# @defreturn (Element, dictionary)
-
-def XMLID(text):
- parser = XMLTreeBuilder()
- parser.feed(text)
- tree = parser.close()
- ids = {}
- for elem in tree.getiterator():
- id = elem.get("id")
- if id:
- ids[id] = elem
- return tree, ids
-
-##
-# Parses an XML document from a string constant. Same as {@link #XML}.
-#
-# @def fromstring(text)
-# @param source A string containing XML data.
-# @return An Element instance.
-# @defreturn Element
-
-fromstring = XML
-
-##
-# Generates a string representation of an XML element, including all
-# subelements.
-#
-# @param element An Element instance.
-# @return An encoded string containing the XML data.
-# @defreturn string
-
-def tostring(element, encoding=None):
- class dummy:
- pass
- data = []
- file = dummy()
- file.write = data.append
- ElementTree(element).write(file, encoding)
- data2 = []
- for item in data:
- if isinstance(item, bytes):
- item = item.decode()
- data2.append(item)
- return "".join(data2)
-
-##
-# Generic element structure builder. This builder converts a sequence
-# of {@link #TreeBuilder.start}, {@link #TreeBuilder.data}, and {@link
-# #TreeBuilder.end} method calls to a well-formed element structure.
-# <p>
-# You can use this class to build an element structure using a custom XML
-# parser, or a parser for some other XML-like format.
-#
-# @param element_factory Optional element factory. This factory
-# is called to create new Element instances, as necessary.
-
-class TreeBuilder:
-
- def __init__(self, element_factory=None):
- self._data = [] # data collector
- self._elem = [] # element stack
- self._last = None # last element
- self._tail = None # true if we're after an end tag
- if element_factory is None:
- element_factory = _ElementInterface
- self._factory = element_factory
-
- ##
- # Flushes the parser buffers, and returns the toplevel documen
- # element.
- #
- # @return An Element instance.
- # @defreturn Element
-
- def close(self):
- assert len(self._elem) == 0, "missing end tags"
- assert self._last != None, "missing toplevel element"
- return self._last
-
- def _flush(self):
- if self._data:
- if self._last is not None:
- text = ""
- for item in self._data:
- try:
- text += item
- except:
- text += item.decode()
- if self._tail:
- assert self._last.tail is None, "internal error (tail)"
- self._last.tail = text
- else:
- assert self._last.text is None, "internal error (text)"
- self._last.text = text
- self._data = []
-
- ##
- # Adds text to the current element.
- #
- # @param data A string. This should be either an 8-bit string
- # containing ASCII text, or a Unicode string.
-
- def data(self, data):
- self._data.append(data)
-
- ##
- # Opens a new element.
- #
- # @param tag The element name.
- # @param attrib A dictionary containing element attributes.
- # @return The opened element.
- # @defreturn Element
-
- def start(self, tag, attrs):
- self._flush()
- self._last = elem = self._factory(tag, attrs)
- if self._elem:
- self._elem[-1].append(elem)
- self._elem.append(elem)
- self._tail = 0
- return elem
-
- ##
- # Closes the current element.
- #
- # @param tag The element name.
- # @return The closed element.
- # @defreturn Element
-
- def end(self, tag):
- self._flush()
- self._last = self._elem.pop()
- assert self._last.tag == tag,\
- "end tag mismatch (expected %s, got %s)" % (
- self._last.tag, tag)
- self._tail = 1
- return self._last
-
-##
-# Element structure builder for XML source data, based on the
-# <b>expat</b> parser.
-#
-# @keyparam target Target object. If omitted, the builder uses an
-# instance of the standard {@link #TreeBuilder} class.
-# @keyparam html Predefine HTML entities. This flag is not supported
-# by the current implementation.
-# @see #ElementTree
-# @see #TreeBuilder
-
-class XMLTreeBuilder:
-
- def __init__(self, html=0, target=None):
- try:
- from xml.parsers import expat
- except ImportError:
- raise ImportError(
- "No module named expat; use SimpleXMLTreeBuilder instead"
- )
- self._parser = parser = expat.ParserCreate(None, "}")
- if target is None:
- target = TreeBuilder()
- self._target = target
- self._names = {} # name memo cache
- # callbacks
- parser.DefaultHandlerExpand = self._default
- parser.StartElementHandler = self._start
- parser.EndElementHandler = self._end
- parser.CharacterDataHandler = self._data
- # let expat do the buffering, if supported
- try:
- self._parser.buffer_text = 1
- except AttributeError:
- pass
- # use new-style attribute handling, if supported
- try:
- self._parser.ordered_attributes = 1
- self._parser.specified_attributes = 1
- parser.StartElementHandler = self._start_list
- except AttributeError:
- pass
- #encoding = None
- #if not parser.returns_unicode:
- # encoding = "utf-8"
- # target.xml(encoding, None)
- self._doctype = None
- self.entity = {}
-
- def _fixtext(self, text):
- # convert text string to ascii, if possible
- try:
- return _encode(text, "ascii")
- except UnicodeError:
- return text
-
- def _fixname(self, key):
- # expand qname, and convert name string to ascii, if possible
- try:
- name = self._names[key]
- except KeyError:
- name = key
- if "}" in name:
- name = "{" + name
- self._names[key] = name = self._fixtext(name)
- return name
-
- def _start(self, tag, attrib_in):
- fixname = self._fixname
- tag = fixname(tag)
- attrib = {}
- for key, value in attrib_in.items():
- attrib[fixname(key)] = self._fixtext(value)
- return self._target.start(tag, attrib)
-
- def _start_list(self, tag, attrib_in):
- fixname = self._fixname
- tag = fixname(tag)
- attrib = {}
- if attrib_in:
- for i in range(0, len(attrib_in), 2):
- attrib[fixname(attrib_in[i])] = self._fixtext(attrib_in[i+1])
- return self._target.start(tag, attrib)
-
- def _data(self, text):
- return self._target.data(self._fixtext(text))
-
- def _end(self, tag):
- return self._target.end(self._fixname(tag))
-
- def _default(self, text):
- prefix = text[:1]
- if prefix == "&":
- # deal with undefined entities
- try:
- self._target.data(self.entity[text[1:-1]])
- except KeyError:
- from xml.parsers import expat
- raise expat.error(
- "undefined entity %s: line %d, column %d" %
- (text, self._parser.ErrorLineNumber,
- self._parser.ErrorColumnNumber)
- )
- elif prefix == "<" and text[:9] == "<!DOCTYPE":
- self._doctype = [] # inside a doctype declaration
- elif self._doctype is not None:
- # parse doctype contents
- if prefix == ">":
- self._doctype = None
- return
- text = string.strip(text)
- if not text:
- return
- self._doctype.append(text)
- n = len(self._doctype)
- if n > 2:
- type = self._doctype[1]
- if type == "PUBLIC" and n == 4:
- name, type, pubid, system = self._doctype
- elif type == "SYSTEM" and n == 3:
- name, type, system = self._doctype
- pubid = None
- else:
- return
- if pubid:
- pubid = pubid[1:-1]
- self.doctype(name, pubid, system[1:-1])
- self._doctype = None
-
- ##
- # Handles a doctype declaration.
- #
- # @param name Doctype name.
- # @param pubid Public identifier.
- # @param system System identifier.
-
- def doctype(self, name, pubid, system):
- pass
-
- ##
- # Feeds data to the parser.
- #
- # @param data Encoded data.
-
- def feed(self, data):
- # OP 14/11/2017 Ajout de traces pour essayer de decouvrir le pb
- # de remontee de log des tests
- #print "TRACES OP - ElementTree.py/XMLTreeBuilder.feed() data = '#%s#'" %data
- self._parser.Parse(data, 0)
-
- ##
- # Finishes feeding data to the parser.
- #
- # @return An element structure.
- # @defreturn Element
+ aStr = etree.tostring(node, encoding='unicode', method="pretty_xml")
+ except:
+ print("*****************************\n problem node", node)
+ # try no pretty
+ aStr = etree.tostring(node, encoding='unicode')
+ # if be byte
+ # aStr = aStr.decode('utf-8')
+ return aStr
+
+# common use
+Element = etree.Element
+parse = etree.parse
- def close(self):
- self._parser.Parse("", 1) # end of data
- tree = self._target.close()
- del self._target, self._parser # get rid of circular references
- return tree
--- /dev/null
+#
+# ElementTree
+# $Id: ElementTree.py 2326 2005-03-17 07:45:21Z fredrik $
+#
+# light-weight XML support for Python 1.5.2 and later.
+#
+# history:
+# 2001-10-20 fl created (from various sources)
+# 2001-11-01 fl return root from parse method
+# 2002-02-16 fl sort attributes in lexical order
+# 2002-04-06 fl TreeBuilder refactoring, added PythonDoc markup
+# 2002-05-01 fl finished TreeBuilder refactoring
+# 2002-07-14 fl added basic namespace support to ElementTree.write
+# 2002-07-25 fl added QName attribute support
+# 2002-10-20 fl fixed encoding in write
+# 2002-11-24 fl changed default encoding to ascii; fixed attribute encoding
+# 2002-11-27 fl accept file objects or file names for parse/write
+# 2002-12-04 fl moved XMLTreeBuilder back to this module
+# 2003-01-11 fl fixed entity encoding glitch for us-ascii
+# 2003-02-13 fl added XML literal factory
+# 2003-02-21 fl added ProcessingInstruction/PI factory
+# 2003-05-11 fl added tostring/fromstring helpers
+# 2003-05-26 fl added ElementPath support
+# 2003-07-05 fl added makeelement factory method
+# 2003-07-28 fl added more well-known namespace prefixes
+# 2003-08-15 fl fixed typo in ElementTree.findtext (Thomas Dartsch)
+# 2003-09-04 fl fall back on emulator if ElementPath is not installed
+# 2003-10-31 fl markup updates
+# 2003-11-15 fl fixed nested namespace bug
+# 2004-03-28 fl added XMLID helper
+# 2004-06-02 fl added default support to findtext
+# 2004-06-08 fl fixed encoding of non-ascii element/attribute names
+# 2004-08-23 fl take advantage of post-2.1 expat features
+# 2005-02-01 fl added iterparse implementation
+# 2005-03-02 fl fixed iterparse support for pre-2.2 versions
+#
+# Copyright (c) 1999-2005 by Fredrik Lundh. All rights reserved.
+#
+# fredrik@pythonware.com
+# http://www.pythonware.com
+#
+# --------------------------------------------------------------------
+# The ElementTree toolkit is
+#
+# Copyright (c) 1999-2005 by Fredrik Lundh
+#
+# By obtaining, using, and/or copying this software and/or its
+# associated documentation, you agree that you have read, understood,
+# and will comply with the following terms and conditions:
+#
+# Permission to use, copy, modify, and distribute this software and
+# its associated documentation for any purpose and without fee is
+# hereby granted, provided that the above copyright notice appears in
+# all copies, and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# Secret Labs AB or the author not be used in advertising or publicity
+# pertaining to distribution of the software without specific, written
+# prior permission.
+#
+# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
+# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
+# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
+# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
+# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+# OF THIS SOFTWARE.
+# --------------------------------------------------------------------
+
+__all__ = [
+ # public symbols
+ "Comment",
+ "dump",
+ "Element", "ElementTree",
+ "fromstring",
+ "iselement", "iterparse",
+ "parse",
+ "PI", "ProcessingInstruction",
+ "QName",
+ "SubElement",
+ "tostring",
+ "TreeBuilder",
+ "VERSION", "XML",
+ "XMLTreeBuilder",
+ ]
+
+##
+# The <b>Element</b> type is a flexible container object, designed to
+# store hierarchical data structures in memory. The type can be
+# described as a cross between a list and a dictionary.
+# <p>
+# Each element has a number of properties associated with it:
+# <ul>
+# <li>a <i>tag</i>. This is a string identifying what kind of data
+# this element represents (the element type, in other words).</li>
+# <li>a number of <i>attributes</i>, stored in a Python dictionary.</li>
+# <li>a <i>text</i> string.</li>
+# <li>an optional <i>tail</i> string.</li>
+# <li>a number of <i>child elements</i>, stored in a Python sequence</li>
+# </ul>
+#
+# To create an element instance, use the {@link #Element} or {@link
+# #SubElement} factory functions.
+# <p>
+# The {@link #ElementTree} class can be used to wrap an element
+# structure, and convert it from and to XML.
+##
+
+import string, sys, re, platform
+
+class _SimpleElementPath:
+ # emulate pre-1.2 find/findtext/findall behaviour
+ def find(self, element, tag):
+ for elem in element:
+ if elem.tag == tag:
+ return elem
+ return None
+ def findtext(self, element, tag, default=None):
+ for elem in element:
+ if elem.tag == tag:
+ return elem.text or ""
+ return default
+ def findall(self, element, tag):
+ if tag[:3] == ".//":
+ return element.getiterator(tag[3:])
+ result = []
+ for elem in element:
+ if elem.tag == tag:
+ result.append(elem)
+ return result
+
+"""
+# obsolete
+# ElementPath.py is for python3 2019
+# file inexisting in sat before 2019
+try:
+ import ElementPath
+except ImportError:
+ # FIXME: issue warning in this case?
+ ElementPath = _SimpleElementPath()
+"""
+ElementPath = _SimpleElementPath() # before 2019 python2 situation sat5.0
+
+# TODO: add support for custom namespace resolvers/default namespaces
+# TODO: add improved support for incremental parsing
+
+VERSION = "1.2.6"
+
+##
+# Internal element class. This class defines the Element interface,
+# and provides a reference implementation of this interface.
+# <p>
+# You should not create instances of this class directly. Use the
+# appropriate factory functions instead, such as {@link #Element}
+# and {@link #SubElement}.
+#
+# @see Element
+# @see SubElement
+# @see Comment
+# @see ProcessingInstruction
+
+class _ElementInterface:
+ # <tag attrib>text<child/>...</tag>tail
+
+ ##
+ # (Attribute) Element tag.
+
+ tag = None
+
+ ##
+ # (Attribute) Element attribute dictionary. Where possible, use
+ # {@link #_ElementInterface.get},
+ # {@link #_ElementInterface.set},
+ # {@link #_ElementInterface.keys}, and
+ # {@link #_ElementInterface.items} to access
+ # element attributes.
+
+ attrib = None
+
+ ##
+ # (Attribute) Text before first subelement. This is either a
+ # string or the value None, if there was no text.
+
+ text = None
+
+ ##
+ # (Attribute) Text after this element's end tag, but before the
+ # next sibling element's start tag. This is either a string or
+ # the value None, if there was no text.
+
+ tail = None # text after end tag, if any
+
+ def __init__(self, tag, attrib):
+ self.tag = tag
+ self.attrib = attrib
+ self._children = []
+
+ def __repr__(self):
+ return "<Element %s at %x>" % (self.tag, id(self))
+
+ ##
+ # Creates a new element object of the same type as this element.
+ #
+ # @param tag Element tag.
+ # @param attrib Element attributes, given as a dictionary.
+ # @return A new element instance.
+
+ def makeelement(self, tag, attrib):
+ return Element(tag, attrib)
+
+ ##
+ # Returns the number of subelements.
+ #
+ # @return The number of subelements.
+
+ def __len__(self):
+ return len(self._children)
+
+ ##
+ # Returns the given subelement.
+ #
+ # @param index What subelement to return.
+ # @return The given subelement.
+ # @exception IndexError If the given element does not exist.
+
+ def __getitem__(self, index):
+ return self._children[index]
+
+ ##
+ # Replaces the given subelement.
+ #
+ # @param index What subelement to replace.
+ # @param element The new element value.
+ # @exception IndexError If the given element does not exist.
+ # @exception AssertionError If element is not a valid object.
+
+ def __setitem__(self, index, element):
+ assert iselement(element)
+ self._children[index] = element
+
+ ##
+ # Deletes the given subelement.
+ #
+ # @param index What subelement to delete.
+ # @exception IndexError If the given element does not exist.
+
+ def __delitem__(self, index):
+ del self._children[index]
+
+ ##
+ # Returns a list containing subelements in the given range.
+ #
+ # @param start The first subelement to return.
+ # @param stop The first subelement that shouldn't be returned.
+ # @return A sequence object containing subelements.
+
+ def __getslice__(self, start, stop):
+ return self._children[start:stop]
+
+ ##
+ # Replaces a number of subelements with elements from a sequence.
+ #
+ # @param start The first subelement to replace.
+ # @param stop The first subelement that shouldn't be replaced.
+ # @param elements A sequence object with zero or more elements.
+ # @exception AssertionError If a sequence member is not a valid object.
+
+ def __setslice__(self, start, stop, elements):
+ for element in elements:
+ assert iselement(element)
+ self._children[start:stop] = list(elements)
+
+ ##
+ # Deletes a number of subelements.
+ #
+ # @param start The first subelement to delete.
+ # @param stop The first subelement to leave in there.
+
+ def __delslice__(self, start, stop):
+ del self._children[start:stop]
+
+ ##
+ # Adds a subelement to the end of this element.
+ #
+ # @param element The element to add.
+ # @exception AssertionError If a sequence member is not a valid object.
+
+ def append(self, element):
+ assert iselement(element)
+ self._children.append(element)
+
+ ##
+ # Inserts a subelement at the given position in this element.
+ #
+ # @param index Where to insert the new subelement.
+ # @exception AssertionError If the element is not a valid object.
+
+ def insert(self, index, element):
+ assert iselement(element)
+ self._children.insert(index, element)
+
+ ##
+ # Removes a matching subelement. Unlike the <b>find</b> methods,
+ # this method compares elements based on identity, not on tag
+ # value or contents.
+ #
+ # @param element What element to remove.
+ # @exception ValueError If a matching element could not be found.
+ # @exception AssertionError If the element is not a valid object.
+
+ def remove(self, element):
+ assert iselement(element)
+ self._children.remove(element)
+
+ ##
+ # Returns all subelements. The elements are returned in document
+ # order.
+ #
+ # @return A list of subelements.
+ # @defreturn list of Element instances
+
+ def getchildren(self):
+ return self._children
+
+ ##
+ # Finds the first matching subelement, by tag name or path.
+ #
+ # @param path What element to look for.
+ # @return The first matching element, or None if no element was found.
+ # @defreturn Element or None
+
+ def find(self, path):
+ if ElementPath.find(self, path) == None:
+ return ElementPath.find(self, path.encode())
+ return ElementPath.find(self, path)
+
+ ##
+ # Finds text for the first matching subelement, by tag name or path.
+ #
+ # @param path What element to look for.
+ # @param default What to return if the element was not found.
+ # @return The text content of the first matching element, or the
+ # default value no element was found. Note that if the element
+ # has is found, but has no text content, this method returns an
+ # empty string.
+ # @defreturn string
+
+ def findtext(self, path, default=None):
+ return ElementPath.findtext(self, path, default)
+
+ ##
+ # Finds all matching subelements, by tag name or path.
+ #
+ # @param path What element to look for.
+ # @return A list or iterator containing all matching elements,
+ # in document order.
+ # @defreturn list of Element instances
+
+ def findall(self, path):
+ return ElementPath.findall(self, path)
+
+ ##
+ # Resets an element. This function removes all subelements, clears
+ # all attributes, and sets the text and tail attributes to None.
+
+ def clear(self):
+ self.attrib.clear()
+ self._children = []
+ self.text = self.tail = None
+
+ ##
+ # Gets an element attribute.
+ #
+ # @param key What attribute to look for.
+ # @param default What to return if the attribute was not found.
+ # @return The attribute value, or the default value, if the
+ # attribute was not found.
+ # @defreturn string or None
+
+ def get(self, key, default=None):
+ res = self.attrib.get(key, default)
+ if not res:
+ res = self.attrib.get(key.encode(), default)
+ if isinstance(res, bytes):
+ return res.decode()
+ else:
+ return res
+
+ ##
+ # Sets an element attribute.
+ #
+ # @param key What attribute to set.
+ # @param value The attribute value.
+
+ def set(self, key, value):
+ self.attrib[key] = value
+
+ ##
+ # Gets a list of attribute names. The names are returned in an
+ # arbitrary order (just like for an ordinary Python dictionary).
+ #
+ # @return A list of element attribute names.
+ # @defreturn list of strings
+
+ def keys(self):
+ res = []
+ for key in self.attrib.keys():
+ if isinstance(key, bytes):
+ res.append(key.decode())
+ else:
+ res.append(key)
+ return res
+
+ ##
+ # Gets element attributes, as a sequence. The attributes are
+ # returned in an arbitrary order.
+ #
+ # @return A list of (name, value) tuples for all attributes.
+ # @defreturn list of (string, string) tuples
+
+ def items(self):
+ return self.attrib.items()
+
+ ##
+ # Creates a tree iterator. The iterator loops over this element
+ # and all subelements, in document order, and returns all elements
+ # with a matching tag.
+ # <p>
+ # If the tree structure is modified during iteration, the result
+ # is undefined.
+ #
+ # @param tag What tags to look for (default is to return all elements).
+ # @return A list or iterator containing all the matching elements.
+ # @defreturn list or iterator
+
+ def getiterator(self, tag=None):
+ nodes = []
+ if tag == "*":
+ tag = None
+ if tag is None or self.tag == tag:
+ nodes.append(self)
+ for node in self._children:
+ nodes.extend(node.getiterator(tag))
+ return nodes
+
+# compatibility
+_Element = _ElementInterface
+
+##
+# Element factory. This function returns an object implementing the
+# standard Element interface. The exact class or type of that object
+# is implementation dependent, but it will always be compatible with
+# the {@link #_ElementInterface} class in this module.
+# <p>
+# The element name, attribute names, and attribute values can be
+# either 8-bit ASCII strings or Unicode strings.
+#
+# @param tag The element name.
+# @param attrib An optional dictionary, containing element attributes.
+# @param **extra Additional attributes, given as keyword arguments.
+# @return An element instance.
+# @defreturn Element
+
+def Element(tag, attrib={}, **extra):
+ attrib = attrib.copy()
+ attrib.update(extra)
+ return _ElementInterface(tag, attrib)
+
+##
+# Subelement factory. This function creates an element instance, and
+# appends it to an existing element.
+# <p>
+# The element name, attribute names, and attribute values can be
+# either 8-bit ASCII strings or Unicode strings.
+#
+# @param parent The parent element.
+# @param tag The subelement name.
+# @param attrib An optional dictionary, containing element attributes.
+# @param **extra Additional attributes, given as keyword arguments.
+# @return An element instance.
+# @defreturn Element
+
+def SubElement(parent, tag, attrib={}, **extra):
+ attrib = attrib.copy()
+ attrib.update(extra)
+ element = parent.makeelement(tag, attrib)
+ parent.append(element)
+ return element
+
+##
+# Comment element factory. This factory function creates a special
+# element that will be serialized as an XML comment.
+# <p>
+# The comment string can be either an 8-bit ASCII string or a Unicode
+# string.
+#
+# @param text A string containing the comment string.
+# @return An element instance, representing a comment.
+# @defreturn Element
+
+def Comment(text=None):
+ element = Element(Comment)
+ element.text = text
+ return element
+
+##
+# PI element factory. This factory function creates a special element
+# that will be serialized as an XML processing instruction.
+#
+# @param target A string containing the PI target.
+# @param text A string containing the PI contents, if any.
+# @return An element instance, representing a PI.
+# @defreturn Element
+
+def ProcessingInstruction(target, text=None):
+ element = Element(ProcessingInstruction)
+ element.text = target
+ if text:
+ element.text = element.text + " " + text
+ return element
+
+PI = ProcessingInstruction
+
+##
+# QName wrapper. This can be used to wrap a QName attribute value, in
+# order to get proper namespace handling on output.
+#
+# @param text A string containing the QName value, in the form {uri}local,
+# or, if the tag argument is given, the URI part of a QName.
+# @param tag Optional tag. If given, the first argument is interpreted as
+# an URI, and this argument is interpreted as a local name.
+# @return An opaque object, representing the QName.
+
+class QName:
+ def __init__(self, text_or_uri, tag=None):
+ if tag:
+ text_or_uri = "{%s}%s" % (text_or_uri, tag)
+ self.text = text_or_uri
+ def __str__(self):
+ return self.text
+ def __hash__(self):
+ return hash(self.text)
+ def __cmp__(self, other):
+ if isinstance(other, QName):
+ return cmp(self.text, other.text)
+ return cmp(self.text, other)
+
+##
+# ElementTree wrapper class. This class represents an entire element
+# hierarchy, and adds some extra support for serialization to and from
+# standard XML.
+#
+# @param element Optional root element.
+# @keyparam file Optional file handle or name. If given, the
+# tree is initialized with the contents of this XML file.
+
+class ElementTree:
+
+ def __init__(self, element=None, file=None):
+ assert element is None or iselement(element)
+ self._root = element # first node
+ if file:
+ self.parse(file)
+
+ ##
+ # Gets the root element for this tree.
+ #
+ # @return An element instance.
+ # @defreturn Element
+
+ def getroot(self):
+ return self._root
+
+ ##
+ # Replaces the root element for this tree. This discards the
+ # current contents of the tree, and replaces it with the given
+ # element. Use with care.
+ #
+ # @param element An element instance.
+
+ def _setroot(self, element):
+ assert iselement(element)
+ self._root = element
+
+ ##
+ # Loads an external XML document into this element tree.
+ #
+ # @param source A file name or file object.
+ # @param parser An optional parser instance. If not given, the
+ # standard {@link XMLTreeBuilder} parser is used.
+ # @return The document root element.
+ # @defreturn Element
+
+ def parse(self, source, parser=None):
+ if not hasattr(source, "read"):
+ source = open(source, "rb")
+ if not parser:
+ parser = XMLTreeBuilder()
+ while 1:
+ data = source.read(32768)
+ if not data:
+ break
+ parser.feed(data)
+ self._root = parser.close()
+ return self._root
+
+ ##
+ # Creates a tree iterator for the root element. The iterator loops
+ # over all elements in this tree, in document order.
+ #
+ # @param tag What tags to look for (default is to return all elements)
+ # @return An iterator.
+ # @defreturn iterator
+
+ def getiterator(self, tag=None):
+ assert self._root is not None
+ return self._root.getiterator(tag)
+
+ ##
+ # Finds the first toplevel element with given tag.
+ # Same as getroot().find(path).
+ #
+ # @param path What element to look for.
+ # @return The first matching element, or None if no element was found.
+ # @defreturn Element or None
+
+ def find(self, path):
+ assert self._root is not None
+ if path[:1] == "/":
+ path = "." + path
+ return self._root.find(path)
+
+ ##
+ # Finds the element text for the first toplevel element with given
+ # tag. Same as getroot().findtext(path).
+ #
+ # @param path What toplevel element to look for.
+ # @param default What to return if the element was not found.
+ # @return The text content of the first matching element, or the
+ # default value no element was found. Note that if the element
+ # has is found, but has no text content, this method returns an
+ # empty string.
+ # @defreturn string
+
+ def findtext(self, path, default=None):
+ assert self._root is not None
+ if path[:1] == "/":
+ path = "." + path
+ return self._root.findtext(path, default)
+
+ ##
+ # Finds all toplevel elements with the given tag.
+ # Same as getroot().findall(path).
+ #
+ # @param path What element to look for.
+ # @return A list or iterator containing all matching elements,
+ # in document order.
+ # @defreturn list of Element instances
+
+ def findall(self, path):
+ assert self._root is not None
+ if path[:1] == "/":
+ path = "." + path
+ return self._root.findall(path)
+
+ ##
+ # Writes the element tree to a file, as XML.
+ #
+ # @param file A file name, or a file object opened for writing.
+ # @param encoding Optional output encoding (default is US-ASCII).
+
+ def write(self, file, encoding="us-ascii"):
+ assert self._root is not None
+ if not hasattr(file, "write"):
+ file = open(file, "wb")
+ if not encoding:
+ encoding = "us-ascii"
+ elif encoding != "utf-8" and encoding != "us-ascii":
+ file.write("<?xml version='1.0' encoding='%s'?>\n" % encoding)
+ self._write(file, self._root, encoding, {})
+
+ def _write(self, file, node, encoding, namespaces, margin=0):
+ # write XML to file
+ tag = node.tag
+ if tag is Comment:
+ file.write("<!-- %s -->\n" % _escape_cdata(node.text, encoding))
+ elif tag is ProcessingInstruction:
+ file.write("<?%s?>\n" % _escape_cdata(node.text, encoding))
+ else:
+ items = node.items()
+ xmlns_items = [] # new namespaces in this scope
+ try:
+ if isinstance(tag, QName) or tag[:1] == "{":
+ tag, xmlns = fixtag(tag, namespaces)
+ if xmlns: xmlns_items.append(xmlns)
+ except TypeError:
+ _raise_serialization_error(tag)
+ file.write(' ' * margin)
+ file.write(_encode("<", encoding) + _encode(tag, encoding))
+ if items or xmlns_items:
+ try:
+ items = sorted(items) # lexical order
+ except:
+ print("*** problem sorting items", items)
+ for k, v in items:
+ try:
+ if isinstance(k, QName) or k[:1] == "{":
+ k, xmlns = fixtag(k, namespaces)
+ if xmlns: xmlns_items.append(xmlns)
+ except TypeError:
+ _raise_serialization_error(k)
+ try:
+ if isinstance(v, QName):
+ v, xmlns = fixtag(v, namespaces)
+ if xmlns: xmlns_items.append(xmlns)
+ except TypeError:
+ _raise_serialization_error(v)
+ file.write(" %s=\"%s\"" % (k,v))
+ for k, v in xmlns_items:
+ file.write(" %s=\"%s\"" % (k,v))
+ if node.text or len(node):
+ file.write(">")
+ if node.text:
+ file.write(_escape_cdata(node.text, encoding))
+ if len(node) > 0: file.write("\n")
+ for n in node:
+ self._write(file, n, encoding, namespaces, margin + 2)
+ if len(node) > 0: file.write(' ' * margin)
+ file.write(_encode("</", encoding) + _encode(tag, encoding) + _encode(">\n", encoding))
+ else:
+ file.write("/>\n")
+ for k, v in xmlns_items:
+ del namespaces[v]
+ if node.tail:
+ file.write(_escape_cdata(node.tail, encoding))
+
+# --------------------------------------------------------------------
+# helpers
+
+##
+# Checks if an object appears to be a valid element object.
+#
+# @param An element instance.
+# @return A true value if this is an element object.
+# @defreturn flag
+
+def iselement(element):
+ # FIXME: not sure about this; might be a better idea to look
+ # for tag/attrib/text attributes
+ return isinstance(element, _ElementInterface) or hasattr(element, "tag")
+
+##
+# Writes an element tree or element structure to sys.stdout. This
+# function should be used for debugging only.
+# <p>
+# The exact output format is implementation dependent. In this
+# version, it's written as an ordinary XML file.
+#
+# @param elem An element tree or an individual element.
+
+def dump(elem):
+ # debugging
+ if not isinstance(elem, ElementTree):
+ elem = ElementTree(elem)
+ elem.write(sys.stdout)
+ tail = elem.getroot().tail
+ if not tail or tail[-1] != "\n":
+ sys.stdout.write("\n")
+
+def _encode(s, encoding):
+ try:
+ return s.encode(encoding)
+ except AttributeError:
+ return s # 1.5.2: assume the string uses the right encoding
+
+if sys.version[:3] == "1.5":
+ _escape = re.compile(r"[&<>\"\x80-\xff]+") # 1.5.2
+else:
+ _escape = re.compile(eval(r'u"[&<>\"\u0080-\uffff]+"'))
+
+_escape_map = {
+ "&": "&",
+ "<": "<",
+ ">": ">",
+ '"': """,
+}
+
+_namespace_map = {
+ # "well-known" namespace prefixes
+ "http://www.w3.org/XML/1998/namespace": "xml",
+ "http://www.w3.org/1999/xhtml": "html",
+ "http://www.w3.org/1999/02/22-rdf-syntax-ns#": "rdf",
+ "http://schemas.xmlsoap.org/wsdl/": "wsdl",
+}
+
+def _raise_serialization_error(text):
+ raise TypeError(
+ "cannot serialize %r (type %s)" % (text, type(text).__name__)
+ )
+
+def _encode_entity(text, pattern=_escape):
+ # map reserved and non-ascii characters to numerical entities
+ def escape_entities(m, map=_escape_map):
+ out = []
+ append = out.append
+ for char in m.group():
+ text = map.get(char)
+ if text is None:
+ text = "&#%d;" % ord(char)
+ append(text)
+ return string.join(out, "")
+ try:
+ return _encode(pattern.sub(escape_entities, text), "ascii")
+ except TypeError:
+ _raise_serialization_error(text)
+
+#
+# the following functions assume an ascii-compatible encoding
+# (or "utf-16")
+
+def _escape_cdata(text, encoding=None, replace=str.replace):
+ # escape character data
+ try:
+ if platform.python_version()[0] == '2': # python 2.x.y
+ if encoding:
+ try:
+ text = _encode(text, encoding)
+ except UnicodeError:
+ return _encode_entity(text)
+
+ text = replace(text, "&", "&")
+ text = replace(text, "<", "<")
+ text = replace(text, ">", ">")
+ text = replace(text, "####newLine####", "<br \>")
+ if encoding:
+ try:
+ text = _encode(text, encoding)
+ except UnicodeError:
+ return _encode_entity(text)
+ return text
+ except (TypeError, AttributeError):
+ _raise_serialization_error(text)
+
+def _escape_attrib(text, encoding=None, replace=str.replace):
+ # escape attribute value
+ try:
+ text = replace(text, "&", "&")
+ text = replace(text, "'", "'") # FIXME: overkill
+ text = replace(text, "\"", """)
+ text = replace(text, "<", "<")
+ text = replace(text, ">", ">")
+ if encoding:
+ try:
+ text = _encode(text, encoding)
+ except UnicodeError:
+ return _encode_entity(text)
+ return text
+ except (TypeError, AttributeError):
+ _raise_serialization_error(text)
+
+def fixtag(tag, namespaces):
+ # given a decorated tag (of the form {uri}tag), return prefixed
+ # tag and namespace declaration, if any
+ if isinstance(tag, QName):
+ tag = tag.text
+ namespace_uri, tag = string.split(tag[1:], "}", 1)
+ prefix = namespaces.get(namespace_uri)
+ if prefix is None:
+ prefix = _namespace_map.get(namespace_uri)
+ if prefix is None:
+ prefix = "ns%d" % len(namespaces)
+ namespaces[namespace_uri] = prefix
+ if prefix == "xml":
+ xmlns = None
+ else:
+ xmlns = ("xmlns:%s" % prefix, namespace_uri)
+ else:
+ xmlns = None
+ return "%s:%s" % (prefix, tag), xmlns
+
+##
+# Parses an XML document into an element tree.
+#
+# @param source A filename or file object containing XML data.
+# @param parser An optional parser instance. If not given, the
+# standard {@link XMLTreeBuilder} parser is used.
+# @return An ElementTree instance
+
+def parse(source, parser=None):
+ tree = ElementTree()
+ tree.parse(source, parser)
+ return tree
+
+##
+# Parses an XML document into an element tree incrementally, and reports
+# what's going on to the user.
+#
+# @param source A filename or file object containing XML data.
+# @param events A list of events to report back. If omitted, only "end"
+# events are reported.
+# @return A (event, elem) iterator.
+
+class iterparse:
+
+ def __init__(self, source, events=None):
+ if not hasattr(source, "read"):
+ # OP TEST
+ print("iterparse.__init__ source = %s" % source)
+ source = open(source, "rb")
+ self._file = source
+ self._events = []
+ self._index = 0
+ self.root = self._root = None
+ self._parser = XMLTreeBuilder()
+ # wire up the parser for event reporting
+ parser = self._parser._parser
+ append = self._events.append
+ if events is None:
+ events = ["end"]
+ for event in events:
+ if event == "start":
+ try:
+ parser.ordered_attributes = 1
+ parser.specified_attributes = 1
+ def handler(tag, attrib_in, event=event, append=append,
+ start=self._parser._start_list):
+ append((event, start(tag, attrib_in)))
+ parser.StartElementHandler = handler
+ except AttributeError:
+ def handler(tag, attrib_in, event=event, append=append,
+ start=self._parser._start):
+ append((event, start(tag, attrib_in)))
+ parser.StartElementHandler = handler
+ elif event == "end":
+ def handler(tag, event=event, append=append,
+ end=self._parser._end):
+ append((event, end(tag)))
+ parser.EndElementHandler = handler
+ elif event == "start-ns":
+ def handler(prefix, uri, event=event, append=append):
+ try:
+ uri = _encode(uri, "ascii")
+ except UnicodeError:
+ pass
+ append((event, (prefix or "", uri)))
+ parser.StartNamespaceDeclHandler = handler
+ elif event == "end-ns":
+ def handler(prefix, event=event, append=append):
+ append((event, None))
+ parser.EndNamespaceDeclHandler = handler
+
+ def next(self):
+ while 1:
+ try:
+ item = self._events[self._index]
+ except IndexError:
+ if self._parser is None:
+ self.root = self._root
+ try:
+ raise StopIteration
+ except NameError:
+ raise IndexError
+ # load event buffer
+ del self._events[:]
+ self._index = 0
+ data = self._file.read(16384)
+ if data:
+ self._parser.feed(data)
+ else:
+ self._root = self._parser.close()
+ self._parser = None
+ else:
+ self._index = self._index + 1
+ return item
+
+ try:
+ iter
+ def __iter__(self):
+ return self
+ except NameError:
+ def __getitem__(self, index):
+ return self.next()
+
+##
+# Parses an XML document from a string constant. This function can
+# be used to embed "XML literals" in Python code.
+#
+# @param source A string containing XML data.
+# @return An Element instance.
+# @defreturn Element
+
+def XML(text):
+ parser = XMLTreeBuilder()
+ parser.feed(text)
+ return parser.close()
+
+##
+# Parses an XML document from a string constant, and also returns
+# a dictionary which maps from element id:s to elements.
+#
+# @param source A string containing XML data.
+# @return A tuple containing an Element instance and a dictionary.
+# @defreturn (Element, dictionary)
+
+def XMLID(text):
+ parser = XMLTreeBuilder()
+ parser.feed(text)
+ tree = parser.close()
+ ids = {}
+ for elem in tree.getiterator():
+ id = elem.get("id")
+ if id:
+ ids[id] = elem
+ return tree, ids
+
+##
+# Parses an XML document from a string constant. Same as {@link #XML}.
+#
+# @def fromstring(text)
+# @param source A string containing XML data.
+# @return An Element instance.
+# @defreturn Element
+
+fromstring = XML
+
+##
+# Generates a string representation of an XML element, including all
+# subelements.
+#
+# @param element An Element instance.
+# @return An encoded string containing the XML data.
+# @defreturn string
+
+def tostring(element, encoding=None):
+ class dummy:
+ pass
+ data = []
+ file = dummy()
+ file.write = data.append
+ ElementTree(element).write(file, encoding)
+ data2 = []
+ for item in data:
+ if isinstance(item, bytes):
+ item = item.decode()
+ data2.append(item)
+ return "".join(data2)
+
+##
+# Generic element structure builder. This builder converts a sequence
+# of {@link #TreeBuilder.start}, {@link #TreeBuilder.data}, and {@link
+# #TreeBuilder.end} method calls to a well-formed element structure.
+# <p>
+# You can use this class to build an element structure using a custom XML
+# parser, or a parser for some other XML-like format.
+#
+# @param element_factory Optional element factory. This factory
+# is called to create new Element instances, as necessary.
+
+class TreeBuilder:
+
+ def __init__(self, element_factory=None):
+ self._data = [] # data collector
+ self._elem = [] # element stack
+ self._last = None # last element
+ self._tail = None # true if we're after an end tag
+ if element_factory is None:
+ element_factory = _ElementInterface
+ self._factory = element_factory
+
+ ##
+ # Flushes the parser buffers, and returns the toplevel documen
+ # element.
+ #
+ # @return An Element instance.
+ # @defreturn Element
+
+ def close(self):
+ assert len(self._elem) == 0, "missing end tags"
+ assert self._last != None, "missing toplevel element"
+ return self._last
+
+ def _flush(self):
+ if self._data:
+ if self._last is not None:
+ text = ""
+ for item in self._data:
+ try:
+ text += item
+ except:
+ text += item.decode()
+ if self._tail:
+ assert self._last.tail is None, "internal error (tail)"
+ self._last.tail = text
+ else:
+ assert self._last.text is None, "internal error (text)"
+ self._last.text = text
+ self._data = []
+
+ ##
+ # Adds text to the current element.
+ #
+ # @param data A string. This should be either an 8-bit string
+ # containing ASCII text, or a Unicode string.
+
+ def data(self, data):
+ self._data.append(data)
+
+ ##
+ # Opens a new element.
+ #
+ # @param tag The element name.
+ # @param attrib A dictionary containing element attributes.
+ # @return The opened element.
+ # @defreturn Element
+
+ def start(self, tag, attrs):
+ self._flush()
+ self._last = elem = self._factory(tag, attrs)
+ if self._elem:
+ self._elem[-1].append(elem)
+ self._elem.append(elem)
+ self._tail = 0
+ return elem
+
+ ##
+ # Closes the current element.
+ #
+ # @param tag The element name.
+ # @return The closed element.
+ # @defreturn Element
+
+ def end(self, tag):
+ self._flush()
+ self._last = self._elem.pop()
+ assert self._last.tag == tag,\
+ "end tag mismatch (expected %s, got %s)" % (
+ self._last.tag, tag)
+ self._tail = 1
+ return self._last
+
+##
+# Element structure builder for XML source data, based on the
+# <b>expat</b> parser.
+#
+# @keyparam target Target object. If omitted, the builder uses an
+# instance of the standard {@link #TreeBuilder} class.
+# @keyparam html Predefine HTML entities. This flag is not supported
+# by the current implementation.
+# @see #ElementTree
+# @see #TreeBuilder
+
+class XMLTreeBuilder:
+
+ def __init__(self, html=0, target=None):
+ try:
+ from xml.parsers import expat
+ except ImportError:
+ raise ImportError(
+ "No module named expat; use SimpleXMLTreeBuilder instead"
+ )
+ self._parser = parser = expat.ParserCreate(None, "}")
+ if target is None:
+ target = TreeBuilder()
+ self._target = target
+ self._names = {} # name memo cache
+ # callbacks
+ parser.DefaultHandlerExpand = self._default
+ parser.StartElementHandler = self._start
+ parser.EndElementHandler = self._end
+ parser.CharacterDataHandler = self._data
+ # let expat do the buffering, if supported
+ try:
+ self._parser.buffer_text = 1
+ except AttributeError:
+ pass
+ # use new-style attribute handling, if supported
+ try:
+ self._parser.ordered_attributes = 1
+ self._parser.specified_attributes = 1
+ parser.StartElementHandler = self._start_list
+ except AttributeError:
+ pass
+ #encoding = None
+ #if not parser.returns_unicode:
+ # encoding = "utf-8"
+ # target.xml(encoding, None)
+ self._doctype = None
+ self.entity = {}
+
+ def _fixtext(self, text):
+ # convert text string to ascii, if possible
+ try:
+ return _encode(text, "ascii")
+ except UnicodeError:
+ return text
+
+ def _fixname(self, key):
+ # expand qname, and convert name string to ascii, if possible
+ try:
+ name = self._names[key]
+ except KeyError:
+ name = key
+ if "}" in name:
+ name = "{" + name
+ self._names[key] = name = self._fixtext(name)
+ return name
+
+ def _start(self, tag, attrib_in):
+ fixname = self._fixname
+ tag = fixname(tag)
+ attrib = {}
+ for key, value in attrib_in.items():
+ attrib[fixname(key)] = self._fixtext(value)
+ return self._target.start(tag, attrib)
+
+ def _start_list(self, tag, attrib_in):
+ fixname = self._fixname
+ tag = fixname(tag)
+ attrib = {}
+ if attrib_in:
+ for i in range(0, len(attrib_in), 2):
+ attrib[fixname(attrib_in[i])] = self._fixtext(attrib_in[i+1])
+ return self._target.start(tag, attrib)
+
+ def _data(self, text):
+ return self._target.data(self._fixtext(text))
+
+ def _end(self, tag):
+ return self._target.end(self._fixname(tag))
+
+ def _default(self, text):
+ prefix = text[:1]
+ if prefix == "&":
+ # deal with undefined entities
+ try:
+ self._target.data(self.entity[text[1:-1]])
+ except KeyError:
+ from xml.parsers import expat
+ raise expat.error(
+ "undefined entity %s: line %d, column %d" %
+ (text, self._parser.ErrorLineNumber,
+ self._parser.ErrorColumnNumber)
+ )
+ elif prefix == "<" and text[:9] == "<!DOCTYPE":
+ self._doctype = [] # inside a doctype declaration
+ elif self._doctype is not None:
+ # parse doctype contents
+ if prefix == ">":
+ self._doctype = None
+ return
+ text = string.strip(text)
+ if not text:
+ return
+ self._doctype.append(text)
+ n = len(self._doctype)
+ if n > 2:
+ type = self._doctype[1]
+ if type == "PUBLIC" and n == 4:
+ name, type, pubid, system = self._doctype
+ elif type == "SYSTEM" and n == 3:
+ name, type, system = self._doctype
+ pubid = None
+ else:
+ return
+ if pubid:
+ pubid = pubid[1:-1]
+ self.doctype(name, pubid, system[1:-1])
+ self._doctype = None
+
+ ##
+ # Handles a doctype declaration.
+ #
+ # @param name Doctype name.
+ # @param pubid Public identifier.
+ # @param system System identifier.
+
+ def doctype(self, name, pubid, system):
+ pass
+
+ ##
+ # Feeds data to the parser.
+ #
+ # @param data Encoded data.
+
+ def feed(self, data):
+ """
+ my_str = "hello world"
+ my_str_as_bytes = str.encode(my_str)
+ type(my_str_as_bytes) # ensure it is byte representation
+ my_decoded_str = my_str_as_bytes.decode()
+ type(my_decoded_str) # ensure it is string representation
+ """
+ try:
+ self._parser.Parse(data, 0)
+ except:
+ print("*** problem feed:\n%s" % data.decode('utf-8'))
+
+ ##
+ # Finishes feeding data to the parser.
+ #
+ # @return An element structure.
+ # @defreturn Element
+
+ def close(self):
+ self._parser.Parse("", 1) # end of data
+ tree = self._target.close()
+ del self._target, self._parser # get rid of circular references
+ return tree
--- /dev/null
+"""Lightweight XML support for Python.
+
+ XML is an inherently hierarchical data format, and the most natural way to
+ represent it is with a tree. This module has two classes for this purpose:
+
+ 1. ElementTree represents the whole XML document as a tree and
+
+ 2. Element represents a single node in this tree.
+
+ Interactions with the whole document (reading and writing to/from files) are
+ usually done on the ElementTree level. Interactions with a single XML element
+ and its sub-elements are done on the Element level.
+
+ Element is a flexible container object designed to store hierarchical data
+ structures in memory. It can be described as a cross between a list and a
+ dictionary. Each Element has a number of properties associated with it:
+
+ 'tag' - a string containing the element's name.
+
+ 'attributes' - a Python dictionary storing the element's attributes.
+
+ 'text' - a string containing the element's text content.
+
+ 'tail' - an optional string containing text after the element's end tag.
+
+ And a number of child elements stored in a Python sequence.
+
+ To create an element instance, use the Element constructor,
+ or the SubElement factory function.
+
+ You can also use the ElementTree class to wrap an element structure
+ and convert it to and from XML.
+
+"""
+
+#---------------------------------------------------------------------
+# Licensed to PSF under a Contributor Agreement.
+# See http://www.python.org/psf/license for licensing details.
+#
+# ElementTree
+# Copyright (c) 1999-2008 by Fredrik Lundh. All rights reserved.
+#
+# fredrik@pythonware.com
+# http://www.pythonware.com
+# --------------------------------------------------------------------
+# The ElementTree toolkit is
+#
+# Copyright (c) 1999-2008 by Fredrik Lundh
+#
+# By obtaining, using, and/or copying this software and/or its
+# associated documentation, you agree that you have read, understood,
+# and will comply with the following terms and conditions:
+#
+# Permission to use, copy, modify, and distribute this software and
+# its associated documentation for any purpose and without fee is
+# hereby granted, provided that the above copyright notice appears in
+# all copies, and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# Secret Labs AB or the author not be used in advertising or publicity
+# pertaining to distribution of the software without specific, written
+# prior permission.
+#
+# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
+# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
+# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
+# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
+# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+# OF THIS SOFTWARE.
+# --------------------------------------------------------------------
+
+__all__ = [
+ # public symbols
+ "Comment",
+ "dump",
+ "Element", "ElementTree",
+ "fromstring", "fromstringlist",
+ "iselement", "iterparse",
+ "parse", "ParseError",
+ "PI", "ProcessingInstruction",
+ "QName",
+ "SubElement",
+ "tostring", "tostringlist",
+ "TreeBuilder",
+ "VERSION",
+ "XML", "XMLID",
+ "XMLParser",
+ "register_namespace",
+ ]
+
+VERSION = "1.3.0"
+
+import sys
+import re
+import warnings
+import io
+import contextlib
+
+import ElementPath
+
+
+class ParseError(SyntaxError):
+ """An error when parsing an XML document.
+
+ In addition to its exception value, a ParseError contains
+ two extra attributes:
+ 'code' - the specific exception code
+ 'position' - the line and column of the error
+
+ """
+ pass
+
+# --------------------------------------------------------------------
+
+
+def iselement(element):
+ """Return True if *element* appears to be an Element."""
+ return hasattr(element, 'tag')
+
+
+class Element:
+ """An XML element.
+
+ This class is the reference implementation of the Element interface.
+
+ An element's length is its number of subelements. That means if you
+ want to check if an element is truly empty, you should check BOTH
+ its length AND its text attribute.
+
+ The element tag, attribute names, and attribute values can be either
+ bytes or strings.
+
+ *tag* is the element name. *attrib* is an optional dictionary containing
+ element attributes. *extra* are additional element attributes given as
+ keyword arguments.
+
+ Example form:
+ <tag attrib>text<child/>...</tag>tail
+
+ """
+
+ tag = None
+ """The element's name."""
+
+ attrib = None
+ """Dictionary of the element's attributes."""
+
+ text = None
+ """
+ Text before first subelement. This is either a string or the value None.
+ Note that if there is no text, this attribute may be either
+ None or the empty string, depending on the parser.
+
+ """
+
+ tail = None
+ """
+ Text after this element's end tag, but before the next sibling element's
+ start tag. This is either a string or the value None. Note that if there
+ was no text, this attribute may be either None or an empty string,
+ depending on the parser.
+
+ """
+
+ def __init__(self, tag, attrib={}, **extra):
+ if not isinstance(attrib, dict):
+ raise TypeError("attrib must be dict, not %s" % (
+ attrib.__class__.__name__,))
+ attrib = attrib.copy()
+ attrib.update(extra)
+ self.tag = tag
+ self.attrib = attrib
+ self._children = []
+
+ def __repr__(self):
+ return "<Element %s at 0x%x>" % (repr(self.tag), id(self))
+
+ def makeelement(self, tag, attrib):
+ """Create a new element with the same type.
+
+ *tag* is a string containing the element name.
+ *attrib* is a dictionary containing the element attributes.
+
+ Do not call this method, use the SubElement factory function instead.
+
+ """
+ return self.__class__(tag, attrib)
+
+ def copy(self):
+ """Return copy of current element.
+
+ This creates a shallow copy. Subelements will be shared with the
+ original tree.
+
+ """
+ elem = self.makeelement(self.tag, self.attrib)
+ elem.text = self.text
+ elem.tail = self.tail
+ elem[:] = self
+ return elem
+
+ def __len__(self):
+ return len(self._children)
+
+ def __bool__(self):
+ warnings.warn(
+ "The behavior of this method will change in future versions. "
+ "Use specific 'len(elem)' or 'elem is not None' test instead.",
+ FutureWarning, stacklevel=2
+ )
+ return len(self._children) != 0 # emulate old behaviour, for now
+
+ def __getitem__(self, index):
+ return self._children[index]
+
+ def __setitem__(self, index, element):
+ # if isinstance(index, slice):
+ # for elt in element:
+ # assert iselement(elt)
+ # else:
+ # assert iselement(element)
+ self._children[index] = element
+
+ def __delitem__(self, index):
+ del self._children[index]
+
+ def append(self, subelement):
+ """Add *subelement* to the end of this element.
+
+ The new element will appear in document order after the last existing
+ subelement (or directly after the text, if it's the first subelement),
+ but before the end tag for this element.
+
+ """
+ self._assert_is_element(subelement)
+ self._children.append(subelement)
+
+ def extend(self, elements):
+ """Append subelements from a sequence.
+
+ *elements* is a sequence with zero or more elements.
+
+ """
+ for element in elements:
+ self._assert_is_element(element)
+ self._children.extend(elements)
+
+ def insert(self, index, subelement):
+ """Insert *subelement* at position *index*."""
+ self._assert_is_element(subelement)
+ self._children.insert(index, subelement)
+
+ def _assert_is_element(self, e):
+ # Need to refer to the actual Python implementation, not the
+ # shadowing C implementation.
+ if not isinstance(e, _Element_Py):
+ raise TypeError('expected an Element, not %s' % type(e).__name__)
+
+ def remove(self, subelement):
+ """Remove matching subelement.
+
+ Unlike the find methods, this method compares elements based on
+ identity, NOT ON tag value or contents. To remove subelements by
+ other means, the easiest way is to use a list comprehension to
+ select what elements to keep, and then use slice assignment to update
+ the parent element.
+
+ ValueError is raised if a matching element could not be found.
+
+ """
+ # assert iselement(element)
+ self._children.remove(subelement)
+
+ def getchildren(self):
+ """(Deprecated) Return all subelements.
+
+ Elements are returned in document order.
+
+ """
+ warnings.warn(
+ "This method will be removed in future versions. "
+ "Use 'list(elem)' or iteration over elem instead.",
+ DeprecationWarning, stacklevel=2
+ )
+ return self._children
+
+ def find(self, path, namespaces=None):
+ """Find first matching element by tag name or path.
+
+ *path* is a string having either an element tag or an XPath,
+ *namespaces* is an optional mapping from namespace prefix to full name.
+
+ Return the first matching element, or None if no element was found.
+
+ """
+ return ElementPath.find(self, path, namespaces)
+
+ def findtext(self, path, default=None, namespaces=None):
+ """Find text for first matching element by tag name or path.
+
+ *path* is a string having either an element tag or an XPath,
+ *default* is the value to return if the element was not found,
+ *namespaces* is an optional mapping from namespace prefix to full name.
+
+ Return text content of first matching element, or default value if
+ none was found. Note that if an element is found having no text
+ content, the empty string is returned.
+
+ """
+ return ElementPath.findtext(self, path, default, namespaces)
+
+ def findall(self, path, namespaces=None):
+ """Find all matching subelements by tag name or path.
+
+ *path* is a string having either an element tag or an XPath,
+ *namespaces* is an optional mapping from namespace prefix to full name.
+
+ Returns list containing all matching elements in document order.
+
+ """
+ return ElementPath.findall(self, path, namespaces)
+
+ def iterfind(self, path, namespaces=None):
+ """Find all matching subelements by tag name or path.
+
+ *path* is a string having either an element tag or an XPath,
+ *namespaces* is an optional mapping from namespace prefix to full name.
+
+ Return an iterable yielding all matching elements in document order.
+
+ """
+ return ElementPath.iterfind(self, path, namespaces)
+
+ def clear(self):
+ """Reset element.
+
+ This function removes all subelements, clears all attributes, and sets
+ the text and tail attributes to None.
+
+ """
+ self.attrib.clear()
+ self._children = []
+ self.text = self.tail = None
+
+ def get(self, key, default=None):
+ """Get element attribute.
+
+ Equivalent to attrib.get, but some implementations may handle this a
+ bit more efficiently. *key* is what attribute to look for, and
+ *default* is what to return if the attribute was not found.
+
+ Returns a string containing the attribute value, or the default if
+ attribute was not found.
+
+ """
+ return self.attrib.get(key, default)
+
+ def set(self, key, value):
+ """Set element attribute.
+
+ Equivalent to attrib[key] = value, but some implementations may handle
+ this a bit more efficiently. *key* is what attribute to set, and
+ *value* is the attribute value to set it to.
+
+ """
+ self.attrib[key] = value
+
+ def keys(self):
+ """Get list of attribute names.
+
+ Names are returned in an arbitrary order, just like an ordinary
+ Python dict. Equivalent to attrib.keys()
+
+ """
+ return self.attrib.keys()
+
+ def items(self):
+ """Get element attributes as a sequence.
+
+ The attributes are returned in arbitrary order. Equivalent to
+ attrib.items().
+
+ Return a list of (name, value) tuples.
+
+ """
+ return self.attrib.items()
+
+ def iter(self, tag=None):
+ """Create tree iterator.
+
+ The iterator loops over the element and all subelements in document
+ order, returning all elements with a matching tag.
+
+ If the tree structure is modified during iteration, new or removed
+ elements may or may not be included. To get a stable set, use the
+ list() function on the iterator, and loop over the resulting list.
+
+ *tag* is what tags to look for (default is to return all elements)
+
+ Return an iterator containing all the matching elements.
+
+ """
+ if tag == "*":
+ tag = None
+ if tag is None or self.tag == tag:
+ yield self
+ for e in self._children:
+ yield from e.iter(tag)
+
+ # compatibility
+ def getiterator(self, tag=None):
+ # Change for a DeprecationWarning in 1.4
+ warnings.warn(
+ "This method will be removed in future versions. "
+ "Use 'elem.iter()' or 'list(elem.iter())' instead.",
+ PendingDeprecationWarning, stacklevel=2
+ )
+ return list(self.iter(tag))
+
+ def itertext(self):
+ """Create text iterator.
+
+ The iterator loops over the element and all subelements in document
+ order, returning all inner text.
+
+ """
+ tag = self.tag
+ if not isinstance(tag, str) and tag is not None:
+ return
+ if self.text:
+ yield self.text
+ for e in self:
+ yield from e.itertext()
+ if e.tail:
+ yield e.tail
+
+
+def SubElement(parent, tag, attrib={}, **extra):
+ """Subelement factory which creates an element instance, and appends it
+ to an existing parent.
+
+ The element tag, attribute names, and attribute values can be either
+ bytes or Unicode strings.
+
+ *parent* is the parent element, *tag* is the subelements name, *attrib* is
+ an optional directory containing element attributes, *extra* are
+ additional attributes given as keyword arguments.
+
+ """
+ attrib = attrib.copy()
+ attrib.update(extra)
+ element = parent.makeelement(tag, attrib)
+ parent.append(element)
+ return element
+
+
+def Comment(text=None):
+ """Comment element factory.
+
+ This function creates a special element which the standard serializer
+ serializes as an XML comment.
+
+ *text* is a string containing the comment string.
+
+ """
+ element = Element(Comment)
+ element.text = text
+ return element
+
+
+def ProcessingInstruction(target, text=None):
+ """Processing Instruction element factory.
+
+ This function creates a special element which the standard serializer
+ serializes as an XML comment.
+
+ *target* is a string containing the processing instruction, *text* is a
+ string containing the processing instruction contents, if any.
+
+ """
+ element = Element(ProcessingInstruction)
+ element.text = target
+ if text:
+ element.text = element.text + " " + text
+ return element
+
+PI = ProcessingInstruction
+
+
+class QName:
+ """Qualified name wrapper.
+
+ This class can be used to wrap a QName attribute value in order to get
+ proper namespace handing on output.
+
+ *text_or_uri* is a string containing the QName value either in the form
+ {uri}local, or if the tag argument is given, the URI part of a QName.
+
+ *tag* is an optional argument which if given, will make the first
+ argument (text_or_uri) be interpreted as a URI, and this argument (tag)
+ be interpreted as a local name.
+
+ """
+ def __init__(self, text_or_uri, tag=None):
+ if tag:
+ text_or_uri = "{%s}%s" % (text_or_uri, tag)
+ self.text = text_or_uri
+ def __str__(self):
+ return self.text
+ def __repr__(self):
+ return '<QName %r>' % (self.text,)
+ def __hash__(self):
+ return hash(self.text)
+ def __le__(self, other):
+ if isinstance(other, QName):
+ return self.text <= other.text
+ return self.text <= other
+ def __lt__(self, other):
+ if isinstance(other, QName):
+ return self.text < other.text
+ return self.text < other
+ def __ge__(self, other):
+ if isinstance(other, QName):
+ return self.text >= other.text
+ return self.text >= other
+ def __gt__(self, other):
+ if isinstance(other, QName):
+ return self.text > other.text
+ return self.text > other
+ def __eq__(self, other):
+ if isinstance(other, QName):
+ return self.text == other.text
+ return self.text == other
+ def __ne__(self, other):
+ if isinstance(other, QName):
+ return self.text != other.text
+ return self.text != other
+
+# --------------------------------------------------------------------
+
+
+class ElementTree:
+ """An XML element hierarchy.
+
+ This class also provides support for serialization to and from
+ standard XML.
+
+ *element* is an optional root element node,
+ *file* is an optional file handle or file name of an XML file whose
+ contents will be used to initialize the tree with.
+
+ """
+ def __init__(self, element=None, file=None):
+ # assert element is None or iselement(element)
+ self._root = element # first node
+ if file:
+ self.parse(file)
+
+ def getroot(self):
+ """Return root element of this tree."""
+ return self._root
+
+ def _setroot(self, element):
+ """Replace root element of this tree.
+
+ This will discard the current contents of the tree and replace it
+ with the given element. Use with care!
+
+ """
+ # assert iselement(element)
+ self._root = element
+
+ def parse(self, source, parser=None):
+ """Load external XML document into element tree.
+
+ *source* is a file name or file object, *parser* is an optional parser
+ instance that defaults to XMLParser.
+
+ ParseError is raised if the parser fails to parse the document.
+
+ Returns the root element of the given source document.
+
+ """
+ close_source = False
+ if not hasattr(source, "read"):
+ source = open(source, "rb")
+ close_source = True
+ try:
+ if parser is None:
+ # If no parser was specified, create a default XMLParser
+ parser = XMLParser()
+ if hasattr(parser, '_parse_whole'):
+ # The default XMLParser, when it comes from an accelerator,
+ # can define an internal _parse_whole API for efficiency.
+ # It can be used to parse the whole source without feeding
+ # it with chunks.
+ self._root = parser._parse_whole(source)
+ return self._root
+ while True:
+ data = source.read(65536)
+ if not data:
+ break
+ parser.feed(data)
+ self._root = parser.close()
+ return self._root
+ finally:
+ if close_source:
+ source.close()
+
+ def iter(self, tag=None):
+ """Create and return tree iterator for the root element.
+
+ The iterator loops over all elements in this tree, in document order.
+
+ *tag* is a string with the tag name to iterate over
+ (default is to return all elements).
+
+ """
+ # assert self._root is not None
+ return self._root.iter(tag)
+
+ # compatibility
+ def getiterator(self, tag=None):
+ # Change for a DeprecationWarning in 1.4
+ warnings.warn(
+ "This method will be removed in future versions. "
+ "Use 'tree.iter()' or 'list(tree.iter())' instead.",
+ PendingDeprecationWarning, stacklevel=2
+ )
+ return list(self.iter(tag))
+
+ def find(self, path, namespaces=None):
+ """Find first matching element by tag name or path.
+
+ Same as getroot().find(path), which is Element.find()
+
+ *path* is a string having either an element tag or an XPath,
+ *namespaces* is an optional mapping from namespace prefix to full name.
+
+ Return the first matching element, or None if no element was found.
+
+ """
+ # assert self._root is not None
+ if path[:1] == "/":
+ path = "." + path
+ warnings.warn(
+ "This search is broken in 1.3 and earlier, and will be "
+ "fixed in a future version. If you rely on the current "
+ "behaviour, change it to %r" % path,
+ FutureWarning, stacklevel=2
+ )
+ return self._root.find(path, namespaces)
+
+ def findtext(self, path, default=None, namespaces=None):
+ """Find first matching element by tag name or path.
+
+ Same as getroot().findtext(path), which is Element.findtext()
+
+ *path* is a string having either an element tag or an XPath,
+ *namespaces* is an optional mapping from namespace prefix to full name.
+
+ Return the first matching element, or None if no element was found.
+
+ """
+ # assert self._root is not None
+ if path[:1] == "/":
+ path = "." + path
+ warnings.warn(
+ "This search is broken in 1.3 and earlier, and will be "
+ "fixed in a future version. If you rely on the current "
+ "behaviour, change it to %r" % path,
+ FutureWarning, stacklevel=2
+ )
+ return self._root.findtext(path, default, namespaces)
+
+ def findall(self, path, namespaces=None):
+ """Find all matching subelements by tag name or path.
+
+ Same as getroot().findall(path), which is Element.findall().
+
+ *path* is a string having either an element tag or an XPath,
+ *namespaces* is an optional mapping from namespace prefix to full name.
+
+ Return list containing all matching elements in document order.
+
+ """
+ # assert self._root is not None
+ if path[:1] == "/":
+ path = "." + path
+ warnings.warn(
+ "This search is broken in 1.3 and earlier, and will be "
+ "fixed in a future version. If you rely on the current "
+ "behaviour, change it to %r" % path,
+ FutureWarning, stacklevel=2
+ )
+ return self._root.findall(path, namespaces)
+
+ def iterfind(self, path, namespaces=None):
+ """Find all matching subelements by tag name or path.
+
+ Same as getroot().iterfind(path), which is element.iterfind()
+
+ *path* is a string having either an element tag or an XPath,
+ *namespaces* is an optional mapping from namespace prefix to full name.
+
+ Return an iterable yielding all matching elements in document order.
+
+ """
+ # assert self._root is not None
+ if path[:1] == "/":
+ path = "." + path
+ warnings.warn(
+ "This search is broken in 1.3 and earlier, and will be "
+ "fixed in a future version. If you rely on the current "
+ "behaviour, change it to %r" % path,
+ FutureWarning, stacklevel=2
+ )
+ return self._root.iterfind(path, namespaces)
+
+ def write(self, file_or_filename,
+ encoding=None,
+ xml_declaration=None,
+ default_namespace=None,
+ method=None, *,
+ short_empty_elements=True):
+ """Write element tree to a file as XML.
+
+ Arguments:
+ *file_or_filename* -- file name or a file object opened for writing
+
+ *encoding* -- the output encoding (default: US-ASCII)
+
+ *xml_declaration* -- bool indicating if an XML declaration should be
+ added to the output. If None, an XML declaration
+ is added if encoding IS NOT either of:
+ US-ASCII, UTF-8, or Unicode
+
+ *default_namespace* -- sets the default XML namespace (for "xmlns")
+
+ *method* -- either "xml" (default), "html, "text", or "c14n"
+
+ *short_empty_elements* -- controls the formatting of elements
+ that contain no content. If True (default)
+ they are emitted as a single self-closed
+ tag, otherwise they are emitted as a pair
+ of start/end tags
+
+ """
+ if not method:
+ method = "xml"
+ elif method not in _serialize:
+ raise ValueError("unknown method %r" % method)
+ if not encoding:
+ if method == "c14n":
+ encoding = "utf-8"
+ else:
+ encoding = "us-ascii"
+ enc_lower = encoding.lower()
+ with _get_writer(file_or_filename, enc_lower) as write:
+ if method == "xml" and (xml_declaration or
+ (xml_declaration is None and
+ enc_lower not in ("utf-8", "us-ascii", "unicode"))):
+ declared_encoding = encoding
+ if enc_lower == "unicode":
+ # Retrieve the default encoding for the xml declaration
+ import locale
+ declared_encoding = locale.getpreferredencoding()
+ write("<?xml version='1.0' encoding='%s'?>\n" % (
+ declared_encoding,))
+ if method == "text":
+ _serialize_text(write, self._root)
+ else:
+ qnames, namespaces = _namespaces(self._root, default_namespace)
+ serialize = _serialize[method]
+ serialize(write, self._root, qnames, namespaces,
+ short_empty_elements=short_empty_elements)
+
+ def write_c14n(self, file):
+ # lxml.etree compatibility. use output method instead
+ return self.write(file, method="c14n")
+
+# --------------------------------------------------------------------
+# serialization support
+
+@contextlib.contextmanager
+def _get_writer(file_or_filename, encoding):
+ # returns text write method and release all resources after using
+ try:
+ write = file_or_filename.write
+ except AttributeError:
+ # file_or_filename is a file name
+ if encoding == "unicode":
+ file = open(file_or_filename, "w")
+ else:
+ file = open(file_or_filename, "w", encoding=encoding,
+ errors="xmlcharrefreplace")
+ with file:
+ yield file.write
+ else:
+ # file_or_filename is a file-like object
+ # encoding determines if it is a text or binary writer
+ if encoding == "unicode":
+ # use a text writer as is
+ yield write
+ else:
+ # wrap a binary writer with TextIOWrapper
+ with contextlib.ExitStack() as stack:
+ if isinstance(file_or_filename, io.BufferedIOBase):
+ file = file_or_filename
+ elif isinstance(file_or_filename, io.RawIOBase):
+ file = io.BufferedWriter(file_or_filename)
+ # Keep the original file open when the BufferedWriter is
+ # destroyed
+ stack.callback(file.detach)
+ else:
+ # This is to handle passed objects that aren't in the
+ # IOBase hierarchy, but just have a write method
+ file = io.BufferedIOBase()
+ file.writable = lambda: True
+ file.write = write
+ try:
+ # TextIOWrapper uses this methods to determine
+ # if BOM (for UTF-16, etc) should be added
+ file.seekable = file_or_filename.seekable
+ file.tell = file_or_filename.tell
+ except AttributeError:
+ pass
+ file = io.TextIOWrapper(file,
+ encoding=encoding,
+ errors="xmlcharrefreplace",
+ newline="\n")
+ # Keep the original file open when the TextIOWrapper is
+ # destroyed
+ stack.callback(file.detach)
+ yield file.write
+
+def _namespaces(elem, default_namespace=None):
+ # identify namespaces used in this tree
+
+ # maps qnames to *encoded* prefix:local names
+ qnames = {None: None}
+
+ # maps uri:s to prefixes
+ namespaces = {}
+ if default_namespace:
+ namespaces[default_namespace] = ""
+
+ def add_qname(qname):
+ # calculate serialized qname representation
+ try:
+ if qname[:1] == "{":
+ uri, tag = qname[1:].rsplit("}", 1)
+ prefix = namespaces.get(uri)
+ if prefix is None:
+ prefix = _namespace_map.get(uri)
+ if prefix is None:
+ prefix = "ns%d" % len(namespaces)
+ if prefix != "xml":
+ namespaces[uri] = prefix
+ if prefix:
+ qnames[qname] = "%s:%s" % (prefix, tag)
+ else:
+ qnames[qname] = tag # default element
+ else:
+ if default_namespace:
+ # FIXME: can this be handled in XML 1.0?
+ raise ValueError(
+ "cannot use non-qualified names with "
+ "default_namespace option"
+ )
+ qnames[qname] = qname
+ except TypeError:
+ _raise_serialization_error(qname)
+
+ # populate qname and namespaces table
+ for elem in elem.iter():
+ tag = elem.tag
+ if isinstance(tag, QName):
+ if tag.text not in qnames:
+ add_qname(tag.text)
+ elif isinstance(tag, str):
+ if tag not in qnames:
+ add_qname(tag)
+ elif tag is not None and tag is not Comment and tag is not PI:
+ _raise_serialization_error(tag)
+ for key, value in elem.items():
+ if isinstance(key, QName):
+ key = key.text
+ if key not in qnames:
+ add_qname(key)
+ if isinstance(value, QName) and value.text not in qnames:
+ add_qname(value.text)
+ text = elem.text
+ if isinstance(text, QName) and text.text not in qnames:
+ add_qname(text.text)
+ return qnames, namespaces
+
+def _serialize_xml(write, elem, qnames, namespaces,
+ short_empty_elements, **kwargs):
+ tag = elem.tag
+ text = elem.text
+ if tag is Comment:
+ write("<!--%s-->" % text)
+ elif tag is ProcessingInstruction:
+ write("<?%s?>" % text)
+ else:
+ tag = qnames[tag]
+ if tag is None:
+ if text:
+ write(_escape_cdata(text))
+ for e in elem:
+ _serialize_xml(write, e, qnames, None,
+ short_empty_elements=short_empty_elements)
+ else:
+ write("<" + tag)
+ items = list(elem.items())
+ if items or namespaces:
+ if namespaces:
+ for v, k in sorted(namespaces.items(),
+ key=lambda x: x[1]): # sort on prefix
+ if k:
+ k = ":" + k
+ write(" xmlns%s=\"%s\"" % (
+ k,
+ _escape_attrib(v)
+ ))
+ for k, v in sorted(items): # lexical order
+ if isinstance(k, QName):
+ k = k.text
+ if isinstance(v, QName):
+ v = qnames[v.text]
+ else:
+ v = _escape_attrib(v)
+ write(" %s=\"%s\"" % (qnames[k], v))
+ if text or len(elem) or not short_empty_elements:
+ write(">")
+ if text:
+ write(_escape_cdata(text))
+ for e in elem:
+ _serialize_xml(write, e, qnames, None,
+ short_empty_elements=short_empty_elements)
+ write("</" + tag + ">")
+ else:
+ write(" />")
+ if elem.tail:
+ write(_escape_cdata(elem.tail))
+
+# add from cvw jan 2019
+def _serialize_pretty_xml(write, elem, qnames, namespaces,
+ short_empty_elements, indent=0):
+ # print("*****pretty***** indent", elem.tag, indent)
+ tag = elem.tag
+ text = elem.text
+ if tag is Comment:
+ write("<!--%s-->" % text)
+ elif tag is ProcessingInstruction:
+ write("<?%s?>" % text)
+ else:
+ tag = qnames[tag]
+ if tag is None:
+ if text:
+ write(_escape_cdata(text))
+ for e in elem:
+ _serialize_pretty_xml(write, e, qnames, None,
+ short_empty_elements=short_empty_elements, indent=indent)
+ else:
+ write(" "*indent + "<" + tag)
+ items = list(elem.items())
+ if items or namespaces:
+ if namespaces:
+ for v, k in sorted(namespaces.items(),
+ key=lambda x: x[1]): # sort on prefix
+ if k:
+ k = ":" + k
+ write(" xmlns%s=\"%s\"" % (
+ k,
+ _escape_attrib(v)
+ ))
+ for k, v in sorted(items): # lexical order
+ # print("atrrib ", k, v)
+ if isinstance(k, QName):
+ k = k.text
+ if isinstance(v, QName):
+ v = qnames[v.text]
+ else:
+ v = _escape_attrib(v)
+ write(" %s=\"%s\"" % (qnames[k], v))
+ if text or len(elem) or not short_empty_elements:
+ if text:
+ write(">")
+ write(_escape_cdata(text))
+ else:
+ write(">\n")
+
+ for e in elem:
+ _serialize_pretty_xml(write, e, qnames, None,
+ short_empty_elements=short_empty_elements, indent=indent+2)
+ write(" "*indent + "</" + tag + ">\n")
+ else:
+ write(" />\n")
+ if elem.tail:
+ write(_escape_cdata(elem.tail))
+
+
+HTML_EMPTY = ("area", "base", "basefont", "br", "col", "frame", "hr",
+ "img", "input", "isindex", "link", "meta", "param")
+
+try:
+ HTML_EMPTY = set(HTML_EMPTY)
+except NameError:
+ pass
+
+def _serialize_html(write, elem, qnames, namespaces, **kwargs):
+ tag = elem.tag
+ text = elem.text
+ if tag is Comment:
+ write("<!--%s-->" % _escape_cdata(text))
+ elif tag is ProcessingInstruction:
+ write("<?%s?>" % _escape_cdata(text))
+ else:
+ tag = qnames[tag]
+ if tag is None:
+ if text:
+ write(_escape_cdata(text))
+ for e in elem:
+ _serialize_html(write, e, qnames, None)
+ else:
+ write("<" + tag)
+ items = list(elem.items())
+ if items or namespaces:
+ if namespaces:
+ for v, k in sorted(namespaces.items(),
+ key=lambda x: x[1]): # sort on prefix
+ if k:
+ k = ":" + k
+ write(" xmlns%s=\"%s\"" % (
+ k,
+ _escape_attrib(v)
+ ))
+ for k, v in sorted(items): # lexical order
+ if isinstance(k, QName):
+ k = k.text
+ if isinstance(v, QName):
+ v = qnames[v.text]
+ else:
+ v = _escape_attrib_html(v)
+ # FIXME: handle boolean attributes
+ write(" %s=\"%s\"" % (qnames[k], v))
+ write(">")
+ ltag = tag.lower()
+ if text:
+ if ltag == "script" or ltag == "style":
+ write(text)
+ else:
+ write(_escape_cdata(text))
+ for e in elem:
+ _serialize_html(write, e, qnames, None)
+ if ltag not in HTML_EMPTY:
+ write("</" + tag + ">")
+ if elem.tail:
+ write(_escape_cdata(elem.tail))
+
+def _serialize_text(write, elem):
+ for part in elem.itertext():
+ write(part)
+ if elem.tail:
+ write(elem.tail)
+
+_serialize = {
+ "xml": _serialize_xml,
+ "pretty_xml": _serialize_pretty_xml,
+ "html": _serialize_html,
+ "text": _serialize_text,
+# this optional method is imported at the end of the module
+# "c14n": _serialize_c14n,
+}
+
+
+def register_namespace(prefix, uri):
+ """Register a namespace prefix.
+
+ The registry is global, and any existing mapping for either the
+ given prefix or the namespace URI will be removed.
+
+ *prefix* is the namespace prefix, *uri* is a namespace uri. Tags and
+ attributes in this namespace will be serialized with prefix if possible.
+
+ ValueError is raised if prefix is reserved or is invalid.
+
+ """
+ if re.match("ns\d+$", prefix):
+ raise ValueError("Prefix format reserved for internal use")
+ for k, v in list(_namespace_map.items()):
+ if k == uri or v == prefix:
+ del _namespace_map[k]
+ _namespace_map[uri] = prefix
+
+_namespace_map = {
+ # "well-known" namespace prefixes
+ "http://www.w3.org/XML/1998/namespace": "xml",
+ "http://www.w3.org/1999/xhtml": "html",
+ "http://www.w3.org/1999/02/22-rdf-syntax-ns#": "rdf",
+ "http://schemas.xmlsoap.org/wsdl/": "wsdl",
+ # xml schema
+ "http://www.w3.org/2001/XMLSchema": "xs",
+ "http://www.w3.org/2001/XMLSchema-instance": "xsi",
+ # dublin core
+ "http://purl.org/dc/elements/1.1/": "dc",
+}
+# For tests and troubleshooting
+register_namespace._namespace_map = _namespace_map
+
+def _raise_serialization_error(text):
+ raise TypeError(
+ "cannot serialize %r (type %s)" % (text, type(text).__name__)
+ )
+
+def _escape_cdata(text):
+ # escape character data
+ try:
+ # it's worth avoiding do-nothing calls for strings that are
+ # shorter than 500 character, or so. assume that's, by far,
+ # the most common case in most applications.
+ if "&" in text:
+ text = text.replace("&", "&")
+ if "<" in text:
+ text = text.replace("<", "<")
+ if ">" in text:
+ text = text.replace(">", ">")
+ return text
+ except (TypeError, AttributeError):
+ _raise_serialization_error(text)
+
+def _escape_attrib(text):
+ # escape attribute value
+ try:
+ if "&" in text:
+ text = text.replace("&", "&")
+ if "<" in text:
+ text = text.replace("<", "<")
+ if ">" in text:
+ text = text.replace(">", ">")
+ if "\"" in text:
+ text = text.replace("\"", """)
+ if "\n" in text:
+ text = text.replace("\n", " ")
+ return text
+ except (TypeError, AttributeError):
+ _raise_serialization_error(text)
+
+def _escape_attrib_html(text):
+ # escape attribute value
+ try:
+ if "&" in text:
+ text = text.replace("&", "&")
+ if ">" in text:
+ text = text.replace(">", ">")
+ if "\"" in text:
+ text = text.replace("\"", """)
+ return text
+ except (TypeError, AttributeError):
+ _raise_serialization_error(text)
+
+# --------------------------------------------------------------------
+
+def tostring(element, encoding=None, method=None, *,
+ short_empty_elements=True):
+ """Generate string representation of XML element.
+
+ All subelements are included. If encoding is "unicode", a string
+ is returned. Otherwise a bytestring is returned.
+
+ *element* is an Element instance, *encoding* is an optional output
+ encoding defaulting to US-ASCII, *method* is an optional output which can
+ be one of "xml" (default), "html", "text" or "c14n".
+
+ Returns an (optionally) encoded string containing the XML data.
+
+ """
+ stream = io.StringIO() if encoding == 'unicode' else io.BytesIO()
+ ElementTree(element).write(stream, encoding, method=method,
+ short_empty_elements=short_empty_elements)
+ return stream.getvalue()
+
+class _ListDataStream(io.BufferedIOBase):
+ """An auxiliary stream accumulating into a list reference."""
+ def __init__(self, lst):
+ self.lst = lst
+
+ def writable(self):
+ return True
+
+ def seekable(self):
+ return True
+
+ def write(self, b):
+ self.lst.append(b)
+
+ def tell(self):
+ return len(self.lst)
+
+def tostringlist(element, encoding=None, method=None, *,
+ short_empty_elements=True):
+ lst = []
+ stream = _ListDataStream(lst)
+ ElementTree(element).write(stream, encoding, method=method,
+ short_empty_elements=short_empty_elements)
+ return lst
+
+
+def dump(elem):
+ """Write element tree or element structure to sys.stdout.
+
+ This function should be used for debugging only.
+
+ *elem* is either an ElementTree, or a single Element. The exact output
+ format is implementation dependent. In this version, it's written as an
+ ordinary XML file.
+
+ """
+ # debugging
+ if not isinstance(elem, ElementTree):
+ elem = ElementTree(elem)
+ elem.write(sys.stdout, encoding="unicode")
+ tail = elem.getroot().tail
+ if not tail or tail[-1] != "\n":
+ sys.stdout.write("\n")
+
+# --------------------------------------------------------------------
+# parsing
+
+
+def parse(source, parser=None):
+ """Parse XML document into element tree.
+
+ *source* is a filename or file object containing XML data,
+ *parser* is an optional parser instance defaulting to XMLParser.
+
+ Return an ElementTree instance.
+
+ """
+ tree = ElementTree()
+ tree.parse(source, parser)
+ return tree
+
+
+def iterparse(source, events=None, parser=None):
+ """Incrementally parse XML document into ElementTree.
+
+ This class also reports what's going on to the user based on the
+ *events* it is initialized with. The supported events are the strings
+ "start", "end", "start-ns" and "end-ns" (the "ns" events are used to get
+ detailed namespace information). If *events* is omitted, only
+ "end" events are reported.
+
+ *source* is a filename or file object containing XML data, *events* is
+ a list of events to report back, *parser* is an optional parser instance.
+
+ Returns an iterator providing (event, elem) pairs.
+
+ """
+ close_source = False
+ if not hasattr(source, "read"):
+ source = open(source, "rb")
+ close_source = True
+ try:
+ return _IterParseIterator(source, events, parser, close_source)
+ except:
+ if close_source:
+ source.close()
+ raise
+
+
+class XMLPullParser:
+
+ def __init__(self, events=None, *, _parser=None):
+ # The _parser argument is for internal use only and must not be relied
+ # upon in user code. It will be removed in a future release.
+ # See http://bugs.python.org/issue17741 for more details.
+
+ # _elementtree.c expects a list, not a deque
+ self._events_queue = []
+ self._index = 0
+ self._parser = _parser or XMLParser(target=TreeBuilder())
+ # wire up the parser for event reporting
+ if events is None:
+ events = ("end",)
+ self._parser._setevents(self._events_queue, events)
+
+ def feed(self, data):
+ """Feed encoded data to parser."""
+ if self._parser is None:
+ raise ValueError("feed() called after end of stream")
+ if data:
+ try:
+ self._parser.feed(data)
+ except SyntaxError as exc:
+ self._events_queue.append(exc)
+
+ def _close_and_return_root(self):
+ # iterparse needs this to set its root attribute properly :(
+ root = self._parser.close()
+ self._parser = None
+ return root
+
+ def close(self):
+ """Finish feeding data to parser.
+
+ Unlike XMLParser, does not return the root element. Use
+ read_events() to consume elements from XMLPullParser.
+ """
+ self._close_and_return_root()
+
+ def read_events(self):
+ """Return an iterator over currently available (event, elem) pairs.
+
+ Events are consumed from the internal event queue as they are
+ retrieved from the iterator.
+ """
+ events = self._events_queue
+ while True:
+ index = self._index
+ try:
+ event = events[self._index]
+ # Avoid retaining references to past events
+ events[self._index] = None
+ except IndexError:
+ break
+ index += 1
+ # Compact the list in a O(1) amortized fashion
+ # As noted above, _elementree.c needs a list, not a deque
+ if index * 2 >= len(events):
+ events[:index] = []
+ self._index = 0
+ else:
+ self._index = index
+ if isinstance(event, Exception):
+ raise event
+ else:
+ yield event
+
+
+class _IterParseIterator:
+
+ def __init__(self, source, events, parser, close_source=False):
+ # Use the internal, undocumented _parser argument for now; When the
+ # parser argument of iterparse is removed, this can be killed.
+ self._parser = XMLPullParser(events=events, _parser=parser)
+ self._file = source
+ self._close_file = close_source
+ self.root = self._root = None
+
+ def __next__(self):
+ try:
+ while 1:
+ for event in self._parser.read_events():
+ return event
+ if self._parser._parser is None:
+ break
+ # load event buffer
+ data = self._file.read(16 * 1024)
+ if data:
+ self._parser.feed(data)
+ else:
+ self._root = self._parser._close_and_return_root()
+ self.root = self._root
+ except:
+ if self._close_file:
+ self._file.close()
+ raise
+ if self._close_file:
+ self._file.close()
+ raise StopIteration
+
+ def __iter__(self):
+ return self
+
+
+def XML(text, parser=None):
+ """Parse XML document from string constant.
+
+ This function can be used to embed "XML Literals" in Python code.
+
+ *text* is a string containing XML data, *parser* is an
+ optional parser instance, defaulting to the standard XMLParser.
+
+ Returns an Element instance.
+
+ """
+ if not parser:
+ parser = XMLParser(target=TreeBuilder())
+ parser.feed(text)
+ return parser.close()
+
+
+def XMLID(text, parser=None):
+ """Parse XML document from string constant for its IDs.
+
+ *text* is a string containing XML data, *parser* is an
+ optional parser instance, defaulting to the standard XMLParser.
+
+ Returns an (Element, dict) tuple, in which the
+ dict maps element id:s to elements.
+
+ """
+ if not parser:
+ parser = XMLParser(target=TreeBuilder())
+ parser.feed(text)
+ tree = parser.close()
+ ids = {}
+ for elem in tree.iter():
+ id = elem.get("id")
+ if id:
+ ids[id] = elem
+ return tree, ids
+
+# Parse XML document from string constant. Alias for XML().
+fromstring = XML
+
+def fromstringlist(sequence, parser=None):
+ """Parse XML document from sequence of string fragments.
+
+ *sequence* is a list of other sequence, *parser* is an optional parser
+ instance, defaulting to the standard XMLParser.
+
+ Returns an Element instance.
+
+ """
+ if not parser:
+ parser = XMLParser(target=TreeBuilder())
+ for text in sequence:
+ parser.feed(text)
+ return parser.close()
+
+# --------------------------------------------------------------------
+
+
+class TreeBuilder:
+ """Generic element structure builder.
+
+ This builder converts a sequence of start, data, and end method
+ calls to a well-formed element structure.
+
+ You can use this class to build an element structure using a custom XML
+ parser, or a parser for some other XML-like format.
+
+ *element_factory* is an optional element factory which is called
+ to create new Element instances, as necessary.
+
+ """
+ def __init__(self, element_factory=None):
+ self._data = [] # data collector
+ self._elem = [] # element stack
+ self._last = None # last element
+ self._tail = None # true if we're after an end tag
+ if element_factory is None:
+ element_factory = Element
+ self._factory = element_factory
+
+ def close(self):
+ """Flush builder buffers and return toplevel document Element."""
+ assert len(self._elem) == 0, "missing end tags"
+ assert self._last is not None, "missing toplevel element"
+ return self._last
+
+ def _flush(self):
+ if self._data:
+ if self._last is not None:
+ text = "".join(self._data)
+ if self._tail:
+ assert self._last.tail is None, "internal error (tail)"
+ self._last.tail = text
+ else:
+ assert self._last.text is None, "internal error (text)"
+ self._last.text = text
+ self._data = []
+
+ def data(self, data):
+ """Add text to current element."""
+ self._data.append(data)
+
+ def start(self, tag, attrs):
+ """Open new element and return it.
+
+ *tag* is the element name, *attrs* is a dict containing element
+ attributes.
+
+ """
+ self._flush()
+ self._last = elem = self._factory(tag, attrs)
+ if self._elem:
+ self._elem[-1].append(elem)
+ self._elem.append(elem)
+ self._tail = 0
+ return elem
+
+ def end(self, tag):
+ """Close and return current Element.
+
+ *tag* is the element name.
+
+ """
+ self._flush()
+ self._last = self._elem.pop()
+ assert self._last.tag == tag,\
+ "end tag mismatch (expected %s, got %s)" % (
+ self._last.tag, tag)
+ self._tail = 1
+ return self._last
+
+
+# also see ElementTree and TreeBuilder
+class XMLParser:
+ """Element structure builder for XML source data based on the expat parser.
+
+ *html* are predefined HTML entities (not supported currently),
+ *target* is an optional target object which defaults to an instance of the
+ standard TreeBuilder class, *encoding* is an optional encoding string
+ which if given, overrides the encoding specified in the XML file:
+ http://www.iana.org/assignments/character-sets
+
+ """
+
+ def __init__(self, html=0, target=None, encoding=None):
+ try:
+ from xml.parsers import expat
+ except ImportError:
+ try:
+ import pyexpat as expat
+ except ImportError:
+ raise ImportError(
+ "No module named expat; use SimpleXMLTreeBuilder instead"
+ )
+ parser = expat.ParserCreate(encoding, "}")
+ if target is None:
+ target = TreeBuilder()
+ # underscored names are provided for compatibility only
+ self.parser = self._parser = parser
+ self.target = self._target = target
+ self._error = expat.error
+ self._names = {} # name memo cache
+ # main callbacks
+ parser.DefaultHandlerExpand = self._default
+ if hasattr(target, 'start'):
+ parser.StartElementHandler = self._start
+ if hasattr(target, 'end'):
+ parser.EndElementHandler = self._end
+ if hasattr(target, 'data'):
+ parser.CharacterDataHandler = target.data
+ # miscellaneous callbacks
+ if hasattr(target, 'comment'):
+ parser.CommentHandler = target.comment
+ if hasattr(target, 'pi'):
+ parser.ProcessingInstructionHandler = target.pi
+ # Configure pyexpat: buffering, new-style attribute handling.
+ parser.buffer_text = 1
+ parser.ordered_attributes = 1
+ parser.specified_attributes = 1
+ self._doctype = None
+ self.entity = {}
+ try:
+ self.version = "Expat %d.%d.%d" % expat.version_info
+ except AttributeError:
+ pass # unknown
+
+ def _setevents(self, events_queue, events_to_report):
+ # Internal API for XMLPullParser
+ # events_to_report: a list of events to report during parsing (same as
+ # the *events* of XMLPullParser's constructor.
+ # events_queue: a list of actual parsing events that will be populated
+ # by the underlying parser.
+ #
+ parser = self._parser
+ append = events_queue.append
+ for event_name in events_to_report:
+ if event_name == "start":
+ parser.ordered_attributes = 1
+ parser.specified_attributes = 1
+ def handler(tag, attrib_in, event=event_name, append=append,
+ start=self._start):
+ append((event, start(tag, attrib_in)))
+ parser.StartElementHandler = handler
+ elif event_name == "end":
+ def handler(tag, event=event_name, append=append,
+ end=self._end):
+ append((event, end(tag)))
+ parser.EndElementHandler = handler
+ elif event_name == "start-ns":
+ def handler(prefix, uri, event=event_name, append=append):
+ append((event, (prefix or "", uri or "")))
+ parser.StartNamespaceDeclHandler = handler
+ elif event_name == "end-ns":
+ def handler(prefix, event=event_name, append=append):
+ append((event, None))
+ parser.EndNamespaceDeclHandler = handler
+ else:
+ raise ValueError("unknown event %r" % event_name)
+
+ def _raiseerror(self, value):
+ err = ParseError(value)
+ err.code = value.code
+ err.position = value.lineno, value.offset
+ raise err
+
+ def _fixname(self, key):
+ # expand qname, and convert name string to ascii, if possible
+ try:
+ name = self._names[key]
+ except KeyError:
+ name = key
+ if "}" in name:
+ name = "{" + name
+ self._names[key] = name
+ return name
+
+ def _start(self, tag, attr_list):
+ # Handler for expat's StartElementHandler. Since ordered_attributes
+ # is set, the attributes are reported as a list of alternating
+ # attribute name,value.
+ fixname = self._fixname
+ tag = fixname(tag)
+ attrib = {}
+ if attr_list:
+ for i in range(0, len(attr_list), 2):
+ attrib[fixname(attr_list[i])] = attr_list[i+1]
+ return self.target.start(tag, attrib)
+
+ def _end(self, tag):
+ return self.target.end(self._fixname(tag))
+
+ def _default(self, text):
+ prefix = text[:1]
+ if prefix == "&":
+ # deal with undefined entities
+ try:
+ data_handler = self.target.data
+ except AttributeError:
+ return
+ try:
+ data_handler(self.entity[text[1:-1]])
+ except KeyError:
+ from xml.parsers import expat
+ err = expat.error(
+ "undefined entity %s: line %d, column %d" %
+ (text, self.parser.ErrorLineNumber,
+ self.parser.ErrorColumnNumber)
+ )
+ err.code = 11 # XML_ERROR_UNDEFINED_ENTITY
+ err.lineno = self.parser.ErrorLineNumber
+ err.offset = self.parser.ErrorColumnNumber
+ raise err
+ elif prefix == "<" and text[:9] == "<!DOCTYPE":
+ self._doctype = [] # inside a doctype declaration
+ elif self._doctype is not None:
+ # parse doctype contents
+ if prefix == ">":
+ self._doctype = None
+ return
+ text = text.strip()
+ if not text:
+ return
+ self._doctype.append(text)
+ n = len(self._doctype)
+ if n > 2:
+ type = self._doctype[1]
+ if type == "PUBLIC" and n == 4:
+ name, type, pubid, system = self._doctype
+ if pubid:
+ pubid = pubid[1:-1]
+ elif type == "SYSTEM" and n == 3:
+ name, type, system = self._doctype
+ pubid = None
+ else:
+ return
+ if hasattr(self.target, "doctype"):
+ self.target.doctype(name, pubid, system[1:-1])
+ elif self.doctype != self._XMLParser__doctype:
+ # warn about deprecated call
+ self._XMLParser__doctype(name, pubid, system[1:-1])
+ self.doctype(name, pubid, system[1:-1])
+ self._doctype = None
+
+ def doctype(self, name, pubid, system):
+ """(Deprecated) Handle doctype declaration
+
+ *name* is the Doctype name, *pubid* is the public identifier,
+ and *system* is the system identifier.
+
+ """
+ warnings.warn(
+ "This method of XMLParser is deprecated. Define doctype() "
+ "method on the TreeBuilder target.",
+ DeprecationWarning,
+ )
+
+ # sentinel, if doctype is redefined in a subclass
+ __doctype = doctype
+
+ def feed(self, data):
+ """Feed encoded data to parser."""
+ try:
+ self.parser.Parse(data, 0)
+ except self._error as v:
+ self._raiseerror(v)
+
+ def close(self):
+ """Finish feeding data to parser and return element structure."""
+ try:
+ self.parser.Parse("", 1) # end of data
+ except self._error as v:
+ self._raiseerror(v)
+ try:
+ close_handler = self.target.close
+ except AttributeError:
+ pass
+ else:
+ return close_handler()
+ finally:
+ # get rid of circular references
+ del self.parser, self._parser
+ del self.target, self._target
+
+
+# Import the C accelerators
+try:
+ # Element is going to be shadowed by the C implementation. We need to keep
+ # the Python version of it accessible for some "creative" by external code
+ # (see tests)
+ _Element_Py = Element
+
+ # Element, SubElement, ParseError, TreeBuilder, XMLParser
+ from _elementtree import *
+except ImportError:
+ pass
\ No newline at end of file
import errno
import stat
import fnmatch
+import pprint as PP
from ftplib import FTP
from . import pyconf
return config[param_name]
return default
+def strSplitN(aList, nb, skip="\n "):
+ """
+ example
+ aStr = 'this-is-a-string'
+ splitN(aStr, 2, '-')
+ split it by every 2nd '-' rather than every '-'
+ """
+ strValue = ""
+ i = 0
+ for v in aList:
+ strValue += "%15s, " % str(v)
+ i += 1
+ if i >= nb:
+ strValue += skip
+ i = 0
+ if len(aList) > nb:
+ strValue = skip + strValue
+ return strValue
def getProductNames(cfg, wildcards, logger):
"""get products names using * or ? as wildcards like shell Linux"""
wilds = wildcards
else:
wilds = [wildcards]
+ notFound = {}
products = cfg.APPLICATION.products.keys()
- for prod in products:
- for wild in wildcards:
+ for wild in wildcards:
+ ok = False
+ for prod in products:
filtered = fnmatch.filter([prod], wild)
# print("filtered", prod, wild, filtered)
if len(filtered) > 0:
res.append(prod)
+ ok = True
break
+ if not ok:
+ notFound[wild] = None
if len(res) == 0:
logger.warning("Empty list of products, from %s" % wilds)
+ if len(notFound.keys()) > 0:
+ strProd = strSplitN( sorted(products), 5)
+ logger.warning("products not found: %s\n availables products are:\n%s" % \
+ (sorted(notFound.keys()), strProd) )
return res
'''
# In windows case, the USERNAME environment variable has to be set
if is_windows():
- if not os.environ.has_key('USERNAME'):
+ if not 'USERNAME' in os.environ:
raise Exception('USERNAME environment variable not set')
return os.environ['USERNAME']
else: # linux
path = lsb_path + ":" + path
from subprocess import Popen, PIPE
- res = Popen(['lsb_release', args], env={'PATH': path},
- stdout=PIPE).communicate()[0][:-1]
+ res = Popen(['lsb_release', args], env={'PATH': path}, stdout=PIPE).communicate()[0][:-1]
# in case of python3, convert byte to str
if isinstance(res, bytes):
res = res.decode()
return res
except OSError:
sys.stderr.write(_(u"lsb_release not installed\n"))
- sys.stderr.write(_(u"You can define $LSB_PATH to give"
- " the path to lsb_release\n"))
+ sys.stderr.write(_(u"You can define $LSB_PATH to give the path to lsb_release\n"))
sys.exit(-1)
def get_distribution(codes):
distrib = codes[distrib]
else:
sys.stderr.write(_(u"Unknown distribution: '%s'\n") % distrib)
- sys.stderr.write(_(u"Please add your distribution to"
- " src/internal_config/distrib.pyconf\n"))
+ sys.stderr.write(_(u"Please add your distribution to src/internal_config/distrib.pyconf\n"))
sys.exit(-1)
return distrib
+def get_infosys():
+ """
+ from a CentOS example,
+ returns '7.6' from command 'lsb_release -ds'
+ extracted from 'CentOS Linux release 7.6.1810 (Core)'
+ and also for RedHat fedora etc.
+ """
+ import re
+ osys = ""
+ version = ""
+ architecture = ""
+ osys_value = "Unknown"
+ os_dict = {"mandrivalinux":"MD",
+ "centos":"CO",
+ "RedHatEnterpriseServer":"CO",
+ "RedHatEnterpriseWorkstation":"CO",
+ "fedora":"FD",
+ "ubuntu":"UB",
+ "debian":"DB",
+ "mageia":"MG",}
+ # lsb_cmd = "lsb_release -ds"
+ args = "-ds"
+ output = _lsb_release(args)
+ # print("lsb_release output %s" % output)
+ regexp = r"(^[0-9]+([.]?[0-9]+)+)"
+ for an_item in output.replace('"','').split():
+ if re.match(regexp, an_item) is not None and not version:
+ version = ".".join(an_item.split(".")[:2])
+ else:
+ for sub_item in os_dict.keys():
+ if sub_item == an_item.lower():
+ osys = an_item
+ osys_value = os_dict[sub_item]
+ if version and osys: break
+ return version
def get_distrib_version(distrib, codes):
'''Gets the version of the distribution
--- /dev/null
+#!/usr/bin/env python
+#-*- coding:utf-8 -*-
+
+# https://gist.github.com/techtonik/2151727
+# Public Domain, i.e. feel free to copy/paste
+# Considered a hack in Python 2
+
+import inspect
+import logging
+import sys
+
+
+def caller_name(skip=1):
+ """Get a name of a caller in the format module.class.method
+
+ `skip` specifies how many levels of stack to skip while getting caller
+ name. skip=1 means "who calls me", skip=2 "who calls my caller" etc.
+
+ An empty string is returned if skipped levels exceed stack height
+ """
+ def stack_(frame):
+ framelist = []
+ while frame:
+ framelist.append(frame)
+ frame = frame.f_back
+ return framelist
+
+ stack = stack_(sys._getframe(1))
+ start = 0 + skip
+ if len(stack) < start + 1:
+ return ''
+ parentframe = stack[start]
+
+ name = []
+ module = inspect.getmodule(parentframe)
+ # `modname` can be None when frame is executed directly in console
+ # TODO(techtonik): consider using __main__
+ if module:
+ name.append(module.__name__)
+ # detect classname
+ if 'self' in parentframe.f_locals:
+ # I don't know any way to detect call from the object method
+ # XXX: there seems to be no way to detect static method call - it will
+ # be just a function call
+ name.append(parentframe.f_locals['self'].__class__.__name__)
+ codename = parentframe.f_code.co_name
+ lineno = inspect.currentframe().f_back.f_back.f_lineno
+ if codename != '<module>': # top level usually
+ name.append(codename + "[%s]" % str(lineno)) # function or a method
+ del parentframe
+ return ".".join(name)
\ No newline at end of file
import os
import sys
import traceback
-import StringIO as SIO
import pprint as PP
+# Compatibility python 2/3 for unicode
+try:
+ _test = unicode
+except:
+ unicode = str
+
+# Compatibility python 2/3 for StringIO
+try:
+ from StringIO import StringIO
+except ImportError:
+ from io import StringIO
+
_debug = [False] #support push/pop for temporary activate debug outputs
_user = os.environ['USER']
# utilitaires divers pour debug
###############################################
-class OutStream(SIO.StringIO):
+class OutStream(StringIO):
"""
utility class for pyconf.Config output iostream
"""
keep value before lost as self.value
"""
self.value = self.getvalue()
- SIO.StringIO.close(self)
+ StringIO.close(self)
-class InStream(SIO.StringIO):
+class InStream(StringIO):
"""utility class for pyconf.Config input iostream"""
pass
src.printcolors.printcInfo(name), sign, value))
def is_defined(self, name):
- return self.defined.has_key(name)
+ return name in self.defined
def get(self, name):
return "${%s}" % name
:param delai int: the number max
:param ss str: the string to display
"""
- logger.write("\r%s\r%s %s / %s " % ((" " * 30), ss, top, (delai - top)), 4,
- False)
+ logger.write("\r%s\r%s timeout %s / %s stay %s s " % ((" " * 30), ss, top, delai, (delai - top)), 4, False)
logger.flush()
def write_back(logger, message, level):
# --------------
def launch_command(cmd, logger, cwd, args=[], log=None):
if log:
- log = file(log, "a")
- logger.write("launch: %s\n" % cmd, 5, screenOnly=True)
+ log = open(log, "a") # python2 file(log, "a")
for arg in args:
cmd += " " + arg
- # OP Add Windows case
+ logger.write("launch_command:\n %s\n" % cmd, 5, screenOnly=True)
+
+ # Add Windows case
if src.architecture.is_windows():
prs = subprocess.Popen(cmd,
shell=True,
stdout=log,
stderr=subprocess.STDOUT,
cwd=cwd)
- pass
+
else:
prs = subprocess.Popen(cmd,
shell=True,
stderr=subprocess.STDOUT,
cwd=cwd,
executable='/bin/bash')
- pass
- # END OP
+
return prs
# Launch a batch
write_back(logger, "batch: exit (%s)\n" % str(proc.returncode), 5)
return (proc.returncode == 0), top
+
# Launch a salome process
# -----------------------
def batch_salome(cmd, logger, cwd, args, getTmpDir,
- pendant="SALOME_Session_Server", fin="killSalome.py",
- log=None, delai=20, sommeil=1, delaiapp=0):
+ pendant="SALOME_Session_Server", fin="killSalome.py",
+ log=None, delai=20, sommeil=1, delaiapp=0):
beginTime = time.time()
launch_command(cmd, logger, cwd, args, log)
if delaiapp == 0:
delaiapp = delai
- # first launch salome (looking for .pidict file)
+ # first launch salome (looking for _pidict file)
top = 0
found = False
+ foundSalome = "batch salome not seen"
tmp_dir = getTmpDir()
+ # print("batch_salome %s %s / %s sommeil %s:\n%s" % (tmp_dir, delai, delaiapp, sommeil, cmd))
while (not found and top < delaiapp):
if os.path.exists(tmp_dir):
listFile = os.listdir(tmp_dir)
+ listFile = [f for f in listFile if f.endswith("_pidict")]
+ # print("listFile %s" % listFile)
else:
listFile = []
for file_name in listFile:
- if file_name.endswith("pidict"):
- # sometime we get a old file that will be removed by runSalome.
- # So we test that we can read it.
- currentTime = None
+ # sometime we get a old file that will be removed by runSalome.
+ # So we test that we can read it.
+ currentTime = None
+ try:
+ statinfo = os.stat(os.path.join(tmp_dir, file_name))
+ currentTime = statinfo.st_mtime
+ except:
+ pass
+
+ if currentTime and currentTime > beginTime:
+ found = True
+ pidictFile = file_name
+
+ """
+ # CVW avoid unsupported pickle protocol: 3 because pidict from python3 KERNEL/bin/salome/addToKillList.py
try:
- statinfo = os.stat(os.path.join(tmp_dir, file_name))
- currentTime = statinfo.st_mtime
- except: pass
-
- if currentTime and currentTime > beginTime:
- try:
- file_ = open(os.path.join(tmp_dir, file_name), "r")
+ with open(os.path.join(tmp_dir, file_name), "r") as file_:
process_ids = pickle.load(file_)
- file_.close()
- for process_id in process_ids:
- for __, cmd in process_id.items():
- if cmd == [pendant]:
- found = True
- pidictFile = file_name
- except:
- file_.close()
+ # print("pidict %s" % process_ids)
+ for process_id in process_ids:
+ for __, cmd in process_id.items():
+ if cmd == [pendant]:
+ foundSalome = "batch salome started"
+ pidictFile = file_name
+ except Exception as e:
+ foundSalome = "python version %s problem reading file: %s" % (sys.version, e)
+ pass
+ """
time.sleep(sommeil)
top += 1
- show_progress(logger, top, delaiapp, "launching salome or appli:")
+ show_progress(logger, top, delaiapp, "launching salome or appli found=%s:" % found)
# continue or not
if found:
- write_back(logger, "batch_salome: started\n", 5)
+ logger.write("\nbatch_salome: supposed started\n", 5)
else:
- logger.write("batch_salome: FAILED to launch salome or appli\n", 3)
+ logger.write("\nbatch_salome: seems FAILED to launch salome or appli\n" % foundSalome, 3)
return False, -1
# salome launched run the script
log_macro_command_file_expression = "^[0-9]{8}_+[0-9]{6}_+.*\.xml$"
log_all_command_file_expression = "^.*[0-9]{8}_+[0-9]{6}_+.*\.xml$"
+verbose = True # cvw TODO
+
class Logger(object):
"""\
Class to handle log mechanism.
self.config.VARS.user})
# The time when command was launched
Y, m, dd, H, M, S = date_to_datetime(self.config.VARS.datehour)
- date_hour = "%2s/%2s/%4s %2sh%2sm%2ss" % (dd, m, Y, H, M, S)
+ date_hour = "%4s/%2s/%2s %2sh%2sm%2ss" % (Y, m, dd, H, M, S)
self.xmlFile.append_node_attrib("Site", attrib={"beginTime" :
date_hour})
# The application if any
"""
xmlLinks = self.xmlFile.xmlroot.find("Links")
flc = src.xmlManager.escapeSequence(full_launched_command)
- att = {"command" : command_name, "passed" : command_res, "launchedCommand" : flc}
+ att = {"command" : command_name, "passed" : str(command_res), "launchedCommand" : flc}
src.xmlManager.add_simple_node(xmlLinks, "link", text = log_file_name, attrib = att)
def write(self, message, level=None, screenOnly=False):
sys.stdout.write(printcolors.printcWarning("%s\n%s\n" % (msg, e)))
return False, None, None
- if 'application' in logFileXml.xmlroot.keys():
- appliLog = logFileXml.xmlroot.get('application')
- launched_cmd = logFileXml.xmlroot.find('Site').attrib['launchedCommand']
- # if it corresponds, then the log has to be shown
- if appliLog == application:
- return True, appliLog, launched_cmd
- elif application != 'None':
- return False, appliLog, launched_cmd
-
- return True, appliLog, launched_cmd
-
+ try:
+ if 'application' in logFileXml.xmlroot.keys():
+ appliLog = logFileXml.xmlroot.get('application')
+ launched_cmd = logFileXml.xmlroot.find('Site').attrib['launchedCommand']
+ # if it corresponds, then the log has to be shown
+ if appliLog == application:
+ return True, appliLog, launched_cmd
+ elif application != 'None':
+ return False, appliLog, launched_cmd
+
+ return True, appliLog, launched_cmd
+ except Exception as e:
+ msg = _("WARNING: the log file %s cannot be parsed:" % logFilePath)
+ sys.stdout.write(printcolors.printcWarning("%s\n%s\n" % (msg, e)))
+ return False, None, None
+
if application == 'None':
return True, None, None
'''In this file is stored the mechanism that manage color prints in the terminal
'''
+
# define constant to use in scripts
COLOR_ERROR = 'ERROR'
COLOR_WARNING = 'WARNING'
:param level int: the level of verboseness.
:param suffix str: the suffix to add at the end.
'''
+ if type(value) is list:
+ skip = "\n "
+ strValue = ""
+ i = 0
+ for v in value:
+ strValue += "%15s, " % str(v)
+ i += 1
+ if i >= 5:
+ strValue += skip
+ i = 0
+ if len(value) > 5:
+ strValue = skip + strValue
+ else:
+ strValue = str(value)
+ strValue = printcInfo(strValue)
if logger is None:
- print(" %s = %s %s" % (label, printcInfo(str(value)), suffix))
+ print(" %s = %s %s" % (label, strValue, suffix))
else:
- logger.write(" %s = %s %s\n" % (label, printcInfo(str(value)),
- suffix), level)
+ logger.write(" %s = %s %s\n" % (label, strValue, suffix), level)
def print_color_range(start, end):
'''print possible range values for colors
with open(aFile, 'w') as f:
res.__save__(f)
- # this file is for human eye reading
+ # this file is not mandatory, is for human eye reading
aFile = os.path.join(p_info.install_dir, PRODUCT_FILENAME)
- with open(aFile, 'w') as f:
- # f.write(DBG.getStrConfigDbg(p_info)) # debug mode
- try:
- p_info.__save__(f, evaluated=True) # evaluated expressions mode
- except:
- # the second file is not mandatory. In the context of non VCS archives, p_info cannot be evaluated
- # because information on git server is not available. In this case, we skip the writing without raising an error.
- #DBG.write("cannot evaluate product info - do not write ", PRODUCT_FILENAME, True)
- pass
-
-
+ try:
+ with open(aFile, 'w') as f:
+ p_info.__save__(f, evaluated=True) # evaluated expressions mode
+ except:
+ DBG.write("cannot evaluate product info - problem in file %s" % aFile, p_info, True)
+ # write DBG mode, as no problem if evaluation not possible
+ msg = """\
+# Some informations cannot be evaluated.
+# for example:
+# In the context of non VCS archives, information on git server is not available.
+
+"""
+ with open(aFile, 'w') as f:
+ f.write(msg)
+ f.write(DBG.getStrConfigDbg(p_info))
def check_config_exists(config, prod_dir, prod_info, verbose=False):
"""\
def __getitem__(self, key):
data = object.__getattribute__(self, 'data')
if key not in data:
- raise AttributeError(key)
+ raise AttributeError("Unknown pyconf key: '%s'" % key)
rv = data[key]
return self.evaluate(rv)
#if name == "__class__":
# return ''
data = object.__getattribute__(self, "data")
- useData = data.has_key(name)
+ useData = name in data
if useData:
rv = getattr(data, name)
else:
try:
rv = data[index]
except (IndexError, KeyError, TypeError):
- raise ConfigResolutionError('%r is not a valid index for %r' % (index, object.__getattribute__(self, 'path')))
+ raise ConfigResolutionError('Invalid pyconf index %r for %r' % (index, object.__getattribute__(self, 'path')))
if not isinstance(rv, list):
rv = self.evaluate(rv)
else:
self.remaindersArgs = remaindersArgs # the command and their options
self.datadir = datadir # default value will be <salomeTools root>/data
self._setCommands(cmdsdir)
+ DBG.write("Sat.options", self.options, self.options.debug_mode)
def getConfig(self):
return self.cfg
# Get error
logger_command.write("\n***** ", 1)
logger_command.write(src.printcolors.printcError(
- "salomeTools ERROR:"), 1)
- logger_command.write("\n" + str(e) + "\n\n", 1)
+ "salomeTools ERROR: sat %s" % __nameCmd__), 1)
+
+ logger_command.write("\n" + DBG.format_exception("") + "\n", 1)
+
+ """
+ # have python 3 problems...
# get stack
__, __, exc_traceback = sys.exc_info()
- fp = tempfile.TemporaryFile()
+ fp = tempfile.TemporaryFile(mode='wt')
traceback.print_tb(exc_traceback, file=fp)
fp.seek(0)
stack = fp.read()
verbosity = 5
if self.options.debug_mode:
verbosity = 1
- logger_command.write("TRACEBACK: %s" % stack.replace('"',"'"),
- verbosity)
+ logger_command.write("TRACEBACK: %s" % stack.replace('"',"'"), verbosity)
+ """
+
finally:
# set res if it is not set in the command
if res is None:
like open a browser or an editor, or call a git command
'''
-import subprocess
import os
+import subprocess
+import time
import tarfile
import debug as DBG
cmd = r"""
set -x
-rmdir %(where)s && \
+rmdir %(where)s
git clone %(remote)s %(where)s && \
git --git-dir=%(where_git)s --work-tree=%(where)s checkout %(tag)s
"""
logger.logTxtFile.flush()
DBG.write("cmd", cmd)
- rc = UTS.Popen(cmd, cwd=str(where.dir()), env=environment.environ.environ, logger=logger)
+
+ for nbtry in range(0,3): # retries case of network problem
+ rc = UTS.Popen(cmd, cwd=str(where.dir()), env=environment.environ.environ, logger=logger)
+ if rc.isOk(): break
+ time.sleep(30) # wait a little
+
return rc.isOk()
"""
cmd = r"""
set -x
export tmpDir=%(tmpWhere)s && \
-rm -rf $tmpDir && \
+rm -rf $tmpDir
git clone %(remote)s $tmpDir && \
cd $tmpDir && \
git checkout %(tag)s && \
rm -rf $tmpDir
""" % aDict
DBG.write("cmd", cmd)
- rc = UTS.Popen(cmd, cwd=parentWhere, env=environment.environ.environ, logger=logger)
- return rc.isOk()
+ for nbtry in range(0,3): # retries case of network problem
+ rc = UTS.Popen(cmd, cwd=parentWhere, env=environment.environ.environ, logger=logger)
+ if rc.isOk(): break
+ time.sleep(30) # wait a little
+
+ return rc.isOk()
def archive_extract(from_what, where, logger):
'''Extracts sources from an archive.
import string
import subprocess
-"""
-Exception class for test errors.
-"""
+
class SatTestError(Exception):
+ """
+ Exception class for test errors.
+ """
def __init__(self, value):
self.value = value
+
def __str__(self):
return repr(self.value)
class SatNotApplicableError(Exception):
+ """
+ Exception class for test errors.
+ """
def __init__(self, value):
self.value = value
+
def __str__(self):
return repr(self.value)
def ERROR(message):
- print message
+ print("ERROR", message)
raise SatTestError(message)
def NOT_APPLICABLE(message):
- print message
+ print("NOT_APPLICABLE", message)
raise SatNotApplicableError(message)
-##
-# Compares 2 numbers with tolerance tol.
def compFloat(f1, f2, tol=10e-10):
+ """Compares 2 numbers with tolerance tol."""
diff = abs(f1 - f2)
- print "|f1-f2| = %s (tol=%s)" % (str(diff), str(tol))
+ print("|f1-f2| = %s (tol=%s)" % (str(diff), str(tol)))
if diff <= tol:
comp = "OK"
else:
comp = "KO"
return comp
-##
-# Compares 2 files.
def compFiles(f1, f2, tol=0):
+ """Compares 2 files."""
assert os.path.exists(f1), "compFiles: file not found: %s" % f1
assert os.path.exists(f2), "compFiles: file not found: %s" % f2
diffLine = os.popen("diff -y --suppress-common-lines %s %s" % (f1, f2))
diff = len(string.split(diffLine.read(), "\n"))
diffLine.close()
- print "nb of diff lines = %s (tol=%s)" % (str(diff), str(tol))
+ print("nb of diff lines = %s (tol=%s)" % (str(diff), str(tol)))
if diff <= tol:
comp = "OK"
else:
comp = "KO"
return comp
-##
-# Uses mdump to dump a med file.
def mdump_med(med_file, dump_file, options):
+ """Uses mdump to dump a med file."""
assert isinstance(options, list), "Bad options for mdump: %s" % options
assert len(options) == 3, "Bad options for mdump: %s" % options
cmd = "mdump %s %s" % (med_file, " ".join(options))
- #print cmd
-
- df = open(dump_file, "w")
- pdump = subprocess.Popen(cmd, shell=True, stdout=df)
- st = pdump.wait()
- df.close()
+ #print(cmd)
+ with open(dump_file, "w") as df:
+ pdump = subprocess.Popen(cmd, shell=True, stdout=df)
+ st = pdump.wait()
return st
-##
-# Compares 2 med files by using mdump.
def compMED(file1, file2, tol=0, diff_flags=""):
- assert os.path.exists(file1), "compMED: file not found: %s" % file1
- assert os.path.exists(file2), "compMED: file not found: %s" % file2
-
- print
- print ">>>> compMED"
- print " file1:", file1
- print " file2:", file2
-
+ """Compares 2 med files by using mdump."""
+
+ # local utility method
def do_dump(med):
dump = os.path.join(os.environ['TT_TMP_RESULT'], os.path.basename(med) + ".mdump")
st = mdump_med(med, dump, ["1", "NODALE", "FULL_INTERLACE"])
raise Exception("Error mpdump %s" % med)
# replace file name with "filename"
- lines = open(dump, "r").readlines()
- dumpfile = open(dump, "w")
- for line in lines:
- try:
- line.index('Nom universel du maillage')
- continue
- except:
- dumpfile.write(line.replace(med, 'filename'))
+ with open(dump, "r") as ff:
+ lines = ff.readlines()
+ with open(dump, "w") as dumpfile:
+ for line in lines:
+ try:
+ line.index('Universal name of mesh')
+ continue
+ except:
+ dumpfile.write(line.replace(med, 'filename'))
return dump
+
+ # begin method
+ print(""">>>> compMED
+ file1: %s
+ file2: %s
+""" % (file1, file2))
+
+ if not os.path.exists(file1):
+ print("compMED: file not found: '%s'" % file1)
+ print("<<<< compMED\n")
+ return 1
+ if not os.path.exists(file2):
+ print("compMED: file not found: '%s'" % file2)
+ print("<<<< compMED\n")
+ return 1
+
dump1 = do_dump(file1)
dump2 = do_dump(file2)
diff_cmd = "diff %s %s %s" % (diff_flags, dump1, dump2)
- print " >" + diff_cmd
+ print(" >" + diff_cmd)
pdiff = subprocess.Popen(diff_cmd, shell=True, stdout=subprocess.PIPE)
status = pdiff.wait()
- print " Diff =", status
+ print(" Diff =", status)
if status != 0:
- print pdiff.stdout.read()
-
- print "<<<< compMED"
- print
+ print(pdiff.stdout.read())
+ print("<<<< compMED\n")
return status
-class TOOLS_class:
+class TOOLS_class(object):
def __init__(self, base_ressources_dir, tmp_dir, test_ressources_dir):
self.base_ressources_dir = base_ressources_dir
self.tmp_dir = tmp_dir
#!/usr/bin/env python
#-*- coding:utf-8 -*-
-import os, sys, traceback
+"""
+This script is automatically generated by 'command sat test etc...'
+from ...salomeTools/src/test/scriptTemplate.py
+"""
+
+import os
+import sys
+import traceback
import os.path
import time as THEBIGTIME
+
# set path
toolsWay = r'${toolsWay}'
resourcesWay = r'${resourcesWay}'
from TOOLS import SatNotApplicableError
-# on set les variables d'environement
+# set environement variables
os.environ['TT_BASE_RESSOURCES'] = resourcesWay
sys.path.append(resourcesWay)
-exec_result = open(r'${resultFile}', 'w')
-exec_result.write('Open\n')
-
__stdout__ = sys.stdout
__stderr__ = sys.stderr
-for test in listTest:
- pylog = open(os.path.join(outWay, test[:-3] + ".result.py"), "w")
- testout = open(os.path.join(outWay, test[:-3] + ".out.py"), "w")
- my_tools.init()
- sys.stdout = testout
- sys.stderr = testout
-
- pylog.write('#-*- coding:utf-8 -*-\n')
- exec_result.write("Run %s " % test)
- exec_result.flush()
-
- try:
- timeStart = THEBIGTIME.time()
- execfile(os.path.join(outWay, test), globals(), locals())
- timeTest = THEBIGTIME.time() - timeStart
- except SatNotApplicableError, ex:
- status = "NA"
- reason = str(ex)
- exec_result.write("NA\n")
- timeTest = THEBIGTIME.time() - timeStart
- pylog.write('status = "NA"\n')
- pylog.write('time = "' + timeTest.__str__() + '"\n')
- pylog.write('callback = "%s"\n' % reason)
- except Exception, ex:
- status = "KO"
- reason = ""
- if ignore.has_key(test):
- status = "KF"
- reason = "Known Failure = %s\n\n" % ignore[test]
- exec_result.write("%s\n" % status)
- timeTest = THEBIGTIME.time() - timeStart
- pylog.write('status = "%s" \n' % status)
- pylog.write('time = "' + timeTest.__str__() + '"\n')
- pylog.write('callback="""' + reason)
- exc_type, exc_value, exc_traceback = sys.exc_info()
- traceback.print_exception(exc_type,
- exc_value,
- exc_traceback,
- None,
- file=pylog)
- pylog.write('"""\n')
- else:
- exec_result.write("OK\n")
- pylog.write('status = "OK"\n')
- pylog.write('time = "' + timeTest.__str__() + '"\n')
-
- testout.close()
- sys.stdout = __stdout__
- sys.stderr = __stderr__
- my_tools.writeInFiles(pylog)
- pylog.close()
-
-exec_result.write('Close\n')
-exec_result.close()
+with open(r'${resultFile}', 'w') as exec_result:
+ exec_result.write('Open\n')
+ print("wrapper ignore tests: %s" % ignore)
+ for test in listTest:
+ fileTest = os.path.join(outWay, test)
+ # print("test file: %s" % fileTest) # cvw TODO
+ with open(os.path.join(outWay, test[:-3] + ".result.py"), "w") as pylog:
+ with open(os.path.join(outWay, test[:-3] + ".out.py"), "w") as testout:
+ my_tools.init()
+ # print("here set sys.stdout")
+ sys.stdout = testout
+ sys.stderr = testout
+
+ # pylog.write('#!/usr/bin/env python\n')
+ exec_result.write("Run %s " % test)
+ exec_result.flush()
+
+ try:
+ timeStart = THEBIGTIME.time()
+ # cd ..print("begin... %s" % fileTest)
+ # execfile(fileTest, globals(), locals()) obsolete python3
+ with open(fileTest) as f:
+ # compile associates the filename with the code object making debugging a little easier
+ code = compile(f.read(), fileTest, 'exec')
+ exec(code, globals(), locals())
+ timeTest = THEBIGTIME.time() - timeStart
+ # print("...done %s" % fileTest)
+ timeTest = THEBIGTIME.time() - timeStart
+ except SatNotApplicableError as ex:
+ # print("here SatNotApplicableError")
+ # pylog.write("here SatNotApplicableError")
+ status = "NA"
+ reason = str(ex)
+ exec_result.write("NA\n")
+ timeTest = THEBIGTIME.time() - timeStart
+ pylog.write('status = "NA"\n')
+ pylog.write('time = "' + timeTest.__str__() + '"\n')
+ pylog.write('callback = "%s"\n' % reason)
+ except Exception as ex:
+ # print("here Exception")
+ # pylog.write("here Exception pylog\n")
+ status = "KO"
+ reason = ""
+ if test in ignore:
+ status = "KF"
+ reason = "Known Failure = %s\n\n" % ignore[test]
+ exec_result.write("%s\n" % status)
+ timeTest = THEBIGTIME.time() - timeStart
+ pylog.write('status = "%s" \n' % status)
+ pylog.write('time = "' + timeTest.__str__() + '"\n')
+ pylog.write('callback="""' + reason)
+ exc_type, exc_value, exc_traceback = sys.exc_info()
+ traceback.print_exception(exc_type,
+ exc_value,
+ exc_traceback,
+ None,
+ file=pylog)
+ pylog.write('"""\n')
+ else:
+ # print("here else")
+ # pylog.write("here else pylog")
+ exec_result.write("OK\n")
+ pylog.write('status = "OK"\n')
+ pylog.write('time = "' + timeTest.__str__() + '"\n')
+
+ pass
+ # print("here testout.flush")
+ testout.flush()
+ # testout.close()
+
+ # print("here restore sys.stdout")
+ sys.stdout = __stdout__
+ sys.stderr = __stderr__
+ my_tools.writeInFiles(pylog)
+ pass
+ pylog.flush()
+ # pylog.close()
+
+ exec_result.write('Close\n')
+ pass
+ # exec_result.close()
if 'PY' not in '${sessionName}':
import salome_utils
# 'salome',
# 'killSalome.py')
#cmd = '{python} {killScript} {port}'.format(python=os.environ['PYTHONBIN'],
- # killScript=killScript,
- # port=salome_utils.getPortNumber())
+ # killScript=killScript,
+ # port=salome_utils.getPortNumber())
cmd = 'killSalome.py {port}'.format( port=salome_utils.getPortNumber())
os.system(cmd)
+
code = compile(f.read(), somefile, 'exec')
exec(code, global_vars, local_vars)
-
import os
import sys
import datetime
import string
import imp
import subprocess
+import glob
+import pprint as PP
+
+verbose = False
from . import fork
import src
results[test] = ["?", -1, "", []]
else:
gdic, ldic = {}, {}
- execfile(resfile, gdic, ldic)
-
- status = src.TIMEOUT_STATUS
- if not has_timed_out:
- status = src.KO_STATUS
-
- if ldic.has_key('status'):
- status = ldic['status']
-
- expected = []
- if status == src.KO_STATUS or status == src.OK_STATUS:
- status, expected = self.search_known_errors(status,
- self.currentgrid,
- self.currentsession,
- test)
-
- callback = ""
- if ldic.has_key('callback'):
- callback = ldic['callback']
- elif status == src.KO_STATUS:
- callback = "CRASH"
-
- exec_time = -1
- if ldic.has_key('time'):
- try:
- exec_time = float(ldic['time'])
- except:
+ if verbose:
+ print("test script: '%s':\n'%s'\n" % (resfile, open(resfile, 'r').read()))
+
+ try:
+ execfile(resfile, gdic, ldic)
+
+ status = src.TIMEOUT_STATUS
+ if not has_timed_out:
+ status = src.KO_STATUS
+
+ if 'status' in ldic:
+ status = ldic['status']
+
+ expected = []
+ if status == src.KO_STATUS or status == src.OK_STATUS:
+ status, expected = self.search_known_errors(status,
+ self.currentgrid,
+ self.currentsession,
+ test)
+
+ callback = ""
+ if 'callback' in ldic:
+ callback = ldic['callback']
+ elif status == src.KO_STATUS:
+ callback = "CRASH"
+ if verbose:
+ print("--- CRASH ldic\n%s" % PP.pformat(ldic)) # cvw TODO
+ print("--- CRASH gdic\n%s" % PP.pformat(gdic))
pass
- results[test] = [status, exec_time, callback, expected]
+ exec_time = -1
+ if 'time' in ldic:
+ try:
+ exec_time = float(ldic['time'])
+ except:
+ pass
+
+ results[test] = [status, exec_time, callback, expected]
+
+ except:
+ results[test] = ["?", -1, "", []]
+ # results[test] = [src.O_STATUS, -1, open(resfile, 'r').read(), []]
# check if <test>.py file exists
testfile = os.path.join(self.currentDir,
# calling all the scripts of a single directory.
def generate_script(self, listTest, script_path, ignoreList):
# open template file
- template_file = open(os.path.join(self.config.VARS.srcDir,
- "test",
- "scriptTemplate.py"), 'r')
- template = string.Template(template_file.read())
+ tFile = os.path.join(self.config.VARS.srcDir, "test", "scriptTemplate.py")
+ with open(tFile, 'r') as f:
+ template = string.Template(f.read())
# create substitution dictionary
d = dict()
d['resourcesWay'] = os.path.join(self.currentDir, 'RESSOURCES')
d['tmpDir'] = os.path.join(self.tmp_working_dir, 'WORK')
d['toolsWay'] = os.path.join(self.config.VARS.srcDir, "test")
- d['sessionDir'] = os.path.join(self.currentDir,
- self.currentgrid,
- self.currentsession)
- d['resultFile'] = os.path.join(self.tmp_working_dir,
- 'WORK',
- 'exec_result')
+ d['sessionDir'] = os.path.join(self.currentDir, self.currentgrid, self.currentsession)
+ d['resultFile'] = os.path.join(self.tmp_working_dir, 'WORK', 'exec_result')
d['listTest'] = listTest
d['sessionName'] = self.currentsession
d['ignore'] = ignoreList
# create script with template
- script = open(script_path, 'w')
- script.write(template.safe_substitute(d))
- script.close()
+ contents = template.safe_substitute(d)
+ if verbose: print("generate_script '%s':\n%s" % (script_path, contents)) # cvw TODO
+ with open(script_path, 'w') as f:
+ f.write(contents)
- # Find the getTmpDir function that gives access to *pidict file directory.
- # (the *pidict file exists when SALOME is launched)
+
+ # Find the getTmpDir function that gives access to *_pidict file directory.
+ # (the *_pidict file exists when SALOME is launched)
def get_tmp_dir(self):
# Rare case where there is no KERNEL in grid list
# (for example MED_STANDALONE)
if 'KERNEL_ROOT_DIR' in os.environ:
root_dir = os.environ['KERNEL_ROOT_DIR']
- if ('APPLICATION' in self.config
- and 'KERNEL' in self.config.APPLICATION.products):
- root_dir = src.product.get_product_config(self.config,
- "KERNEL").install_dir
+ if ('APPLICATION' in self.config and
+ 'KERNEL' in self.config.APPLICATION.products):
+ root_dir = src.product.get_product_config(self.config, "KERNEL").install_dir
# Case where there the appli option is called (with path to launcher)
if len(self.launcher) > 0:
launcherDir = os.path.dirname(self.launcher)
if launcherName == 'runAppli':
# Old application
- cmd = ("for i in " + launcherDir + "/env.d/*.sh; do source ${i};"
- " done ; echo $KERNEL_ROOT_DIR")
+ cmd = """
+for i in %s/env.d/*.sh;
+ do source ${i};
+done
+echo $KERNEL_ROOT_DIR
+""" % launcherDir
else:
# New application
- cmd = ("echo -e 'import os\nprint os.environ[\"KERNEL_" +
- "ROOT_DIR\"]' > tmpscript.py; %s shell" +
- " tmpscript.py") % self.launcher
-
- # OP 14/11/2017 Ajout de traces pour essayer de decouvrir le pb
- # de remontee de log des tests
- #root_dir = subprocess.Popen(cmd,
- # stdout=subprocess.PIPE,
- # shell=True,
- # executable='/bin/bash').communicate()[0].split()[-1]
- # OP Add Windows case
+ cmd = """
+echo -e 'import os\nprint(os.environ[\"KERNEL_ROOT_DIR\"])' > tmpscript.py
+%s shell tmpscript.py
+""" % self.launcher
+
if src.architecture.is_windows():
subproc_res = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
shell=True,
executable='/bin/bash').communicate()
pass
- #print "TRACES OP - test_module.py/Test.get_tmp_dir() subproc_res = "
- #for resLine in subproc_res:
- # print "- '#%s#'" %resLine
root_dir = subproc_res[0].split()[-1]
-
- # OP 14/11/2017 Ajout de traces pour essayer de decouvrir le pb
- # de remontee de log des tests
- #print "TRACES OP - test_module.py/Test.get_tmp_dir() root_dir = '#%s#'" %root_dir
# import grid salome_utils from KERNEL that gives
# the right getTmpDir function
- (file_, pathname, description) = imp.find_module("salome_utils",
- [os.path.join(root_dir,
- 'bin',
- 'salome')])
+ root_dir = root_dir.decode('utf-8')
+ aPath = [os.path.join(root_dir, 'bin', 'salome')]
+ sal_uts = "salome_utils"
+ try:
+ (file_, pathname, description) = imp.find_module(sal_uts, aPath )
+ except Exception:
+ msg = "inexisting %s.py in %s" % (sal_uts, aPath)
+ raise Exception(msg)
+
try:
- grid = imp.load_module("salome_utils",
- file_,
- pathname,
- description)
+ grid = imp.load_module(sal_uts, file_, pathname, description)
return grid.getLogDir
except:
- grid = imp.load_module("salome_utils",
- file_,
- pathname,
- description)
+ grid = imp.load_module(sal_uts, file_, pathname, description)
return grid.getTmpDir
finally:
if file_:
out_path = os.path.join(self.currentDir,
self.currentgrid,
self.currentsession)
+ if verbose: print("run_tests '%s'\nlistTest: %s\nignoreList: %s" %
+ (self.currentDir, PP.pformat(listTest), PP.pformat(ignoreList))) # cvw TODO
sessionname = "%s/%s" % (self.currentgrid, self.currentsession)
time_out = self.get_test_timeout(sessionname,
DEFAULT_TIMEOUT)
tmpDir = self.get_tmp_dir()
binSalome, binPython, killSalome = self.generate_launching_commands()
- if self.settings.has_key("run_with_grids") \
- and self.settings["run_with_grids"].has_key(sessionname):
- binSalome = (binSalome +
- " -m %s" % self.settings["run_with_grids"][sessionname])
+ if "run_with_grids" in self.settings and \
+ sessionname in self.settings["run_with_grids"]:
+ binSalome = (binSalome + " -m %s" % self.settings["run_with_grids"][sessionname])
logWay = os.path.join(self.tmp_working_dir, "WORK", "log_cxx")
elapsed = -1
if self.currentsession.startswith("NOGUI_"):
# runSalome -t (bash)
- status, elapsed = fork.batch(binSalome, self.logger,
- os.path.join(self.tmp_working_dir,
- "WORK"),
- [ "-t",
- "--shutdown-server=1",
- script_path ],
- delai=time_out,
- log=logWay)
+ status, elapsed = fork.batch(
+ binSalome,
+ self.logger,
+ os.path.join(self.tmp_working_dir, "WORK"),
+ [ "-t", "--shutdown-server=1", script_path ],
+ delai=time_out,
+ log=logWay)
elif self.currentsession.startswith("PY_"):
# python script.py
- status, elapsed = fork.batch(binPython, self.logger,
- os.path.join(self.tmp_working_dir,
- "WORK"),
- [script_path],
- delai=time_out, log=logWay)
+ status, elapsed = fork.batch(
+ binPython,
+ self.logger,
+ os.path.join(self.tmp_working_dir, "WORK"),
+ [script_path],
+ delai=time_out,
+ log=logWay)
else:
opt = "-z 0"
if self.show_desktop: opt = "--show-desktop=0"
- status, elapsed = fork.batch_salome(binSalome,
- self.logger,
- os.path.join(
- self.tmp_working_dir,
- "WORK"),
- [ opt,
- "--shutdown-server=1",
- script_path ],
- getTmpDir=tmpDir,
- fin=killSalome,
- delai=time_out,
- log=logWay,
- delaiapp=time_out_salome)
-
- self.logger.write("status = %s, elapsed = %s\n" % (status, elapsed),
- 5)
+ status, elapsed = fork.batch_salome(
+ binSalome,
+ self.logger,
+ os.path.join( self.tmp_working_dir, "WORK"),
+ [ opt, "--shutdown-server=1", script_path ],
+ getTmpDir=tmpDir,
+ fin=killSalome,
+ delai=time_out,
+ log=logWay,
+ delaiapp=time_out_salome)
+
+ self.logger.write("status = %s, elapsed = %s\n" % (status, elapsed), 5)
# create the test result to add in the config object
test_info = src.pyconf.Mapping(self.config)
tests = os.listdir(os.path.join(self.currentDir,
self.currentgrid,
self.currentsession))
- tests = filter(lambda l: l.endswith(".py"), tests)
+ # avoid result files of previous tests, if presents
+ # tests = filter(lambda l: l.endswith(".py"), tests)
+ tests = [t for t in tests if t.endswith(".py") \
+ and not ( t.endswith(".out.py") or \
+ t.endswith(".result.py") or \
+ t.endswith("wrapperScript.py") \
+ ) ]
tests = sorted(tests, key=str.lower)
# build list of known failures
l)), sessions)
sessions = sorted(sessions, key=str.lower)
+ existingSessions = self.getSubDirectories(grid_path)
for session_ in sessions:
if not os.path.exists(os.path.join(grid_path, session_)):
self.logger.write(self.write_test_margin(2), 3)
- self.logger.write(src.printcolors.printcWarning("Session %s not"
- " found" % session_) + "\n", 3, False)
+ msg = """\
+Session '%s' not found
+Existing sessions are:
+%s
+""" % (session_, PP.pformat(sorted(existingSessions)))
+ self.logger.write(src.printcolors.printcWarning(msg), 3, False)
else:
self.currentsession = session_
self.run_session_tests()
+ def getSubDirectories(self, aDir):
+ """
+ get names of first level of sub directories in aDir
+ excluding '.git' etc as beginning with '.'
+ """
+ res = os.listdir(aDir)
+ res = [d for d in res if os.path.isdir(os.path.join(aDir, d)) and d[0] != '.']
+ # print("getSubDirectories %s are:\n%s" % (aDir, PP.pformat(res)))
+ return res
+
##
# Runs test testbase.
def run_testbase_tests(self):
if os.path.exists(settings_file):
gdic, ldic = {}, {}
execfile(settings_file, gdic, ldic)
- self.logger.write(_("Load test settings\n"), 3)
+ self.logger.write("Load test settings '%s'\n" % settings_file, 5)
self.settings = ldic['settings_dic']
self.ignore_tests = ldic['known_failures_list']
if isinstance(self.ignore_tests, list):
self.ignore_tests = {}
- self.logger.write(src.printcolors.printcWarning("known_failur"
- "es_list must be a dictionary (not a list)") + "\n", 1, False)
+ self.logger.write(src.printcolors.printcWarning(
+ "known_failures_list must be a dictionary (not a list)") + "\n", 1, False)
else:
self.ignore_tests = {}
self.settings.clear()
grids)
grids = sorted(grids, key=str.lower)
+ existingGrids = self.getSubDirectories(self.currentDir)
for grid in grids:
if not os.path.exists(os.path.join(self.currentDir, grid)):
self.logger.write(self.write_test_margin(1), 3)
- self.logger.write(src.printcolors.printcWarning(
- "grid %s does not exist\n" % grid), 3, False)
+ msg = """\
+Grid '%s' does not exist
+Existing grids are:
+%s
+""" % (grid, PP.pformat(sorted(existingGrids)))
+ self.logger.write(src.printcolors.printcWarning(msg), 3, False)
else:
self.currentgrid = grid
self.run_grid_tests()
def run_script(self, script_name):
- if ('APPLICATION' in self.config and
+ if ('APPLICATION' in self.config and
script_name in self.config.APPLICATION):
script = self.config.APPLICATION[script_name]
if len(script) == 0:
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import os
+import shutil
+
try: # For python2
import sys
reload(sys)
pass
import src
-from . import ElementTree as etree
+import src.ElementTree as etree
+
+verbose = False
class XmlLogFile(object):
'''Class to manage writing in salomeTools xml log file
'''
log_file_path = self.logFile
if file_path:
- log_file_path = file_path
+ log_file_path = file_path
try:
- f = open(log_file_path, 'w')
+ with open(log_file_path, 'w') as f:
f.write("<?xml version='1.0' encoding='utf-8'?>\n")
if stylesheet:
- f.write("<?xml-stylesheet type='text/xsl' href='%s'?>\n" %
- stylesheet)
- f.write(etree.tostring(self.xmlroot, encoding='utf-8'))
- f.close()
+ f.write("<?xml-stylesheet type='text/xsl' href='%s'?>\n" % stylesheet)
+ pass
+ res= etree.tostring(self.xmlroot, encoding='utf-8')
+ f.write(res)
except IOError:
pass
:param xmlroot etree.Element: the Etree element to write to the file
:param stylesheet str: The stylesheet to add to the begin of the file
"""
- if not os.path.exists(os.path.dirname(filename)):
- os.makedirs(os.path.dirname(filename))
-
- f = open(filename, "w")
- f.write("<?xml version='1.0' encoding='utf-8'?>\n")
+ dirname = os.path.dirname(filename)
+ if not os.path.exists(dirname):
+ os.makedirs(dirname)
if len(stylesheet) > 0:
- f.write("<?xml-stylesheet type='text/xsl' href='%s'?>\n" % stylesheet)
- f.write(etree.tostring(xmlroot, encoding='utf-8'))
- f.close()
+ styleName = stylesheet
+ else:
+ styleName = None
+
+ with open(filename, "w") as f:
+ f.write("<?xml version='1.0' encoding='utf-8'?>\n")
+ if styleName is not None:
+ f.write("<?xml-stylesheet type='text/xsl' href='%s'?>\n" % styleName)
+ res = etree.tostring(xmlroot, encoding='utf-8')
+ # print("********** etree.tostring %s" % res)
+ f.write(res)
+
+ # create fileStyle in dirname if not existing
+ if styleName is not None:
+ styleFile = os.path.join(dirname, styleName)
+ if not os.path.exists(styleFile):
+ # copy if from "salomeTools/src/xsl"
+ srcdir = os.path.dirname(src.__file__)
+ srcFile = os.path.join(srcdir, "xsl", styleName)
+ if verbose: print("write_report %s style %s" % (srcFile, styleFile))
+ shutil.copy(srcFile, dirname)
def escapeSequence(aStr):
"""
classes = []
for n,t,o,e in result_list:
cls = t.__class__
- if not rmap.has_key(cls):
+ if not cls in rmap:
rmap[cls] = []
classes.append(cls)
rmap[cls].append((n,t,o,e))
