From 48ab180a922bcd22437d77330a5664cacf7d9648 Mon Sep 17 00:00:00 2001 From: Konstantin Leontev Date: Tue, 29 Nov 2022 14:57:56 +0300 Subject: [PATCH] First integration of Salome On Demand --- bin/CMakeLists.txt | 2 + bin/SalomeOnDemandTK/CMakeLists.txt | 34 + bin/SalomeOnDemandTK/__init__.py | 30 + bin/SalomeOnDemandTK/extension_builder.py | 166 +++++ bin/SalomeOnDemandTK/extension_query.py | 466 ++++++++++++++ bin/SalomeOnDemandTK/extension_remover.py | 193 ++++++ bin/SalomeOnDemandTK/extension_unpacker.py | 153 +++++ bin/SalomeOnDemandTK/extension_utilities.py | 669 ++++++++++++++++++++ bin/launchConfigureParser.py | 66 +- bin/runSalome.py | 23 +- bin/runSalomeCommon.py | 37 +- bin/runSalomeOnDemand.py | 80 +++ bin/setenv.py | 1 - 13 files changed, 1871 insertions(+), 49 deletions(-) create mode 100644 bin/SalomeOnDemandTK/CMakeLists.txt create mode 100644 bin/SalomeOnDemandTK/__init__.py create mode 100644 bin/SalomeOnDemandTK/extension_builder.py create mode 100644 bin/SalomeOnDemandTK/extension_query.py create mode 100644 bin/SalomeOnDemandTK/extension_remover.py create mode 100644 bin/SalomeOnDemandTK/extension_unpacker.py create mode 100644 bin/SalomeOnDemandTK/extension_utilities.py create mode 100755 bin/runSalomeOnDemand.py diff --git a/bin/CMakeLists.txt b/bin/CMakeLists.txt index 23136e5b6..f36b09885 100644 --- a/bin/CMakeLists.txt +++ b/bin/CMakeLists.txt @@ -18,6 +18,7 @@ # ADD_SUBDIRECTORY(appliskel) +ADD_SUBDIRECTORY(SalomeOnDemandTK) SALOME_CONFIGURE_FILE(VERSION.in VERSION INSTALL ${SALOME_INSTALL_BINS}) SALOME_CONFIGURE_FILE(salomeContextUtils.py.in salomeContextUtils.py) @@ -48,6 +49,7 @@ SET(SCRIPTS runSalomeCommon.py runSalome.py runSalomeOld.py + runSalomeOnDemand.py runSalomeNoServer.py runSession.py runConsole.py diff --git a/bin/SalomeOnDemandTK/CMakeLists.txt b/bin/SalomeOnDemandTK/CMakeLists.txt new file mode 100644 index 000000000..04408b8a2 --- /dev/null +++ b/bin/SalomeOnDemandTK/CMakeLists.txt @@ -0,0 +1,34 @@ +# Copyright (C) 2012-2022 CEA/DEN, EDF R&D, OPEN CASCADE +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See https://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +# =============================================================== +# Files to be installed +# =============================================================== + +# These files are part of SalomeOnDemandTK package +SET(SCRIPTS + __init__.py + extension_builder.py + extension_query.py + extension_remover.py + extension_unpacker.py + extension_utilities.py +) + +SALOME_INSTALL_SCRIPTS("${SCRIPTS}" ${SALOME_INSTALL_SCRIPT_SCRIPTS}/SalomeOnDemandTK) diff --git a/bin/SalomeOnDemandTK/__init__.py b/bin/SalomeOnDemandTK/__init__.py new file mode 100644 index 000000000..5a5954897 --- /dev/null +++ b/bin/SalomeOnDemandTK/__init__.py @@ -0,0 +1,30 @@ +# -*- coding: iso-8859-1 -*- +# Copyright (C) 2007-2022 CEA/DEN, EDF R&D, OPEN CASCADE +# +# Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +# CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See https://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +# File : __init__.py +# Author : Konstantin Leontev, Open Cascade +# +""" + + +""" diff --git a/bin/SalomeOnDemandTK/extension_builder.py b/bin/SalomeOnDemandTK/extension_builder.py new file mode 100644 index 000000000..cb62a87f8 --- /dev/null +++ b/bin/SalomeOnDemandTK/extension_builder.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (C) 2007-2022 CEA/DEN, EDF R&D, OPEN CASCADE +# +# Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +# CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See https://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +# File : extension_builder.py +# Author : Konstantin Leontev, Open Cascade +# +# @package SalomeOnDemandTK +# @brief Set of utility functions those help to build SALOME python extensions. + +"""Build salome extension archive in tar.gz format. +""" + +import tarfile +import os +import sys +import io +from traceback import format_exc + +from .extension_utilities import logger, \ + BFILE_EXT, DFILE_EXT, PYFILE_EXT, EXTNAME_KEY, ARCFILE_EXT, SALOME_EXTDIR, CFILE_EXT, \ + isvalid_filename, isvalid_dirname, read_salomexd, read_salomexb, list_files_filter, \ + list_tonewline_str + + +def add_files(ext, files_abs, files_rel): + """ + Add selected files into salome extension archive. + + Args: + ext - a file oject to pack in. + files_abs - a list of the files with abs file names. + files_rel - a list of the files with rel file names. + + Returns: + None. + """ + + logger.debug('Add selected files into archive %s directory...', SALOME_EXTDIR) + + # Set progress bar, because it can get some time for large archives + progress_count = 0 + total_count = len(files_abs) + default_terminator = logger.handlers[0].terminator + logger.handlers[0].terminator = '' + for (file_abs, file_rel) in zip(files_abs, files_rel): + ext.add(file_abs, os.path.normpath(SALOME_EXTDIR + '/' + file_rel)) + + # Progress bar's length is 100 symbols. + progress_count += 1 + logger.debug('\r|%-100s|', '=' * int(100 * progress_count/(total_count - 1))) + + # Reset terminator to default value, otherwise all the followed logs will be in one line + logger.debug('\n') + logger.handlers[0].terminator = default_terminator + + +def create_salomex(salomexb, salomexd, env_py, top_repository): + """ + Makes salome extension archive from provided in salomexb file directories. + + Args: + salomexb - a path to the .salomexb file. + salomexd - a path to the .salomexd file. + env_py - a path to the _env.py file. + top_repository - a root directory for all the paths listed inside salomexb file. + + Returns: + None. + """ + + logger.debug('Starting create an salome extension file') + + # Check if provided filenames are valid + if not isvalid_filename(salomexb, BFILE_EXT) or \ + not isvalid_filename(salomexd, DFILE_EXT) or \ + not isvalid_filename(env_py, PYFILE_EXT) or \ + not isvalid_dirname(top_repository): + return + + # Try to get info from salomexd file + salome_ext_name = '' + salomexd_content = read_salomexd(salomexd) + if EXTNAME_KEY in salomexd_content and salomexd_content[EXTNAME_KEY]: + salome_ext_name = salomexd_content[EXTNAME_KEY] + else: + # Now as a last resort we get a name right from salomexd filename + # We need to decide if we can handle this case handsomely + salome_ext_name = os.path.basename(salomexd) + logger.warning('Cannot get a SALOME extension name from salomexd file! \ + Use salomexd file name as an extension name.') + + logger.debug('Set an extension name as: %s', salome_ext_name) + + try: + with tarfile.open(salome_ext_name + '.' + ARCFILE_EXT, "w:gz") as ext: + # Write all included files to the extension's dir + + # Get the file's matching pattern in the first place + included_files_patterns = read_salomexb(salomexb) + logger.debug('Included files pattern: %s', included_files_patterns) + if not included_files_patterns: + # We don't have any pattern, so we don't know what we must put inside an archive + logger.error('Cannot create salomex file: \ + a list of included files patterns is empty.') + return + + # List of the files those actually written to the archive. + # It goes to the salomexc file then. + files_abs, files_rel = list_files_filter(top_repository, included_files_patterns) + id = 0 + for f in files_rel: + fsplit = f.split('/') + del fsplit[0] + files_rel[id] = '/'.join(fsplit) + id +=1 + add_files(ext, files_abs, files_rel) + + # Write the control file - list of the files inside extension's dir + logger.debug('Write the %s control file into archive root...', CFILE_EXT) + included_files_data = list_tonewline_str(files_rel).encode('utf8') + info = tarfile.TarInfo(salome_ext_name + '.' + CFILE_EXT) + info.size = len(included_files_data) + ext.addfile(info, io.BytesIO(included_files_data)) + + # Write the description file as is + logger.debug('Copy the %s file into archive root...', salomexd) + ext.add(salomexd, os.path.basename(salomexd)) + + # Write the env_py file as is + logger.debug('Copy the %s file into archive root...', env_py) + ext.add(env_py, os.path.basename(env_py)) + + logger.debug('SALOME extension %s was created.', salome_ext_name) + + except OSError: + logger.error(format_exc()) + + +if __name__ == '__main__': + if len(sys.argv) == 5: + arg_1, arg_2, arg_3, arg_4 = sys.argv[1:5] + create_salomex(arg_1, arg_2, arg_3, arg_4) + else: + logger.error('You must provide all the arguments!') + logger.info(create_salomex.__doc__) diff --git a/bin/SalomeOnDemandTK/extension_query.py b/bin/SalomeOnDemandTK/extension_query.py new file mode 100644 index 000000000..aab41bbb3 --- /dev/null +++ b/bin/SalomeOnDemandTK/extension_query.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (C) 2007-2022 CEA/DEN, EDF R&D, OPEN CASCADE +# +# Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +# CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See https://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +# File : extension_query.py +# Author : Konstantin Leontev, Open Cascade +# +# @package SalomeOnDemandTK +# @brief An utility package that will allow you to know the size of an extension +# and generate a dependency tree. + +""" +An utility package that will allow you to know the size of an extension +and generate a dependency tree. +""" + +import os +import sys +from traceback import format_exc + +from .extension_utilities import logger, \ + SALOME_EXTDIR, DFILE_EXT, EXTDEPENDSON_KEY, EXTDESCR_KEY, EXTAUTHOR_KEY, EXTCOMPONENT_KEY, \ + isvalid_dirname, find_salomexc, list_files_ext, read_salomexd + + +def size_to_str(size, format_in_bytes=False): + """ + Returns a string describing a size. + + Args: + size - the size to represent as a string. + format_in_bytes - if True, size is expressed in bytes, + otherwise human readable units are used. + + Returns: + The size as a string with units. + """ + + if format_in_bytes: + return '%s' % size + + __units__ = ["", "Ki", "Mi", "Gi", "Ti"] + + kilo = 1024.0 + prefix_index = 0 + prefix_index_max = len(__units__) - 1 + while (size > kilo and prefix_index < prefix_index_max): + prefix_index += 1 + size /= kilo + + unit = __units__[prefix_index] + return '%.2f %sB' % (size, unit) + + +def dir_size(directory): + """ + Calculate a total size of the given directory. + + Args: + directory - the given directory + + Returns: + Size of the directory. + """ + + logger.debug('Get the size of %s', directory) + + total_size = 0 + for root, _, files in os.walk(directory): + for file in files: + itempath = os.path.join(root, file) + if os.path.islink(itempath): + continue + + total_size += os.path.getsize(itempath) + + logger.debug('The size of %s: %s', directory, total_size) + return total_size + + +def dir_size_str(directory, format_in_bytes=False): + """ + Calculate a total size of the given directory and format as a string. + + Args: + directory - the given directory + format_in_bytes - if True, size is expressed in bytes, + otherwise human readable units are used. + + Returns: + Size of the directory as a formatted string. + """ + + size = dir_size(directory) + size_str = size_to_str(size, format_in_bytes) + + logger.debug('The size of %s: %s', directory, size_str) + return size_str + + +def size_bylist(root_dir, salomexc): + """ + Calcualate the total size of files listed in the given salomexc file. + + Args: + root_dir - a root dir for listed files + salomexc - file that contents a list of files. + + Returns: + The total size of listed files. + """ + + logger.debug('Calcualate the total size of files inside %s listed in %s...', + root_dir, salomexc) + + try: + with open(salomexc, 'r', encoding='UTF-8') as file: + total_size = 0 + for line in file: + path_to_file = os.path.join(root_dir, line.strip()) + #logger.debug('Check the file %s...', path_to_file) + + if os.path.isfile(path_to_file): + size = os.path.getsize(path_to_file) + total_size += size + logger.debug('%s size: %s', path_to_file, size) + + elif os.path.islink(path_to_file): + logger.debug('%s is link. Skip.', path_to_file) + continue + + elif os.path.isdir(path_to_file): + logger.warning('Directories are not expected to be listed in %s file! ' + 'Skip.', + salomexc) + continue + + else: + logger.warning('Unexpected path %s ' + 'is not a file, link or directory. Skip.', + path_to_file) + + return total_size + + except OSError: + logger.error(format_exc()) + + return None + + +def ext_size(install_dir, salomex_name): + """ + Calculate a total size of a salome extension from SALOME install root. + + Args: + salome_root - path to SALOME install root directory. + salomex_name - a name of the given salome extension. + + Returns: + Size of the directory in bytes. + """ + + # Check if provided dirname is valid + if not isvalid_dirname(install_dir): + return None + + # Check if an extension with this name is installed + salomexc = find_salomexc(install_dir, salomex_name) + if not salomexc: + logger.error('Cannot calculate the size of %s extension!', + salomex_name) + return None + + # Calculate the size + return size_bylist(os.path.join(install_dir, SALOME_EXTDIR), salomexc) + + +def ext_size_str(install_dir, salomex_name, format_in_bytes=False): + """ + Calculate a total size of the given extension and format as a string. + + Args: + install_dir - directory where the given extension is installed. + salomex_name - the given extension's name. + format_in_bytes - if True, size is expressed in bytes, + otherwise human readable units are used. + + Returns: + Size of the extension as a formatted string. + """ + + size = ext_size(install_dir, salomex_name) + if size is None: + return '' + + size_str = size_to_str(size, format_in_bytes) + + logger.debug('The size of %s: %s', salomex_name, size_str) + return size_str + + +def dependency_tree(directory): + r""" + Create a dependency tree for all salome extensions + installed in the given directory. + + Args: + directory - the given directory + + Returns: + A dictionary like that for extensions A, B, C, D and E: + A + /|\ + / B D + \/ \/ + C E + + { 'A': ['B', 'C', 'D'], 'B': ['C', 'E'], 'C': [], 'D': ['E'], 'E': [] }. + """ + + logger.debug('Build dependency tree for extensions in %s', directory) + + tree = {} + salomexd_files = list_files_ext(directory, DFILE_EXT) + logger.debug('There are %s extensions in %s', len(salomexd_files), directory) + + for salomexd_file in salomexd_files: + ext_name, _ = os.path.splitext(os.path.basename(salomexd_file)) + + logger.debug('Check dependencies for %s...', salomexd_file) + salomexd_content = read_salomexd(salomexd_file) + + if EXTDEPENDSON_KEY in salomexd_content: + depends_on = salomexd_content[EXTDEPENDSON_KEY] + logger.debug('List of dependencies: %s', depends_on) + + tree[ext_name] = depends_on + + logger.debug('Dependency tree: %s', tree) + return tree + + +def ext_info_dict(directory): + r""" + Get installed salome extensions info. + + Args: + directory - the given ext install directory. + + Returns: + A dictionary {name: [descr, author, components, size]}. + """ + + logger.debug('Build info dictionary for extensions in %s', directory) + + ext_info = {} + salomexd_files = list_files_ext(directory, DFILE_EXT) + logger.debug('There are %s extensions in %s', len(salomexd_files), directory) + + for salomexd_file in salomexd_files: + # Collect size info + ext_name, _ = os.path.splitext(os.path.basename(salomexd_file)) + size = ext_size_str(directory, ext_name) + + # Collect salomexd info + salomexd_content = read_salomexd(salomexd_file) + + descr = '' + if EXTDESCR_KEY in salomexd_content: + descr = salomexd_content[EXTDESCR_KEY] + logger.debug('descr: %s', descr) + + author = '' + if EXTAUTHOR_KEY in salomexd_content: + author = salomexd_content[EXTAUTHOR_KEY] + logger.debug('author: %s', author) + + components = '' + if EXTCOMPONENT_KEY in salomexd_content: + components = ', '.join(salomexd_content[EXTCOMPONENT_KEY]) + logger.debug('components: %s', components) + + ext_info[ext_name] = [descr, author, components, size] + + logger.debug('Installed extensions info: %s', ext_info) + return ext_info + + +def ext_by_dependants(dep_tree, depends_on=None): + r""" + Calcualate a list of extensions names sorted by dependencies. + + Args: + dep_tree - a dependecy tree of all considered extensions. + depends_on - a set of extensions names to check if the current one depends on them. + + Returns: + A list of of extensions from dep_tree sorted by dependencies. + For example, dictionary like that for extensions A, B, C, D and E: + A + /|\ + / B D + \/ \/ + C E + + represented { 'A': ['B', 'C', 'D'], 'B': ['C', 'E'], 'C': [], 'D': ['E'], 'E': [] } + returns ['A', 'B', 'D', 'C', 'E']. + """ + + logger.debug('dep_tree: %s, depends_on: %s', dep_tree, depends_on) + + if not dep_tree: + logger.debug('Dependency tree is empty! Return') + return [] + + # Check if we've got a default value + if not depends_on: + depends_on = set() + + # Collect all dependencies for the given dependency level in the tree + cur_depends_on = set() + for ext, dependendants in dep_tree.items(): + # Select all components with empty dependendants + is_selected = not dependendants + + # Check dependendants intersection if the case + if not is_selected: + dep_set = set(dependendants) + intersection = dep_set & depends_on + logger.debug('ext: %s, dep_set: %s, intersection: %s', ext, dep_set, intersection) + is_selected = intersection == dep_set + + if is_selected: + cur_depends_on.add(ext) + + # Check collected dependencies + logger.debug('cur_depends_on: %s', cur_depends_on) + if not cur_depends_on: + logger.warning( + 'Extensions in the tree doesnt rely on any other extensions! Return all of them') + return [ext for ext, _ in dep_tree.items()] + + # Collect all extension for this level + res_extensions = [] + for ext in cur_depends_on: + res_extensions.append(ext) + del dep_tree[ext] + + logger.debug('res_extensions: %s', res_extensions) + + # Get extensions from the next dependency level of the tree + cur_depends_on |= depends_on + return res_extensions + ext_by_dependants(dep_tree, cur_depends_on) + + +def ext_by_name(directory): + """ + Calcualate a list of extensions installed in the given directory. + + Args: + directory - a directory where extensions are installed. + + Returns: + A list of extensions names sorted by name. + """ + + logger.debug('directory: %s', directory) + + # Get ext files + salomexd_files = list_files_ext(directory, DFILE_EXT) + logger.debug('There are %s extensions in %s', len(salomexd_files), directory) + + # Get ext names + res_names = [] + for salomexd_file in salomexd_files: + ext_name, _ = os.path.splitext(os.path.basename(salomexd_file)) + res_names.append(ext_name) + + # Sort + res_names.sort() + logger.debug('Installed extensions: %s', res_names) + + return res_names + + +def ext_canremove_flags(directory): + r""" + Calcualate a dict of extensions names paired with bool flag if able to be removed. + + Args: + directory - the given ext install directory. + + Returns: + For dependency tree for extensions A, B, C, D and E: + A + /|\ + / B D + \/ \/ + C E + + represented { 'A': ['B', 'C', 'D'], 'B': ['C', 'E'], 'C': [], 'D': ['E'], 'E': [] } + returns ['A': True, 'B': False, 'D': False, 'C': False, 'E': False]. + We can remove only A here, because we don't have any other modules depend on it. + """ + + logger.debug('directory: %s', directory) + + dep_tree = dependency_tree(directory) + + res_dict = {} + for ext in dep_tree: + can_remove = True + for dep in dep_tree.values(): + if ext in dep: + # At least one ext depends on it, so can't remove + can_remove = False + break + + res_dict[ext] = can_remove + + # We shouldn't remove Base in any case + if 'Base' in dep_tree: + res_dict['Base'] = False + + logger.debug('res_dict: %s', res_dict) + + return res_dict + + +if __name__ == '__main__': + if len(sys.argv) == 2: + dir_size_str(sys.argv[1]) + ext_tree = dependency_tree(sys.argv[1]) + ext_list = ext_by_dependants(ext_tree) + logger.info('ext_list: %s', ext_list) + ext_by_name(sys.argv[1]) + ext_info_dict(sys.argv[1]) + ext_canremove_flags(sys.argv[1]) + elif len(sys.argv) == 3: + arg_1, arg_2 = sys.argv[1:] # pylint: disable=unbalanced-tuple-unpacking + ext_size_str(arg_1, arg_2) + else: + logger.error('You must provide all the arguments!') + logger.info(dir_size_str.__doc__) + logger.info(dependency_tree.__doc__) + logger.info(ext_size_str.__doc__) diff --git a/bin/SalomeOnDemandTK/extension_remover.py b/bin/SalomeOnDemandTK/extension_remover.py new file mode 100644 index 000000000..7533436c8 --- /dev/null +++ b/bin/SalomeOnDemandTK/extension_remover.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (C) 2007-2022 CEA/DEN, EDF R&D, OPEN CASCADE +# +# Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +# CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See https://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +# File : extension_remover.py +# Author : Konstantin Leontev, Open Cascade +# +# @package SalomeOnDemandTK +# @brief Set of utility to remove SALOME python extensions. + +"""Set of utility to remove SALOME python extensions. +""" + +import os +import sys +import shutil +from traceback import format_exc + +from .extension_utilities import logger, \ + SALOME_EXTDIR, EXTCOMPONENT_KEY, \ + isvalid_dirname, list_dependants, is_empty_dir, \ + find_envpy, value_from_salomexd, check_if_installed + + +def remove_if_empty(top_dir, directory): + """ + Recursively remove empty directories from the given one to the top. + + Args: + top_dir - top parent directory that can be removed as well + directory - the given directory + + Returns: + None. + """ + + #logger.debug('Check if %s is empty...', directory) + if not is_empty_dir(directory): + return + + logger.debug('Directory %s is empty. Remove it.', directory) + os.rmdir(directory) + + # Don't go up than top root + if top_dir == directory: + return + + # Remove the parent dir as well + parent_dir = os.path.abspath(os.path.join(directory, os.pardir)) + remove_if_empty(top_dir, parent_dir) + + +def remove_bylist(root_dir, salomexc): + """ + Remove files and directories listed in the given salomexc file. + + Args: + root_dir - a root dir for listed files + salomexc - file that contents a list of files to remove. + + Returns: + True if all the files were deleted without critical errors. + """ + + logger.debug('Remove files from %s dir listed in %s...', + root_dir, salomexc) + + try: + with open(salomexc, 'r', encoding='UTF-8') as file: + for line in file: + path_to_remove = os.path.join(root_dir, line.strip()) + logger.debug('Remove file %s...', path_to_remove) + + if os.path.isfile(path_to_remove): + os.remove(path_to_remove) + + # Remove the parent folder if empty + parent_dir = os.path.dirname(path_to_remove) + remove_if_empty(root_dir, parent_dir) + + elif os.path.islink(path_to_remove): + os.unlink(path_to_remove) + + # Remove the parent folder if empty + parent_dir = os.path.dirname(path_to_remove) + remove_if_empty(root_dir, parent_dir) + + elif os.path.isdir(path_to_remove): + logger.warning('Directories are not expected to be listed in %s file! ' + 'Remove %s anyway.', + salomexc, path_to_remove) + # Use instead of rmdir here, because dir can be not empty + shutil.rmtree(path_to_remove) + + else: + logger.warning('Unexpected path %s!' + 'It is not a file or directory. Skip.', + path_to_remove) + + except OSError: + logger.error(format_exc()) + return False + + return True + +def remove_salomex(install_dir, salomex_name): + """ + Remove a salome extension from SALOME install root. + + Args: + salome_root - path to SALOME install root directory. + salomex_name - a name of salome extension to remove. + + Returns: + List of deleted components or None if the functions fails. + """ + + logger.debug('Starting remove a salome extension %s', salomex_name) + + # Return value + components = None + + # Check if provided dirname is valid + if not isvalid_dirname(install_dir): + return components + + # Check if the given extension is installed + salomexd, salomexc = check_if_installed(install_dir, salomex_name) + if not salomexc: + logger.debug('Going to exit from extension removing process.') + return components + + # Check if we cannot remove an extension because of dependencies + dependants = list_dependants(install_dir, salomex_name) + if len(dependants) > 0: + logger.error('Cannot remove an extension %s because followed extensions depend on it: %s! ' + 'Going to exit from extension removing process.', + salomex_name, dependants) + return components + + # Try to remove all the files listed in the control file + if not remove_bylist(os.path.join(install_dir, SALOME_EXTDIR), salomexc): + return components + + # Remove control file + os.remove(salomexc) + + # Remove env file + env_py = find_envpy(install_dir, salomex_name) + if env_py: + os.remove(env_py) + else: + logger.warning('Cannot find and remove %s file! ', env_py) + + # Remove description file + if salomexd: + # Get components to deactivate in UI if the case + components = value_from_salomexd(salomexd, EXTCOMPONENT_KEY) + os.remove(salomexd) + + logger.debug('An extension %s was removed from %s', + salomex_name, install_dir) + + return components if components else [] + + +if __name__ == '__main__': + if len(sys.argv) == 3: + arg_1, arg_2 = sys.argv[1:] # pylint: disable=unbalanced-tuple-unpacking + remove_salomex(arg_1, arg_2) + else: + logger.error('You must provide all the arguments!') + logger.info(remove_salomex.__doc__) diff --git a/bin/SalomeOnDemandTK/extension_unpacker.py b/bin/SalomeOnDemandTK/extension_unpacker.py new file mode 100644 index 000000000..0f7f1d1a2 --- /dev/null +++ b/bin/SalomeOnDemandTK/extension_unpacker.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (C) 2007-2022 CEA/DEN, EDF R&D, OPEN CASCADE +# +# Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +# CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See https://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +# File : extension_unpacker.py +# Author : Konstantin Leontev, Open Cascade +# +# @package SalomeOnDemandTK +# @brief Set of utility to unpack SALOME python extensions. + +"""Set of utility to unpack SALOME python extensions. +""" + +import tarfile +import os +import sys +import json +from traceback import format_exc + +from .extension_utilities import logger, \ + DFILE_EXT, ARCFILE_EXT, EXTDEPENDSON_KEY, EXTCOMPONENT_KEY, \ + isvalid_filename, isvalid_dirname, ext_info_bykey, set_selext_env, get_app_root, \ + check_if_installed + +def unpack_salomex(salome_root, salomex): + """ + Unpack a given salome extension archive into SALOME install root. + + Args: + salome_root - path to SALOME install root directory. + salomex - a given salomex file to unpack. + + Returns: + True if an ext was successfully unpacked. + """ + + logger.debug('Starting unpack a salome extension file') + + # Check if provided filenames are valid + if not isvalid_dirname(salome_root) or \ + not isvalid_filename(salomex, ARCFILE_EXT): + return False + + # Check if the given extension is already installed + salome_ext_name, _ = os.path.splitext(os.path.basename(salomex)) + _, salomexc = check_if_installed(salome_root, salome_ext_name) + if salomexc: + logger.debug('To reinstall an extension you need to remove it first!') + return False + + try: + with tarfile.open(salomex) as ext: + + # Read a list of dependencies, so let's check if they are present in salome_root + logger.debug('Try to read %s.%s file...', salome_ext_name, DFILE_EXT) + salomexd_mem = ext.getmember(salome_ext_name + '.' + DFILE_EXT) + salomexd_file = ext.extractfile(salomexd_mem) + if not salomexd_file: + logger.error('Cannot extract %s.%s file!', salome_ext_name, DFILE_EXT) + return False + + salomexd_content = json.load(salomexd_file) + + logger.debug('Check dependencies...') + if EXTDEPENDSON_KEY in salomexd_content and salomexd_content[EXTDEPENDSON_KEY]: + depends_on = salomexd_content[EXTDEPENDSON_KEY] + + # Check every module if it's in the salome_root + for depends in depends_on: + depends_filename = os.path.join(salome_root, depends + '.' + DFILE_EXT) + if not os.path.isfile(depends_filename): + logger.error('Cannot find %s for a module that extension depends on!', + depends_filename) + return False + + # Unpack archive in the salome_root + logger.debug('Extract all the files into %s...', salome_root) + ext.extractall(salome_root) + + logger.debug('SALOME extension %s was installed.', salome_ext_name) + + except (OSError, KeyError): + logger.error(format_exc()) + return False + + return True + + +def install_salomex(salomex): + """ + Install a given salome extension into SALOME_APPLICATION_DIR. + + Args: + salomex - a given salomex file to unpack. + + Returns: + A list of components to be activated later or None if the function failed. + """ + + logger.debug('Starting install a salome extension from %s', salomex) + + # Check if we have the salome root path + app_root = os.environ.get('SALOME_APPLICATION_DIR', '') + if not app_root: + # It should be set on the app start, but leave it here to run as a standalone script + logger.warning( + 'Env var SALOME_APPLICATION_DIR is not set! Try to set it going up from cur location.') + app_root = get_app_root() + + # Unpack an archive + if not unpack_salomex(app_root, salomex): + return None + + # Set up an environment + # It's not clear at the moment what to do if it fails + ext_name, _ = os.path.splitext(os.path.basename(salomex)) + set_selext_env(app_root, ext_name) + + # Get components to activate later + components = ext_info_bykey(app_root, ext_name, EXTCOMPONENT_KEY) + + return components if components else [] + + +if __name__ == '__main__': + if len(sys.argv) == 3: + arg_1, arg_2 = sys.argv[1:] # pylint: disable=unbalanced-tuple-unpacking + unpack_salomex(arg_1, arg_2) + elif len(sys.argv) == 2: + install_salomex(sys.argv[1]) + else: + logger.error('You must provide all the arguments!') + logger.info(unpack_salomex.__doc__) diff --git a/bin/SalomeOnDemandTK/extension_utilities.py b/bin/SalomeOnDemandTK/extension_utilities.py new file mode 100644 index 000000000..9031b1a3b --- /dev/null +++ b/bin/SalomeOnDemandTK/extension_utilities.py @@ -0,0 +1,669 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (C) 2007-2022 CEA/DEN, EDF R&D, OPEN CASCADE +# +# Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +# CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See https://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +# File : extension_utilities.py +# Author : Konstantin Leontev, Open Cascade +# +# @package SalomeOnDemandTK +# @brief Utilities and constants those help to deal with salome extension files. + +""" +Utilities and constants those help to deal with salome extension files. +""" + +import os +import sys +import logging +import json +from traceback import format_exc +from pathlib import Path +import importlib.util +import fnmatch +import re + +# Usually logging verbosity is set inside bin/runSalomeCommon.py when salome is starting. +# Here we do just the same for a case if we call this package stand alone. +FORMAT = '%(levelname)s : %(asctime)s : [%(filename)s:%(funcName)s:%(lineno)s] : %(message)s' +logging.basicConfig(format=FORMAT, level=logging.DEBUG) +logger = logging.getLogger() + +# SalomeContext sets the logging verbosity level on its own, +# and we put it here, so it doesn't override the local format above. +#pylint: disable=wrong-import-position +import salomeContext +#pylint: enable=wrong-import-position + +SALOME_EXTDIR = '__SALOME_EXT__' +ARCFILE_EXT = 'salomex' +BFILE_EXT = 'salomexb' +CFILE_EXT = 'salomexc' +DFILE_EXT = 'salomexd' +PYFILE_EXT = 'py' +ENVPYFILE_SUF = '_env.py' + +EXTNAME_KEY = 'name' +EXTDESCR_KEY = 'descr' +EXTDEPENDSON_KEY = 'depends_on' +EXTAUTHOR_KEY = 'author' +EXTCOMPONENT_KEY = 'components' + + +def create_salomexd(name, descr='', depends_on=None, author='', components=None): + """ + Create a basic salomexd file from provided args. + Current version is a json file with function args as the keys. + + Args: + name - the name of the corresponding salome extension. + depends_on - list of the modules that current extension depends on. + author - an author of the extension. + components - the names of the modules those current extension was built from. + + Returns: + None + """ + + logger.debug('Create salomexd file...') + + if depends_on is None: + depends_on = [] + + if components is None: + components = [] + + json_string = json.dumps( + { + EXTNAME_KEY: name, + EXTDESCR_KEY: descr, + EXTDEPENDSON_KEY: depends_on, + EXTAUTHOR_KEY: author, + EXTCOMPONENT_KEY: components + }, + indent=4 + ) + + try: + with open(name + '.' + DFILE_EXT, "w", encoding="utf-8") as file: + file.write(json_string) + + except OSError: + logger.error(format_exc()) + + +def read_salomexd(file_path): + """ + Reads a content of a salomexd file. Current version is a json file. + There's no check if the file_path is a valid salomexd file name. + It's expected that user call isvalid_filename() in advance. + + Args: + file_path - the path to the salomexd file. + + Returns: + A dictionary that represents the content of the salomexd file. + """ + + logger.debug('Read salomexd file %s', file_path) + + try: + with open(file_path, 'r', encoding='UTF-8') as file: + return json.load(file) + + except OSError: + logger.error(format_exc()) + return {} + + +def value_from_salomexd(file_path, key): + """ + Reads a content of a salomexd file and return a value for the given key. + + Args: + file_path - the path to the salomexd file. + key - the key to search an assigned value. + + Returns: + A value assigned to the given key if exist, otherwise None. + """ + + file_content = read_salomexd(file_path) + if key in file_content and file_content[key]: + logger.debug('Key: %s, value: %s', key, file_content[key]) + return file_content[key] + + logger.warning('Cannot get a value for key %s in salomexd file %s', key, file_path) + return None + + +def ext_info_bykey(salome_root, salomex_name, key): + """ + Search a salomexd file by ext name and return a value for the given key. + + Args: + install_dir - directory where the given extension is installed. + salomex_name - the given extension's name. + key - the key to search an assigned value. + + Returns: + A value for key in the given ext salomexd file. + """ + + salomexd = find_salomexd(salome_root, salomex_name) + if salomexd: + return value_from_salomexd(salomexd, key) + + return None + + +def create_salomexb(name, included): + """ + Create a salomexb file from a list of included file names. + For example: + */*.py + doc/*.html + doc/*.jp + + Args: + name - the name of the corresponding salome extension. + included - list of the directories those must be included inside a salomex. + + Returns: + None + """ + + logger.debug('Create salomexb file...') + + try: + with open(name + '.' + BFILE_EXT, "w", encoding="utf-8") as file: + file.write('\n'.join(included[0:])) + + except OSError: + logger.error(format_exc()) + + +def read_salomexb(file_path): + """ + Returns a content af a salomexb file as a list of strings. + There's no check if the file_path is a valid salomexb file name. + It's expected that user call isvalid_filename() in advance. + + Args: + file_path - the path to the salomexb file. + + Returns: + List of strings - paths to the directories those must be included in + corresponding salomex archive file. + """ + + logger.debug('Read salomexb file %s', file_path) + + try: + with open(file_path, 'r', encoding='UTF-8') as file: + return [line.rstrip() for line in file] + + except OSError: + logger.error(format_exc()) + return [] + + +def list_files(dir_path): + """ + Returns the recursive list of relative paths to files as strings + in the dir_path root directory and all subdirectories. + + Args: + dir_path - the path to the directory where you search for files. + + Raises: + Raises OSError exception. + + Returns: + A list of relative paths to files inside the given directory. + """ + + files_list = [] + for root, _, files in os.walk(dir_path): + for file in files: + files_list += os.path.relpath(os.path.join(root, file), dir_path) + + return files_list + + +def filter_to_regex(dir_path, filter_patterns): + r""" + Makes a regex pattern from a given filter. + + Args: + dir_path - the path to the directory where you search for files. + filter_patterns - list of expressions for matching file names. + + Returns: + A regex string translated from the filter. + For example: + Filter: ['SMESH/**.cmake', 'SMESH/share/s*.med'] + Regex: (?s:SMESH/.*\.cmake)\Z|(?s:SMESH/share/s.*\.med)\Z + Matches: SMESH/adm_local/cmake_files/SalomeSMESHConfig.cmake + SMESH/share/salome/resources/smesh/padderdata/ferraill.med + """ + + logger.debug('Convert given filter to regex...') + + # On Windows, it converts forward slashes to backward slashes. + norm_filter = [os.path.normpath(pat) for pat in filter_patterns] + + # Make a regex pattern + # Adding '*' at the end of the folders names to match all the files inside. + regex_pattern = r'|'.join( + [fnmatch.translate(pat + '*' if os.path.isdir(os.path.join(dir_path, pat)) else pat) + for pat + in norm_filter]) + + logger.debug('Regex pattern: %s', regex_pattern) + + return regex_pattern + + +def list_files_filter(dir_path, filter_patterns): + """ + Returns the recursive list of relative paths to files as strings + in the dir_path root directory and all subdirectories. + + Args: + dir_path - the path to the directory where you search for files. + filter_patterns - list of expressions for matching file names. + + Returns: + files_abs - a list of absolute paths to selected files. + files_rel - a list of relative paths to selected files. + """ + + logger.debug('Get list of files to add into archive...') + + regex_pattern = filter_to_regex(dir_path, filter_patterns) + + files_abs = [] + files_rel = [] + + for root, _, files in os.walk(dir_path): + for file in files: + filename_abs = os.path.join(root, file) + filename_rel = os.path.relpath(filename_abs, dir_path) + + if re.match(regex_pattern, filename_rel): + logger.debug('File name %s matches pattern', filename_rel) + files_abs.append(filename_abs) + files_rel.append(filename_rel) + + return files_abs, files_rel + + +def list_files_ext(dir_path, ext): + """ + Returns a list of abs paths to files with a given extension + in the dir_path directory. + + Args: + dir_path - the path to the directory where you search for files. + ext - a given extension. + + Returns: + A list of absolute paths to selected files. + """ + + logger.debug('Get list of files with extension %s...', ext) + + dot_ext = '.' + ext + return [os.path.join(dir_path, f) for f in os.listdir(dir_path) if f.endswith(dot_ext)] + + +def list_tonewline_str(str_list): + """ + Converts the given list of strings to a newline separated string. + + Args: + dir_path - the path to the directory where you search for files. + + Returns: + A newline separated string. + """ + return '\n'.join(file for file in str_list) + + +def isvalid_filename(filename, extension): + """ + Checks if a given filename is valid in a sense that it exists and have a given extension. + + Args: + filename - the name of the file to check. + extension - expected file name extension. + + Returns: + True if the given filename is valid for given extension. + """ + + logger.debug('Check if the filename %s exists and has an extension %s', filename, extension) + + # First do we even have something to check here + if filename == '' or extension == '': + logger.error('A filename and extension cannot be empty! Args: filename=%s, extension=%s', + filename, extension) + return False + + # Check if the filename matchs the provided extension + _, ext = os.path.splitext(filename) + ext = ext.lstrip('.') + if ext != extension: + logger.error('The filename %s doesnt have a valid extension! \ + The valid extension must be: %s, but get: %s', + filename, extension, ext) + return False + + # Check if the file base name is not empty + base_name = os.path.basename(filename) + if base_name == '': + logger.error('The file name %s has an empty base name!', filename) + return False + + # Check if a file with given filename exists + if not os.path.isfile(filename): + logger.error('The filename %s is not an existing regular file!', filename) + return False + + logger.debug('Filename %s exists and has an extension %s', filename, extension) + return True + + +def isvalid_dirname(dirname): + """ + Checks if a given directory name exists. + + Args: + dirname - the name of the directory to check. + + Returns: + True if the given dirname is valid. + """ + + logger.debug('Check if the dirname %s exists', dirname) + + # First do we even have something to check here + if dirname == '': + logger.error('A dirname argument cannot be empty! dirname=%s', dirname) + return False + + # Check if a file with given filename exists + if not os.path.isdir(dirname): + logger.error('The dirname %s is not an existing regular file!', dirname) + return False + + logger.debug('Directory %s exists', dirname) + return True + + +def list_dependants(install_dir, salomex_name): + """ + Checks if we have installed extensions those depend on a given extension. + + Args: + install_dir - path to SALOME install root directory. + salomex_name - a name of salome extension to check. + + Returns: + True if the given extension has dependants. + """ + + logger.debug('Check if there are other extensions that depends on %s...', salomex_name) + dependants = [] + salomexd_files = list_files_ext(install_dir, DFILE_EXT) + + for salomexd_file in salomexd_files: + dependant_name, _ = os.path.splitext(os.path.basename(salomexd_file)) + + # Don't process extension itself + if dependant_name == salomex_name: + continue + + logger.debug('Check dependencies for %s...', salomexd_file) + salomexd_content = read_salomexd(salomexd_file) + + if EXTDEPENDSON_KEY in salomexd_content and salomexd_content[EXTDEPENDSON_KEY]: + depends_on = salomexd_content[EXTDEPENDSON_KEY] + logger.debug('List of dependencies: %s', depends_on) + + if salomex_name in depends_on: + dependants.append(dependant_name) + + if len(dependants) > 0: + logger.debug('Followed extensions %s depend on %s', + dependants, salomex_name) + + return dependants + + +def is_empty_dir(directory): + """ + Checks if the given directory is empty. + + Args: + directory - path to directory to check. + + Returns: + True if the given directory is empty. + """ + + return not next(os.scandir(directory), None) + + +def find_file(directory, file_name): + """ + Finds a file in the given directory. + + Args: + directory - path to directory to check. + file_name - given base filename with extension + + Returns: + Abs path if the file exist, otherwise None. + """ + + logger.debug('Try to find %s file in %s...', file_name, directory) + file = os.path.join(directory, file_name) + if os.path.isfile(file): + logger.debug('File %s exists.', file) + return file + + logger.debug('File %s doesnt\'t exist. Return None.', file) + return None + + +def find_salomexd(install_dir, salomex_name): + """ + Finds a salomexd file for the given extension. + + Args: + install_dir - path to directory to check. + salomex_name - extension's name. + + Returns: + Abs path if the file exist, otherwise None. + """ + + return find_file(install_dir, salomex_name + '.' + DFILE_EXT) + + +def find_salomexc(install_dir, salomex_name): + """ + Finds a salomexc file for the given extension. + + Args: + install_dir - path to directory to check. + salomex_name - extension's name. + + Returns: + Abs path if the file exist, otherwise None. + """ + + return find_file(install_dir, salomex_name + '.' + CFILE_EXT) + + +def find_envpy(install_dir, salomex_name): + """ + Finds a _env.py file for the given extension. + + Args: + install_dir - path to directory to check. + salomex_name - extension's name. + + Returns: + Abs path if the file exist, otherwise None. + """ + + return find_file(install_dir, salomex_name + ENVPYFILE_SUF) + + +def module_from_filename(filename): + """ + Create and execute a module by filename. + + Args: + filename - a given python filename. + + Returns: + Module. + """ + + # Get the module from the filename + basename = os.path.basename(filename) + module_name, _ = os.path.splitext(basename) + + spec = importlib.util.spec_from_file_location(module_name, filename) + if not spec: + logger.error('Could not get a spec for %s file!') + return None + + module = importlib.util.module_from_spec(spec) + if not module: + logger.error('Could not get a module for %s file!') + return None + + sys.modules[module_name] = module + + logger.debug('Execute %s module', module_name) + if not spec.loader: + logger.error('spec.loader is None for %s file!') + return None + + spec.loader.exec_module(module) + + return module + + +def set_selext_env(install_dir, salomex_name, context=None): + """ + Finds and run _env.py file for the given extension. + + Args: + install_dir - path to directory to check. + salomex_name - extension's name. + context - SalomeContext object. + + Returns: + True if an envpy file was found and run its init func. + """ + + logger.debug('Set an env for salome extension: %s...', salomex_name) + + # Set the root dir as env variable + if not context: + context = salomeContext.SalomeContext(None) + context.setVariable('SALOME_APPLICATION_DIR', install_dir, overwrite=False) + + # Find env file + ext_envpy = find_envpy(install_dir, salomex_name) + if not ext_envpy: + return False + + # Get a module + envpy_module = module_from_filename(ext_envpy) + if not envpy_module: + return False + + # Set env if we have something to set + ext_dir = os.path.join(install_dir, SALOME_EXTDIR) + if hasattr(envpy_module, 'init'): + envpy_module.init(context, ext_dir) + return True + else: + logger.warning('Env file %s doesnt have init func:!', ext_envpy) + + logger.warning('Setting an env for salome extension %s failed!', salomex_name) + return False + + +def get_app_root(levels_up=5): + """ + Finds an app root by going up on the given steps. + + Args: + levels_up - steps up in dir hierarchy relative to the current file. + + Returns: + Path to the app root. + """ + + app_root = str(Path(__file__).resolve().parents[levels_up - 1]) + logger.debug('App root: %s', app_root) + + return app_root + + +def check_if_installed(install_dir, salomex_name): + """ + Check if a given salome extension is installed in install_dir. + Now for install|remove process we consider an ext is installed + if we have at least salomexc file with list of files to remove + if we need to clean up. + + Args: + install_dir - path to SALOME install root directory. + salomex_name - a given ext name. + + Returns: + salomexd, salomexc file names. + """ + + logger.debug('Check if %s extension is installed in %s...', salomex_name, install_dir) + + salomexd = find_salomexd(install_dir, salomex_name) + if not salomexd: + logger.debug('Extension has been already removed or %s file was deleted by mistake. ' + 'In the former case we can use %s file to clean up.', DFILE_EXT, CFILE_EXT) + + salomexc = find_salomexc(install_dir, salomex_name) + if salomexc: + logger.debug('An extension %s IS installed.', salomex_name) + else: + logger.debug('An extension %s IS NOT installed.', salomex_name) + + return salomexd, salomexc diff --git a/bin/launchConfigureParser.py b/bin/launchConfigureParser.py index 42c90e6f4..2d754a396 100644 --- a/bin/launchConfigureParser.py +++ b/bin/launchConfigureParser.py @@ -85,6 +85,7 @@ salomecfgname = "salome" salomeappname = "SalomeApp" script_nam = "pyscript" verbosity_nam = "verbosity" +on_demand_nam = "on_demand" # possible choices for the "embedded" and "standalone" parameters embedded_choices = [ "registry", "study", "moduleCatalog", "cppContainer", "SalomeAppEngine" ] @@ -92,7 +93,7 @@ standalone_choices = [ "registry", "study", "moduleCatalog", "cppContainer"] # values of boolean type (must be '0' or '1'). # xml_parser.boolValue() is used for correct setting -boolKeys = ( gui_nam, splash_nam, logger_nam, file_nam, xterm_nam, portkill_nam, killall_nam, except_nam, pinter_nam, shutdown_servers_nam, launcher_only_nam ) +boolKeys = ( gui_nam, splash_nam, logger_nam, file_nam, xterm_nam, portkill_nam, killall_nam, except_nam, pinter_nam, shutdown_servers_nam, launcher_only_nam, on_demand_nam ) intKeys = ( interp_nam, ) strKeys = ( launcher_nam ) @@ -827,6 +828,17 @@ Python file arguments, if any, must be comma-separated (without blank characters default="0", help=help_str) + # On demand + help_str = "Use installed salome on-demand extensions." + help_str += "0 to run without salome extensions [default], " + help_str += "1 to run only installed salome extensions. " + pars.add_argument("--on-demand", + dest="on_demand", + metavar="<0/1>", + action=StoreBooleanAction, + default=False, + help=help_str) + # Positional arguments (hdf file, python file) pars.add_argument("arguments", nargs=argparse.REMAINDER) @@ -868,21 +880,21 @@ def get_env(appname=salomeappname, cfgname=salomecfgname, exeName=None, keepEnvi global args config_var = appname+'Config' + ############################ + # parse command line options + pars = CreateOptionParser(exeName=exeName) + cmd_opts = pars.parse_args(sys.argv[1:]) + ############################ + # check KERNEL_ROOT_DIR kernel_root_dir = os.environ.get("KERNEL_ROOT_DIR", None) - if kernel_root_dir is None: + if kernel_root_dir is None and not cmd_opts.on_demand: print(""" For each SALOME module, the environment variable _ROOT_DIR must be set. KERNEL_ROOT_DIR is mandatory. """) sys.exit(1) - ############################ - # parse command line options - pars = CreateOptionParser(exeName=exeName) - cmd_opts = pars.parse_args(sys.argv[1:]) - ############################ - # Process --print-port option if cmd_opts.print_port: from searchFreePort import searchFreePort @@ -906,7 +918,7 @@ def get_env(appname=salomeappname, cfgname=salomecfgname, exeName=None, keepEnvi else: dirs += re.split('[;|:]', os.getenv(config_var)) - if not keepEnvironment: + if not keepEnvironment and not cmd_opts.on_demand: if os.getenv("GUI_ROOT_DIR") and os.path.isdir(os.getenv("GUI_ROOT_DIR")): gui_resources_dir = os.path.join(os.getenv("GUI_ROOT_DIR"),'share','salome','resources','gui') if os.path.isdir(gui_resources_dir): @@ -1002,7 +1014,7 @@ def get_env(appname=salomeappname, cfgname=salomecfgname, exeName=None, keepEnvi if cmd_opts.batch is not None: args[batch_nam] = True - if not os.getenv("GUI_ROOT_DIR") or not os.path.isdir(os.getenv("GUI_ROOT_DIR")): + if ( not os.getenv("GUI_ROOT_DIR") or not os.path.isdir(os.getenv("GUI_ROOT_DIR")) ) and not cmd_opts.on_demand: args[gui_nam] = False if args[gui_nam]: @@ -1046,6 +1058,7 @@ def get_env(appname=salomeappname, cfgname=salomecfgname, exeName=None, keepEnvi args[script_nam] = new_args args[verbosity_nam] = cmd_opts.verbosity + args[on_demand_nam] = cmd_opts.on_demand # xterm if cmd_opts.xterm is not None: @@ -1145,23 +1158,24 @@ def get_env(appname=salomeappname, cfgname=salomecfgname, exeName=None, keepEnvi # now modify SalomeAppConfig environment variable # to take into account the SALOME modules - if os.sys.platform == 'win32': - dirs = re.split('[;]', os.environ[config_var] ) - else: - dirs = re.split('[;|:]', os.environ[config_var] ) - for module in args[modules_nam]: - if module not in ["KERNEL", "GUI", ""] and os.getenv("{0}_ROOT_DIR".format(module)): - d1 = os.path.join(os.getenv("{0}_ROOT_DIR".format(module)),"share","salome","resources",module.lower()) - d2 = os.path.join(os.getenv("{0}_ROOT_DIR".format(module)),"share","salome","resources") - #if os.path.exists( "%s/%s.xml"%(d1, appname) ): - if os.path.exists( os.path.join(d1,"{0}.xml".format(salomeappname)) ): - dirs.append( d1 ) - #elif os.path.exists( "%s/%s.xml"%(d2, appname) ): - elif os.path.exists( os.path.join(d2,"{0}.xml".format(salomeappname)) ): - dirs.append( d2 ) + if not args[on_demand_nam]: + if os.sys.platform == 'win32': + dirs = re.split('[;]', os.environ[config_var] ) else: - # print("* '"+m+"' should be deleted from ",args[modules_nam]) - pass + dirs = re.split('[;|:]', os.environ[config_var] ) + for module in args[modules_nam]: + if module not in ["KERNEL", "GUI", ""] and os.getenv("{0}_ROOT_DIR".format(module)): + d1 = os.path.join(os.getenv("{0}_ROOT_DIR".format(module)),"share","salome","resources",module.lower()) + d2 = os.path.join(os.getenv("{0}_ROOT_DIR".format(module)),"share","salome","resources") + #if os.path.exists( "%s/%s.xml"%(d1, appname) ): + if os.path.exists( os.path.join(d1,"{0}.xml".format(salomeappname)) ): + dirs.append( d1 ) + #elif os.path.exists( "%s/%s.xml"%(d2, appname) ): + elif os.path.exists( os.path.join(d2,"{0}.xml".format(salomeappname)) ): + dirs.append( d2 ) + else: + # print("* '"+m+"' should be deleted from ",args[modules_nam]) + pass # Test if cmd_opts.test_script_file is not None: diff --git a/bin/runSalome.py b/bin/runSalome.py index eb6fa9a1b..1a0e1134b 100755 --- a/bin/runSalome.py +++ b/bin/runSalome.py @@ -24,7 +24,6 @@ # import sys, os, string, glob, time, pickle, re -import orbmodule import setenv from server import process_id, Server import json @@ -33,6 +32,8 @@ from salomeContextUtils import ScriptAndArgsObjectEncoder import runSalomeNoServer import runSalomeCommon import platform +import runSalomeOnDemand +from launchConfigureParser import verbosity_nam, on_demand_nam import logging logger = logging.getLogger() @@ -81,7 +82,7 @@ def startSalome(args, modules_list, modules_root_dir): mySessionServ.run() ior_fakens_filename = mySessionServ.iorfakens logger.debug("Rendez-vous file for to retrieve IOR of session is \"{}\"".format(ior_fakens_filename)) - + end_time = os.times() # @@ -175,9 +176,18 @@ def main(exeName=None): """Salome launch as a main application""" keep_env = not os.getenv('SALOME_PLEASE_SETUP_ENVIRONMENT_AS_BEFORE') args, modules_list, modules_root_dir = setenv.get_config(exeName=exeName, keepEnvironment=keep_env) - runSalomeCommon.setVerbose(args["verbosity"]) + runSalomeCommon.setVerbose(args[verbosity_nam]) + kill_salome(args) # -- + + # Setup extension's env in salome on demand case + if args[on_demand_nam]: + runSalomeOnDemand.set_ext_env() + # Reset up module_list and modules_root_dir, if we dont want to define SalomeAppConfig on salome.py. + # We must remove the "else" on "if os.sys.platform == 'win32':" in launcheConfigureParser.py too. + args, _, __ = setenv.get_config(exeName=exeName, keepEnvironment=keep_env) + setenv.set_env(args, modules_list, modules_root_dir, keepEnvironment=keep_env) ior_fakens_filename = useSalome(args, modules_list, modules_root_dir) # Management of -t @@ -200,7 +210,7 @@ def main(exeName=None): proc = subprocess.Popen(command, shell=True, env = env) addToKillList(proc.pid, command) res = proc.wait() - if res: sys.exit(1) + if res: sys.exit(1) return args, ior_fakens_filename # ----------------------------------------------------------------------------- @@ -219,8 +229,8 @@ def foreGround(args, ior_fakens_filename): logger.warn("No file {} set to host IOR of the fake naming server does not exit !") return import CORBA - import Engines - import SALOME + #import Engines + #import SALOME from time import sleep orb = CORBA.ORB_init([''], CORBA.ORB_ID) ior_fakens = None @@ -297,6 +307,7 @@ def runSalome(): args, ior_fakens_filename = main() # -- test = args['gui'] and args['session_gui'] + test = test and not args[on_demand_nam] test = test or args['wake_up_session'] # -- # The next test covers the --pinter option or if var PYTHONINSPECT is set diff --git a/bin/runSalomeCommon.py b/bin/runSalomeCommon.py index 7ff22727d..b0d307d3f 100755 --- a/bin/runSalomeCommon.py +++ b/bin/runSalomeCommon.py @@ -34,6 +34,11 @@ import subprocess from salomeContextUtils import ScriptAndArgsObjectEncoder import platform import logging + +# Setting formatter in setVerbose() was commented because adding of handler +# breaks using of root logger in other modules and cause many double lines in logs. +FORMAT = '%(levelname)s : %(asctime)s : [%(filename)s:%(funcName)s:%(lineno)s] : %(message)s' +logging.basicConfig(format=FORMAT) logger = logging.getLogger() class ColoredFormatter(logging.Formatter): @@ -66,20 +71,20 @@ class BackTraceFormatter(logging.Formatter): return logging.Formatter.format(self, record) def setVerbose(verbose): - from packaging import version - current_version = version.parse("{}.{}".format(sys.version_info.major,sys.version_info.minor)) - version_ref = version.parse("3.5.0") - global logger - formatter = None - if current_version >= version_ref: - formatter = BackTraceFormatter('%(levelname)s : %(asctime)s : %(message)s ',style='%') - else: - formatter = logging.Formatter('%(levelname)s : %(asctime)s : %(message)s ',style='%') - formatter.default_time_format = '%H:%M:%S' - formatter.default_msec_format = "%s.%03d" - stream_handler = logging.StreamHandler() - stream_handler.setFormatter(formatter) - logger.addHandler(stream_handler) + # from packaging import version + # current_version = version.parse("{}.{}".format(sys.version_info.major,sys.version_info.minor)) + # version_ref = version.parse("3.5.0") + # global logger + # formatter = None + # if current_version >= version_ref: + # formatter = BackTraceFormatter('%(levelname)s : %(asctime)s : %(message)s ',style='%') + # else: + # formatter = logging.Formatter('%(levelname)s : %(asctime)s : %(message)s ',style='%') + # formatter.default_time_format = '%H:%M:%S' + # formatter.default_msec_format = "%s.%03d" + # stream_handler = logging.StreamHandler() + # stream_handler.setFormatter(formatter) + # logger.addHandler(stream_handler) verbose_map = { "0": logging.WARNING, "1": logging.INFO, "2": logging.DEBUG} if verbose in verbose_map: @@ -356,7 +361,7 @@ class CommonSessionServer(Server): @abc.abstractmethod def getSessionServerExe(self): pass - + def setpath(self,modules_list,modules_root_dir): list_modules = modules_list[:] list_modules.reverse() @@ -407,7 +412,7 @@ class SessionServer(CommonSessionServer): super().__init__(args,modules_list,modules_root_dir) import KernelBasis KernelBasis.setSSLMode(False) - + def getSessionServerExe(self): return "SALOME_Session_Server" # --- diff --git a/bin/runSalomeOnDemand.py b/bin/runSalomeOnDemand.py new file mode 100755 index 000000000..3047b209c --- /dev/null +++ b/bin/runSalomeOnDemand.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (C) 2022 CEA/DEN, EDF R&D, OPEN CASCADE +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See https://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +# File : runSalomeOnDemand.py +# Author : Konstantin Leontev, Open Cascade +# +## @package runSalomeOnDemand +# \brief Module that provides services to launch SALOME with custom set of modules +# + +"""Run SALOME app in the context of adding modules as extensions. +""" + +import os,sys +import salomeContext +from SalomeOnDemandTK.extension_utilities import logger, \ + set_selext_env, get_app_root, find_file +from SalomeOnDemandTK.extension_query import ext_by_dependants, dependency_tree + + +def set_ext_env(app_name='', version=''): + """ + Set an environment to start SALOME as a set of extensions. + + Args: + app_name - an application's name. + version - a version of an application. + + Returns: + None. + """ + + logger.debug('Set an env for app: %s, version: %s...', app_name, version) + + # Get the root directory + app_root = get_app_root() + + # Set the root dir as env variable + context = salomeContext.SalomeContext(None) + context.setVariable('SALOME_APPLICATION_DIR', app_root, overwrite=True) + + # Find and source all _env.py files for installed extensions + tree = dependency_tree(app_root) + installed_ext = ext_by_dependants(tree) + logger.debug('Installed extensions: %s', installed_ext) + if not installed_ext: + logger.debug('There are not any extensions in %s!', app_root) + return + + # Execute env file as a module + for ext_name in installed_ext: + set_selext_env(app_root, ext_name, context) + for python_path in os.environ["PYTHONPATH"].split(':'): + sys.path[:0] = [python_path] + +if __name__ == "__main__": + if len(sys.argv) == 3: + arg_1, arg_2 = sys.argv[1:] + set_ext_env(arg_1, arg_2) + else: + logger.error('You must provide all the arguments!') + logger.info(set_ext_env.__doc__) diff --git a/bin/setenv.py b/bin/setenv.py index 824e6a962..2e5474be4 100755 --- a/bin/setenv.py +++ b/bin/setenv.py @@ -23,7 +23,6 @@ # import sys, os, string, glob, time, pickle -import orbmodule from launchConfigureParser import verbose # this file is extraction of set_env from runSalome.py -- 2.39.2