From 27b0c3eb1c83ae1752c8dc1e4ec580ef18d55fc4 Mon Sep 17 00:00:00 2001 From: Konstantin Leontev Date: Wed, 14 Dec 2022 12:30:39 +0300 Subject: [PATCH] [bos #32522][EDF] SALOME on Demand. Added extension_query module. Some refactoring in remover and utilities modules. --- bin/SalomeOnDemandTK/extension_query.py | 230 ++++++++++++++++++++ bin/SalomeOnDemandTK/extension_remover.py | 27 +-- bin/SalomeOnDemandTK/extension_utilities.py | 102 ++++++--- 3 files changed, 319 insertions(+), 40 deletions(-) create mode 100644 bin/SalomeOnDemandTK/extension_query.py diff --git a/bin/SalomeOnDemandTK/extension_query.py b/bin/SalomeOnDemandTK/extension_query.py new file mode 100644 index 000000000..a56e32303 --- /dev/null +++ b/bin/SalomeOnDemandTK/extension_query.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (C) 2007-2022 CEA/DEN, EDF R&D, OPEN CASCADE +# +# Copyright (C) 2003-2007 OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN, +# CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See https://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +# File : extension_query.py +# Author : Konstantin Leontev, Open Cascade +# +# @package SalomeOnDemandTK +# @brief An utility package that will allow you to know the size of an extension +# and generate a dependency tree. + +""" +An utility package that will allow you to know the size of an extension +and generate a dependency tree. +""" + +import os +import sys +from traceback import format_exc + +from .extension_utilities import logger, \ + SALOME_EXTDIR, \ + isvalid_dirname, find_salomexc + + +def size_to_str(size, format_in_bytes=False): + """ + Returns a string describing a size. + + Args: + size - the size to represent as a string. + format_in_bytes - if True, size is expressed in bytes, + otherwise human readable units are used. + + Returns: + The size as a string with units. + """ + + if format_in_bytes: + return '%s' % size + + __units__ = ["", "Ki", "Mi", "Gi", "Ti"] + + kilo = 1024.0 + prefix_index = 0 + prefix_index_max = len(__units__) - 1 + while (size > kilo and prefix_index < prefix_index_max): + prefix_index += 1 + size /= kilo + + unit = __units__[prefix_index] + return '%.2f %sB' % (size, unit) + + +def dir_size(directory): + """ + Calculate a total size of the given directory. + + Args: + directory - the given directory + + Returns: + Size of the directory. + """ + + logger.debug('Get the size of %s', directory) + + total_size = 0 + for root, _, files in os.walk(directory): + for file in files: + itempath = os.path.join(root, file) + if os.path.islink(itempath): + continue + + total_size += os.path.getsize(itempath) + + logger.debug('The size of %s: %s', directory, total_size) + return total_size + + +def dir_size_str(directory, format_in_bytes=False): + """ + Calculate a total size of the given directory and format as a string. + + Args: + directory - the given directory + format_in_bytes - if True, size is expressed in bytes, + otherwise human readable units are used. + + Returns: + Size of the directory as a formatted string. + """ + + size = dir_size(directory) + size_str = size_to_str(size, format_in_bytes) + + logger.debug('The size of %s: %s', directory, size_str) + return size_str + + +def size_bylist(root_dir, salomexc): + """ + Calcualate the total size of files listed in the given salomexc file. + + Args: + root_dir - a root dir for listed files + salomexc - file that contents a list of files. + + Returns: + The total size of listed files. + """ + + logger.debug('Calcualate the total size of files inside %s listed in %s...', + root_dir, salomexc) + + try: + with open(salomexc, 'r', encoding='UTF-8') as file: + total_size = 0 + for line in file: + path_to_file = os.path.join(root_dir, line.strip()) + #logger.debug('Check the file %s...', path_to_file) + + if os.path.isfile(path_to_file): + size = os.path.getsize(path_to_file) + total_size += size + logger.debug('%s size: %s', path_to_file, size) + + elif os.path.islink(path_to_file): + logger.debug('%s is link. Skip.', path_to_file) + continue + + elif os.path.isdir(path_to_file): + logger.warning('Directories are not expected to be listed in %s file! ' + 'Skip.', + salomexc) + continue + + else: + logger.error('Unexpected path %s ' + 'is not a file, link or directory. Skip.', + path_to_file) + + return total_size + + except OSError: + logger.error(format_exc()) + + return None + + +def ext_size(install_dir, salomex_name): + """ + Calculate a total size of a salome extension from SALOME install root. + + Args: + salome_root - path to SALOME install root directory. + salomex_name - a name of the given salome extension. + + Returns: + Size of the directory in bytes. + """ + + # Check if provided dirname is valid + if not isvalid_dirname(install_dir): + return None + + # Check if an extension with this name is installed + salomexc = find_salomexc(install_dir, salomex_name) + if not salomexc: + logger.error('Cannot calculate the size of %s extension!', + salomex_name) + return None + + # Calculate the size + return size_bylist(os.path.join(install_dir, SALOME_EXTDIR), salomexc) + + +def ext_size_str(install_dir, salomex_name, format_in_bytes=False): + """ + Calculate a total size of the given extension and format as a string. + + Args: + install_dir - directory where the given extension is installed. + salomex_name - the given extension's name. + format_in_bytes - if True, size is expressed in bytes, + otherwise human readable units are used. + + Returns: + Size of the extension as a formatted string. + """ + + size = ext_size(install_dir, salomex_name) + if size is None: + return '' + + size_str = size_to_str(size, format_in_bytes) + + logger.debug('The size of %s: %s', salomex_name, size_str) + return size_str + + +if __name__ == '__main__': + if len(sys.argv) == 2: + dir_size_str(sys.argv[1]) + elif len(sys.argv) == 3: + arg_1, arg_2 = sys.argv[1:] # pylint: disable=unbalanced-tuple-unpacking + ext_size_str(arg_1, arg_2) + else: + logger.error('You must provide all the arguments!') + logger.info(dir_size.__doc__) diff --git a/bin/SalomeOnDemandTK/extension_remover.py b/bin/SalomeOnDemandTK/extension_remover.py index af2291917..d0b6d1dc8 100644 --- a/bin/SalomeOnDemandTK/extension_remover.py +++ b/bin/SalomeOnDemandTK/extension_remover.py @@ -26,7 +26,7 @@ # Author : Konstantin Leontev, Open Cascade # # @package SalomeOnDemandTK -# @brief Set of utility to unpack SALOME python extensions. +# @brief Set of utility to remove SALOME python extensions. """Set of utility to remove SALOME python extensions. """ @@ -37,8 +37,8 @@ import shutil from traceback import format_exc from .extension_utilities import logger, \ - DFILE_EXT, CFILE_EXT, SALOME_EXTDIR, ENVPYFILE_SUF, \ - isvalid_dirname, list_dependants, is_empty_dir + DFILE_EXT, CFILE_EXT, SALOME_EXTDIR, \ + isvalid_dirname, list_dependants, is_empty_dir, find_salomexd, find_salomexc, find_envpy def remove_if_empty(top_dir, directory): @@ -143,18 +143,15 @@ def remove_salomex(install_dir, salomex_name): # Check if the given extension is installed logger.debug('Check if an extension %s is installed:', salomex_name) - logger.debug('Try to find %s file...', DFILE_EXT) - salomexd = os.path.join(install_dir, salomex_name + '.' + DFILE_EXT) - has_salomexd = os.path.isfile(salomexd) - if not has_salomexd: - logger.debug('Cannot find a description file %s for extension %s! ' + salomexd = find_salomexd(install_dir, salomex_name) + if not salomexd: + logger.debug('Cannot find a description file for extension %s! ' 'Extension has been already removed or %s file was deleted by mistake. ' 'In the former case we can use %s file to clean up.', - salomexd, salomex_name, DFILE_EXT, CFILE_EXT) + salomex_name, DFILE_EXT, CFILE_EXT) - logger.debug('Try to find %s file...', CFILE_EXT) - salomexc = os.path.join(install_dir, salomex_name + '.' + CFILE_EXT) - if not os.path.isfile(salomexc): + salomexc = find_salomexc(install_dir, salomex_name) + if not salomexc: logger.debug('Cannot find %s for extension %s! ' 'Going to exit from extension removing process.', salomexc, salomex_name) @@ -176,14 +173,14 @@ def remove_salomex(install_dir, salomex_name): os.remove(salomexc) # Remove env file - env_py = os.path.join(install_dir, salomex_name + ENVPYFILE_SUF) - if os.path.isfile(env_py): + env_py = find_envpy(install_dir, salomex_name) + if env_py: os.remove(env_py) else: logger.error('Cannot find and remove %s file! ', env_py) # Remove description file - if has_salomexd: + if salomexd: os.remove(salomexd) logger.debug('An extension %s was removed from %s', diff --git a/bin/SalomeOnDemandTK/extension_utilities.py b/bin/SalomeOnDemandTK/extension_utilities.py index c0c367298..ebfa9c519 100644 --- a/bin/SalomeOnDemandTK/extension_utilities.py +++ b/bin/SalomeOnDemandTK/extension_utilities.py @@ -26,10 +26,10 @@ # Author : Konstantin Leontev, Open Cascade # # @package SalomeOnDemandTK -# @brief Set of utility functions those help to build SALOME python extensions. +# @brief Utilities and constants those help to deal with salome extension files. """ -Utilities and constants those help deal with salome extension files. +Utilities and constants those help to deal with salome extension files. """ import os @@ -68,9 +68,6 @@ def create_salomexd(name, descr='', depends_on=None, author='', components=None) author - an author of the extension. components - the names of the modules those current extension was built from. - Raises: - Raises OSError exception. - Returns: None """ @@ -111,9 +108,6 @@ def read_salomexd(file_path): Args: file_path - the path to the salomexd file. - Raises: - Raises OSError exception. - Returns: A dictionary that represents the content of the salomexd file. """ @@ -141,9 +135,6 @@ def create_salomexb(name, included): name - the name of the corresponding salome extension. included - list of the directories those must be included inside a salomex. - Raises: - Raises OSError exception. - Returns: None """ @@ -167,9 +158,6 @@ def read_salomexb(file_path): Args: file_path - the path to the salomexb file. - Raises: - Raises OSError exception. - Returns: List of strings - paths to the directories those must be included in corresponding salomex archive file. @@ -362,6 +350,12 @@ def list_dependants(install_dir, salomex_name): salomexd_files = list_files_ext(install_dir, DFILE_EXT) for salomexd_file in salomexd_files: + dependant_name, _ = os.path.splitext(os.path.basename(salomexd_file)) + + # Don't process extension itself + if dependant_name == salomex_name: + continue + logger.debug('Check dependencies for %s...', salomexd_file) salomexd_content = read_salomexd(salomexd_file) @@ -370,20 +364,11 @@ def list_dependants(install_dir, salomex_name): logger.debug('List of dependencies: %s', depends_on) if salomex_name in depends_on: - dependant_name = None - if EXTNAME_KEY in salomexd_content and salomexd_content[EXTNAME_KEY]: - dependant_name = salomexd_content[EXTNAME_KEY] - else: - logger.warning('%s file doesn\'t have %s key! ' - 'Get an extension name from the filename.', - salomexd_file, EXTNAME_KEY) - dependant_name, _ = os.path.splitext(os.path.basename(salomexd_file)) - dependants.append(dependant_name) if len(dependants) > 0: - logger.debug('An extension %s has followed extensions those depend on it: %s', - salomex_name, dependants) + logger.debug('Followed extensions %s depend on %s', + dependants, salomex_name) return dependants @@ -400,3 +385,70 @@ def is_empty_dir(directory): """ return not next(os.scandir(directory), None) + + +def find_file(directory, file_name): + """ + Finds a file in the given directory. + + Args: + directory - path to directory to check. + file_name - given base filename with extension + + Returns: + Abs path if the file exist, otherwise None. + """ + + logger.debug('Try to find %s file in %s...', file_name, directory) + file = os.path.join(directory, file_name) + if os.path.isfile(file): + logger.debug('File %s exists.', file) + return file + + logger.debug('File %s doesnt\'t exist. Return None.', file) + return None + + +def find_salomexd(install_dir, salomex_name): + """ + Finds a salomexd file for the given extension. + + Args: + install_dir - path to directory to check. + salomex_name - extension's name. + + Returns: + Abs path if the file exist, otherwise None. + """ + + return find_file(install_dir, salomex_name + '.' + DFILE_EXT) + + +def find_salomexc(install_dir, salomex_name): + """ + Finds a salomexc file for the given extension. + + Args: + install_dir - path to directory to check. + salomex_name - extension's name. + + Returns: + Abs path if the file exist, otherwise None. + """ + + return find_file(install_dir, salomex_name + '.' + CFILE_EXT) + + +def find_envpy(install_dir, salomex_name): + """ + Finds a _env.py file for the given extension. + + Args: + install_dir - path to directory to check. + salomex_name - extension's name. + + Returns: + Abs path if the file exist, otherwise None. + """ + + return find_file(install_dir, salomex_name + ENVPYFILE_SUF) -- 2.39.2