From c4f41ced7e1c31b5f8683b1ea0da91718f47fce6 Mon Sep 17 00:00:00 2001 From: crouzet Date: Wed, 17 Jul 2019 17:26:25 +0200 Subject: [PATCH] sat #17206 pip management : management of environment and packages --- commands/compile.py | 37 +++++++++++++++++++++---------------- commands/package.py | 31 ++++++++++++++++++++++++++++--- commands/source.py | 3 ++- src/environment.py | 6 ++++++ 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/commands/compile.py b/commands/compile.py index fb1fee0..7bf4b76 100644 --- a/commands/compile.py +++ b/commands/compile.py @@ -197,7 +197,10 @@ def compile_all_products(sat, config, options, products_infos, all_products_dict # Check if sources was already successfully installed check_source = src.product.check_source(p_info) - if not options.no_compile: # don't check sources with option --show! + is_pip= (src.appli_test_property(config,"pip", "yes") and src.product.product_test_property(p_info,"pip", "yes")) + # don't check sources with option --show + # or for products managed by pip (there sources are in wheels stored in LOCAL.ARCHIVE + if not (options.no_compile or is_pip): if not check_source: logger.write(_("Sources of product not found (try 'sat -h prepare') \n")) res += 1 # one more error @@ -414,17 +417,18 @@ def compile_product_pip(sat, res = 0 error_step = "" pip_install_in_python=False + pip_wheels_dir=os.path.join(config.LOCAL.archive_dir,"wheels") if src.appli_test_property(config,"pip_install_dir", "python"): # pip will install product in python directory" pip_install_cmd="pip3 install --disable-pip-version-check --no-index --find-links=%s --build %s %s==%s" %\ - (config.LOCAL.archive_dir, p_info.build_dir, p_name, p_info.version) + (pip_wheels_dir, p_info.build_dir, p_info.name, p_info.version) pip_install_in_python=True else: # pip will install product in product install_dir pip_install_dir=os.path.join(p_info.install_dir, "lib", "python${PYTHON_VERSION:0:3}", "site-packages") pip_install_cmd="pip3 install --disable-pip-version-check --no-index --find-links=%s --build %s --target %s %s==%s" %\ - (config.LOCAL.archive_dir, p_info.build_dir, pip_install_dir, p_name, p_info.version) + (pip_wheels_dir, p_info.build_dir, pip_install_dir, p_info.name, p_info.version) log_step(logger, header, "PIP") logger.write("\n"+pip_install_cmd+"\n", 4) len_end_line = len_end + 3 @@ -437,19 +441,20 @@ def compile_product_pip(sat, build_environ.silent = (config.USER.output_verbose_level < 5) build_environ.set_full_environ(logger, environ_info) - if pip_install_in_python and (options.clean_install or options.clean_all): - # for products installed by pip inside python install dir - # finish the clean by uninstalling the product from python install dir - pip_clean_cmd="pip3 uninstall -y %s==%s" % (p_name, p_info.version) - res_pipclean = (subprocess.call(pip_clean_cmd, - shell=True, - cwd=config.LOCAL.workdir, - env=build_environ.environ.environ, - stdout=logger.logTxtFile, - stderr=subprocess.STDOUT) == 0) - if not res_pipclean: - logger.write("\n",1) - logger.warning("pip3 uninstall failed!") + # useless - pip uninstall himself when wheel is alredy installed + #if pip_install_in_python and (options.clean_install or options.clean_all): + # # for products installed by pip inside python install dir + # # finish the clean by uninstalling the product from python install dir + # pip_clean_cmd="pip3 uninstall -y %s==%s" % (p_name, p_info.version) + # res_pipclean = (subprocess.call(pip_clean_cmd, + # shell=True, + # cwd=config.LOCAL.workdir, + # env=build_environ.environ.environ, + # stdout=logger.logTxtFile, + # stderr=subprocess.STDOUT) == 0) + # if not res_pipclean: + # logger.write("\n",1) + # logger.warning("pip3 uninstall failed!") res_pip = (subprocess.call(pip_install_cmd, shell=True, diff --git a/commands/package.py b/commands/package.py index 68817fb..8c75f35 100644 --- a/commands/package.py +++ b/commands/package.py @@ -23,6 +23,7 @@ import datetime import tarfile import codecs import string +import glob import pprint as PP import src @@ -789,11 +790,35 @@ def get_archives(config, logger): if p_info.get_source == "archive": archive_path = p_info.archive_info.archive_name archive_name = os.path.basename(archive_path) + d_archives[p_name] = (archive_path, + os.path.join(ARCHIVE_DIR, archive_name)) + if (src.appli_test_property(config,"pip", "yes") and + src.product.product_test_property(p_info,"pip", "yes")): + # if pip mode is activated, and product is managed by pip + pip_wheels_dir=os.path.join(config.LOCAL.archive_dir,"wheels") + pip_wheel_pattern=os.path.join(pip_wheels_dir, + "%s-%s*" % (p_info.name, p_info.version)) + print "CNC pip_wheel_pattern = ",pip_wheel_pattern + pip_wheel_path=glob.glob(pip_wheel_pattern) + msg_pip_not_found="Error in get_archive, pip wheel for "\ + "product %s-%s was not found in %s directory" + msg_pip_two_or_more="Error in get_archive, several pip wheels for "\ + "product %s-%s were found in %s directory" + if len(pip_wheel_path)==0: + raise src.SatException(msg_pip_not_found %\ + (p_info.name, p_info.version, pip_wheels_dir)) + if len(pip_wheel_path)>1: + raise src.SatException(msg_pip_two_or_more %\ + (p_info.name, p_info.version, pip_wheels_dir)) + + pip_wheel_name=os.path.basename(pip_wheel_path[0]) + d_archives[p_name+" (pip wheel)"]=(pip_wheel_path[0], + os.path.join(ARCHIVE_DIR, "wheels", pip_wheel_name)) else: - l_pinfo_vcs.append((p_name, p_info)) + # this product is not managed by archive, + # an archive of the vcs directory will be created by get_archive_vcs + l_pinfo_vcs.append((p_name, p_info)) - d_archives[p_name] = (archive_path, - os.path.join(ARCHIVE_DIR, archive_name)) return d_archives, l_pinfo_vcs def add_salomeTools(config, tmp_working_dir): diff --git a/commands/source.py b/commands/source.py index 54bd9f0..0df9e67 100644 --- a/commands/source.py +++ b/commands/source.py @@ -159,8 +159,9 @@ def get_source_from_archive(config, product_info, source_dir, logger): if (src.appli_test_property(config,"pip", "yes") and src.product.product_test_property(product_info,"pip", "yes")): # download whl in local archive dir + pip_wheels_dir=os.path.join(config.LOCAL.archive_dir,"wheels") pip_download_cmd="pip download --disable-pip-version-check --destination-directory %s --no-deps %s==%s " %\ - (config.LOCAL.archive_dir, product_info.name, product_info.version) + (pip_wheels_dir, product_info.name, product_info.version) logger.write(pip_download_cmd, 3, False) res_pip = (subprocess.call(pip_download_cmd, shell=True, diff --git a/src/environment.py b/src/environment.py index 20a5c1e..898e07b 100644 --- a/src/environment.py +++ b/src/environment.py @@ -597,6 +597,12 @@ class SalomeEnviron: if src.product.product_is_compile_time(pi): return + # skip pip products when pip is activated and installation is done in python + if (src.appli_test_property(self.cfg,"pip", "yes") and + src.product.product_test_property(pi,"pip", "yes") and + src.appli_test_property(self.cfg,"pip_install_dir", "python") ): + return + # skip mesa products (if any) at run time, # unless use_mesa property was activated if not self.forBuild: -- 2.39.2