From 9e0e887f4e964e4b723ab0ad457681f8c8177c78 Mon Sep 17 00:00:00 2001 From: crouzet Date: Wed, 19 Dec 2018 15:04:43 +0100 Subject: [PATCH] =?utf8?q?Ajout=20d'un=20mode=20ftp=20pour=20les=20archive?= =?utf8?q?s=20de=20pr=C3=A9requis?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit --- commands/config.py | 2 ++ commands/source.py | 18 ++++++++++++++---- src/__init__.py | 43 +++++++++++++++++++++++++++++++++++++++++++ src/product.py | 40 +++++++++++++++------------------------- 4 files changed, 74 insertions(+), 29 deletions(-) diff --git a/commands/config.py b/commands/config.py index 768a2f2..8b4859f 100644 --- a/commands/config.py +++ b/commands/config.py @@ -350,6 +350,7 @@ class ConfigManager: cfg.PATHS.PRODUCTPATH.append(cfg.VARS.personal_products_dir, "") cfg.PATHS["ARCHIVEPATH"] = src.pyconf.Sequence(cfg.PATHS) cfg.PATHS.ARCHIVEPATH.append(cfg.VARS.personal_archives_dir, "") + cfg.PATHS["ARCHIVEFTP"] = src.pyconf.Sequence(cfg.PATHS) cfg.PATHS["JOBPATH"] = src.pyconf.Sequence(cfg.PATHS) cfg.PATHS.JOBPATH.append(cfg.VARS.personal_jobs_dir, "") cfg.PATHS["MACHINEPATH"] = src.pyconf.Sequence(cfg.PATHS) @@ -364,6 +365,7 @@ class ConfigManager: for PATH in ["APPLICATIONPATH", "PRODUCTPATH", "ARCHIVEPATH", #comment this for default archive #8646 + "ARCHIVEFTP", "JOBPATH", "MACHINEPATH"]: if PATH not in cfg.PROJECTS.projects[project]: diff --git a/commands/source.py b/commands/source.py index 7b2131e..23b41d7 100644 --- a/commands/source.py +++ b/commands/source.py @@ -130,9 +130,10 @@ def get_source_from_git(product_info, return retcode -def get_source_from_archive(product_info, source_dir, logger): +def get_source_from_archive(config, product_info, source_dir, logger): '''The method called if the product is to be get in archive mode + :param config Config: The global configuration :param product_info Config: The configuration specific to the product to be prepared :param source_dir Path: The Path instance corresponding to the @@ -143,8 +144,17 @@ def get_source_from_archive(product_info, source_dir, logger): ''' # check archive exists if not os.path.exists(product_info.archive_info.archive_name): - raise src.SatException(_("Archive not found in config.PATHS.ARCHIVEPATH: '%s'") % - product_info.archive_info.archive_name) + # The archive is not found on local file system (ARCHIVEPATH) + # We try ftp! + logger.write("\n The archive is not found on local file system, we try ftp\n", 3) + ret=src.find_file_in_ftppath(product_info.archive_info.archive_name, + config.PATHS.ARCHIVEFTP, config.LOCAL.archive_dir, logger) + if ret: + # archive was found on ftp and stored in ret + product_info.archive_info.archive_name=ret + else: + raise src.SatException(_("Archive not found in ARCHIVEPATH, nor on ARCHIVEFTP: '%s'") % + product_info.archive_info.archive_name) logger.write('arc:%s ... ' % src.printcolors.printcInfo(product_info.archive_info.archive_name), @@ -346,7 +356,7 @@ def get_product_sources(config, is_dev,env_appli) if product_info.get_source == "archive": - return get_source_from_archive(product_info, source_dir, logger) + return get_source_from_archive(config, product_info, source_dir, logger) if product_info.get_source == "dir": return get_source_from_dir(product_info, source_dir, logger) diff --git a/src/__init__.py b/src/__init__.py index 52ea8a8..2a4a742 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -26,6 +26,7 @@ import shutil import errno import stat import fnmatch +from ftplib import FTP from . import pyconf from . import architecture @@ -375,6 +376,48 @@ def find_file_in_lpath(file_name, lpath, additional_dir = ""): return os.path.join(dir_complete, file_name) return False +def find_file_in_ftppath(file_name, ftppath, installation_dir, logger): + """\ + Find in all ftp servers in ftppath the file called file_name + If it is found then return the destination path of the file + (the place where the file was downloaded" + else return False. + + :param file_name str: The file name to search + :param ftppath, List: The list of ftp servers where to search + :param installation_dir str: The name of the installation directory + :return: the full path of the file or False if not found + :param logger Logger: The logging instance to use for the prints. + :rtype: str + """ + destination=os.path.join(installation_dir, file_name) + for ftp_archive in ftppath: + try: + # ftp_archive has the form ftp.xxx.yyy/dir1/dir2/... + ftp_archive_split=ftp_archive.split("/") + ftp_server=ftp_archive_split[0] + ftp = FTP(ftp_server) + logger.write(" Connect to ftp server %s\n" % ftp_server, 3) + ftp.login() + for directory in ftp_archive_split[1:]: + logger.write(" Change directory to %s\n" % directory, 3) + ftp.cwd(directory) + except: + logger.error("while connecting to ftp server %s\n" % ftp_server) + + try: + if ftp.size(file_name) > 0: + # if file exists and is non empty + with open(destination,'wb') as dest_file: + ftp.retrbinary("RETR "+file_name, dest_file.write) + logger.write(" Archive %s was retrieved and stored in %s\n" % (file_name, destination), 3) + return destination + except: + logger.error("File not found in ftp_archive %s\n" % ftp_server) + pass + + return False + def handleRemoveReadonly(func, path, exc): excvalue = exc[1] if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES: diff --git a/src/product.py b/src/product.py index a06b5b8..04ade2e 100644 --- a/src/product.py +++ b/src/product.py @@ -182,33 +182,23 @@ Please add a section in it.""") % {"1" : vv, "2" : prod_pyconf_path} prod_info.addMapping("archive_info", src.pyconf.Mapping(prod_info), "") - if "archive_name" not in prod_info.archive_info: + if "archive_name" in prod_info.archive_info: + arch_name = prod_info.archive_info.archive_name + else: + # standard name arch_name = product_name + "-" + version + ".tar.gz" - arch_path = src.find_file_in_lpath(arch_name, - config.PATHS.ARCHIVEPATH) - if not arch_path: - msg = _("Archive %(1)s for %(2)s not found in config.PATHS.ARCHIVEPATH") % \ - {"1" : arch_name, "2" : prod_info.name} - DBG.tofix(msg, config.PATHS.ARCHIVEPATH) - prod_info.archive_info.archive_name = arch_name #without path - # raise src.SatException(msg) #may be a warning, continue #8646 - else: - prod_info.archive_info.archive_name = arch_path + + arch_path = src.find_file_in_lpath(arch_name, + config.PATHS.ARCHIVEPATH) + if not arch_path: + # arch_path is not found. It may generate an error in sat source, + # unless the archive is found in ftp serveur + msg = _("Archive %(1)s for %(2)s not found in config.PATHS.ARCHIVEPATH") % \ + {"1" : arch_name, "2" : prod_info.name} + DBG.tofix(msg, config.PATHS.ARCHIVEPATH) + prod_info.archive_info.archive_name = arch_name #without path else: - if (os.path.basename(prod_info.archive_info.archive_name) == - prod_info.archive_info.archive_name): - arch_name = prod_info.archive_info.archive_name - arch_path = src.find_file_in_lpath( - arch_name, - config.PATHS.ARCHIVEPATH) - if not arch_path: - msg = _("Archive %(1)s for %(2)s not found in config.PATHS.ARCHIVEPATH") % \ - {"1" : arch_name, "2" : prod_info.name} - DBG.tofix(msg, config.PATHS.ARCHIVEPATH) #avoid 2 messages in compile - prod_info.archive_info.archive_name = arch_name #without path - # raise src.SatException(msg) #may be a warning, continue #8646 - else: - prod_info.archive_info.archive_name = arch_path + prod_info.archive_info.archive_name = arch_path # If the product compiles with a script, check the script existence -- 2.30.2