3 # Copyright (C) 2010-2013 CEA/DEN
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Lesser General Public
7 # License as published by the Free Software Foundation; either
8 # version 2.1 of the License.
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # Lesser General Public License for more details.
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with this library; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 initial imports and utilities methods for salomeTools
30 from ftplib import FTP
33 from . import architecture
34 from . import printcolors
37 from . import ElementTree
40 from . import environment
41 from . import fileEnviron
42 from . import compilation
43 from . import test_module
44 from . import template
47 if platform.system() == "Windows" :
54 KNOWNFAILURE_STATUS = "KF"
55 TIMEOUT_STATUS = "TIMEOUT"
56 class SatException(Exception):
57 """sat exception class"""
58 def message(self, arg):
59 if sys.version_info[0] >= 3:
60 # message method is not available for python 3.8+
61 return super().msg(arg)
63 return super(SatException,self).message(arg)
65 def ensure_path_exists(p):
66 """Create a path if not existing
68 :param p str: The path.
70 if not os.path.exists(p):
73 def check_config_has_application( config, details = None ):
74 """check that the config has the key APPLICATION. Else raise an exception.
76 :param config class 'common.pyconf.Config': The config.
78 if 'APPLICATION' not in config:
79 message = _("An APPLICATION is required. Use 'config --list' to get the list of available applications.\n")
81 details.append(message)
82 raise SatException( message )
84 def check_platform_is_supported( config, logger ):
85 """check that the platform is supported, write warning if not.
87 :param config class 'common.pyconf.Config': The config.
89 if 'platform' in config.APPLICATION and config.VARS.dist not in config.APPLICATION.platform:
90 msg = "WARNING: Your application configuration is not supported on this platform (%s)\n"\
91 " Please consider using the native application!" % config.VARS.dist
92 logger.write("\n%s\n\n" % printcolors.printcWarning(msg), 1)
95 def check_config_has_profile( config, details = None ):
97 check that the config has the key APPLICATION.profile.
98 else, raise an exception.
100 :param config class 'common.pyconf.Config': The config.
102 check_config_has_application(config)
103 if 'profile' not in config.APPLICATION:
104 message = _("A profile section is required in your application.\n")
106 details.append(message)
107 raise SatException( message )
109 def check_application_syntax_deprecated(config, logger):
111 check that the application has the key git_server.
112 else, raise a warning
114 :param config class 'common.pyconf.Config': The config.
115 :param logger Logger: The logging instance to use for the prints.
117 if 'APPLICATION' in config and 'properties' in config.APPLICATION and not 'git_server' in config.APPLICATION.properties :
118 msg = 'WARNING: Your application is using repo_dev key which is deprecated and will be removed from future SAT releases!\n'
119 msg+= ' Please upgrade your application configuration file and add a valid git_server key.\n'
120 msg+= ' git_server key values need to be defined in the project file (e.g. salome.pyconf)!'
121 logger.write("\n%s\n\n" % printcolors.printcWarning(msg), 1)
123 def appli_test_property(config,property_name, property_value):
124 """Generic function to test if an application has a property set to a value
125 :param config class 'common.pyconf.Config': The config.
126 :param property_name : The name of the property to check
127 :param property_value : The value of the property to test
128 :return: True if the application has the property set to property_value
131 # first check if application has property_value
132 if not ("APPLICATION" in config and
133 "properties" in config.APPLICATION and
134 property_name in config.APPLICATION.properties):
137 # then check to the property is set to property_value
138 eval_expression = 'config.APPLICATION.properties.%s == "%s"' %\
139 (property_name,property_value)
140 result = eval(eval_expression)
143 def config_has_application( config ):
144 return 'APPLICATION' in config
146 def get_cfg_param(config, param_name, default):
148 eearch for param_name value in config.
149 if param_name is not in config
151 else return the found value
153 :param config class 'common.pyconf.Config': The config.
154 :param param_name str: the name of the parameter to get the value
155 :param default str: The value to return if param_name is not in config
156 :return: see initial description of the function
159 if param_name in config:
160 return config[param_name]
163 def strSplitN(aList, nb, skip="\n "):
166 aStr = 'this-is-a-string'
168 split it by every 2nd '-' rather than every '-'
173 strValue += "%15s, " % str(v)
179 strValue = skip + strValue
182 def getProductNames(cfg, wildcards, logger):
183 """get products names using * or ? as wildcards like shell Linux"""
185 if type(wildcards) is list:
190 products = cfg.APPLICATION.products.keys()
191 for wild in wildcards:
193 for prod in products:
194 filtered = fnmatch.filter([prod], wild)
195 # print("filtered", prod, wild, filtered)
196 if len(filtered) > 0:
201 notFound[wild] = None
203 logger.warning("Empty list of products, from %s" % wilds)
204 if len(notFound.keys()) > 0:
205 strProd = strSplitN( sorted(products), 5)
206 logger.warning("products not found: %s\n availables products are:\n%s" % \
207 (sorted(notFound.keys()), strProd) )
211 def print_info(logger, info):
213 Prints the tuples that are in info variable in a formatted way.
215 :param logger Logger: The logging instance to use for the prints.
216 :param info list: The list of tuples to display
218 # find the maximum length of the first value of the tuples in info
219 smax = max(map(lambda l: len(l[0]), info))
220 # Print each item of info with good indentation
222 sp = " " * (smax - len(i[0]))
223 printcolors.print_value(logger, sp + i[0], i[1], 2)
224 logger.write("\n", 2)
226 def get_base_path(config):
228 Returns the path of the products base.
230 :param config Config: The global Config instance.
231 :return: The path of the products base.
234 if "base" not in config.LOCAL:
235 local_file_path = os.path.join(config.VARS.salometoolsway,
238 msg = _("Please define a base path in the file %s" % local_file_path)
239 raise SatException(msg)
241 base_path = os.path.abspath(config.LOCAL.base)
245 def get_launcher_name(config):
247 Returns the name of salome launcher.
249 :param config Config: The global Config instance.
250 :return: The name of salome launcher.
253 check_config_has_application(config)
254 if 'profile' in config.APPLICATION and 'launcher_name' in config.APPLICATION.profile:
255 launcher_name = config.APPLICATION.profile.launcher_name
257 launcher_name = 'salome'
261 def get_launcher_exe(config):
263 Returns the name of exe defined in profile section.
265 :param config Config: The global Config instance.
266 :return: The name of the exe to use in launcher.
269 check_config_has_application(config)
270 if 'profile' in config.APPLICATION and 'exe' in config.APPLICATION.profile:
271 exe_name = config.APPLICATION.profile.exe
278 def get_log_path(config):
280 Returns the path of the logs.
282 :param config Config: The global Config instance.
283 :return: The path of the logs.
286 if "log_dir" not in config.LOCAL:
287 local_file_path = os.path.join(config.VARS.salometoolsway,
290 msg = _("Please define a log_dir in the file %s" % local_file_path)
291 raise SatException(msg)
293 log_dir_path = os.path.abspath(config.LOCAL.log_dir)
297 def get_salometool_version(config):
298 """Return the salomeTool version.
300 :param config Config: The global Config instance.
301 :return: the description of this version of sat in terms of tag and commit
303 return config.LOCAL.tag
306 def get_salome_version(config):
307 import versionMinorMajorPatch as VMMP
309 if hasattr(config.APPLICATION, 'version_salome'):
310 version = VMMP.MinorMajorPatch(config.APPLICATION.version_salome)
312 kernel_info = product.get_product_config(config, "KERNEL")
313 aFile = os.path.join(
314 kernel_info.install_dir,
318 if not os.path.isfile(aFile):
320 with open(aFile) as f:
321 line = f.readline() # example: '[SALOME KERNEL] : 8.4.0'
322 version = VMMP.MinorMajorPatch(line.split(":")[1])
324 # from nov. 2023 and SALOME 9.10.0 forbid test(s) on integer, use MajorMinorPatch class tests
327 def read_config_from_a_file(filePath):
329 cfg_file = pyconf.Config(filePath)
330 except pyconf.ConfigError as e:
331 raise SatException(_("Error in configuration file: %(file)s\n %(error)s") % \
332 { 'file': filePath, 'error': str(e) })
335 def get_tmp_filename(cfg, name):
336 if not os.path.exists(cfg.VARS.tmp_root):
337 os.makedirs(cfg.VARS.tmp_root)
339 return os.path.join(cfg.VARS.tmp_root, name)
342 # Utils class to simplify path manipulations.
344 def __init__(self, path):
345 self.path = str(path)
347 def __add__(self, other):
348 return Path(os.path.join(self.path, str(other)))
351 return Path(os.path.abspath(self.path))
356 def __eq__(self, other):
357 return self.path == other.path
360 return self.islink() or os.path.exists(self.path)
363 return os.path.islink(self.path)
366 return os.path.isdir(self.path)
369 return os.path.isfile(self.path)
372 return [Path(p) for p in os.listdir(self.path)]
375 return Path(os.path.dirname(self.path))
378 return Path(os.path.basename(self.path))
380 def make(self, mode=None):
381 os.makedirs(self.path)
383 os.chmod(self.path, mode)
385 def chmod(self, mode):
386 os.chmod(self.path, mode)
392 shutil.rmtree( self.path, onerror = handleRemoveReadonly )
394 def copy(self, path, smart=False):
395 if not isinstance(path, Path):
398 if os.path.islink(self.path):
399 return self.copylink(path)
400 elif os.path.isdir(self.path):
401 return self.copydir(path, smart)
403 return self.copyfile(path)
405 def smartcopy(self, path):
406 return self.copy(path, True)
410 return os.readlink(self.path)
414 def symlink(self, path):
416 os.symlink(str(path), self.path)
421 def copylink(self, path):
423 os.symlink(os.readlink(self.path), str(path))
428 def copydir(self, dst, smart=False):
438 if smart and (str(name) in [".git", "CVS", ".svn"]):
440 srcname = self + name
442 srcname.copy(dstname, smart)
447 def copyfile(self, path):
449 shutil.copy2(self.path, str(path))
454 def find_file_in_lpath(file_name, lpath, additional_dir = ""):
456 Find in all the directories in lpath list the file that has the same name
459 then return the full path of the file
462 The additional_dir (optional) is the name of the directory to add to all
465 :param file_name str: The file name to search
466 :param lpath List: The list of directories where to search
467 :param additional_dir str: The name of the additional directory
468 :return: the full path of the file or False if not found
471 for directory in lpath:
472 dir_complete = os.path.join(directory, additional_dir)
473 if not os.path.isdir(directory) or not os.path.isdir(dir_complete):
475 l_files = os.listdir(dir_complete)
476 for file_n in l_files:
477 if file_n == file_name:
478 return os.path.join(dir_complete, file_name)
481 def find_file_in_ftppath(file_name, ftppath, installation_dir, logger, additional_dir = ""):
483 Find in all ftp servers in ftppath the file called file_name
484 If it is found then return the destination path of the file
485 (the place where the file was downloaded"
488 :param file_name str: The file name to search
489 :param ftppath, List: The list of ftp servers where to search
490 :param installation_dir str: The name of the installation directory
491 :return: the full path of the file or False if not found
492 :param logger Logger: The logging instance to use for the prints.
496 # make sure installation_dir exists
497 if not os.path.exists(installation_dir):
498 os.makedirs(installation_dir)
500 destination=os.path.join(installation_dir, file_name)
502 # paths in ftppath may contain several paths separated by ":"
503 # we plit them, and push all paths in bigftppath
505 for ipath in ftppath:
506 splpath=ipath.split(":")
509 for ftp_archive in bigftppath:
511 # ftp_archive has the form ftp.xxx.yyy/dir1/dir2/...
512 ftp_archive_split=ftp_archive.split("/")
513 ftp_server=ftp_archive_split[0]
514 ftp = FTP(ftp_server)
515 logger.write(" Connect to ftp server %s\n" % ftp_server, 3)
517 for directory in ftp_archive_split[1:]:
518 logger.write(" Change directory to %s\n" % directory, 3)
521 ftp.cwd(additional_dir)
523 logger.error("while connecting to ftp server %s\n" % ftp_server)
526 try: # get md5 file if it exists
527 file_name_md5=file_name + ".md5"
528 destination_md5=destination + ".md5"
529 if ftp.size(file_name_md5) > 0:
530 with open(destination_md5,'wb') as dest_file_md5:
531 ftp.retrbinary("RETR "+file_name_md5, dest_file_md5.write)
536 if ftp.size(file_name) > 0:
537 # if file exists and is non empty
538 with open(destination,'wb') as dest_file:
539 ftp.retrbinary("RETR "+file_name, dest_file.write)
540 logger.write(" Archive %s was retrieved and stored in %s\n" % (file_name, destination), 3)
543 logger.error("File not found in ftp_archive %s\n" % ftp_server)
547 def handleRemoveReadonly(func, path, exc):
549 if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
550 os.chmod(path, stat.S_IRWXU| stat.S_IRWXG| stat.S_IRWXO) # 0777
555 def deepcopy_list(input_list):
557 Do a deep copy of a list
559 :param input_list List: The list to copy
560 :return: The copy of the list
564 for elem in input_list:
568 def remove_item_from_list(input_list, item):
570 Remove all occurences of item from input_list
572 :param input_list List: The list to modify
573 :return: The without any item
577 for elem in input_list:
583 def parse_date(date):
585 Transform YYYYMMDD_hhmmss into YYYY-MM-DD hh:mm:ss.
587 :param date str: The date to transform
588 :return: The date in the new format
593 res = "%s-%s-%s %s:%s:%s" % (date[0:4],
601 def merge_dicts(*dict_args):
603 Given any number of dicts, shallow copy and merge into a new dict,
604 precedence goes to key value pairs in latter dicts.
607 for dictionary in dict_args:
608 result.update(dictionary)
611 def replace_in_file(filein, strin, strout):
612 """Replace <strin> by <strout> in file <filein>"""
613 shutil.move(filein, filein + "_old")
615 filein = filein + "_old"
616 fin = open(filein, "r")
617 fout = open(fileout, "w")
619 fout.write(line.replace(strin, strout))
621 def get_property_in_product_cfg(product_cfg, pprty):
622 if not "properties" in product_cfg:
624 if not pprty in product_cfg.properties:
626 return product_cfg.properties[pprty]
628 def activate_mesa_property(config):
629 """Add mesa property into application properties
631 :param config Config: The global configuration. It must have an application!
633 # Verify the existence of the file
634 if not 'properties' in config.APPLICATION:
635 config.APPLICATION.addMapping( 'properties', pyconf.Mapping(), None )
636 config.APPLICATION.properties.use_mesa="yes"
638 def git_server_has_all_repositories( config, the_git_server):
639 """check that the git server contains all repositories (closed and open)
640 :param config class 'common.pyconf.Config': The config.
641 :param logger Logger: The logging instance to use for the prints.
643 if 'opensource_git_servers' in config.VARS:
644 for git_server in config.VARS['opensource_git_servers']:
645 if git_server == the_git_server:
649 def get_git_server(config, logger):
651 has_properties = 'properties' in config.APPLICATION
652 if has_properties and "git_server" in config.APPLICATION.properties:
653 the_git_server = config.APPLICATION.properties.git_server
654 elif has_properties and "repo_dev" in config.APPLICATION.properties:
655 # Fall back to deprecated approach but issue a warning that this approach is deprecated
656 if config.APPLICATION.properties.repo_dev == 'yes':
657 the_git_server = [ git_server for git_server in config.VARS.git_servers if git_server not in config.VARS.opensource_git_servers][0]
659 the_git_server = [ git_server for git_server in config.VARS.git_servers if git_server in config.VARS.opensource_git_servers][0]
660 return the_git_server