3 # Copyright (C) 2010-2012 CEA/DEN
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Lesser General Public
7 # License as published by the Free Software Foundation; either
8 # version 2.1 of the License.
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # Lesser General Public License for more details.
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with this library; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 import src.debug as DBG
28 # Define all possible option for patch command : sat patch <options>
29 parser = src.options.Options()
30 parser.add_option('p', 'products', 'list2', 'products',
31 _('Optional: products from which to get the sources. This option accepts a comma separated list.'))
33 def get_source_for_dev(config, product_info, source_dir, logger, pad):
34 '''The method called if the product is in development mode
36 :param config Config: The global configuration
37 :param product_info Config: The configuration specific to
38 the product to be prepared
39 :param source_dir Path: The Path instance corresponding to the
40 directory where to put the sources
41 :param logger Logger: The logger instance to use for the display and logging
42 :param pad int: The gap to apply for the terminal display
43 :return: True if it succeed, else False
47 # Call the function corresponding to get the sources with True checkout
48 retcode = get_product_sources(config,
55 logger.write("\n", 3, False)
56 # +2 because product name is followed by ': '
57 logger.write(" " * (pad+2), 3, False)
59 logger.write('dev: %s ... ' %
60 src.printcolors.printcInfo(product_info.source_dir), 3, False)
65 def get_source_from_git(config,
72 '''The method called if the product is to be get in git mode
74 :param product_info Config: The configuration specific to
75 the product to be prepared
76 :param source_dir Path: The Path instance corresponding to the
77 directory where to put the sources
78 :param logger Logger: The logger instance to use for the display and logging
79 :param pad int: The gap to apply for the terminal display
80 :param is_dev boolean: True if the product is in development mode
81 :param environ src.environment.Environ: The environment to source when
83 :return: True if it succeed, else False
88 # Get the repository address.
89 # If the application has the repo_dev property
90 # Or if the product is in dev mode
91 # Then we use repo_dev if the key exists
92 coflag = src.printcolors.printcHighlight(coflag.upper())
94 git_server = src.get_git_server(config,logger)
95 product_file = product_info.from_file.split('/').pop()
96 if 'git_info' in product_info and 'repositories' in product_info.git_info:
97 if git_server in product_info.git_info.repositories.keys(): # keys are git servers
98 repo_git = product_info.git_info.repositories[git_server]
99 elif 'properties' in product_info and 'is_opensource' in product_info.properties and product_info.properties.is_opensource == 'yes' :
100 for git_server in product_info.git_info.repositories.keys():
101 if git_server in config.VARS.opensource_git_servers:
102 repo_git = product_info.git_info.repositories[git_server]
104 elif 'properties' in product_info and not 'is_opensource' in product_info.properties:
105 for git_server in product_info.git_info.repositories.keys():
106 if git_server in config.VARS.opensource_git_servers:
107 repo_git = product_info.git_info.repositories[git_server]
108 logger.warning("Using opensource repository ({}) for product {}".format(git_server, product_info.name))
112 logger.error("Error in configuration file: define git repository for product: {} in file {}".format(product_info.name, product_file))
115 elif 'repo_dev' in product_info.git_info:
116 repo_git = product_info.git_info.repo_dev
118 logger.error("Error in configuration file: define git repository for product: {}".format(product_info.name))
122 logger.error("Error in configuration file: define git repository for product: {} in file {}.".format(product_info.name, product_file))
125 # Display informations
126 logger.write('%s:%s' % (coflag, src.printcolors.printcInfo(repo_git)), 3,
128 logger.write(' ' * (pad + 50 - len(repo_git)), 3, False)
129 logger.write(' tag:%s' % src.printcolors.printcInfo(
130 product_info.git_info.tag),
133 logger.write(' %s. ' % ('.' * (10 - len(product_info.git_info.tag))), 3,
136 logger.write('\n', 5, False)
139 if "git_options" in product_info.git_info:
140 git_options = product_info.git_info.git_options
143 # what do we do with git tree structure and history
144 if is_dev and "sub_dir" in product_info.git_info:
145 logger.error("dev mode for product is incompatible with 'sub_dir' option")
148 if not is_dev and "sub_dir" in product_info.git_info:
149 sub_dir = product_info.git_info.sub_dir
152 # Call the system function that do the extraction in git mode
153 retcode = src.system.git_extract(repo_git,
154 product_info.git_info.tag, git_options,
155 source_dir, logger, environ)
157 # Call the system function that do the extraction of a sub_dir in git mode
158 logger.write("sub_dir:%s " % sub_dir, 3)
159 retcode = src.system.git_extract_sub_dir(repo_git,
160 product_info.git_info.tag,git_options,
161 source_dir, sub_dir, logger, environ)
166 def get_source_from_archive(config, product_info, source_dir, logger):
167 '''The method called if the product is to be get in archive mode
169 :param config Config: The global configuration
170 :param product_info Config: The configuration specific to
171 the product to be prepared
172 :param source_dir Path: The Path instance corresponding to the
173 directory where to put the sources
174 :param logger Logger: The logger instance to use for the display and logging
175 :return: True if it succeed, else False
179 # check if pip should be used : pip mode id activated if the application and product have pip property
180 if (src.appli_test_property(config,"pip", "yes") and
181 src.product.product_test_property(product_info,"pip", "yes")):
182 pip_msg = "PIP : do nothing, product will be downloaded later at compile time "
183 logger.write(pip_msg, 3)
186 # check archive exists
187 if not os.path.exists(product_info.archive_info.archive_name):
188 # The archive is not found on local file system (ARCHIVEPATH)
190 logger.write("\n The archive is not found on local file system, we try ftp\n", 3)
191 ret=src.find_file_in_ftppath(product_info.archive_info.archive_name,
192 config.PATHS.ARCHIVEFTP, config.LOCAL.archive_dir, logger)
194 # archive was found on ftp and stored in ret
195 product_info.archive_info.archive_name=ret
197 raise src.SatException(_("Archive not found in ARCHIVEPATH, nor on ARCHIVEFTP: '%s'") %
198 product_info.archive_info.archive_name)
200 logger.write('arc:%s ... ' %
201 src.printcolors.printcInfo(product_info.archive_info.archive_name),
205 # Call the system function that do the extraction in archive mode
206 retcode, NameExtractedDirectory = src.system.archive_extract(
207 product_info.archive_info.archive_name,
208 source_dir.dir(), logger)
210 # Rename the source directory if
211 # it does not match with product_info.source_dir
212 if (NameExtractedDirectory.replace('/', '') !=
213 os.path.basename(product_info.source_dir)):
214 shutil.move(os.path.join(os.path.dirname(product_info.source_dir),
215 NameExtractedDirectory),
216 product_info.source_dir)
220 def get_source_from_dir(product_info, source_dir, logger):
222 if "dir_info" not in product_info:
223 msg = _("Error: you must put a dir_info section"
224 " in the file %s.pyconf" % product_info.name)
225 logger.write("\n%s\n" % src.printcolors.printcError(msg), 1)
228 if "dir" not in product_info.dir_info:
229 msg = _("Error: you must put a dir in the dir_info section"
230 " in the file %s.pyconf" % product_info.name)
231 logger.write("\n%s\n" % src.printcolors.printcError(msg), 1)
234 # check that source exists
235 if not os.path.exists(product_info.dir_info.dir):
236 msg = _("Error: the dir %s defined in the file"
237 " %s.pyconf does not exists" % (product_info.dir_info.dir,
239 logger.write("\n%s\n" % src.printcolors.printcError(msg), 1)
242 logger.write('DIR: %s ... ' % src.printcolors.printcInfo(
243 product_info.dir_info.dir), 3)
246 retcode = src.Path(product_info.dir_info.dir).copy(source_dir)
250 def get_source_from_cvs(user,
257 '''The method called if the product is to be get in cvs mode
259 :param user str: The user to use in for the cvs command
260 :param product_info Config: The configuration specific to
261 the product to be prepared
262 :param source_dir Path: The Path instance corresponding to the
263 directory where to put the sources
264 :param checkout boolean: If True, get the source in checkout mode
265 :param logger Logger: The logger instance to use for the display and logging
266 :param pad int: The gap to apply for the terminal display
267 :param environ src.environment.Environ: The environment to source when
269 :return: True if it succeed, else False
272 # Get the protocol to use in the command
273 if "protocol" in product_info.cvs_info:
274 protocol = product_info.cvs_info.protocol
278 # Construct the line to display
279 if "protocol" in product_info.cvs_info:
280 cvs_line = "%s:%s@%s:%s" % \
281 (protocol, user, product_info.cvs_info.server,
282 product_info.cvs_info.product_base)
284 cvs_line = "%s / %s" % (product_info.cvs_info.server,
285 product_info.cvs_info.product_base)
288 if checkout: coflag = src.printcolors.printcHighlight(coflag.upper())
290 logger.write('%s:%s' % (coflag, src.printcolors.printcInfo(cvs_line)),
293 logger.write(' ' * (pad + 50 - len(cvs_line)), 3, False)
294 logger.write(' src:%s' %
295 src.printcolors.printcInfo(product_info.cvs_info.source),
298 logger.write(' ' * (pad + 1 - len(product_info.cvs_info.source)), 3, False)
299 logger.write(' tag:%s' %
300 src.printcolors.printcInfo(product_info.cvs_info.tag),
303 # at least one '.' is visible
304 logger.write(' %s. ' % ('.' * (10 - len(product_info.cvs_info.tag))),
308 logger.write('\n', 5, False)
310 # Call the system function that do the extraction in cvs mode
311 retcode = src.system.cvs_extract(protocol, user,
312 product_info.cvs_info.server,
313 product_info.cvs_info.product_base,
314 product_info.cvs_info.tag,
315 product_info.cvs_info.source,
316 source_dir, logger, checkout, environ)
319 def get_source_from_svn(user,
325 '''The method called if the product is to be get in svn mode
327 :param user str: The user to use in for the svn command
328 :param product_info Config: The configuration specific to
329 the product to be prepared
330 :param source_dir Path: The Path instance corresponding to the
331 directory where to put the sources
332 :param checkout boolean: If True, get the source in checkout mode
333 :param logger Logger: The logger instance to use for the display and logging
334 :param environ src.environment.Environ: The environment to source when
336 :return: True if it succeed, else False
340 if checkout: coflag = src.printcolors.printcHighlight(coflag.upper())
342 logger.write('%s:%s ... ' % (coflag,
343 src.printcolors.printcInfo(
344 product_info.svn_info.repo)),
348 logger.write('\n', 5, False)
349 # Call the system function that do the extraction in svn mode
350 retcode = src.system.svn_extract(user,
351 product_info.svn_info.repo,
352 product_info.svn_info.tag,
359 def get_product_sources(config,
366 '''Get the product sources.
368 :param config Config: The global configuration
369 :param product_info Config: The configuration specific to
370 the product to be prepared
371 :param is_dev boolean: True if the product is in development mode
372 :param source_dir Path: The Path instance corresponding to the
373 directory where to put the sources
374 :param logger Logger: The logger instance to use for the display and logging
375 :param pad int: The gap to apply for the terminal display
376 :param checkout boolean: If True, get the source in checkout mode
377 :return: True if it succeed, else False
381 # Get the application environment
382 logger.write(_("Set the application environment\n"), 5)
383 env_appli = src.environment.SalomeEnviron(config,
384 src.environment.Environ(dict(os.environ)))
385 env_appli.set_application_env(logger)
387 # Call the right function to get sources regarding the product settings
388 if not checkout and is_dev:
389 return get_source_for_dev(config,
395 if product_info.get_source == "git":
396 return get_source_from_git(config, product_info, source_dir, logger, pad,
399 if product_info.get_source == "archive":
400 return get_source_from_archive(config, product_info, source_dir, logger)
402 if product_info.get_source == "dir":
403 return get_source_from_dir(product_info, source_dir, logger)
405 if product_info.get_source == "cvs":
406 cvs_user = config.USER.cvs_user
407 return get_source_from_cvs(cvs_user, product_info, source_dir,
408 checkout, logger, pad, env_appli)
410 if product_info.get_source == "svn":
411 svn_user = config.USER.svn_user
412 return get_source_from_svn(svn_user, product_info, source_dir,
413 checkout, logger, env_appli)
415 if product_info.get_source == "native":
416 # for native products we check the corresponding system packages are installed
417 logger.write("Native : Checking system packages are installed\n" , 3)
418 check_cmd=src.system.get_pkg_check_cmd(config.VARS.dist_name) # (either rmp or apt)
419 run_pkg,build_pkg=src.product.check_system_dep(config.VARS.dist, check_cmd, product_info)
422 logger.write(" - " + pkg + " : " + run_pkg[pkg] + '\n', 1)
423 if "KO" in run_pkg[pkg]:
425 for pkg in build_pkg:
426 logger.write(" - " + pkg + " : " + build_pkg[pkg] + '\n', 1)
427 if "KO" in build_pkg[pkg]:
430 logger.error("some system dependencies are missing, please install them with "+\
434 if product_info.get_source == "fixed":
436 logger.write('%s ' % src.printcolors.printc(src.OK_STATUS),
439 msg = "FIXED : %s\n" % product_info.install_dir
441 if not os.path.isdir(product_info.install_dir):
442 logger.warning("The fixed path do not exixts!! Please check it : %s\n" % product_info.install_dir)
447 # if the get_source is not in [git, archive, cvs, svn, fixed, native]
448 logger.write(_("Unknown get source method \"%(get)s\" for product %(product)s") % \
449 { 'get': product_info.get_source, 'product': product_info.name }, 3, False)
450 logger.write(" ... ", 3, False)
454 def get_all_product_sources(config, products, logger):
455 '''Get all the product sources.
457 :param config Config: The global configuration
458 :param products List: The list of tuples (product name, product informations)
459 :param logger Logger: The logger instance to be used for the logging
460 :return: the tuple (number of success, dictionary product_name/success_fail)
464 # Initialize the variables that will count the fails and success
468 # Get the maximum name length in order to format the terminal display
469 max_product_name_len = 1
470 if len(products) > 0:
471 max_product_name_len = max(map(lambda l: len(l), products[0])) + 4
473 # The loop on all the products from which to get the sources
474 # DBG.write("source.get_all_product_sources config id", id(config), True)
475 for product_name, product_info in products:
476 # get product name, product informations and the directory where to put
478 if (not (src.product.product_is_fixed(product_info) or
479 src.product.product_is_native(product_info))):
480 source_dir = src.Path(product_info.source_dir)
482 source_dir = src.Path('')
485 logger.write('%s: ' % src.printcolors.printcLabel(product_name), 3)
486 logger.write(' ' * (max_product_name_len - len(product_name)), 3, False)
487 logger.write("\n", 4, False)
489 # Remove the existing source directory if
490 # the product is not in development mode
491 is_dev = src.product.product_is_dev(product_info)
492 if source_dir.exists():
493 logger.write('%s ' % src.printcolors.printc(src.OK_STATUS), 3, False)
494 msg = _("INFO : Not doing anything because the source directory already exists:\n %s\n") % source_dir
496 good_result = good_result + 1
497 # Do not get the sources and go to next product
500 # Call to the function that get the sources for one product
501 retcode = get_product_sources(config, product_info, is_dev,
502 source_dir, logger, max_product_name_len,
506 if 'no_rpath' in product_info.keys():
507 if product_info.no_rpath:
508 hack_no_rpath(config, product_info, logger)
511 # Check that the sources are correctly get using the files to be tested
512 # in product information
514 check_OK, wrong_path = check_sources(product_info, logger)
516 # Print the missing file path
517 msg = _("The required file %s does not exists. " % wrong_path)
518 logger.write(src.printcolors.printcError("\nERROR: ") + msg, 3)
522 results[product_name] = retcode
524 # The case where it succeed
526 good_result = good_result + 1
528 # The case where it failed
532 if not(src.product.product_is_fixed(product_info) or
533 src.product.product_is_native(product_info)):
534 logger.write('%s\n' % src.printcolors.printc(res), 3, False)
536 return good_result, results
538 def check_sources(product_info, logger):
539 '''Check that the sources are correctly get, using the files to be tested
540 in product information
542 :param product_info Config: The configuration specific to
543 the product to be prepared
544 :return: True if the files exists (or no files to test is provided).
547 # Get the files to test if there is any
548 if ("present_files" in product_info and
549 "source" in product_info.present_files):
550 l_files_to_be_tested = product_info.present_files.source
551 for file_path in l_files_to_be_tested:
552 # The path to test is the source directory
553 # of the product joined the file path provided
554 path_to_test = os.path.join(product_info.source_dir, file_path)
555 logger.write(_("\nTesting existence of file: \n"), 5)
556 logger.write(path_to_test, 5)
557 if not os.path.exists(path_to_test):
558 return False, path_to_test
559 logger.write(src.printcolors.printcSuccess(" OK\n"), 5)
563 '''method that is called when salomeTools is called with --help option.
565 :return: The text to display for the source command description.
568 return _("The source command gets the sources of the application products "
569 "from cvs, git or an archive.\n\nexample:"
570 "\nsat source SALOME-master --products KERNEL,GUI")
572 def run(args, runner, logger):
573 '''method that is called when salomeTools is called with source parameter.
575 DBG.write("source.run()", args)
577 (options, args) = parser.parse_args(args)
579 # check that the command has been called with an application
580 src.check_config_has_application( runner.cfg )
582 # Print some informations
583 logger.write(_('Getting sources of the application %s\n') %
584 src.printcolors.printcLabel(runner.cfg.VARS.application), 1)
585 src.printcolors.print_value(logger, 'workdir',
586 runner.cfg.APPLICATION.workdir, 2)
587 logger.write("\n", 2, False)
589 # Get the products list with products informations regarding the options
590 products_infos = src.product.get_products_list(options, runner.cfg, logger)
592 # Call to the function that gets all the sources
593 good_result, results = get_all_product_sources(runner.cfg, products_infos, logger)
595 # Display the results (how much passed, how much failed, etc...)
596 status = src.OK_STATUS
599 logger.write("\n", 2, False)
600 if good_result == len(products_infos):
601 res_count = "%d / %d" % (good_result, good_result)
603 status = src.KO_STATUS
604 res_count = "%d / %d" % (good_result, len(products_infos))
606 for product in results:
607 if results[product] == 0 or results[product] is None:
608 details.append(product)
610 result = len(products_infos) - good_result
613 logger.write(_("Getting sources of the application:"), 1)
614 logger.write(" " + src.printcolors.printc(status), 1, False)
615 logger.write(" (%s)\n" % res_count, 1, False)
618 logger.write(_("Following sources haven't been get:\n"), 2)
619 logger.write(" ".join(details), 2)
620 logger.write("\n", 2, False)