3 # Copyright (C) 2010-2012 CEA/DEN
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Lesser General Public
7 # License as published by the Free Software Foundation; either
8 # version 2.1 of the License.
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # Lesser General Public License for more details.
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with this library; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 import src.debug as DBG
25 # Compatibility python 2/3 for input function
26 # input stays input for python 3 and input = raw_input for python 2
33 # Define all possible option for the compile command : sat compile <options>
34 parser = src.options.Options()
35 parser.add_option('p', 'products', 'list2', 'products',
36 _('Optional: products to compile. This option accepts a comma separated list.'))
37 parser.add_option('', 'with_fathers', 'boolean', 'fathers',
38 _("Optional: build all necessary products to the given product (KERNEL is "
39 "build before building GUI)."), False)
40 parser.add_option('', 'with_children', 'boolean', 'children',
41 _("Optional: build all products using the given product (all SMESH plugins"
42 " are build after SMESH)."), False)
43 parser.add_option('', 'clean_all', 'boolean', 'clean_all',
44 _("Optional: clean BUILD dir and INSTALL dir before building product."),
46 parser.add_option('', 'clean_install', 'boolean', 'clean_install',
47 _("Optional: clean INSTALL dir before building product."), False)
48 parser.add_option('', 'make_flags', 'string', 'makeflags',
49 _("Optional: add extra options to the 'make' command."))
50 parser.add_option('', 'show', 'boolean', 'no_compile',
51 _("Optional: DO NOT COMPILE just show if products are installed or not."),
53 parser.add_option('', 'stop_first_fail', 'boolean', 'stop_first_fail', _(
54 "Optional: Stops the command at first product compilation"
56 parser.add_option('', 'check', 'boolean', 'check', _(
57 "Optional: execute the unit tests after compilation"), False)
59 parser.add_option('', 'clean_build_after', 'boolean', 'clean_build_after',
60 _('Optional: remove the build directory after successful compilation'), False)
63 # from sat product infos, represent the product dependencies in a simple python graph
64 # keys are nodes, the list of dependencies are values
65 def get_dependencies_graph(p_infos):
67 for (p_name,p_info) in p_infos:
68 graph[p_name]=p_info.depend
71 # this recursive function calculates all the dependencies of node start
72 def depth_search_graph(graph, start, visited=[]):
73 visited= visited+ [start]
74 for node in graph[start]: # for all nodes in start dependencies
75 if node not in visited:
76 visited=depth_search_graph(graph, node, visited)
79 # find a path from start node to end (a group of nodes)
80 def find_path_graph(graph, start, end, path=[]):
84 if start not in graph:
86 for node in graph[start]:
88 newpath = find_path_graph(graph, node, end, path)
89 if newpath: return newpath
92 # Topological sorting algo
93 # return in sorted_nodes the list of sorted nodes
94 def depth_first_topo_graph(graph, start, visited=[], sorted_nodes=[]):
95 visited = visited + [start]
96 if start not in graph:
97 raise src.SatException('Error in product dependencies : %s product is referenced in products dependencies but is not present in the application !' % start)
98 for node in graph[start]:
99 if node not in visited:
100 visited,sorted_nodes=depth_first_topo_graph(graph, node, visited,sorted_nodes)
102 if node not in sorted_nodes:
103 raise src.SatException('Error in product dependencies : cycle detection for node %s and %s !' % (start,node))
105 sorted_nodes = sorted_nodes + [start]
106 return visited,sorted_nodes
109 # check for p_name that all dependencies are installed
110 def check_dependencies(config, p_name_p_info, all_products_dict):
111 l_depends_not_installed = []
112 for prod in p_name_p_info[1]["depend_all"]:
113 # for each dependency, check the install
114 prod_name, prod_info=all_products_dict[prod]
115 if not(src.product.check_installation(config, prod_info)):
116 l_depends_not_installed.append(prod_name)
117 return l_depends_not_installed # non installed deps
119 def log_step(logger, header, step):
120 logger.write("\r%s%s" % (header, " " * 30), 3)
121 logger.write("\r%s%s" % (header, step), 3)
124 def log_res_step(logger, res):
126 logger.write("%s \n" % src.printcolors.printcSuccess("OK"), 4)
129 logger.write("%s \n" % src.printcolors.printcError("KO"), 4)
132 def compile_all_products(sat, config, options, products_infos, all_products_dict, logger):
133 '''Execute the proper configuration commands
134 in each product build directory.
136 :param config Config: The global configuration
137 :param products_info list: List of
138 (str, Config) => (product_name, product_info)
139 :param all_products_dict: Dict of all products
140 :param logger Logger: The logger instance to use for the display and logging
141 :return: the number of failing commands.
145 for p_name_info in products_infos:
147 p_name, p_info = p_name_info
151 header = _("Compilation of %s") % src.printcolors.printcLabel(p_name)
152 header += " %s " % ("." * (len_end_line - len(p_name)))
153 logger.write(header, 3)
156 # Do nothing if the product is not compilable
157 if not src.product.product_compiles(p_info):
158 log_step(logger, header, "ignored")
159 logger.write("\n", 3, False)
162 # Do nothing if the product is native
163 if src.product.product_is_native(p_info):
164 log_step(logger, header, "native")
165 logger.write("\n", 3, False)
168 # Do nothing if the product is fixed (already compiled by third party)
169 if src.product.product_is_fixed(p_info):
170 log_step(logger, header, "native")
171 logger.write("\n", 3, False)
174 # Clean the build and the install directories
175 # if the corresponding options was called
176 if options.clean_all:
177 log_step(logger, header, "CLEAN BUILD AND INSTALL ")
178 sat.clean(config.VARS.application +
179 " --products " + p_name +
180 " --build --install",
183 logger_add_link = logger)
186 # Clean the the install directory
187 # if the corresponding option was called
188 if options.clean_install and not options.clean_all:
189 log_step(logger, header, "CLEAN INSTALL ")
190 sat.clean(config.VARS.application +
191 " --products " + p_name +
195 logger_add_link = logger)
197 # Recompute the product information to get the right install_dir
198 # (it could change if there is a clean of the install directory)
199 p_info = src.product.get_product_config(config, p_name)
201 # Check if sources was already successfully installed
202 check_source = src.product.check_source(p_info)
203 is_pip= (src.appli_test_property(config,"pip", "yes") and src.product.product_test_property(p_info,"pip", "yes"))
204 # don't check sources with option --show
205 # or for products managed by pip (there sources are in wheels stored in LOCAL.ARCHIVE
206 if not (options.no_compile or is_pip):
208 logger.write(_("Sources of product not found (try 'sat -h prepare') \n"))
209 res += 1 # one more error
212 if src.product.product_is_salome(p_info):
213 # For salome modules, we check if the sources of configuration modules are present
214 # configuration modules have the property "configure_dependency"
216 # get the list of all modules in application
217 all_products_infos = src.product.get_products_infos(config.APPLICATION.products,
220 # for configuration modules, check if sources are present
221 for prod in all_products_dict:
222 product_name, product_info = all_products_dict[prod]
223 if ("properties" in product_info and
224 "configure_dependency" in product_info.properties and
225 product_info.properties.configure_dependency == "yes"):
226 check_source = check_source and src.product.check_source(product_info)
228 logger.write(_("\nERROR : SOURCES of %s not found! It is required for"
229 " the configuration\n" % product_name))
230 logger.write(_(" Get it with the command : sat prepare %s -p %s \n" %
231 (config.APPLICATION.name, product_name)))
233 # if at least one configuration module is not present, we stop compilation
237 # Check if it was already successfully installed
238 if src.product.check_installation(config, p_info):
239 logger.write(_("Already installed"))
240 logger.write(_(" in %s" % p_info.install_dir), 4)
241 logger.write(_("\n"))
244 # If the show option was called, do not launch the compilation
245 if options.no_compile:
246 logger.write(_("Not installed in %s\n" % p_info.install_dir))
249 # Check if the dependencies are installed
250 l_depends_not_installed = check_dependencies(config, p_name_info, all_products_dict)
251 if len(l_depends_not_installed) > 0:
252 log_step(logger, header, "")
253 logger.write(src.printcolors.printcError(
254 _("ERROR : the following mandatory product(s) is(are) not installed: ")))
255 for prod_name in l_depends_not_installed:
256 logger.write(src.printcolors.printcError(prod_name + " "))
260 # Call the function to compile the product
261 res_prod, len_end_line, error_step = compile_product(
262 sat, p_name_info, config, options, logger, header, len_end_line)
266 # there was an error, we clean install dir, unless :
267 # - the error step is "check", or
268 # - the product is managed by pip and installed in python dir
269 do_not_clean_install=False
270 if (error_step == "CHECK") or (is_pip and \
271 src.appli_test_property(config,"pip_install_dir", "python")) :
272 do_not_clean_install=True
273 if not do_not_clean_install:
274 # Clean the install directory if there is any
276 "Cleaning the install directory if there is any\n"),
278 sat.clean(config.VARS.application +
279 " --products " + p_name +
283 logger_add_link = logger)
285 # Clean the build directory if the compilation and tests succeed
286 if options.clean_build_after:
287 log_step(logger, header, "CLEAN BUILD")
288 sat.clean(config.VARS.application +
289 " --products " + p_name +
293 logger_add_link = logger)
297 logger.write("\r%s%s" % (header, " " * len_end_line), 3)
298 logger.write("\r" + header + src.printcolors.printcError("KO ") + error_step)
299 logger.write("\n==== %(KO)s in compile of %(name)s \n" %
300 { "name" : p_name , "KO" : src.printcolors.printcInfo("ERROR")}, 4)
301 if error_step == "CHECK":
302 logger.write(_("\nINSTALL directory = %s" %
303 src.printcolors.printcInfo(p_info.install_dir)), 3)
306 logger.write("\r%s%s" % (header, " " * len_end_line), 3)
307 logger.write("\r" + header + src.printcolors.printcSuccess("OK"))
308 logger.write(_("\nINSTALL directory = %s" %
309 src.printcolors.printcInfo(p_info.install_dir)), 3)
310 logger.write("\n==== %s \n" % src.printcolors.printcInfo("OK"), 4)
311 logger.write("\n==== Compilation of %(name)s %(OK)s \n" %
312 { "name" : p_name , "OK" : src.printcolors.printcInfo("OK")}, 4)
314 logger.write("\n", 3, False)
317 if res_prod != 0 and options.stop_first_fail:
322 def compile_product(sat, p_name_info, config, options, logger, header, len_end):
323 '''Execute the proper configuration command(s)
324 in the product build directory.
326 :param p_name_info tuple: (str, Config) => (product_name, product_info)
327 :param config Config: The global configuration
328 :param logger Logger: The logger instance to use for the display
330 :param header Str: the header to display when logging
331 :param len_end Int: the lenght of the the end of line (used in display)
332 :return: 1 if it fails, else 0.
336 p_name, p_info = p_name_info
338 # Get the build procedure from the product configuration.
340 # build_sources : autotools -> build_configure, configure, make, make install
341 # build_sources : cmake -> cmake, make, make install
342 # build_sources : script -> script executions
346 # check if pip should be used : the application and product have pip property
347 if (src.appli_test_property(config,"pip", "yes") and
348 src.product.product_test_property(p_info,"pip", "yes")):
349 res, len_end_line, error_step = compile_product_pip(sat,
357 if (src.product.product_is_autotools(p_info) or
358 src.product.product_is_cmake(p_info)):
359 res, len_end_line, error_step = compile_product_cmake_autotools(sat,
366 if src.product.product_has_script(p_info):
367 res, len_end_line, error_step = compile_product_script(sat,
375 # Check that the install directory exists
376 if res==0 and not(os.path.exists(p_info.install_dir)):
378 error_step = "NO INSTALL DIR"
379 msg = _("Error: despite the fact that all the steps ended successfully,"
380 " no install directory was found !")
381 logger.write(src.printcolors.printcError(msg), 4)
382 logger.write("\n", 4)
383 return res, len_end, error_step
385 # Add the config file corresponding to the dependencies/versions of the
386 # product that have been successfully compiled
388 logger.write(_("Add the config file in installation directory\n"), 5)
389 src.product.add_compile_config_file(p_info, config)
392 # Do the unit tests (call the check command)
393 log_step(logger, header, "CHECK")
394 res_check = sat.check(
395 config.VARS.application + " --products " + p_name,
397 logger_add_link = logger)
403 return res, len_end_line, error_step
406 def compile_product_pip(sat,
413 '''Execute the proper build procedure for pip products
414 :param p_name_info tuple: (str, Config) => (product_name, product_info)
415 :param config Config: The global configuration
416 :param logger Logger: The logger instance to use for the display
418 :param header Str: the header to display when logging
419 :param len_end Int: the lenght of the the end of line (used in display)
420 :return: 1 if it fails, else 0.
424 p_name, p_info = p_name_info
427 pip_install_in_python=False
428 pip_wheels_dir=os.path.join(config.LOCAL.archive_dir,"wheels")
429 pip_install_cmd=config.INTERNAL.command.pip_install # parametrized in src/internal
431 # b) get the build environment (useful to get the installed python & pip3)
432 build_environ = src.environment.SalomeEnviron(config,
433 src.environment.Environ(dict(os.environ)),
435 environ_info = src.product.get_product_dependencies(config,
437 build_environ.silent = (config.USER.output_verbose_level < 5)
438 build_environ.set_full_environ(logger, environ_info)
440 # c- download : check/get pip wheel in pip_wheels_dir
441 pip_download_cmd=config.INTERNAL.command.pip_download +\
442 " --destination-directory %s --no-deps %s==%s " %\
443 (pip_wheels_dir, p_info.name, p_info.version)
444 logger.write("\n"+pip_download_cmd+"\n", 4, False)
445 res_pip_dwl = (subprocess.call(pip_download_cmd,
447 cwd=config.LOCAL.workdir,
448 env=build_environ.environ.environ,
449 stdout=logger.logTxtFile,
450 stderr=subprocess.STDOUT) == 0)
451 # error is not managed at the stage. error will be handled by pip install
452 # here we just print a message
454 logger.write("Error in pip download\n", 4, False)
457 # d- install (in python or in separate product directory)
458 if src.appli_test_property(config,"pip_install_dir", "python"):
459 # pip will install product in python directory"
460 pip_install_cmd+=" --find-links=%s --build %s %s==%s" %\
461 (pip_wheels_dir, p_info.build_dir, p_info.name, p_info.version)
462 pip_install_in_python=True
465 # pip will install product in product install_dir
466 pip_install_dir=os.path.join(p_info.install_dir, "lib", "python${PYTHON_VERSION:0:3}", "site-packages")
467 pip_install_cmd+=" --find-links=%s --build %s --target %s %s==%s" %\
468 (pip_wheels_dir, p_info.build_dir, pip_install_dir, p_info.name, p_info.version)
469 log_step(logger, header, "PIP")
470 logger.write("\n"+pip_install_cmd+"\n", 4)
471 len_end_line = len_end + 3
474 res_pip = (subprocess.call(pip_install_cmd,
476 cwd=config.LOCAL.workdir,
477 env=build_environ.environ.environ,
478 stdout=logger.logTxtFile,
479 stderr=subprocess.STDOUT) == 0)
483 #log_res_step(logger, res)
486 logger.write("\nError in pip command, please consult details with sat log command's internal traces\n", 3)
488 return res, len_end_line, error_step
492 def compile_product_cmake_autotools(sat,
499 '''Execute the proper build procedure for autotools or cmake
500 in the product build directory.
502 :param p_name_info tuple: (str, Config) => (product_name, product_info)
503 :param config Config: The global configuration
504 :param logger Logger: The logger instance to use for the display
506 :param header Str: the header to display when logging
507 :param len_end Int: the lenght of the the end of line (used in display)
508 :return: 1 if it fails, else 0.
511 p_name, p_info = p_name_info
513 # Execute "sat configure", "sat make" and "sat install"
517 # Logging and sat command call for configure step
518 len_end_line = len_end
519 log_step(logger, header, "CONFIGURE")
520 res_c = sat.configure(config.VARS.application + " --products " + p_name,
522 logger_add_link = logger)
523 log_res_step(logger, res_c)
527 error_step = "CONFIGURE"
529 # Logging and sat command call for make step
530 # Logging take account of the fact that the product has a compilation
532 if src.product.product_has_script(p_info):
533 # if the product has a compilation script,
534 # it is executed during make step
535 scrit_path_display = src.printcolors.printcLabel(
536 p_info.compil_script)
537 log_step(logger, header, "SCRIPT " + scrit_path_display)
538 len_end_line = len(scrit_path_display)
540 log_step(logger, header, "MAKE")
541 make_arguments = config.VARS.application + " --products " + p_name
542 # Get the make_flags option if there is any
543 if options.makeflags:
544 make_arguments += " --option -j" + options.makeflags
545 res_m = sat.make(make_arguments,
547 logger_add_link = logger)
548 log_res_step(logger, res_m)
554 # Logging and sat command call for make install step
555 log_step(logger, header, "MAKE INSTALL")
556 res_mi = sat.makeinstall(config.VARS.application +
560 logger_add_link = logger)
562 log_res_step(logger, res_mi)
566 error_step = "MAKE INSTALL"
568 return res, len_end_line, error_step
570 def compile_product_script(sat,
577 '''Execute the script build procedure in the product build directory.
579 :param p_name_info tuple: (str, Config) => (product_name, product_info)
580 :param config Config: The global configuration
581 :param logger Logger: The logger instance to use for the display
583 :param header Str: the header to display when logging
584 :param len_end Int: the lenght of the the end of line (used in display)
585 :return: 1 if it fails, else 0.
588 p_name, p_info = p_name_info
590 # Execute "sat configure", "sat make" and "sat install"
593 # Logging and sat command call for the script step
594 scrit_path_display = src.printcolors.printcLabel(p_info.compil_script)
595 log_step(logger, header, "SCRIPT " + scrit_path_display)
596 len_end_line = len_end + len(scrit_path_display)
597 res = sat.script(config.VARS.application + " --products " + p_name,
599 logger_add_link = logger)
600 log_res_step(logger, res)
602 return res, len_end_line, error_step
606 '''method that is called when salomeTools is called with --help option.
608 :return: The text to display for the compile command description.
611 return _("The compile command constructs the products of the application"
612 "\n\nexample:\nsat compile SALOME-master --products KERNEL,GUI,"
613 "MEDCOUPLING --clean_all")
615 def run(args, runner, logger):
616 '''method that is called when salomeTools is called with compile parameter.
619 (options, args) = parser.parse_args(args)
621 # Warn the user if he invoked the clean_all option
622 # without --products option
623 if (options.clean_all and
624 options.products is None and
625 not runner.options.batch):
626 rep = input(_("You used --clean_all without specifying a product"
627 " are you sure you want to continue? [Yes/No] "))
628 if rep.upper() != _("YES").upper():
631 # check that the command has been called with an application
632 src.check_config_has_application( runner.cfg )
634 # Print some informations
635 logger.write(_('Executing the compile commands in the build '
636 'directories of the products of '
637 'the application %s\n') %
638 src.printcolors.printcLabel(runner.cfg.VARS.application), 1)
641 (_("SOURCE directory"),
642 os.path.join(runner.cfg.APPLICATION.workdir, 'SOURCES')),
643 (_("BUILD directory"),
644 os.path.join(runner.cfg.APPLICATION.workdir, 'BUILD'))
646 src.print_info(logger, info)
648 # Get the list of all application products, and create its dependency graph
649 all_products_infos = src.product.get_products_infos(runner.cfg.APPLICATION.products,
651 all_products_graph=get_dependencies_graph(all_products_infos)
652 #logger.write("Dependency graph of all application products : %s\n" % all_products_graph, 6)
653 DBG.write("Dependency graph of all application products : ", all_products_graph)
655 # Get the list of products we have to compile
656 products_infos = src.product.get_products_list(options, runner.cfg, logger)
657 products_list = [pi[0] for pi in products_infos]
659 logger.write("Product we have to compile (as specified by user) : %s\n" % products_list, 5)
661 # Extend the list with all recursive dependencies of the given products
663 for p_name in products_list:
664 visited=depth_search_graph(all_products_graph, p_name, visited)
665 products_list = visited
667 logger.write("Product list to compile with fathers : %s\n" % products_list, 5)
669 # Extend the list with all products that depends upon the given products
671 for n in all_products_graph:
672 # for all products (that are not in products_list):
673 # if we we find a path from the product to the product list,
674 # then we product is a child and we add it to the children list
675 if (n not in children) and (n not in products_list):
676 if find_path_graph(all_products_graph, n, products_list):
677 children = children + [n]
678 # complete products_list (the products we have to compile) with the list of children
679 products_list = products_list + children
680 logger.write("Product list to compile with children : %s\n" % products_list, 5)
682 # Sort the list of all products (topological sort).
683 # the products listed first do not depend upon products listed after
686 for n in all_products_graph:
687 if n not in visited_nodes:
688 visited_nodes,sorted_nodes=depth_first_topo_graph(all_products_graph, n, visited_nodes,sorted_nodes)
689 logger.write("Complete dependency graph topological search (sorting): %s\n" % sorted_nodes, 6)
691 # use the sorted list of all products to sort the list of products we have to compile
692 sorted_product_list=[]
693 for n in sorted_nodes:
694 if n in products_list:
695 sorted_product_list.append(n)
696 logger.write("Sorted list of products to compile : %s\n" % sorted_product_list, 5)
699 # from the sorted list of products to compile, build a sorted list of products infos
700 # a- create a dict to facilitate products_infos sorting
702 for (pname,pinfo) in all_products_infos:
703 all_products_dict[pname]=(pname,pinfo)
704 # b- build a sorted list of products infos in products_infos
706 for product in sorted_product_list:
707 products_infos.append(all_products_dict[product])
709 # for all products to compile, store in "depend_all" field the complete dependencies (recursive)
710 # (will be used by check_dependencies funvtion)
711 for pi in products_infos:
713 dep_prod=depth_search_graph(all_products_graph,pi[0], dep_prod)
714 pi[1]["depend_all"]=dep_prod[1:]
717 # Call the function that will loop over all the products and execute
718 # the right command(s)
719 res = compile_all_products(runner, runner.cfg, options, products_infos, all_products_dict, logger)
721 # Print the final state
722 nb_products = len(products_infos)
728 logger.write(_("\nCompilation: %(status)s (%(valid_result)d/%(nb_products)d)\n") % \
729 { 'status': src.printcolors.printc(final_status),
730 'valid_result': nb_products - res,
731 'nb_products': nb_products }, 1)