src/compilation.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import subprocess
  21 import sys
  22 import shutil
  23 import glob
  24
  25 import src
  26
  27 C_COMPILE_ENV_LIST = ["CC",
  28                       "CXX",
  29                       "F77",
  30                       "CFLAGS",
  31                       "CXXFLAGS",
  32                       "LIBS",
  33                       "LDFLAGS"]
  34
  35 class Builder:
  36     """Class to handle all construction steps, like cmake, configure, make, ...
  37     """
  38     def __init__(self,
  39                  config,
  40                  logger,
  41                  product_name,
  42                  product_info,
  43                  options = src.options.OptResult(),
  44                  check_src=True):
  45         self.config = config
  46         self.logger = logger
  47         self.options = options
  48         self.product_name = product_name
  49         self.product_info = product_info
  50         self.build_dir = src.Path(self.product_info.build_dir)
  51         self.source_dir = src.Path(self.product_info.source_dir)
  52         self.install_dir = src.Path(self.product_info.install_dir)
  53         self.header = ""
  54         self.debug_mode = False
  55         self.cmake_build_type = 'Release'
  56         if "cmake_build_type" in self.config.APPLICATION:
  57             self.set_cmake_build_type(self.config.APPLICATION.cmake_build_type)
  58         # keep backward compatibility
  59         if "debug" in self.config.APPLICATION and self.config.APPLICATION.debug == "yes":
  60             self.debug_mode = True
  61             self.cmake_build_type = 'Debug'
  62
  63         # in case a product defines its own configuration, then use it
  64         if "cmake_build_type" in self.product_info:
  65             self.set_cmake_build_type(self.product_info.cmake_build_type)
  66         # keep backward compatibility
  67         if "debug" in self.product_info and self.product_info.debug == "yes":
  68             self.debug_mode = True
  69             self.cmake_build_type = 'Debug'
  70
  71         self.verbose_mode = False
  72         if "verbose" in self.product_info and self.product_info.verbose == "yes":
  73             self.verbose_mode = True
  74
  75
  76     # set cmake build type
  77     def set_cmake_build_type(self, build_type):
  78         if build_type.lower() in ['debug', 'relwithdebinfo', 'release', 'minsizerel']:
  79             if build_type.lower() ==  'debug':
  80                 self.cmake_build_type = 'Debug'
  81                 self.debug_mode = True
  82             elif build_type.lower() ==  'relwithdebinfo':
  83                 self.cmake_build_type = 'RelWithDebInfo'
  84             elif build_type.lower() ==  'release':
  85                 self.cmake_build_type = 'Release'
  86             elif build_type.lower() ==  'minsizerel':
  87                 self.cmake_build_type = 'MinSizeRel'
  88             else:
  89                 raise src.SatException("Unknown cmake build mode: {}. Supported values are: Debug, RelWithDebInfo, Release or MinSizeRel".format(build_type))
  90         return
  91
  92     # Shortcut method to log in log file.
  93     def log(self, text, level, showInfo=True):
  94         self.logger.write(text, level, showInfo)
  95         self.logger.logTxtFile.write(src.printcolors.cleancolor(text))
  96         self.logger.flush()
  97
  98     ##
  99     # Shortcut method to log a command.
 100     def log_command(self, command):
 101         self.log("> %s\n" % command, 5)
 102
 103     ##
 104     # Prepares the environment.
 105     # Build two environment: one for building and one for testing (launch).
 106     def prepare(self, add_env_launch=False):
 107
 108         if not self.build_dir.exists():
 109             # create build dir
 110             self.build_dir.make()
 111
 112         self.log('  build_dir   = %s\n' % str(self.build_dir), 4)
 113         self.log('  install_dir = %s\n' % str(self.install_dir), 4)
 114         self.log('\n', 4)
 115
 116         # add products in depend and opt_depend list recursively
 117         environ_info = src.product.get_product_dependencies(self.config,
 118                                                             self.product_name,
 119                                                             self.product_info)
 120         #environ_info.append(self.product_info.name)
 121
 122         # create build environment
 123         self.build_environ = src.environment.SalomeEnviron(self.config,
 124                                       src.environment.Environ(dict(os.environ)),
 125                                       True)
 126         self.build_environ.silent = (self.config.USER.output_verbose_level < 5)
 127         self.build_environ.set_full_environ(self.logger, environ_info)
 128
 129         if add_env_launch:
 130         # create runtime environment
 131             self.launch_environ = src.environment.SalomeEnviron(self.config,
 132                                       src.environment.Environ(dict(os.environ)),
 133                                       False)
 134             self.launch_environ.silent = True # no need to show here
 135             self.launch_environ.set_full_environ(self.logger, environ_info)
 136
 137         for ee in C_COMPILE_ENV_LIST:
 138             vv = self.build_environ.get(ee)
 139             if len(vv) > 0:
 140                 self.log("  %s = %s\n" % (ee, vv), 4, False)
 141
 142         return 0
 143
 144     ##
 145     # Runs cmake with the given options.
 146     def cmake(self, options=""):
 147
 148         cmake_option = options
 149         if "cmake_build_type" in self.config.APPLICATION:
 150             self.set_cmake_build_type(self.config.APPLICATION.cmake_build_type)
 151         # keep backward compatibility
 152         if "debug" in self.config.APPLICATION and self.config.APPLICATION.debug == "yes":
 153             self.debug_mode = True
 154             self.cmake_build_type = 'Debug'
 155
 156         if 'cmake_options' in self.product_info:
 157             cmake_option += " %s " % " ".join(
 158                                         self.product_info.cmake_options.split())
 159
 160         # add cmake build_type options
 161         cmake_option += " -DCMAKE_BUILD_TYPE=" + self.cmake_build_type
 162
 163         # add verbose option if specified in application for this product.
 164         if self.verbose_mode:
 165             cmake_option += " -DCMAKE_VERBOSE_MAKEFILE=ON"
 166
 167         # In case CMAKE_GENERATOR is defined in environment,
 168         # use it in spite of automatically detect it
 169         if 'cmake_generator' in self.config.APPLICATION:
 170             cmake_option += " -G \"%s\" -A x64 " % self.config.APPLICATION.cmake_generator
 171         command = ("cmake %s -DCMAKE_INSTALL_PREFIX=%s %s" %
 172                             (cmake_option, self.install_dir, self.source_dir))
 173
 174         self.log_command(command)
 175         # for key in sorted(self.build_environ.environ.environ.keys()):
 176             # print key, "  ", self.build_environ.environ.environ[key]
 177         res = subprocess.call(command,
 178                               shell=True,
 179                               cwd=str(self.build_dir),
 180                               env=self.build_environ.environ.environ,
 181                               stdout=self.logger.logTxtFile,
 182                               stderr=subprocess.STDOUT)
 183
 184         self.put_txt_log_in_appli_log_dir("cmake")
 185         if res == 0:
 186             return res
 187         else:
 188             return 1
 189
 190     ##
 191     # Runs build_configure with the given options.
 192     def build_configure(self, options=""):
 193
 194         if 'buildconfigure_options' in self.product_info:
 195             options += " %s " % self.product_info.buildconfigure_options
 196
 197         command = str('%s/build_configure') % (self.source_dir)
 198         command = command + " " + options
 199         self.log_command(command)
 200
 201         res = subprocess.call(command,
 202                               shell=True,
 203                               cwd=str(self.build_dir),
 204                               env=self.build_environ.environ.environ,
 205                               stdout=self.logger.logTxtFile,
 206                               stderr=subprocess.STDOUT)
 207         self.put_txt_log_in_appli_log_dir("build_configure")
 208         if res == 0:
 209             return res
 210         else:
 211             return 1
 212
 213     ##
 214     # Runs configure with the given options.
 215     def configure(self, options=""):
 216
 217         if 'configure_options' in self.product_info:
 218             options += " %s " % self.product_info.configure_options
 219
 220         command = "%s/configure --prefix=%s" % (self.source_dir,
 221                                                 str(self.install_dir))
 222
 223         command = command + " " + options
 224         self.log_command(command)
 225
 226         res = subprocess.call(command,
 227                               shell=True,
 228                               cwd=str(self.build_dir),
 229                               env=self.build_environ.environ.environ,
 230                               stdout=self.logger.logTxtFile,
 231                               stderr=subprocess.STDOUT)
 232
 233         self.put_txt_log_in_appli_log_dir("configure")
 234         if res == 0:
 235             return res
 236         else:
 237             return 1
 238
 239     def hack_libtool(self):
 240         if not os.path.exists(str(self.build_dir + 'libtool')):
 241             return
 242
 243         lf = open(os.path.join(str(self.build_dir), "libtool"), 'r')
 244         for line in lf.readlines():
 245             if 'hack_libtool' in line:
 246                 return
 247
 248         # fix libtool by replacing CC="<compil>" with hack_libtool function
 249         hack_command='''sed -i "s%^CC=\\"\(.*\)\\"%hack_libtool() { \\n\\
 250 if test \\"\$(echo \$@ | grep -E '\\\\\\-L/usr/lib(/../lib)?(64)? ')\\" == \\\"\\\" \\n\\
 251   then\\n\\
 252     cmd=\\"\\1 \$@\\"\\n\\
 253   else\\n\\
 254     cmd=\\"\\1 \\"\`echo \$@ | sed -r -e 's|(.*)-L/usr/lib(/../lib)?(64)? (.*)|\\\\\\1\\\\\\4 -L/usr/lib\\\\\\3|g'\`\\n\\
 255   fi\\n\\
 256   \$cmd\\n\\
 257 }\\n\\
 258 CC=\\"hack_libtool\\"%g" libtool'''
 259
 260         self.log_command(hack_command)
 261         subprocess.call(hack_command,
 262                         shell=True,
 263                         cwd=str(self.build_dir),
 264                         env=self.build_environ.environ.environ,
 265                         stdout=self.logger.logTxtFile,
 266                         stderr=subprocess.STDOUT)
 267
 268
 269     ##
 270     # Runs make to build the module.
 271     def make(self, nb_proc, make_opt=""):
 272
 273         # make
 274         command = 'make'
 275         command = command + " -j" + str(nb_proc)
 276         command = command + " " + make_opt
 277         self.log_command(command)
 278         res = subprocess.call(command,
 279                               shell=True,
 280                               cwd=str(self.build_dir),
 281                               env=self.build_environ.environ.environ,
 282                               stdout=self.logger.logTxtFile,
 283                               stderr=subprocess.STDOUT)
 284         self.put_txt_log_in_appli_log_dir("make")
 285         if res == 0:
 286             return res
 287         else:
 288             return 1
 289
 290     ##
 291     # Runs msbuild to build the module.
 292     def wmake(self,nb_proc, opt_nb_proc = None):
 293
 294         hh = 'MSBUILD /m:%s' % str(nb_proc)
 295         if self.debug_mode:
 296             hh += " " + src.printcolors.printcWarning("DEBUG")
 297         # make
 298         command = 'msbuild'
 299         command+= " /maxcpucount:" + str(nb_proc)
 300         command+= " /p:Configuration={}  /p:Platform=x64 ".format(self.cmake_build_type)
 301         command = command + " ALL_BUILD.vcxproj"
 302
 303         self.log_command(command)
 304         res = subprocess.call(command,
 305                               shell=True,
 306                               cwd=str(self.build_dir),
 307                               env=self.build_environ.environ.environ,
 308                               stdout=self.logger.logTxtFile,
 309                               stderr=subprocess.STDOUT)
 310
 311         self.put_txt_log_in_appli_log_dir("make")
 312         if res == 0:
 313             return res
 314         else:
 315             return 1
 316
 317     ##
 318     # Runs 'make install'.
 319     def install(self):
 320         if src.architecture.is_windows():
 321             command = 'msbuild INSTALL.vcxproj'
 322             command+= ' /p:Configuration={}  /p:Platform=x64 '.format(self.cmake_build_type)
 323         else :
 324             command = 'make install'
 325         self.log_command(command)
 326
 327         res = subprocess.call(command,
 328                               shell=True,
 329                               cwd=str(self.build_dir),
 330                               env=self.build_environ.environ.environ,
 331                               stdout=self.logger.logTxtFile,
 332                               stderr=subprocess.STDOUT)
 333
 334         res_check=self.check_install()
 335         if res_check > 0 :
 336             self.log_command("Error in sat check install - some files are not installed!")
 337         self.put_txt_log_in_appli_log_dir("makeinstall")
 338
 339         res+=res_check
 340         if res == 0:
 341             return res
 342         else:
 343             return 1
 344
 345     # this function checks wether a list of file patterns (specified by check_install keyword)
 346     # exixts after the make install. The objective is to ensure the installation is complete.
 347     # patterns are given relatively to the install dir of the product
 348     def check_install(self):
 349         res=0
 350         if "check_install" in self.product_info:
 351             self.log_command("Check installation of files")
 352             for pattern in self.product_info.check_install:
 353                 # pattern is given relatively to the install dir
 354                 complete_pattern=os.path.join(self.product_info.install_dir, pattern)
 355                 self.log_command("    -> check %s" % complete_pattern)
 356                 # expansion of pattern : takes into account environment variables and unix shell rules
 357                 list_of_path=glob.glob(os.path.expandvars(complete_pattern))
 358                 if not list_of_path:
 359                     # we expect to find at least one entry, if not we consider the test failed
 360                     res+=1
 361                     self.logger.write("Error, sat check install failed for file pattern %s\n" % complete_pattern, 1)
 362                     self.log_command("Error, sat check install failed for file pattern %s" % complete_pattern)
 363         return res
 364
 365     ##
 366     # Runs 'make_check'.
 367     def check(self, command=""):
 368         if src.architecture.is_windows():
 369             cmd = 'msbuild RUN_TESTS.vcxproj /p:Configuration={}  /p:Platform=x64 '.format(self.cmake_build_type)
 370         else :
 371             if self.product_info.build_source=="autotools" :
 372                 cmd = 'make check'
 373             else:
 374                 cmd = 'make test'
 375
 376         if command:
 377             cmd = command
 378
 379         self.log_command(cmd)
 380         self.log_command("For more detailed logs, see test logs in %s" % self.build_dir)
 381
 382         res = subprocess.call(cmd,
 383                               shell=True,
 384                               cwd=str(self.build_dir),
 385                               env=self.launch_environ.environ.environ,
 386                               stdout=self.logger.logTxtFile,
 387                               stderr=subprocess.STDOUT)
 388
 389         self.put_txt_log_in_appli_log_dir("makecheck")
 390         if res == 0:
 391             return res
 392         else:
 393             return 1
 394
 395     ##
 396     # Performs a default build for this module.
 397     def do_default_build(self,
 398                          build_conf_options="",
 399                          configure_options="",
 400                          show_warning=True):
 401         use_autotools = False
 402         if 'use_autotools' in self.product_info:
 403             uc = self.product_info.use_autotools
 404             if uc in ['always', 'yes']:
 405                 use_autotools = True
 406             elif uc == 'option':
 407                 use_autotools = self.options.autotools
 408
 409
 410         self.use_autotools = use_autotools
 411
 412         use_ctest = False
 413         if 'use_ctest' in self.product_info:
 414             uc = self.product_info.use_ctest
 415             if uc in ['always', 'yes']:
 416                 use_ctest = True
 417             elif uc == 'option':
 418                 use_ctest = self.options.ctest
 419
 420         self.use_ctest = use_ctest
 421
 422         if show_warning:
 423             cmd = ""
 424             if use_autotools: cmd = "(autotools)"
 425             if use_ctest: cmd = "(ctest)"
 426
 427             self.log("\n", 4, False)
 428             self.log("%(module)s: Run default compilation method %(cmd)s\n" % \
 429                 { "module": self.module, "cmd": cmd }, 4)
 430
 431         if use_autotools:
 432             if not self.prepare(): return self.get_result()
 433             if not self.build_configure(
 434                                    build_conf_options): return self.get_result()
 435             if not self.configure(configure_options): return self.get_result()
 436             if not self.make(): return self.get_result()
 437             if not self.install(): return self.get_result()
 438             if not self.clean(): return self.get_result()
 439
 440         else: # CMake
 441             if self.config.VARS.dist_name=='Win':
 442                 if not self.wprepare(): return self.get_result()
 443                 if not self.cmake(): return self.get_result()
 444                 if not self.wmake(): return self.get_result()
 445                 if not self.install(): return self.get_result()
 446                 if not self.clean(): return self.get_result()
 447             else :
 448                 if not self.prepare(): return self.get_result()
 449                 if not self.cmake(): return self.get_result()
 450                 if not self.make(): return self.get_result()
 451                 if not self.install(): return self.get_result()
 452                 if not self.clean(): return self.get_result()
 453
 454         return self.get_result()
 455
 456     ##
 457     # Performs a build with a script.
 458     def do_python_script_build(self, script, nb_proc):
 459         # script found
 460         self.logger.write(_("Compile %(product)s using script %(script)s\n") % \
 461             { 'product': self.product_info.name,
 462              'script': src.printcolors.printcLabel(script) }, 4)
 463         try:
 464             import imp
 465             product = self.product_info.name
 466             pymodule = imp.load_source(product + "_compile_script", script)
 467             self.nb_proc = nb_proc
 468             retcode = pymodule.compil(self.config, self, self.logger)
 469         except Exception:
 470             __, exceptionValue, exceptionTraceback = sys.exc_info()
 471             self.logger.write(str(exceptionValue), 1)
 472             import traceback
 473             traceback.print_tb(exceptionTraceback)
 474             traceback.print_exc()
 475             retcode = 1
 476         finally:
 477             self.put_txt_log_in_appli_log_dir("script")
 478
 479         return retcode
 480
 481     def complete_environment(self, make_options):
 482         assert self.build_environ is not None
 483         # pass additional variables to environment
 484         # (may be used by the build script)
 485         self.build_environ.set("APPLICATION_NAME", self.config.APPLICATION.name)
 486         self.build_environ.set("SOURCE_DIR", str(self.source_dir))
 487         self.build_environ.set("INSTALL_DIR", str(self.install_dir))
 488         self.build_environ.set("PRODUCT_INSTALL", str(self.install_dir))
 489         self.build_environ.set("BUILD_DIR", str(self.build_dir))
 490         self.build_environ.set("PRODUCT_BUILD", str(self.build_dir))
 491         self.build_environ.set("MAKE_OPTIONS", make_options)
 492         self.build_environ.set("DIST_NAME", self.config.VARS.dist_name)
 493         self.build_environ.set("DIST_VERSION", self.config.VARS.dist_version)
 494         self.build_environ.set("DIST", self.config.VARS.dist)
 495         self.build_environ.set("VERSION", self.product_info.version)
 496         # if product is in hpc mode, set SAT_HPC to 1
 497         # in order for the compilation script to take it into account
 498         if src.product.product_is_hpc(self.product_info):
 499             self.build_environ.set("SAT_HPC", "1")
 500         if self.debug_mode:
 501             self.build_environ.set("SAT_DEBUG", "1")
 502         if "cmake_build_type" in self.config.APPLICATION:
 503             self.set_cmake_build_type(self.config.APPLICATION.cmake_build_type)
 504             self.build_environ.set("SAT_CMAKE_BUILD_TYPE", self.cmake_build_type)
 505         if self.verbose_mode:
 506             self.build_environ.set("SAT_VERBOSE", "1")
 507
 508
 509     def do_batch_script_build(self, script, nb_proc):
 510
 511         if src.architecture.is_windows():
 512             make_options = "/maxcpucount:%s" % nb_proc
 513         else :
 514             make_options = "-j%s" % nb_proc
 515
 516         self.log_command("  " + _("Run build script %s\n") % script)
 517         self.complete_environment(make_options)
 518
 519         res = subprocess.call(script,
 520                               shell=True,
 521                               stdout=self.logger.logTxtFile,
 522                               stderr=subprocess.STDOUT,
 523                               cwd=str(self.build_dir),
 524                               env=self.build_environ.environ.environ)
 525
 526         res_check=self.check_install()
 527         if res_check > 0 :
 528             self.log_command("Error in sat check install - some files are not installed!")
 529
 530         self.put_txt_log_in_appli_log_dir("script")
 531         res += res_check
 532         if res == 0:
 533             return res
 534         else:
 535             return 1
 536
 537     def do_script_build(self, script, number_of_proc=0):
 538         # define make options (may not be used by the script)
 539         if number_of_proc==0:
 540             nb_proc = src.get_cfg_param(self.product_info,"nb_proc", 0)
 541             if nb_proc == 0:
 542                 nb_proc = self.config.VARS.nb_proc
 543         else:
 544             nb_proc = min(number_of_proc, self.config.VARS.nb_proc)
 545
 546         extension = script.split('.')[-1]
 547         if extension in ["bat","sh"]:
 548             return self.do_batch_script_build(script, nb_proc)
 549         if extension == "py":
 550             return self.do_python_script_build(script, nb_proc)
 551
 552         msg = _("The script %s must have .sh, .bat or .py extension." % script)
 553         raise src.SatException(msg)
 554
 555     def put_txt_log_in_appli_log_dir(self, file_name):
 556         '''Put the txt log (that contain the system logs, like make command
 557            output) in the directory <APPLICATION DIR>/LOGS/<product_name>/
 558
 559         :param file_name Str: the name of the file to write
 560         '''
 561         if self.logger.logTxtFile == sys.__stdout__:
 562             return
 563         dir_where_to_put = os.path.join(self.config.APPLICATION.workdir,
 564                                         "LOGS",
 565                                         self.product_info.name)
 566         file_path = os.path.join(dir_where_to_put, file_name)
 567         src.ensure_path_exists(dir_where_to_put)
 568         # write the logTxtFile copy it to the destination, and then recreate
 569         # it as it was
 570         self.logger.logTxtFile.close()
 571         shutil.move(self.logger.txtFilePath, file_path)
 572         self.logger.logTxtFile = open(str(self.logger.txtFilePath), 'w')
 573         self.logger.logTxtFile.write(open(file_path, "r").read())
 574