src/compilation.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import subprocess
  21 import sys
  22 import shutil
  23 import glob
  24
  25 import src
  26
  27 C_COMPILE_ENV_LIST = ["CC",
  28                       "CXX",
  29                       "F77",
  30                       "CFLAGS",
  31                       "CXXFLAGS",
  32                       "LIBS",
  33                       "LDFLAGS"]
  34
  35 class Builder:
  36     """Class to handle all construction steps, like cmake, configure, make, ...
  37     """
  38     def __init__(self,
  39                  config,
  40                  logger,
  41                  product_name,
  42                  product_info,
  43                  options = src.options.OptResult(),
  44                  check_src=True):
  45         self.config = config
  46         self.logger = logger
  47         self.options = options
  48         self.product_name = product_name
  49         self.product_info = product_info
  50         self.build_dir = src.Path(self.product_info.build_dir)
  51         self.source_dir = src.Path(self.product_info.source_dir)
  52         self.install_dir = src.Path(self.product_info.install_dir)
  53         self.header = ""
  54         self.debug_mode = False
  55         self.cmake_build_type = 'Release'
  56         if "cmake_build_type" in self.config.APPLICATION:
  57             self.set_cmake_build_type(self.config.APPLICATION.cmake_build_type)
  58         # keep backward compatibility
  59         if "debug" in self.config.APPLICATION and self.config.APPLICATION.debug == "yes":
  60             self.debug_mode = True
  61             self.cmake_build_type = 'Debug'
  62
  63         # in case a product defines its own configuration, then use it
  64         if "cmake_build_type" in self.product_info:
  65             self.set_cmake_build_type(self.product_info.cmake_build_type)
  66         # keep backward compatibility
  67         if "debug" in self.product_info and self.product_info.debug == "yes":
  68             self.debug_mode = True
  69             self.cmake_build_type = 'Debug'
  70
  71         self.verbose_mode = False
  72         if "verbose" in self.product_info and self.product_info.verbose == "yes":
  73             self.verbose_mode = True
  74
  75
  76     # set cmake build type
  77     def set_cmake_build_type(self, build_type):
  78         btype = build_type.lower()
  79         if btype in ['debug', 'relwithdebinfo', 'release', 'minsizerel']:
  80             if btype ==  'debug':
  81                 self.cmake_build_type = 'Debug'
  82                 self.debug_mode = True
  83             elif btype ==  'relwithdebinfo':
  84                 self.cmake_build_type = 'RelWithDebInfo'
  85             elif btype ==  'release':
  86                 self.cmake_build_type = 'Release'
  87             elif btype ==  'minsizerel':
  88                 self.cmake_build_type = 'MinSizeRel'
  89         else:
  90             raise src.SatException("Unknown cmake build mode: {}. Supported values are: Debug, RelWithDebInfo, Release or MinSizeRel".format(build_type))
  91         return
  92
  93     # Shortcut method to log in log file.
  94     def log(self, text, level, showInfo=True):
  95         self.logger.write(text, level, showInfo)
  96         self.logger.logTxtFile.write(src.printcolors.cleancolor(text))
  97         self.logger.flush()
  98
  99     ##
 100     # Shortcut method to log a command.
 101     def log_command(self, command):
 102         self.log("> %s\n" % command, 5)
 103
 104     ##
 105     # Prepares the environment.
 106     # Build two environment: one for building and one for testing (launch).
 107     def prepare(self, add_env_launch=False):
 108
 109         if not self.build_dir.exists():
 110             # create build dir
 111             self.build_dir.make()
 112
 113         self.log('  build_dir   = %s\n' % str(self.build_dir), 4)
 114         self.log('  install_dir = %s\n' % str(self.install_dir), 4)
 115         self.log('\n', 4)
 116
 117         # add products in depend and opt_depend list recursively
 118         environ_info = src.product.get_product_dependencies(self.config,
 119                                                             self.product_name,
 120                                                             self.product_info)
 121         #environ_info.append(self.product_info.name)
 122
 123         # create build environment
 124         self.build_environ = src.environment.SalomeEnviron(self.config,
 125                                       src.environment.Environ(dict(os.environ)),
 126                                       True)
 127         self.build_environ.silent = (self.config.USER.output_verbose_level < 5)
 128         self.build_environ.set_full_environ(self.logger, environ_info)
 129
 130         if add_env_launch:
 131         # create runtime environment
 132             self.launch_environ = src.environment.SalomeEnviron(self.config,
 133                                       src.environment.Environ(dict(os.environ)),
 134                                       False)
 135             self.launch_environ.silent = True # no need to show here
 136             self.launch_environ.set_full_environ(self.logger, environ_info)
 137
 138         for ee in C_COMPILE_ENV_LIST:
 139             vv = self.build_environ.get(ee)
 140             if len(vv) > 0:
 141                 self.log("  %s = %s\n" % (ee, vv), 4, False)
 142
 143         return 0
 144
 145     ##
 146     # Runs cmake with the given options.
 147     def cmake(self, options=""):
 148
 149         cmake_option = options
 150         if "cmake_build_type" in self.config.APPLICATION:
 151             self.set_cmake_build_type(self.config.APPLICATION.cmake_build_type)
 152         # keep backward compatibility
 153         if "debug" in self.config.APPLICATION and self.config.APPLICATION.debug == "yes":
 154             self.debug_mode = True
 155             self.cmake_build_type = 'Debug'
 156
 157         if 'cmake_options' in self.product_info:
 158             cmake_option += " %s " % " ".join(
 159                                         self.product_info.cmake_options.split())
 160
 161         # add cmake build_type options
 162         cmake_option += " -DCMAKE_BUILD_TYPE=" + self.cmake_build_type
 163
 164         # add verbose option if specified in application for this product.
 165         if self.verbose_mode:
 166             cmake_option += " -DCMAKE_VERBOSE_MAKEFILE=ON"
 167
 168         # In case CMAKE_GENERATOR is defined in environment,
 169         # use it in spite of automatically detect it
 170         if 'cmake_generator' in self.config.APPLICATION:
 171             cmake_option += " -G \"%s\" -A x64 " % self.config.APPLICATION.cmake_generator
 172         command = ("cmake %s -DCMAKE_INSTALL_PREFIX=%s %s" %
 173                             (cmake_option, self.install_dir, self.source_dir))
 174
 175         self.log_command(command)
 176         # for key in sorted(self.build_environ.environ.environ.keys()):
 177             # print key, "  ", self.build_environ.environ.environ[key]
 178         res = subprocess.call(command,
 179                               shell=True,
 180                               cwd=str(self.build_dir),
 181                               env=self.build_environ.environ.environ,
 182                               stdout=self.logger.logTxtFile,
 183                               stderr=subprocess.STDOUT)
 184
 185         self.put_txt_log_in_appli_log_dir("cmake")
 186         if res == 0:
 187             return res
 188         else:
 189             return 1
 190
 191     ##
 192     # Runs build_configure with the given options.
 193     def build_configure(self, options=""):
 194
 195         if 'buildconfigure_options' in self.product_info:
 196             options += " %s " % self.product_info.buildconfigure_options
 197
 198         command = str('%s/build_configure') % (self.source_dir)
 199         command = command + " " + options
 200         self.log_command(command)
 201
 202         res = subprocess.call(command,
 203                               shell=True,
 204                               cwd=str(self.build_dir),
 205                               env=self.build_environ.environ.environ,
 206                               stdout=self.logger.logTxtFile,
 207                               stderr=subprocess.STDOUT)
 208         self.put_txt_log_in_appli_log_dir("build_configure")
 209         if res == 0:
 210             return res
 211         else:
 212             return 1
 213
 214     ##
 215     # Runs configure with the given options.
 216     def configure(self, options=""):
 217
 218         if 'configure_options' in self.product_info:
 219             options += " %s " % self.product_info.configure_options
 220
 221         command = "%s/configure --prefix=%s" % (self.source_dir,
 222                                                 str(self.install_dir))
 223
 224         command = command + " " + options
 225         self.log_command(command)
 226
 227         res = subprocess.call(command,
 228                               shell=True,
 229                               cwd=str(self.build_dir),
 230                               env=self.build_environ.environ.environ,
 231                               stdout=self.logger.logTxtFile,
 232                               stderr=subprocess.STDOUT)
 233
 234         self.put_txt_log_in_appli_log_dir("configure")
 235         if res == 0:
 236             return res
 237         else:
 238             return 1
 239
 240     def hack_libtool(self):
 241         if not os.path.exists(str(self.build_dir + 'libtool')):
 242             return
 243
 244         lf = open(os.path.join(str(self.build_dir), "libtool"), 'r')
 245         for line in lf.readlines():
 246             if 'hack_libtool' in line:
 247                 return
 248
 249         # fix libtool by replacing CC="<compil>" with hack_libtool function
 250         hack_command='''sed -i "s%^CC=\\"\(.*\)\\"%hack_libtool() { \\n\\
 251 if test \\"\$(echo \$@ | grep -E '\\\\\\-L/usr/lib(/../lib)?(64)? ')\\" == \\\"\\\" \\n\\
 252   then\\n\\
 253     cmd=\\"\\1 \$@\\"\\n\\
 254   else\\n\\
 255     cmd=\\"\\1 \\"\`echo \$@ | sed -r -e 's|(.*)-L/usr/lib(/../lib)?(64)? (.*)|\\\\\\1\\\\\\4 -L/usr/lib\\\\\\3|g'\`\\n\\
 256   fi\\n\\
 257   \$cmd\\n\\
 258 }\\n\\
 259 CC=\\"hack_libtool\\"%g" libtool'''
 260
 261         self.log_command(hack_command)
 262         subprocess.call(hack_command,
 263                         shell=True,
 264                         cwd=str(self.build_dir),
 265                         env=self.build_environ.environ.environ,
 266                         stdout=self.logger.logTxtFile,
 267                         stderr=subprocess.STDOUT)
 268
 269
 270     ##
 271     # Runs make to build the module.
 272     def make(self, nb_proc, make_opt=""):
 273
 274         # make
 275         command = 'make'
 276         command = command + " -j" + str(nb_proc)
 277         command = command + " " + make_opt
 278         self.log_command(command)
 279         res = subprocess.call(command,
 280                               shell=True,
 281                               cwd=str(self.build_dir),
 282                               env=self.build_environ.environ.environ,
 283                               stdout=self.logger.logTxtFile,
 284                               stderr=subprocess.STDOUT)
 285         self.put_txt_log_in_appli_log_dir("make")
 286         if res == 0:
 287             return res
 288         else:
 289             return 1
 290
 291     ##
 292     # Runs msbuild to build the module.
 293     def wmake(self,nb_proc, opt_nb_proc = None):
 294
 295         hh = 'MSBUILD /m:%s' % str(nb_proc)
 296         if self.debug_mode:
 297             hh += " " + src.printcolors.printcWarning("DEBUG")
 298         # make
 299         command = 'msbuild'
 300         command+= " /maxcpucount:" + str(nb_proc)
 301         command+= " /p:Configuration={}  /p:Platform=x64 ".format(self.cmake_build_type)
 302         command = command + " ALL_BUILD.vcxproj"
 303
 304         self.log_command(command)
 305         res = subprocess.call(command,
 306                               shell=True,
 307                               cwd=str(self.build_dir),
 308                               env=self.build_environ.environ.environ,
 309                               stdout=self.logger.logTxtFile,
 310                               stderr=subprocess.STDOUT)
 311
 312         self.put_txt_log_in_appli_log_dir("make")
 313         if res == 0:
 314             return res
 315         else:
 316             return 1
 317
 318     ##
 319     # Runs 'make install'.
 320     def install(self):
 321         if src.architecture.is_windows():
 322             command = 'msbuild INSTALL.vcxproj'
 323             command+= ' /p:Configuration={}  /p:Platform=x64 '.format(self.cmake_build_type)
 324         else :
 325             command = 'make install'
 326         self.log_command(command)
 327
 328         res = subprocess.call(command,
 329                               shell=True,
 330                               cwd=str(self.build_dir),
 331                               env=self.build_environ.environ.environ,
 332                               stdout=self.logger.logTxtFile,
 333                               stderr=subprocess.STDOUT)
 334
 335         res_check=self.check_install()
 336         if res_check > 0 :
 337             self.log_command("Error in sat check install - some files are not installed!")
 338         self.put_txt_log_in_appli_log_dir("makeinstall")
 339
 340         res+=res_check
 341         if res == 0:
 342             return res
 343         else:
 344             return 1
 345
 346     # this function checks wether a list of file patterns (specified by check_install keyword)
 347     # exixts after the make install. The objective is to ensure the installation is complete.
 348     # patterns are given relatively to the install dir of the product
 349     def check_install(self):
 350         res=0
 351         if "check_install" in self.product_info:
 352             self.log_command("Check installation of files")
 353             for pattern in self.product_info.check_install:
 354                 # pattern is given relatively to the install dir
 355                 complete_pattern=os.path.join(self.product_info.install_dir, pattern)
 356                 self.log_command("    -> check %s" % complete_pattern)
 357                 # expansion of pattern : takes into account environment variables and unix shell rules
 358                 list_of_path=glob.glob(os.path.expandvars(complete_pattern))
 359                 if not list_of_path:
 360                     # we expect to find at least one entry, if not we consider the test failed
 361                     res+=1
 362                     self.logger.write("Error, sat check install failed for file pattern %s\n" % complete_pattern, 1)
 363                     self.log_command("Error, sat check install failed for file pattern %s" % complete_pattern)
 364         return res
 365
 366     ##
 367     # Runs 'make_check'.
 368     def check(self, command=""):
 369         if src.architecture.is_windows():
 370             cmd = 'msbuild RUN_TESTS.vcxproj /p:Configuration={}  /p:Platform=x64 '.format(self.cmake_build_type)
 371         else :
 372             if self.product_info.build_source=="autotools" :
 373                 cmd = 'make check'
 374             else:
 375                 cmd = 'make test'
 376
 377         if command:
 378             cmd = command
 379
 380         self.log_command(cmd)
 381         self.log_command("For more detailed logs, see test logs in %s" % self.build_dir)
 382
 383         res = subprocess.call(cmd,
 384                               shell=True,
 385                               cwd=str(self.build_dir),
 386                               env=self.launch_environ.environ.environ,
 387                               stdout=self.logger.logTxtFile,
 388                               stderr=subprocess.STDOUT)
 389
 390         self.put_txt_log_in_appli_log_dir("makecheck")
 391         if res == 0:
 392             return res
 393         else:
 394             return 1
 395
 396     ##
 397     # Performs a default build for this module.
 398     def do_default_build(self,
 399                          build_conf_options="",
 400                          configure_options="",
 401                          show_warning=True):
 402         use_autotools = False
 403         if 'use_autotools' in self.product_info:
 404             uc = self.product_info.use_autotools
 405             if uc in ['always', 'yes']:
 406                 use_autotools = True
 407             elif uc == 'option':
 408                 use_autotools = self.options.autotools
 409
 410
 411         self.use_autotools = use_autotools
 412
 413         use_ctest = False
 414         if 'use_ctest' in self.product_info:
 415             uc = self.product_info.use_ctest
 416             if uc in ['always', 'yes']:
 417                 use_ctest = True
 418             elif uc == 'option':
 419                 use_ctest = self.options.ctest
 420
 421         self.use_ctest = use_ctest
 422
 423         if show_warning:
 424             cmd = ""
 425             if use_autotools: cmd = "(autotools)"
 426             if use_ctest: cmd = "(ctest)"
 427
 428             self.log("\n", 4, False)
 429             self.log("%(module)s: Run default compilation method %(cmd)s\n" % \
 430                 { "module": self.module, "cmd": cmd }, 4)
 431
 432         if use_autotools:
 433             if not self.prepare(): return self.get_result()
 434             if not self.build_configure(
 435                                    build_conf_options): return self.get_result()
 436             if not self.configure(configure_options): return self.get_result()
 437             if not self.make(): return self.get_result()
 438             if not self.install(): return self.get_result()
 439             if not self.clean(): return self.get_result()
 440
 441         else: # CMake
 442             if self.config.VARS.dist_name=='Win':
 443                 if not self.wprepare(): return self.get_result()
 444                 if not self.cmake(): return self.get_result()
 445                 if not self.wmake(): return self.get_result()
 446                 if not self.install(): return self.get_result()
 447                 if not self.clean(): return self.get_result()
 448             else :
 449                 if not self.prepare(): return self.get_result()
 450                 if not self.cmake(): return self.get_result()
 451                 if not self.make(): return self.get_result()
 452                 if not self.install(): return self.get_result()
 453                 if not self.clean(): return self.get_result()
 454
 455         return self.get_result()
 456
 457     ##
 458     # Performs a build with a script.
 459     def do_python_script_build(self, script, nb_proc):
 460         # script found
 461         self.logger.write(_("Compile %(product)s using script %(script)s\n") % \
 462             { 'product': self.product_info.name,
 463              'script': src.printcolors.printcLabel(script) }, 4)
 464         try:
 465             import imp
 466             product = self.product_info.name
 467             pymodule = imp.load_source(product + "_compile_script", script)
 468             self.nb_proc = nb_proc
 469             retcode = pymodule.compil(self.config, self, self.logger)
 470         except Exception:
 471             __, exceptionValue, exceptionTraceback = sys.exc_info()
 472             self.logger.write(str(exceptionValue), 1)
 473             import traceback
 474             traceback.print_tb(exceptionTraceback)
 475             traceback.print_exc()
 476             retcode = 1
 477         finally:
 478             self.put_txt_log_in_appli_log_dir("script")
 479
 480         return retcode
 481
 482     def complete_environment(self, make_options):
 483         assert self.build_environ is not None
 484         # pass additional variables to environment
 485         # (may be used by the build script)
 486         self.build_environ.set("APPLICATION_NAME", self.config.APPLICATION.name)
 487         self.build_environ.set("SOURCE_DIR", str(self.source_dir))
 488         self.build_environ.set("INSTALL_DIR", str(self.install_dir))
 489         self.build_environ.set("PRODUCT_INSTALL", str(self.install_dir))
 490         self.build_environ.set("BUILD_DIR", str(self.build_dir))
 491         self.build_environ.set("PRODUCT_BUILD", str(self.build_dir))
 492         self.build_environ.set("MAKE_OPTIONS", make_options)
 493         self.build_environ.set("DIST_NAME", self.config.VARS.dist_name)
 494         self.build_environ.set("DIST_VERSION", self.config.VARS.dist_version)
 495         self.build_environ.set("DIST", self.config.VARS.dist)
 496         self.build_environ.set("VERSION", self.product_info.version)
 497         # if product is in hpc mode, set SAT_HPC to 1
 498         # in order for the compilation script to take it into account
 499         if src.product.product_is_hpc(self.product_info):
 500             self.build_environ.set("SAT_HPC", "1")
 501         if self.debug_mode:
 502             self.build_environ.set("SAT_DEBUG", "1")
 503         if "cmake_build_type" in self.config.APPLICATION:
 504             self.set_cmake_build_type(self.config.APPLICATION.cmake_build_type)
 505             self.build_environ.set("SAT_CMAKE_BUILD_TYPE", self.cmake_build_type)
 506         if self.verbose_mode:
 507             self.build_environ.set("SAT_VERBOSE", "1")
 508
 509
 510     def do_batch_script_build(self, script, nb_proc):
 511
 512         if src.architecture.is_windows():
 513             make_options = "/maxcpucount:%s" % nb_proc
 514         else :
 515             make_options = "-j%s" % nb_proc
 516
 517         self.log_command("  " + _("Run build script %s\n") % script)
 518         self.complete_environment(make_options)
 519
 520         res = subprocess.call(script,
 521                               shell=True,
 522                               stdout=self.logger.logTxtFile,
 523                               stderr=subprocess.STDOUT,
 524                               cwd=str(self.build_dir),
 525                               env=self.build_environ.environ.environ)
 526
 527         res_check=self.check_install()
 528         if res_check > 0 :
 529             self.log_command("Error in sat check install - some files are not installed!")
 530
 531         self.put_txt_log_in_appli_log_dir("script")
 532         res += res_check
 533         if res == 0:
 534             return res
 535         else:
 536             return 1
 537
 538     def do_script_build(self, script, number_of_proc=0):
 539         # define make options (may not be used by the script)
 540         if number_of_proc==0:
 541             nb_proc = src.get_cfg_param(self.product_info,"nb_proc", 0)
 542             if nb_proc == 0:
 543                 nb_proc = self.config.VARS.nb_proc
 544         else:
 545             nb_proc = min(number_of_proc, self.config.VARS.nb_proc)
 546
 547         extension = script.split('.')[-1]
 548         if extension in ["bat","sh"]:
 549             return self.do_batch_script_build(script, nb_proc)
 550         if extension == "py":
 551             return self.do_python_script_build(script, nb_proc)
 552
 553         msg = _("The script %s must have .sh, .bat or .py extension." % script)
 554         raise src.SatException(msg)
 555
 556     def put_txt_log_in_appli_log_dir(self, file_name):
 557         '''Put the txt log (that contain the system logs, like make command
 558            output) in the directory <APPLICATION DIR>/LOGS/<product_name>/
 559
 560         :param file_name Str: the name of the file to write
 561         '''
 562         if self.logger.logTxtFile == sys.__stdout__:
 563             return
 564         dir_where_to_put = os.path.join(self.config.APPLICATION.workdir,
 565                                         "LOGS",
 566                                         self.product_info.name)
 567         file_path = os.path.join(dir_where_to_put, file_name)
 568         src.ensure_path_exists(dir_where_to_put)
 569         # write the logTxtFile copy it to the destination, and then recreate
 570         # it as it was
 571         self.logger.logTxtFile.close()
 572         shutil.move(self.logger.txtFilePath, file_path)
 573         self.logger.logTxtFile = open(str(self.logger.txtFilePath), 'w')
 574         self.logger.logTxtFile.write(open(file_path, "r").read())
 575