src/compilation.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import subprocess
  21 import sys
  22 import shutil
  23 import glob
  24
  25 import src
  26
  27 C_COMPILE_ENV_LIST = ["CC",
  28                       "CXX",
  29                       "F77",
  30                       "CFLAGS",
  31                       "CXXFLAGS",
  32                       "LIBS",
  33                       "LDFLAGS"]
  34
  35 class Builder:
  36     """Class to handle all construction steps, like cmake, configure, make, ...
  37     """
  38     def __init__(self,
  39                  config,
  40                  logger,
  41                  product_name,
  42                  product_info,
  43                  options = src.options.OptResult(),
  44                  check_src=True):
  45         self.config = config
  46         self.logger = logger
  47         self.options = options
  48         self.product_name = product_name
  49         self.product_info = product_info
  50         self.build_dir = src.Path(self.product_info.build_dir)
  51         self.source_dir = src.Path(self.product_info.source_dir)
  52         self.install_dir = src.Path(self.product_info.install_dir)
  53         self.header = ""
  54         self.debug_mode = False
  55         self.cmake_build_type = 'Release'
  56         if "cmake_build_type" in self.product_info and self.product_info.cmake_build_type.lower() in ['debug', 'relwithdebinfo', 'release', 'minsizerel']:
  57             if self.product_info.cmake_build_type.lower() ==  'debug':
  58                 self.cmake_build_type = 'Debug'
  59                 self.debug_mode = True
  60             elif self.product_info.cmake_build_type.lower() ==  'relwithdebinfo':
  61                 self.cmake_build_type = 'RelWithDebInfo'
  62             elif self.product_info.cmake_build_type.lower() ==  'release':
  63                 self.cmake_build_type = 'Release'
  64             elif self.product_info.cmake_build_type.lower() ==  'minsizerel':
  65                 self.cmake_build_type = 'MinSizeRel'
  66             else:
  67                 raise src.SatException("Unknown cmake build mode: {}. Supported values are: Debug, RelWithDebInfo, Release or MinSizeRel".format(self.product_info.cmake_build_type))
  68         # keep backward compatibility
  69         if "debug" in self.product_info and self.product_info.debug == "yes":
  70             self.debug_mode = True
  71             self.cmake_build_type = 'Debug'
  72
  73         self.verbose_mode = False
  74         if "verbose" in self.product_info and self.product_info.verbose == "yes":
  75             self.verbose_mode = True
  76
  77     ##
  78     # Shortcut method to log in log file.
  79     def log(self, text, level, showInfo=True):
  80         self.logger.write(text, level, showInfo)
  81         self.logger.logTxtFile.write(src.printcolors.cleancolor(text))
  82         self.logger.flush()
  83
  84     ##
  85     # Shortcut method to log a command.
  86     def log_command(self, command):
  87         self.log("> %s\n" % command, 5)
  88
  89     ##
  90     # Prepares the environment.
  91     # Build two environment: one for building and one for testing (launch).
  92     def prepare(self, add_env_launch=False):
  93
  94         if not self.build_dir.exists():
  95             # create build dir
  96             self.build_dir.make()
  97
  98         self.log('  build_dir   = %s\n' % str(self.build_dir), 4)
  99         self.log('  install_dir = %s\n' % str(self.install_dir), 4)
 100         self.log('\n', 4)
 101
 102         # add products in depend and opt_depend list recursively
 103         environ_info = src.product.get_product_dependencies(self.config,
 104                                                             self.product_name,
 105                                                             self.product_info)
 106         #environ_info.append(self.product_info.name)
 107
 108         # create build environment
 109         self.build_environ = src.environment.SalomeEnviron(self.config,
 110                                       src.environment.Environ(dict(os.environ)),
 111                                       True)
 112         self.build_environ.silent = (self.config.USER.output_verbose_level < 5)
 113         self.build_environ.set_full_environ(self.logger, environ_info)
 114
 115         if add_env_launch:
 116         # create runtime environment
 117             self.launch_environ = src.environment.SalomeEnviron(self.config,
 118                                       src.environment.Environ(dict(os.environ)),
 119                                       False)
 120             self.launch_environ.silent = True # no need to show here
 121             self.launch_environ.set_full_environ(self.logger, environ_info)
 122
 123         for ee in C_COMPILE_ENV_LIST:
 124             vv = self.build_environ.get(ee)
 125             if len(vv) > 0:
 126                 self.log("  %s = %s\n" % (ee, vv), 4, False)
 127
 128         return 0
 129
 130     ##
 131     # Runs cmake with the given options.
 132     def cmake(self, options=""):
 133
 134         cmake_option = options
 135         if "cmake_build_type" in self.config.APPLICATION and self.config.APPLICATION.cmake_build_type.lower() in ['debug', 'relwithdebinfo', 'release', 'minsizerel']:
 136             if self.config.APPLICATION.cmake_build_type.lower() ==  'debug':
 137                 self.cmake_build_type = 'Debug'
 138                 self.debug_mode = True
 139             elif self.config.APPLICATION.cmake_build_type.lower() ==  'relwithdebinfo':
 140                 self.cmake_build_type = 'RelWithDebInfo'
 141             elif self.config.APPLICATION.cmake_build_type.lower() ==  'release':
 142                 self.cmake_build_type = 'Release'
 143             elif self.config.APPLICATION.cmake_build_type.lower() ==  'minsizerel':
 144                 self.cmake_build_type = 'MinSizeRel'
 145             else:
 146                 raise src.SatException("Unknown cmake build mode: {}. Supported values are: Debug, RelWithDebInfo, Release or MinSizeRel".format(self.config.APPLICATION.cmake_build_type))
 147         # keep backward compatibility
 148         if "debug" in self.config.APPLICATION and self.config.APPLICATION.debug == "yes":
 149             self.debug_mode = True
 150             self.cmake_build_type = 'Debug'
 151
 152         if 'cmake_options' in self.product_info:
 153             cmake_option += " %s " % " ".join(
 154                                         self.product_info.cmake_options.split())
 155
 156         # add cmake build_type options
 157         cmake_option += " -DCMAKE_BUILD_TYPE=" + self.cmake_build_type
 158
 159         # add verbose option if specified in application for this product.
 160         if self.verbose_mode:
 161             cmake_option += " -DCMAKE_VERBOSE_MAKEFILE=ON"
 162
 163         # In case CMAKE_GENERATOR is defined in environment,
 164         # use it in spite of automatically detect it
 165         if 'cmake_generator' in self.config.APPLICATION:
 166             cmake_option += " -G \"%s\" -A x64 " % self.config.APPLICATION.cmake_generator
 167         command = ("cmake %s -DCMAKE_INSTALL_PREFIX=%s %s" %
 168                             (cmake_option, self.install_dir, self.source_dir))
 169
 170         self.log_command(command)
 171         # for key in sorted(self.build_environ.environ.environ.keys()):
 172             # print key, "  ", self.build_environ.environ.environ[key]
 173         res = subprocess.call(command,
 174                               shell=True,
 175                               cwd=str(self.build_dir),
 176                               env=self.build_environ.environ.environ,
 177                               stdout=self.logger.logTxtFile,
 178                               stderr=subprocess.STDOUT)
 179
 180         self.put_txt_log_in_appli_log_dir("cmake")
 181         if res == 0:
 182             return res
 183         else:
 184             return 1
 185
 186     ##
 187     # Runs build_configure with the given options.
 188     def build_configure(self, options=""):
 189
 190         if 'buildconfigure_options' in self.product_info:
 191             options += " %s " % self.product_info.buildconfigure_options
 192
 193         command = str('%s/build_configure') % (self.source_dir)
 194         command = command + " " + options
 195         self.log_command(command)
 196
 197         res = subprocess.call(command,
 198                               shell=True,
 199                               cwd=str(self.build_dir),
 200                               env=self.build_environ.environ.environ,
 201                               stdout=self.logger.logTxtFile,
 202                               stderr=subprocess.STDOUT)
 203         self.put_txt_log_in_appli_log_dir("build_configure")
 204         if res == 0:
 205             return res
 206         else:
 207             return 1
 208
 209     ##
 210     # Runs configure with the given options.
 211     def configure(self, options=""):
 212
 213         if 'configure_options' in self.product_info:
 214             options += " %s " % self.product_info.configure_options
 215
 216         command = "%s/configure --prefix=%s" % (self.source_dir,
 217                                                 str(self.install_dir))
 218
 219         command = command + " " + options
 220         self.log_command(command)
 221
 222         res = subprocess.call(command,
 223                               shell=True,
 224                               cwd=str(self.build_dir),
 225                               env=self.build_environ.environ.environ,
 226                               stdout=self.logger.logTxtFile,
 227                               stderr=subprocess.STDOUT)
 228
 229         self.put_txt_log_in_appli_log_dir("configure")
 230         if res == 0:
 231             return res
 232         else:
 233             return 1
 234
 235     def hack_libtool(self):
 236         if not os.path.exists(str(self.build_dir + 'libtool')):
 237             return
 238
 239         lf = open(os.path.join(str(self.build_dir), "libtool"), 'r')
 240         for line in lf.readlines():
 241             if 'hack_libtool' in line:
 242                 return
 243
 244         # fix libtool by replacing CC="<compil>" with hack_libtool function
 245         hack_command='''sed -i "s%^CC=\\"\(.*\)\\"%hack_libtool() { \\n\\
 246 if test \\"\$(echo \$@ | grep -E '\\\\\\-L/usr/lib(/../lib)?(64)? ')\\" == \\\"\\\" \\n\\
 247   then\\n\\
 248     cmd=\\"\\1 \$@\\"\\n\\
 249   else\\n\\
 250     cmd=\\"\\1 \\"\`echo \$@ | sed -r -e 's|(.*)-L/usr/lib(/../lib)?(64)? (.*)|\\\\\\1\\\\\\4 -L/usr/lib\\\\\\3|g'\`\\n\\
 251   fi\\n\\
 252   \$cmd\\n\\
 253 }\\n\\
 254 CC=\\"hack_libtool\\"%g" libtool'''
 255
 256         self.log_command(hack_command)
 257         subprocess.call(hack_command,
 258                         shell=True,
 259                         cwd=str(self.build_dir),
 260                         env=self.build_environ.environ.environ,
 261                         stdout=self.logger.logTxtFile,
 262                         stderr=subprocess.STDOUT)
 263
 264
 265     ##
 266     # Runs make to build the module.
 267     def make(self, nb_proc, make_opt=""):
 268
 269         # make
 270         command = 'make'
 271         command = command + " -j" + str(nb_proc)
 272         command = command + " " + make_opt
 273         self.log_command(command)
 274         res = subprocess.call(command,
 275                               shell=True,
 276                               cwd=str(self.build_dir),
 277                               env=self.build_environ.environ.environ,
 278                               stdout=self.logger.logTxtFile,
 279                               stderr=subprocess.STDOUT)
 280         self.put_txt_log_in_appli_log_dir("make")
 281         if res == 0:
 282             return res
 283         else:
 284             return 1
 285
 286     ##
 287     # Runs msbuild to build the module.
 288     def wmake(self,nb_proc, opt_nb_proc = None):
 289
 290         hh = 'MSBUILD /m:%s' % str(nb_proc)
 291         if self.debug_mode:
 292             hh += " " + src.printcolors.printcWarning("DEBUG")
 293         # make
 294         command = 'msbuild'
 295         command+= " /maxcpucount:" + str(nb_proc)
 296         command+= " /p:Configuration={}  /p:Platform=x64 ".format(self.cmake_build_type)
 297         command = command + " ALL_BUILD.vcxproj"
 298
 299         self.log_command(command)
 300         res = subprocess.call(command,
 301                               shell=True,
 302                               cwd=str(self.build_dir),
 303                               env=self.build_environ.environ.environ,
 304                               stdout=self.logger.logTxtFile,
 305                               stderr=subprocess.STDOUT)
 306
 307         self.put_txt_log_in_appli_log_dir("make")
 308         if res == 0:
 309             return res
 310         else:
 311             return 1
 312
 313     ##
 314     # Runs 'make install'.
 315     def install(self):
 316         if src.architecture.is_windows():
 317             command = 'msbuild INSTALL.vcxproj'
 318             command+= ' /p:Configuration={}  /p:Platform=x64 '.format(self.cmake_build_type)
 319         else :
 320             command = 'make install'
 321         self.log_command(command)
 322
 323         res = subprocess.call(command,
 324                               shell=True,
 325                               cwd=str(self.build_dir),
 326                               env=self.build_environ.environ.environ,
 327                               stdout=self.logger.logTxtFile,
 328                               stderr=subprocess.STDOUT)
 329
 330         res_check=self.check_install()
 331         if res_check > 0 :
 332             self.log_command("Error in sat check install - some files are not installed!")
 333         self.put_txt_log_in_appli_log_dir("makeinstall")
 334
 335         res+=res_check
 336         if res == 0:
 337             return res
 338         else:
 339             return 1
 340
 341     # this function checks wether a list of file patterns (specified by check_install keyword)
 342     # exixts after the make install. The objective is to ensure the installation is complete.
 343     # patterns are given relatively to the install dir of the product
 344     def check_install(self):
 345         res=0
 346         if "check_install" in self.product_info:
 347             self.log_command("Check installation of files")
 348             for pattern in self.product_info.check_install:
 349                 # pattern is given relatively to the install dir
 350                 complete_pattern=os.path.join(self.product_info.install_dir, pattern)
 351                 self.log_command("    -> check %s" % complete_pattern)
 352                 # expansion of pattern : takes into account environment variables and unix shell rules
 353                 list_of_path=glob.glob(os.path.expandvars(complete_pattern))
 354                 if not list_of_path:
 355                     # we expect to find at least one entry, if not we consider the test failed
 356                     res+=1
 357                     self.logger.write("Error, sat check install failed for file pattern %s\n" % complete_pattern, 1)
 358                     self.log_command("Error, sat check install failed for file pattern %s" % complete_pattern)
 359         return res
 360
 361     ##
 362     # Runs 'make_check'.
 363     def check(self, command=""):
 364         if src.architecture.is_windows():
 365             cmd = 'msbuild RUN_TESTS.vcxproj /p:Configuration={}  /p:Platform=x64 '.format(self.cmake_build_type)
 366         else :
 367             if self.product_info.build_source=="autotools" :
 368                 cmd = 'make check'
 369             else:
 370                 cmd = 'make test'
 371
 372         if command:
 373             cmd = command
 374
 375         self.log_command(cmd)
 376         self.log_command("For more detailed logs, see test logs in %s" % self.build_dir)
 377
 378         res = subprocess.call(cmd,
 379                               shell=True,
 380                               cwd=str(self.build_dir),
 381                               env=self.launch_environ.environ.environ,
 382                               stdout=self.logger.logTxtFile,
 383                               stderr=subprocess.STDOUT)
 384
 385         self.put_txt_log_in_appli_log_dir("makecheck")
 386         if res == 0:
 387             return res
 388         else:
 389             return 1
 390
 391     ##
 392     # Performs a default build for this module.
 393     def do_default_build(self,
 394                          build_conf_options="",
 395                          configure_options="",
 396                          show_warning=True):
 397         use_autotools = False
 398         if 'use_autotools' in self.product_info:
 399             uc = self.product_info.use_autotools
 400             if uc in ['always', 'yes']:
 401                 use_autotools = True
 402             elif uc == 'option':
 403                 use_autotools = self.options.autotools
 404
 405
 406         self.use_autotools = use_autotools
 407
 408         use_ctest = False
 409         if 'use_ctest' in self.product_info:
 410             uc = self.product_info.use_ctest
 411             if uc in ['always', 'yes']:
 412                 use_ctest = True
 413             elif uc == 'option':
 414                 use_ctest = self.options.ctest
 415
 416         self.use_ctest = use_ctest
 417
 418         if show_warning:
 419             cmd = ""
 420             if use_autotools: cmd = "(autotools)"
 421             if use_ctest: cmd = "(ctest)"
 422
 423             self.log("\n", 4, False)
 424             self.log("%(module)s: Run default compilation method %(cmd)s\n" % \
 425                 { "module": self.module, "cmd": cmd }, 4)
 426
 427         if use_autotools:
 428             if not self.prepare(): return self.get_result()
 429             if not self.build_configure(
 430                                    build_conf_options): return self.get_result()
 431             if not self.configure(configure_options): return self.get_result()
 432             if not self.make(): return self.get_result()
 433             if not self.install(): return self.get_result()
 434             if not self.clean(): return self.get_result()
 435
 436         else: # CMake
 437             if self.config.VARS.dist_name=='Win':
 438                 if not self.wprepare(): return self.get_result()
 439                 if not self.cmake(): return self.get_result()
 440                 if not self.wmake(): return self.get_result()
 441                 if not self.install(): return self.get_result()
 442                 if not self.clean(): return self.get_result()
 443             else :
 444                 if not self.prepare(): return self.get_result()
 445                 if not self.cmake(): return self.get_result()
 446                 if not self.make(): return self.get_result()
 447                 if not self.install(): return self.get_result()
 448                 if not self.clean(): return self.get_result()
 449
 450         return self.get_result()
 451
 452     ##
 453     # Performs a build with a script.
 454     def do_python_script_build(self, script, nb_proc):
 455         # script found
 456         self.logger.write(_("Compile %(product)s using script %(script)s\n") % \
 457             { 'product': self.product_info.name,
 458              'script': src.printcolors.printcLabel(script) }, 4)
 459         try:
 460             import imp
 461             product = self.product_info.name
 462             pymodule = imp.load_source(product + "_compile_script", script)
 463             self.nb_proc = nb_proc
 464             retcode = pymodule.compil(self.config, self, self.logger)
 465         except Exception:
 466             __, exceptionValue, exceptionTraceback = sys.exc_info()
 467             self.logger.write(str(exceptionValue), 1)
 468             import traceback
 469             traceback.print_tb(exceptionTraceback)
 470             traceback.print_exc()
 471             retcode = 1
 472         finally:
 473             self.put_txt_log_in_appli_log_dir("script")
 474
 475         return retcode
 476
 477     def complete_environment(self, make_options):
 478         assert self.build_environ is not None
 479         # pass additional variables to environment
 480         # (may be used by the build script)
 481         self.build_environ.set("APPLICATION_NAME", self.config.APPLICATION.name)
 482         self.build_environ.set("SOURCE_DIR", str(self.source_dir))
 483         self.build_environ.set("INSTALL_DIR", str(self.install_dir))
 484         self.build_environ.set("PRODUCT_INSTALL", str(self.install_dir))
 485         self.build_environ.set("BUILD_DIR", str(self.build_dir))
 486         self.build_environ.set("PRODUCT_BUILD", str(self.build_dir))
 487         self.build_environ.set("MAKE_OPTIONS", make_options)
 488         self.build_environ.set("DIST_NAME", self.config.VARS.dist_name)
 489         self.build_environ.set("DIST_VERSION", self.config.VARS.dist_version)
 490         self.build_environ.set("DIST", self.config.VARS.dist)
 491         self.build_environ.set("VERSION", self.product_info.version)
 492         # if product is in hpc mode, set SAT_HPC to 1
 493         # in order for the compilation script to take it into account
 494         if src.product.product_is_hpc(self.product_info):
 495             self.build_environ.set("SAT_HPC", "1")
 496         if self.debug_mode:
 497             self.build_environ.set("SAT_DEBUG", "1")
 498         self.build_environ.set("SAT_CMAKE_BUILD_TYPE", self.cmake_build_type)
 499         if self.verbose_mode:
 500             self.build_environ.set("SAT_VERBOSE", "1")
 501
 502
 503     def do_batch_script_build(self, script, nb_proc):
 504
 505         if src.architecture.is_windows():
 506             make_options = "/maxcpucount:%s" % nb_proc
 507         else :
 508             make_options = "-j%s" % nb_proc
 509
 510         self.log_command("  " + _("Run build script %s\n") % script)
 511         self.complete_environment(make_options)
 512
 513         res = subprocess.call(script,
 514                               shell=True,
 515                               stdout=self.logger.logTxtFile,
 516                               stderr=subprocess.STDOUT,
 517                               cwd=str(self.build_dir),
 518                               env=self.build_environ.environ.environ)
 519
 520         res_check=self.check_install()
 521         if res_check > 0 :
 522             self.log_command("Error in sat check install - some files are not installed!")
 523
 524         self.put_txt_log_in_appli_log_dir("script")
 525         res += res_check
 526         if res == 0:
 527             return res
 528         else:
 529             return 1
 530
 531     def do_script_build(self, script, number_of_proc=0):
 532         # define make options (may not be used by the script)
 533         if number_of_proc==0:
 534             nb_proc = src.get_cfg_param(self.product_info,"nb_proc", 0)
 535             if nb_proc == 0:
 536                 nb_proc = self.config.VARS.nb_proc
 537         else:
 538             nb_proc = min(number_of_proc, self.config.VARS.nb_proc)
 539
 540         extension = script.split('.')[-1]
 541         if extension in ["bat","sh"]:
 542             return self.do_batch_script_build(script, nb_proc)
 543         if extension == "py":
 544             return self.do_python_script_build(script, nb_proc)
 545
 546         msg = _("The script %s must have .sh, .bat or .py extension." % script)
 547         raise src.SatException(msg)
 548
 549     def put_txt_log_in_appli_log_dir(self, file_name):
 550         '''Put the txt log (that contain the system logs, like make command
 551            output) in the directory <APPLICATION DIR>/LOGS/<product_name>/
 552
 553         :param file_name Str: the name of the file to write
 554         '''
 555         if self.logger.logTxtFile == sys.__stdout__:
 556             return
 557         dir_where_to_put = os.path.join(self.config.APPLICATION.workdir,
 558                                         "LOGS",
 559                                         self.product_info.name)
 560         file_path = os.path.join(dir_where_to_put, file_name)
 561         src.ensure_path_exists(dir_where_to_put)
 562         # write the logTxtFile copy it to the destination, and then recreate
 563         # it as it was
 564         self.logger.logTxtFile.close()
 565         shutil.move(self.logger.txtFilePath, file_path)
 566         self.logger.logTxtFile = open(str(self.logger.txtFilePath), 'w')
 567         self.logger.logTxtFile.write(open(file_path, "r").read())
 568