src/compilation.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import subprocess
  21 import sys
  22 import shutil
  23 import glob
  24
  25 import src
  26
  27 C_COMPILE_ENV_LIST = ["CC",
  28                       "CXX",
  29                       "F77",
  30                       "CFLAGS",
  31                       "CXXFLAGS",
  32                       "LIBS",
  33                       "LDFLAGS"]
  34
  35 class Builder:
  36     """Class to handle all construction steps, like cmake, configure, make, ...
  37     """
  38     def __init__(self,
  39                  config,
  40                  logger,
  41                  product_name,
  42                  product_info,
  43                  options = src.options.OptResult(),
  44                  check_src=True):
  45         self.config = config
  46         self.logger = logger
  47         self.options = options
  48         self.product_name = product_name
  49         self.product_info = product_info
  50         self.build_dir = src.Path(self.product_info.build_dir)
  51         self.source_dir = src.Path(self.product_info.source_dir)
  52         self.install_dir = src.Path(self.product_info.install_dir)
  53         self.header = ""
  54         self.debug_mode = False
  55         if "debug" in self.product_info and self.product_info.debug == "yes":
  56             self.debug_mode = True
  57         self.verbose_mode = False
  58         if "verbose" in self.product_info and self.product_info.verbose == "yes":
  59             self.verbose_mode = True
  60
  61     ##
  62     # Shortcut method to log in log file.
  63     def log(self, text, level, showInfo=True):
  64         self.logger.write(text, level, showInfo)
  65         self.logger.logTxtFile.write(src.printcolors.cleancolor(text))
  66         self.logger.flush()
  67
  68     ##
  69     # Shortcut method to log a command.
  70     def log_command(self, command):
  71         self.log("> %s\n" % command, 5)
  72
  73     ##
  74     # Prepares the environment.
  75     # Build two environment: one for building and one for testing (launch).
  76     def prepare(self, add_env_launch=False):
  77
  78         if not self.build_dir.exists():
  79             # create build dir
  80             self.build_dir.make()
  81
  82         self.log('  build_dir   = %s\n' % str(self.build_dir), 4)
  83         self.log('  install_dir = %s\n' % str(self.install_dir), 4)
  84         self.log('\n', 4)
  85
  86         # add products in depend and opt_depend list recursively
  87         environ_info = src.product.get_product_dependencies(self.config,
  88                                                             self.product_name,
  89                                                             self.product_info)
  90         #environ_info.append(self.product_info.name)
  91
  92         # create build environment
  93         self.build_environ = src.environment.SalomeEnviron(self.config,
  94                                       src.environment.Environ(dict(os.environ)),
  95                                       True)
  96         self.build_environ.silent = (self.config.USER.output_verbose_level < 5)
  97         self.build_environ.set_full_environ(self.logger, environ_info)
  98
  99         if add_env_launch:
 100         # create runtime environment
 101             self.launch_environ = src.environment.SalomeEnviron(self.config,
 102                                       src.environment.Environ(dict(os.environ)),
 103                                       False)
 104             self.launch_environ.silent = True # no need to show here
 105             self.launch_environ.set_full_environ(self.logger, environ_info)
 106
 107         for ee in C_COMPILE_ENV_LIST:
 108             vv = self.build_environ.get(ee)
 109             if len(vv) > 0:
 110                 self.log("  %s = %s\n" % (ee, vv), 4, False)
 111
 112         return 0
 113
 114     ##
 115     # Runs cmake with the given options.
 116     def cmake(self, options=""):
 117
 118         cmake_option = options
 119         # cmake_option +=' -DCMAKE_VERBOSE_MAKEFILE=ON -DSALOME_CMAKE_DEBUG=ON'
 120         if 'cmake_options' in self.product_info:
 121             cmake_option += " %s " % " ".join(
 122                                         self.product_info.cmake_options.split())
 123
 124         # add debug option
 125         if self.debug_mode:
 126             cmake_option += " -DCMAKE_BUILD_TYPE=Debug"
 127         else :
 128             cmake_option += " -DCMAKE_BUILD_TYPE=Release"
 129
 130         # add verbose option if specified in application for this product.
 131         if self.verbose_mode:
 132             cmake_option += " -DCMAKE_VERBOSE_MAKEFILE=ON"
 133
 134         # In case CMAKE_GENERATOR is defined in environment,
 135         # use it in spite of automatically detect it
 136         if 'cmake_generator' in self.config.APPLICATION:
 137             cmake_option += " -DCMAKE_GENERATOR=\"%s\"" \
 138                                        % self.config.APPLICATION.cmake_generator
 139         command = ("cmake %s -DCMAKE_INSTALL_PREFIX=%s %s" %
 140                             (cmake_option, self.install_dir, self.source_dir))
 141
 142         self.log_command(command)
 143         # for key in sorted(self.build_environ.environ.environ.keys()):
 144             # print key, "  ", self.build_environ.environ.environ[key]
 145         res = subprocess.call(command,
 146                               shell=True,
 147                               cwd=str(self.build_dir),
 148                               env=self.build_environ.environ.environ,
 149                               stdout=self.logger.logTxtFile,
 150                               stderr=subprocess.STDOUT)
 151
 152         self.put_txt_log_in_appli_log_dir("cmake")
 153         if res == 0:
 154             return res
 155         else:
 156             return 1
 157
 158     ##
 159     # Runs build_configure with the given options.
 160     def build_configure(self, options=""):
 161
 162         if 'buildconfigure_options' in self.product_info:
 163             options += " %s " % self.product_info.buildconfigure_options
 164
 165         command = str('%s/build_configure') % (self.source_dir)
 166         command = command + " " + options
 167         self.log_command(command)
 168
 169         res = subprocess.call(command,
 170                               shell=True,
 171                               cwd=str(self.build_dir),
 172                               env=self.build_environ.environ.environ,
 173                               stdout=self.logger.logTxtFile,
 174                               stderr=subprocess.STDOUT)
 175         self.put_txt_log_in_appli_log_dir("build_configure")
 176         if res == 0:
 177             return res
 178         else:
 179             return 1
 180
 181     ##
 182     # Runs configure with the given options.
 183     def configure(self, options=""):
 184
 185         if 'configure_options' in self.product_info:
 186             options += " %s " % self.product_info.configure_options
 187
 188         command = "%s/configure --prefix=%s" % (self.source_dir,
 189                                                 str(self.install_dir))
 190
 191         command = command + " " + options
 192         self.log_command(command)
 193
 194         res = subprocess.call(command,
 195                               shell=True,
 196                               cwd=str(self.build_dir),
 197                               env=self.build_environ.environ.environ,
 198                               stdout=self.logger.logTxtFile,
 199                               stderr=subprocess.STDOUT)
 200
 201         self.put_txt_log_in_appli_log_dir("configure")
 202         if res == 0:
 203             return res
 204         else:
 205             return 1
 206
 207     def hack_libtool(self):
 208         if not os.path.exists(str(self.build_dir + 'libtool')):
 209             return
 210
 211         lf = open(os.path.join(str(self.build_dir), "libtool"), 'r')
 212         for line in lf.readlines():
 213             if 'hack_libtool' in line:
 214                 return
 215
 216         # fix libtool by replacing CC="<compil>" with hack_libtool function
 217         hack_command='''sed -i "s%^CC=\\"\(.*\)\\"%hack_libtool() { \\n\\
 218 if test \\"\$(echo \$@ | grep -E '\\\\\\-L/usr/lib(/../lib)?(64)? ')\\" == \\\"\\\" \\n\\
 219   then\\n\\
 220     cmd=\\"\\1 \$@\\"\\n\\
 221   else\\n\\
 222     cmd=\\"\\1 \\"\`echo \$@ | sed -r -e 's|(.*)-L/usr/lib(/../lib)?(64)? (.*)|\\\\\\1\\\\\\4 -L/usr/lib\\\\\\3|g'\`\\n\\
 223   fi\\n\\
 224   \$cmd\\n\\
 225 }\\n\\
 226 CC=\\"hack_libtool\\"%g" libtool'''
 227
 228         self.log_command(hack_command)
 229         subprocess.call(hack_command,
 230                         shell=True,
 231                         cwd=str(self.build_dir),
 232                         env=self.build_environ.environ.environ,
 233                         stdout=self.logger.logTxtFile,
 234                         stderr=subprocess.STDOUT)
 235
 236
 237     ##
 238     # Runs make to build the module.
 239     def make(self, nb_proc, make_opt=""):
 240
 241         # make
 242         command = 'make'
 243         command = command + " -j" + str(nb_proc)
 244         command = command + " " + make_opt
 245         self.log_command(command)
 246         res = subprocess.call(command,
 247                               shell=True,
 248                               cwd=str(self.build_dir),
 249                               env=self.build_environ.environ.environ,
 250                               stdout=self.logger.logTxtFile,
 251                               stderr=subprocess.STDOUT)
 252         self.put_txt_log_in_appli_log_dir("make")
 253         if res == 0:
 254             return res
 255         else:
 256             return 1
 257
 258     ##
 259     # Runs msbuild to build the module.
 260     def wmake(self,nb_proc, opt_nb_proc = None):
 261
 262         hh = 'MSBUILD /m:%s' % str(nb_proc)
 263         if self.debug_mode:
 264             hh += " " + src.printcolors.printcWarning("DEBUG")
 265         # make
 266         command = 'msbuild'
 267         command = command + " /maxcpucount:" + str(nb_proc)
 268         if self.debug_mode:
 269             command = command + " /p:Configuration=Debug  /p:Platform=x64 "
 270         else:
 271             command = command + " /p:Configuration=Release /p:Platform=x64 "
 272         command = command + " ALL_BUILD.vcxproj"
 273
 274         self.log_command(command)
 275         res = subprocess.call(command,
 276                               shell=True,
 277                               cwd=str(self.build_dir),
 278                               env=self.build_environ.environ.environ,
 279                               stdout=self.logger.logTxtFile,
 280                               stderr=subprocess.STDOUT)
 281
 282         self.put_txt_log_in_appli_log_dir("make")
 283         if res == 0:
 284             return res
 285         else:
 286             return 1
 287
 288     ##
 289     # Runs 'make install'.
 290     def install(self):
 291         if src.architecture.is_windows():
 292             command = 'msbuild INSTALL.vcxproj'
 293             if self.debug_mode:
 294                 command = command + " /p:Configuration=Debug  /p:Platform=x64 "
 295             else:
 296                 command = command + " /p:Configuration=Release  /p:Platform=x64 "
 297         else :
 298             command = 'make install'
 299         self.log_command(command)
 300
 301         res = subprocess.call(command,
 302                               shell=True,
 303                               cwd=str(self.build_dir),
 304                               env=self.build_environ.environ.environ,
 305                               stdout=self.logger.logTxtFile,
 306                               stderr=subprocess.STDOUT)
 307
 308         res_check=self.check_install()
 309         if res_check > 0 :
 310             self.log_command("Error in sat check install - some files are not installed!")
 311         self.put_txt_log_in_appli_log_dir("makeinstall")
 312
 313         res+=res_check
 314         if res == 0:
 315             return res
 316         else:
 317             return 1
 318
 319     # this function checks wether a list of file patterns (specified by check_install keyword)
 320     # exixts after the make install. The objective is to ensure the installation is complete.
 321     # patterns are given relatively to the install dir of the product
 322     def check_install(self):
 323         res=0
 324         if "check_install" in self.product_info:
 325             self.log_command("Check installation of files")
 326             for pattern in self.product_info.check_install:
 327                 # pattern is given relatively to the install dir
 328                 complete_pattern=os.path.join(self.product_info.install_dir, pattern)
 329                 self.log_command("    -> check %s" % complete_pattern)
 330                 # expansion of pattern : takes into account environment variables and unix shell rules
 331                 list_of_path=glob.glob(os.path.expandvars(complete_pattern))
 332                 if not list_of_path:
 333                     # we expect to find at least one entry, if not we consider the test failed
 334                     res+=1
 335                     self.logger.write("Error, sat check install failed for file pattern %s\n" % complete_pattern, 1)
 336                     self.log_command("Error, sat check install failed for file pattern %s" % complete_pattern)
 337         return res
 338
 339     ##
 340     # Runs 'make_check'.
 341     def check(self, command=""):
 342         if src.architecture.is_windows():
 343             cmd = 'msbuild RUN_TESTS.vcxproj /p:Configuration=Release  /p:Platform=x64 '
 344         else :
 345             if self.product_info.build_source=="autotools" :
 346                 cmd = 'make check'
 347             else:
 348                 cmd = 'make test'
 349
 350         if command:
 351             cmd = command
 352
 353         self.log_command(cmd)
 354         self.log_command("For more detailed logs, see test logs in %s" % self.build_dir)
 355
 356         res = subprocess.call(cmd,
 357                               shell=True,
 358                               cwd=str(self.build_dir),
 359                               env=self.launch_environ.environ.environ,
 360                               stdout=self.logger.logTxtFile,
 361                               stderr=subprocess.STDOUT)
 362
 363         self.put_txt_log_in_appli_log_dir("makecheck")
 364         if res == 0:
 365             return res
 366         else:
 367             return 1
 368
 369     ##
 370     # Performs a default build for this module.
 371     def do_default_build(self,
 372                          build_conf_options="",
 373                          configure_options="",
 374                          show_warning=True):
 375         use_autotools = False
 376         if 'use_autotools' in self.product_info:
 377             uc = self.product_info.use_autotools
 378             if uc in ['always', 'yes']:
 379                 use_autotools = True
 380             elif uc == 'option':
 381                 use_autotools = self.options.autotools
 382
 383
 384         self.use_autotools = use_autotools
 385
 386         use_ctest = False
 387         if 'use_ctest' in self.product_info:
 388             uc = self.product_info.use_ctest
 389             if uc in ['always', 'yes']:
 390                 use_ctest = True
 391             elif uc == 'option':
 392                 use_ctest = self.options.ctest
 393
 394         self.use_ctest = use_ctest
 395
 396         if show_warning:
 397             cmd = ""
 398             if use_autotools: cmd = "(autotools)"
 399             if use_ctest: cmd = "(ctest)"
 400
 401             self.log("\n", 4, False)
 402             self.log("%(module)s: Run default compilation method %(cmd)s\n" % \
 403                 { "module": self.module, "cmd": cmd }, 4)
 404
 405         if use_autotools:
 406             if not self.prepare(): return self.get_result()
 407             if not self.build_configure(
 408                                    build_conf_options): return self.get_result()
 409             if not self.configure(configure_options): return self.get_result()
 410             if not self.make(): return self.get_result()
 411             if not self.install(): return self.get_result()
 412             if not self.clean(): return self.get_result()
 413
 414         else: # CMake
 415             if self.config.VARS.dist_name=='Win':
 416                 if not self.wprepare(): return self.get_result()
 417                 if not self.cmake(): return self.get_result()
 418                 if not self.wmake(): return self.get_result()
 419                 if not self.install(): return self.get_result()
 420                 if not self.clean(): return self.get_result()
 421             else :
 422                 if not self.prepare(): return self.get_result()
 423                 if not self.cmake(): return self.get_result()
 424                 if not self.make(): return self.get_result()
 425                 if not self.install(): return self.get_result()
 426                 if not self.clean(): return self.get_result()
 427
 428         return self.get_result()
 429
 430     ##
 431     # Performs a build with a script.
 432     def do_python_script_build(self, script, nb_proc):
 433         # script found
 434         self.logger.write(_("Compile %(product)s using script %(script)s\n") % \
 435             { 'product': self.product_info.name,
 436              'script': src.printcolors.printcLabel(script) }, 4)
 437         try:
 438             import imp
 439             product = self.product_info.name
 440             pymodule = imp.load_source(product + "_compile_script", script)
 441             self.nb_proc = nb_proc
 442             retcode = pymodule.compil(self.config, self, self.logger)
 443         except:
 444             __, exceptionValue, exceptionTraceback = sys.exc_info()
 445             self.logger.write(str(exceptionValue), 1)
 446             import traceback
 447             traceback.print_tb(exceptionTraceback)
 448             traceback.print_exc()
 449             retcode = 1
 450         finally:
 451             self.put_txt_log_in_appli_log_dir("script")
 452
 453         return retcode
 454
 455     def complete_environment(self, make_options):
 456         assert self.build_environ is not None
 457         # pass additional variables to environment
 458         # (may be used by the build script)
 459         self.build_environ.set("APPLICATION_NAME", self.config.APPLICATION.name)
 460         self.build_environ.set("SOURCE_DIR", str(self.source_dir))
 461         self.build_environ.set("INSTALL_DIR", str(self.install_dir))
 462         self.build_environ.set("PRODUCT_INSTALL", str(self.install_dir))
 463         self.build_environ.set("BUILD_DIR", str(self.build_dir))
 464         self.build_environ.set("PRODUCT_BUILD", str(self.build_dir))
 465         self.build_environ.set("MAKE_OPTIONS", make_options)
 466         self.build_environ.set("DIST_NAME", self.config.VARS.dist_name)
 467         self.build_environ.set("DIST_VERSION", self.config.VARS.dist_version)
 468         self.build_environ.set("DIST", self.config.VARS.dist)
 469         self.build_environ.set("VERSION", self.product_info.version)
 470         # if product is in hpc mode, set SAT_HPC to 1
 471         # in order for the compilation script to take it into account
 472         if src.product.product_is_hpc(self.product_info):
 473             self.build_environ.set("SAT_HPC", "1")
 474         if self.debug_mode:
 475             self.build_environ.set("SAT_DEBUG", "1")
 476         if self.verbose_mode:
 477             self.build_environ.set("SAT_VERBOSE", "1")
 478
 479
 480     def do_batch_script_build(self, script, nb_proc):
 481
 482         if src.architecture.is_windows():
 483             make_options = "/maxcpucount:%s" % nb_proc
 484         else :
 485             make_options = "-j%s" % nb_proc
 486
 487         self.log_command("  " + _("Run build script %s\n") % script)
 488         self.complete_environment(make_options)
 489
 490         res = subprocess.call(script,
 491                               shell=True,
 492                               stdout=self.logger.logTxtFile,
 493                               stderr=subprocess.STDOUT,
 494                               cwd=str(self.build_dir),
 495                               env=self.build_environ.environ.environ)
 496
 497         res_check=self.check_install()
 498         if res_check > 0 :
 499             self.log_command("Error in sat check install - some files are not installed!")
 500
 501         self.put_txt_log_in_appli_log_dir("script")
 502         res += res_check
 503         if res == 0:
 504             return res
 505         else:
 506             return 1
 507
 508     def do_script_build(self, script, number_of_proc=0):
 509         # define make options (may not be used by the script)
 510         if number_of_proc==0:
 511             nb_proc = src.get_cfg_param(self.product_info,"nb_proc", 0)
 512             if nb_proc == 0:
 513                 nb_proc = self.config.VARS.nb_proc
 514         else:
 515             nb_proc = min(number_of_proc, self.config.VARS.nb_proc)
 516
 517         extension = script.split('.')[-1]
 518         if extension in ["bat","sh"]:
 519             return self.do_batch_script_build(script, nb_proc)
 520         if extension == "py":
 521             return self.do_python_script_build(script, nb_proc)
 522
 523         msg = _("The script %s must have .sh, .bat or .py extension." % script)
 524         raise src.SatException(msg)
 525
 526     def put_txt_log_in_appli_log_dir(self, file_name):
 527         '''Put the txt log (that contain the system logs, like make command
 528            output) in the directory <APPLICATION DIR>/LOGS/<product_name>/
 529
 530         :param file_name Str: the name of the file to write
 531         '''
 532         if self.logger.logTxtFile == sys.__stdout__:
 533             return
 534         dir_where_to_put = os.path.join(self.config.APPLICATION.workdir,
 535                                         "LOGS",
 536                                         self.product_info.name)
 537         file_path = os.path.join(dir_where_to_put, file_name)
 538         src.ensure_path_exists(dir_where_to_put)
 539         # write the logTxtFile copy it to the destination, and then recreate
 540         # it as it was
 541         self.logger.logTxtFile.close()
 542         shutil.move(self.logger.txtFilePath, file_path)
 543         self.logger.logTxtFile = open(str(self.logger.txtFilePath), 'w')
 544         self.logger.logTxtFile.write(open(file_path, "r").read())
 545