src/compilation.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import subprocess
  21 import sys
  22 import shutil
  23 import glob
  24
  25 import src
  26
  27 C_COMPILE_ENV_LIST = ["CC",
  28                       "CXX",
  29                       "F77",
  30                       "CFLAGS",
  31                       "CXXFLAGS",
  32                       "LIBS",
  33                       "LDFLAGS"]
  34
  35 class Builder:
  36     """Class to handle all construction steps, like cmake, configure, make, ...
  37     """
  38     def __init__(self,
  39                  config,
  40                  logger,
  41                  product_name,
  42                  product_info,
  43                  options = src.options.OptResult(),
  44                  check_src=True):
  45         self.config = config
  46         self.logger = logger
  47         self.options = options
  48         self.product_name = product_name
  49         self.product_info = product_info
  50         self.build_dir = src.Path(self.product_info.build_dir)
  51         self.source_dir = src.Path(self.product_info.source_dir)
  52         self.install_dir = src.Path(self.product_info.install_dir)
  53         self.header = ""
  54         self.debug_mode = False
  55         self.cmake_build_type = 'Release'
  56         if "build_type" in self.product_info and self.product_info.build_type.lower() in ['debug', 'relwithdebinfo', 'release', 'minsizerel']:
  57             if self.product_info.build_type.lower() ==  'debug':
  58                 self.cmake_build_type = 'Debug'
  59                 self.debug_mode = True
  60             elif self.product_info.build_type.lower() ==  'relwithdebinfo':
  61                 self.cmake_build_type = 'RelWithDebInfo'
  62             elif self.product_info.build_type.lower() ==  'release':
  63                 self.cmake_build_type = 'Release'
  64             elif self.product_info.build_type.lower() ==  'minsizerel':
  65                 self.cmake_build_type = 'MinSizeRel'
  66             else:
  67                 raise src.SatException("Unknown cmake build mode: {}. Supported values are: Debug, RelWithDebInfo, Release or MinSizeRel".format(self.product_info.build_type))
  68         # keep backward compatibility
  69         if "debug" in self.product_info and self.product_info.debug == "yes":
  70             self.debug_mode = True
  71             self.cmake_build_type = 'Debug'
  72
  73         self.verbose_mode = False
  74         if "verbose" in self.product_info and self.product_info.verbose == "yes":
  75             self.verbose_mode = True
  76
  77     ##
  78     # Shortcut method to log in log file.
  79     def log(self, text, level, showInfo=True):
  80         self.logger.write(text, level, showInfo)
  81         self.logger.logTxtFile.write(src.printcolors.cleancolor(text))
  82         self.logger.flush()
  83
  84     ##
  85     # Shortcut method to log a command.
  86     def log_command(self, command):
  87         self.log("> %s\n" % command, 5)
  88
  89     ##
  90     # Prepares the environment.
  91     # Build two environment: one for building and one for testing (launch).
  92     def prepare(self, add_env_launch=False):
  93
  94         if not self.build_dir.exists():
  95             # create build dir
  96             self.build_dir.make()
  97
  98         self.log('  build_dir   = %s\n' % str(self.build_dir), 4)
  99         self.log('  install_dir = %s\n' % str(self.install_dir), 4)
 100         self.log('\n', 4)
 101
 102         # add products in depend and opt_depend list recursively
 103         environ_info = src.product.get_product_dependencies(self.config,
 104                                                             self.product_name,
 105                                                             self.product_info)
 106         #environ_info.append(self.product_info.name)
 107
 108         # create build environment
 109         self.build_environ = src.environment.SalomeEnviron(self.config,
 110                                       src.environment.Environ(dict(os.environ)),
 111                                       True)
 112         self.build_environ.silent = (self.config.USER.output_verbose_level < 5)
 113         self.build_environ.set_full_environ(self.logger, environ_info)
 114
 115         if add_env_launch:
 116         # create runtime environment
 117             self.launch_environ = src.environment.SalomeEnviron(self.config,
 118                                       src.environment.Environ(dict(os.environ)),
 119                                       False)
 120             self.launch_environ.silent = True # no need to show here
 121             self.launch_environ.set_full_environ(self.logger, environ_info)
 122
 123         for ee in C_COMPILE_ENV_LIST:
 124             vv = self.build_environ.get(ee)
 125             if len(vv) > 0:
 126                 self.log("  %s = %s\n" % (ee, vv), 4, False)
 127
 128         return 0
 129
 130     ##
 131     # Runs cmake with the given options.
 132     def cmake(self, options=""):
 133
 134         cmake_option = options
 135         # cmake_option +=' -DCMAKE_VERBOSE_MAKEFILE=ON -DSALOME_CMAKE_DEBUG=ON'
 136         if 'cmake_options' in self.product_info:
 137             cmake_option += " %s " % " ".join(
 138                                         self.product_info.cmake_options.split())
 139
 140         # add cmake build_type options
 141         cmake_option += " -DCMAKE_BUILD_TYPE=" + self.cmake_build_type
 142
 143         # add verbose option if specified in application for this product.
 144         if self.verbose_mode:
 145             cmake_option += " -DCMAKE_VERBOSE_MAKEFILE=ON"
 146
 147         # In case CMAKE_GENERATOR is defined in environment,
 148         # use it in spite of automatically detect it
 149         if 'cmake_generator' in self.config.APPLICATION:
 150             cmake_option += " -G \"%s\" -A x64 " % self.config.APPLICATION.cmake_generator
 151         command = ("cmake %s -DCMAKE_INSTALL_PREFIX=%s %s" %
 152                             (cmake_option, self.install_dir, self.source_dir))
 153
 154         self.log_command(command)
 155         # for key in sorted(self.build_environ.environ.environ.keys()):
 156             # print key, "  ", self.build_environ.environ.environ[key]
 157         res = subprocess.call(command,
 158                               shell=True,
 159                               cwd=str(self.build_dir),
 160                               env=self.build_environ.environ.environ,
 161                               stdout=self.logger.logTxtFile,
 162                               stderr=subprocess.STDOUT)
 163
 164         self.put_txt_log_in_appli_log_dir("cmake")
 165         if res == 0:
 166             return res
 167         else:
 168             return 1
 169
 170     ##
 171     # Runs build_configure with the given options.
 172     def build_configure(self, options=""):
 173
 174         if 'buildconfigure_options' in self.product_info:
 175             options += " %s " % self.product_info.buildconfigure_options
 176
 177         command = str('%s/build_configure') % (self.source_dir)
 178         command = command + " " + options
 179         self.log_command(command)
 180
 181         res = subprocess.call(command,
 182                               shell=True,
 183                               cwd=str(self.build_dir),
 184                               env=self.build_environ.environ.environ,
 185                               stdout=self.logger.logTxtFile,
 186                               stderr=subprocess.STDOUT)
 187         self.put_txt_log_in_appli_log_dir("build_configure")
 188         if res == 0:
 189             return res
 190         else:
 191             return 1
 192
 193     ##
 194     # Runs configure with the given options.
 195     def configure(self, options=""):
 196
 197         if 'configure_options' in self.product_info:
 198             options += " %s " % self.product_info.configure_options
 199
 200         command = "%s/configure --prefix=%s" % (self.source_dir,
 201                                                 str(self.install_dir))
 202
 203         command = command + " " + options
 204         self.log_command(command)
 205
 206         res = subprocess.call(command,
 207                               shell=True,
 208                               cwd=str(self.build_dir),
 209                               env=self.build_environ.environ.environ,
 210                               stdout=self.logger.logTxtFile,
 211                               stderr=subprocess.STDOUT)
 212
 213         self.put_txt_log_in_appli_log_dir("configure")
 214         if res == 0:
 215             return res
 216         else:
 217             return 1
 218
 219     def hack_libtool(self):
 220         if not os.path.exists(str(self.build_dir + 'libtool')):
 221             return
 222
 223         lf = open(os.path.join(str(self.build_dir), "libtool"), 'r')
 224         for line in lf.readlines():
 225             if 'hack_libtool' in line:
 226                 return
 227
 228         # fix libtool by replacing CC="<compil>" with hack_libtool function
 229         hack_command='''sed -i "s%^CC=\\"\(.*\)\\"%hack_libtool() { \\n\\
 230 if test \\"\$(echo \$@ | grep -E '\\\\\\-L/usr/lib(/../lib)?(64)? ')\\" == \\\"\\\" \\n\\
 231   then\\n\\
 232     cmd=\\"\\1 \$@\\"\\n\\
 233   else\\n\\
 234     cmd=\\"\\1 \\"\`echo \$@ | sed -r -e 's|(.*)-L/usr/lib(/../lib)?(64)? (.*)|\\\\\\1\\\\\\4 -L/usr/lib\\\\\\3|g'\`\\n\\
 235   fi\\n\\
 236   \$cmd\\n\\
 237 }\\n\\
 238 CC=\\"hack_libtool\\"%g" libtool'''
 239
 240         self.log_command(hack_command)
 241         subprocess.call(hack_command,
 242                         shell=True,
 243                         cwd=str(self.build_dir),
 244                         env=self.build_environ.environ.environ,
 245                         stdout=self.logger.logTxtFile,
 246                         stderr=subprocess.STDOUT)
 247
 248
 249     ##
 250     # Runs make to build the module.
 251     def make(self, nb_proc, make_opt=""):
 252
 253         # make
 254         command = 'make'
 255         command = command + " -j" + str(nb_proc)
 256         command = command + " " + make_opt
 257         self.log_command(command)
 258         res = subprocess.call(command,
 259                               shell=True,
 260                               cwd=str(self.build_dir),
 261                               env=self.build_environ.environ.environ,
 262                               stdout=self.logger.logTxtFile,
 263                               stderr=subprocess.STDOUT)
 264         self.put_txt_log_in_appli_log_dir("make")
 265         if res == 0:
 266             return res
 267         else:
 268             return 1
 269
 270     ##
 271     # Runs msbuild to build the module.
 272     def wmake(self,nb_proc, opt_nb_proc = None):
 273
 274         hh = 'MSBUILD /m:%s' % str(nb_proc)
 275         if self.debug_mode:
 276             hh += " " + src.printcolors.printcWarning("DEBUG")
 277         # make
 278         command = 'msbuild'
 279         command+= " /maxcpucount:" + str(nb_proc)
 280         command+= " /p:Configuration={}  /p:Platform=x64 ".format(self.cmake_build_type)
 281         command = command + " ALL_BUILD.vcxproj"
 282
 283         self.log_command(command)
 284         res = subprocess.call(command,
 285                               shell=True,
 286                               cwd=str(self.build_dir),
 287                               env=self.build_environ.environ.environ,
 288                               stdout=self.logger.logTxtFile,
 289                               stderr=subprocess.STDOUT)
 290
 291         self.put_txt_log_in_appli_log_dir("make")
 292         if res == 0:
 293             return res
 294         else:
 295             return 1
 296
 297     ##
 298     # Runs 'make install'.
 299     def install(self):
 300         if src.architecture.is_windows():
 301             command = 'msbuild INSTALL.vcxproj'
 302             command+= ' /p:Configuration={}  /p:Platform=x64 '.format(self.cmake_build_type)
 303         else :
 304             command = 'make install'
 305         self.log_command(command)
 306
 307         res = subprocess.call(command,
 308                               shell=True,
 309                               cwd=str(self.build_dir),
 310                               env=self.build_environ.environ.environ,
 311                               stdout=self.logger.logTxtFile,
 312                               stderr=subprocess.STDOUT)
 313
 314         res_check=self.check_install()
 315         if res_check > 0 :
 316             self.log_command("Error in sat check install - some files are not installed!")
 317         self.put_txt_log_in_appli_log_dir("makeinstall")
 318
 319         res+=res_check
 320         if res == 0:
 321             return res
 322         else:
 323             return 1
 324
 325     # this function checks wether a list of file patterns (specified by check_install keyword)
 326     # exixts after the make install. The objective is to ensure the installation is complete.
 327     # patterns are given relatively to the install dir of the product
 328     def check_install(self):
 329         res=0
 330         if "check_install" in self.product_info:
 331             self.log_command("Check installation of files")
 332             for pattern in self.product_info.check_install:
 333                 # pattern is given relatively to the install dir
 334                 complete_pattern=os.path.join(self.product_info.install_dir, pattern)
 335                 self.log_command("    -> check %s" % complete_pattern)
 336                 # expansion of pattern : takes into account environment variables and unix shell rules
 337                 list_of_path=glob.glob(os.path.expandvars(complete_pattern))
 338                 if not list_of_path:
 339                     # we expect to find at least one entry, if not we consider the test failed
 340                     res+=1
 341                     self.logger.write("Error, sat check install failed for file pattern %s\n" % complete_pattern, 1)
 342                     self.log_command("Error, sat check install failed for file pattern %s" % complete_pattern)
 343         return res
 344
 345     ##
 346     # Runs 'make_check'.
 347     def check(self, command=""):
 348         if src.architecture.is_windows():
 349             cmd = 'msbuild RUN_TESTS.vcxproj /p:Configuration={}  /p:Platform=x64 '.format(self.cmake_build_type)
 350         else :
 351             if self.product_info.build_source=="autotools" :
 352                 cmd = 'make check'
 353             else:
 354                 cmd = 'make test'
 355
 356         if command:
 357             cmd = command
 358
 359         self.log_command(cmd)
 360         self.log_command("For more detailed logs, see test logs in %s" % self.build_dir)
 361
 362         res = subprocess.call(cmd,
 363                               shell=True,
 364                               cwd=str(self.build_dir),
 365                               env=self.launch_environ.environ.environ,
 366                               stdout=self.logger.logTxtFile,
 367                               stderr=subprocess.STDOUT)
 368
 369         self.put_txt_log_in_appli_log_dir("makecheck")
 370         if res == 0:
 371             return res
 372         else:
 373             return 1
 374
 375     ##
 376     # Performs a default build for this module.
 377     def do_default_build(self,
 378                          build_conf_options="",
 379                          configure_options="",
 380                          show_warning=True):
 381         use_autotools = False
 382         if 'use_autotools' in self.product_info:
 383             uc = self.product_info.use_autotools
 384             if uc in ['always', 'yes']:
 385                 use_autotools = True
 386             elif uc == 'option':
 387                 use_autotools = self.options.autotools
 388
 389
 390         self.use_autotools = use_autotools
 391
 392         use_ctest = False
 393         if 'use_ctest' in self.product_info:
 394             uc = self.product_info.use_ctest
 395             if uc in ['always', 'yes']:
 396                 use_ctest = True
 397             elif uc == 'option':
 398                 use_ctest = self.options.ctest
 399
 400         self.use_ctest = use_ctest
 401
 402         if show_warning:
 403             cmd = ""
 404             if use_autotools: cmd = "(autotools)"
 405             if use_ctest: cmd = "(ctest)"
 406
 407             self.log("\n", 4, False)
 408             self.log("%(module)s: Run default compilation method %(cmd)s\n" % \
 409                 { "module": self.module, "cmd": cmd }, 4)
 410
 411         if use_autotools:
 412             if not self.prepare(): return self.get_result()
 413             if not self.build_configure(
 414                                    build_conf_options): return self.get_result()
 415             if not self.configure(configure_options): return self.get_result()
 416             if not self.make(): return self.get_result()
 417             if not self.install(): return self.get_result()
 418             if not self.clean(): return self.get_result()
 419
 420         else: # CMake
 421             if self.config.VARS.dist_name=='Win':
 422                 if not self.wprepare(): return self.get_result()
 423                 if not self.cmake(): return self.get_result()
 424                 if not self.wmake(): return self.get_result()
 425                 if not self.install(): return self.get_result()
 426                 if not self.clean(): return self.get_result()
 427             else :
 428                 if not self.prepare(): return self.get_result()
 429                 if not self.cmake(): return self.get_result()
 430                 if not self.make(): return self.get_result()
 431                 if not self.install(): return self.get_result()
 432                 if not self.clean(): return self.get_result()
 433
 434         return self.get_result()
 435
 436     ##
 437     # Performs a build with a script.
 438     def do_python_script_build(self, script, nb_proc):
 439         # script found
 440         self.logger.write(_("Compile %(product)s using script %(script)s\n") % \
 441             { 'product': self.product_info.name,
 442              'script': src.printcolors.printcLabel(script) }, 4)
 443         try:
 444             import imp
 445             product = self.product_info.name
 446             pymodule = imp.load_source(product + "_compile_script", script)
 447             self.nb_proc = nb_proc
 448             retcode = pymodule.compil(self.config, self, self.logger)
 449         except Exception:
 450             __, exceptionValue, exceptionTraceback = sys.exc_info()
 451             self.logger.write(str(exceptionValue), 1)
 452             import traceback
 453             traceback.print_tb(exceptionTraceback)
 454             traceback.print_exc()
 455             retcode = 1
 456         finally:
 457             self.put_txt_log_in_appli_log_dir("script")
 458
 459         return retcode
 460
 461     def complete_environment(self, make_options):
 462         assert self.build_environ is not None
 463         # pass additional variables to environment
 464         # (may be used by the build script)
 465         self.build_environ.set("APPLICATION_NAME", self.config.APPLICATION.name)
 466         self.build_environ.set("SOURCE_DIR", str(self.source_dir))
 467         self.build_environ.set("INSTALL_DIR", str(self.install_dir))
 468         self.build_environ.set("PRODUCT_INSTALL", str(self.install_dir))
 469         self.build_environ.set("BUILD_DIR", str(self.build_dir))
 470         self.build_environ.set("PRODUCT_BUILD", str(self.build_dir))
 471         self.build_environ.set("MAKE_OPTIONS", make_options)
 472         self.build_environ.set("DIST_NAME", self.config.VARS.dist_name)
 473         self.build_environ.set("DIST_VERSION", self.config.VARS.dist_version)
 474         self.build_environ.set("DIST", self.config.VARS.dist)
 475         self.build_environ.set("VERSION", self.product_info.version)
 476         # if product is in hpc mode, set SAT_HPC to 1
 477         # in order for the compilation script to take it into account
 478         if src.product.product_is_hpc(self.product_info):
 479             self.build_environ.set("SAT_HPC", "1")
 480         if self.debug_mode:
 481             self.build_environ.set("SAT_DEBUG", "1")
 482         self.build_environ.set("SAT_CMAKE_BUILD_TYPE", self.cmake_build_type)
 483         if self.verbose_mode:
 484             self.build_environ.set("SAT_VERBOSE", "1")
 485
 486
 487     def do_batch_script_build(self, script, nb_proc):
 488
 489         if src.architecture.is_windows():
 490             make_options = "/maxcpucount:%s" % nb_proc
 491         else :
 492             make_options = "-j%s" % nb_proc
 493
 494         self.log_command("  " + _("Run build script %s\n") % script)
 495         self.complete_environment(make_options)
 496
 497         res = subprocess.call(script,
 498                               shell=True,
 499                               stdout=self.logger.logTxtFile,
 500                               stderr=subprocess.STDOUT,
 501                               cwd=str(self.build_dir),
 502                               env=self.build_environ.environ.environ)
 503
 504         res_check=self.check_install()
 505         if res_check > 0 :
 506             self.log_command("Error in sat check install - some files are not installed!")
 507
 508         self.put_txt_log_in_appli_log_dir("script")
 509         res += res_check
 510         if res == 0:
 511             return res
 512         else:
 513             return 1
 514
 515     def do_script_build(self, script, number_of_proc=0):
 516         # define make options (may not be used by the script)
 517         if number_of_proc==0:
 518             nb_proc = src.get_cfg_param(self.product_info,"nb_proc", 0)
 519             if nb_proc == 0:
 520                 nb_proc = self.config.VARS.nb_proc
 521         else:
 522             nb_proc = min(number_of_proc, self.config.VARS.nb_proc)
 523
 524         extension = script.split('.')[-1]
 525         if extension in ["bat","sh"]:
 526             return self.do_batch_script_build(script, nb_proc)
 527         if extension == "py":
 528             return self.do_python_script_build(script, nb_proc)
 529
 530         msg = _("The script %s must have .sh, .bat or .py extension." % script)
 531         raise src.SatException(msg)
 532
 533     def put_txt_log_in_appli_log_dir(self, file_name):
 534         '''Put the txt log (that contain the system logs, like make command
 535            output) in the directory <APPLICATION DIR>/LOGS/<product_name>/
 536
 537         :param file_name Str: the name of the file to write
 538         '''
 539         if self.logger.logTxtFile == sys.__stdout__:
 540             return
 541         dir_where_to_put = os.path.join(self.config.APPLICATION.workdir,
 542                                         "LOGS",
 543                                         self.product_info.name)
 544         file_path = os.path.join(dir_where_to_put, file_name)
 545         src.ensure_path_exists(dir_where_to_put)
 546         # write the logTxtFile copy it to the destination, and then recreate
 547         # it as it was
 548         self.logger.logTxtFile.close()
 549         shutil.move(self.logger.txtFilePath, file_path)
 550         self.logger.logTxtFile = open(str(self.logger.txtFilePath), 'w')
 551         self.logger.logTxtFile.write(open(file_path, "r").read())
 552