src/compilation.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import subprocess
  21 import sys
  22 import shutil
  23
  24 import src
  25
  26 C_COMPILE_ENV_LIST = ["CC",
  27                       "CXX",
  28                       "F77",
  29                       "CFLAGS",
  30                       "CXXFLAGS",
  31                       "LIBS",
  32                       "LDFLAGS"]
  33
  34 class Builder:
  35     """Class to handle all construction steps, like cmake, configure, make, ...
  36     """
  37     def __init__(self,
  38                  config,
  39                  logger,
  40                  product_info,
  41                  options = src.options.OptResult(),
  42                  check_src=True):
  43         self.config = config
  44         self.logger = logger
  45         self.options = options
  46         self.product_info = product_info
  47         self.build_dir = src.Path(self.product_info.build_dir)
  48         self.source_dir = src.Path(self.product_info.source_dir)
  49         self.install_dir = src.Path(self.product_info.install_dir)
  50         self.header = ""
  51         self.debug_mode = False
  52         if "debug" in self.product_info and self.product_info.debug == "yes":
  53             self.debug_mode = True
  54         self.verbose_mode = False
  55         if "verbose" in self.product_info and self.product_info.verbose == "yes":
  56             self.verbose_mode = True
  57
  58     ##
  59     # Shortcut method to log in log file.
  60     def log(self, text, level, showInfo=True):
  61         self.logger.write(text, level, showInfo)
  62         self.logger.logTxtFile.write(src.printcolors.cleancolor(text))
  63         self.logger.flush()
  64
  65     ##
  66     # Shortcut method to log a command.
  67     def log_command(self, command):
  68         self.log("> %s\n" % command, 5)
  69
  70     ##
  71     # Prepares the environment.
  72     # Build two environment: one for building and one for testing (launch).
  73     def prepare(self):
  74
  75         if not self.build_dir.exists():
  76             # create build dir
  77             self.build_dir.make()
  78
  79         self.log('  build_dir   = %s\n' % str(self.build_dir), 4)
  80         self.log('  install_dir = %s\n' % str(self.install_dir), 4)
  81         self.log('\n', 4)
  82
  83         # add products in depend and opt_depend list recursively
  84         environ_info = src.product.get_product_dependencies(self.config,
  85                                                             self.product_info)
  86         #environ_info.append(self.product_info.name)
  87
  88         # create build environment
  89         self.build_environ = src.environment.SalomeEnviron(self.config,
  90                                       src.environment.Environ(dict(os.environ)),
  91                                       True)
  92         self.build_environ.silent = (self.config.USER.output_verbose_level < 5)
  93         self.build_environ.set_full_environ(self.logger, environ_info)
  94
  95         # create runtime environment
  96         self.launch_environ = src.environment.SalomeEnviron(self.config,
  97                                       src.environment.Environ(dict(os.environ)),
  98                                       False)
  99         self.launch_environ.silent = True # no need to show here
 100         self.launch_environ.set_full_environ(self.logger, environ_info)
 101
 102         for ee in C_COMPILE_ENV_LIST:
 103             vv = self.build_environ.get(ee)
 104             if len(vv) > 0:
 105                 self.log("  %s = %s\n" % (ee, vv), 4, False)
 106
 107         return 0
 108
 109     ##
 110     # Runs cmake with the given options.
 111     def cmake(self, options=""):
 112
 113         cmake_option = options
 114         # cmake_option +=' -DCMAKE_VERBOSE_MAKEFILE=ON -DSALOME_CMAKE_DEBUG=ON'
 115         if 'cmake_options' in self.product_info:
 116             cmake_option += " %s " % " ".join(
 117                                         self.product_info.cmake_options.split())
 118
 119         # add debug option
 120         if self.debug_mode:
 121             cmake_option += " -DCMAKE_BUILD_TYPE=Debug"
 122         else :
 123             cmake_option += " -DCMAKE_BUILD_TYPE=Release"
 124
 125         # add verbose option if specified in application for this product.
 126         if self.verbose_mode:
 127             cmake_option += " -DCMAKE_VERBOSE_MAKEFILE=ON"
 128
 129         # In case CMAKE_GENERATOR is defined in environment,
 130         # use it in spite of automatically detect it
 131         if 'cmake_generator' in self.config.APPLICATION:
 132             cmake_option += " -DCMAKE_GENERATOR=%s" \
 133                                        % self.config.APPLICATION.cmake_generator
 134
 135         command = ("cmake %s -DCMAKE_INSTALL_PREFIX=%s %s" %
 136                             (cmake_option, self.install_dir, self.source_dir))
 137
 138         self.log_command(command)
 139         # for key in sorted(self.build_environ.environ.environ.keys()):
 140             # print key, "  ", self.build_environ.environ.environ[key]
 141         res = subprocess.call(command,
 142                               shell=True,
 143                               cwd=str(self.build_dir),
 144                               env=self.build_environ.environ.environ,
 145                               stdout=self.logger.logTxtFile,
 146                               stderr=subprocess.STDOUT)
 147
 148         self.put_txt_log_in_appli_log_dir("cmake")
 149         if res == 0:
 150             return res
 151         else:
 152             return 1
 153
 154     ##
 155     # Runs build_configure with the given options.
 156     def build_configure(self, options=""):
 157
 158         if 'buildconfigure_options' in self.product_info:
 159             options += " %s " % self.product_info.buildconfigure_options
 160
 161         command = str('%s/build_configure') % (self.source_dir)
 162         command = command + " " + options
 163         self.log_command(command)
 164
 165         res = subprocess.call(command,
 166                               shell=True,
 167                               cwd=str(self.build_dir),
 168                               env=self.build_environ.environ.environ,
 169                               stdout=self.logger.logTxtFile,
 170                               stderr=subprocess.STDOUT)
 171         self.put_txt_log_in_appli_log_dir("build_configure")
 172         if res == 0:
 173             return res
 174         else:
 175             return 1
 176
 177     ##
 178     # Runs configure with the given options.
 179     def configure(self, options=""):
 180
 181         if 'configure_options' in self.product_info:
 182             options += " %s " % self.product_info.configure_options
 183
 184         command = "%s/configure --prefix=%s" % (self.source_dir,
 185                                                 str(self.install_dir))
 186
 187         command = command + " " + options
 188         self.log_command(command)
 189
 190         res = subprocess.call(command,
 191                               shell=True,
 192                               cwd=str(self.build_dir),
 193                               env=self.build_environ.environ.environ,
 194                               stdout=self.logger.logTxtFile,
 195                               stderr=subprocess.STDOUT)
 196
 197         self.put_txt_log_in_appli_log_dir("configure")
 198         if res == 0:
 199             return res
 200         else:
 201             return 1
 202
 203     def hack_libtool(self):
 204         if not os.path.exists(str(self.build_dir + 'libtool')):
 205             return
 206
 207         lf = open(os.path.join(str(self.build_dir), "libtool"), 'r')
 208         for line in lf.readlines():
 209             if 'hack_libtool' in line:
 210                 return
 211
 212         # fix libtool by replacing CC="<compil>" with hack_libtool function
 213         hack_command='''sed -i "s%^CC=\\"\(.*\)\\"%hack_libtool() { \\n\\
 214 if test \\"\$(echo \$@ | grep -E '\\\\\\-L/usr/lib(/../lib)?(64)? ')\\" == \\\"\\\" \\n\\
 215   then\\n\\
 216     cmd=\\"\\1 \$@\\"\\n\\
 217   else\\n\\
 218     cmd=\\"\\1 \\"\`echo \$@ | sed -r -e 's|(.*)-L/usr/lib(/../lib)?(64)? (.*)|\\\\\\1\\\\\\4 -L/usr/lib\\\\\\3|g'\`\\n\\
 219   fi\\n\\
 220   \$cmd\\n\\
 221 }\\n\\
 222 CC=\\"hack_libtool\\"%g" libtool'''
 223
 224         self.log_command(hack_command)
 225         subprocess.call(hack_command,
 226                         shell=True,
 227                         cwd=str(self.build_dir),
 228                         env=self.build_environ.environ.environ,
 229                         stdout=self.logger.logTxtFile,
 230                         stderr=subprocess.STDOUT)
 231
 232
 233     ##
 234     # Runs make to build the module.
 235     def make(self, nb_proc, make_opt=""):
 236
 237         # make
 238         command = 'make'
 239         command = command + " -j" + str(nb_proc)
 240         command = command + " " + make_opt
 241         self.log_command(command)
 242         res = subprocess.call(command,
 243                               shell=True,
 244                               cwd=str(self.build_dir),
 245                               env=self.build_environ.environ.environ,
 246                               stdout=self.logger.logTxtFile,
 247                               stderr=subprocess.STDOUT)
 248         self.put_txt_log_in_appli_log_dir("make")
 249         if res == 0:
 250             return res
 251         else:
 252             return 1
 253
 254     ##
 255     # Runs msbuild to build the module.
 256     def wmake(self,nb_proc, opt_nb_proc = None):
 257
 258         hh = 'MSBUILD /m:%s' % str(nb_proc)
 259         if self.debug_mode:
 260             hh += " " + src.printcolors.printcWarning("DEBUG")
 261         # make
 262         command = 'msbuild'
 263         command = command + " /maxcpucount:" + str(nb_proc)
 264         if self.debug_mode:
 265             command = command + " /p:Configuration=Debug"
 266         else:
 267             command = command + " /p:Configuration=Release"
 268         command = command + " ALL_BUILD.vcxproj"
 269
 270         self.log_command(command)
 271         res = subprocess.call(command,
 272                               shell=True,
 273                               cwd=str(self.build_dir),
 274                               env=self.build_environ.environ.environ,
 275                               stdout=self.logger.logTxtFile,
 276                               stderr=subprocess.STDOUT)
 277
 278         self.put_txt_log_in_appli_log_dir("make")
 279         if res == 0:
 280             return res
 281         else:
 282             return 1
 283
 284     ##
 285     # Runs 'make install'.
 286     def install(self):
 287         if self.config.VARS.dist_name=="Win":
 288             command = 'msbuild INSTALL.vcxproj'
 289             if self.debug_mode:
 290                 command = command + " /p:Configuration=Debug"
 291             else:
 292                 command = command + " /p:Configuration=Release"
 293         else :
 294             command = 'make install'
 295
 296         self.log_command(command)
 297
 298         res = subprocess.call(command,
 299                               shell=True,
 300                               cwd=str(self.build_dir),
 301                               env=self.build_environ.environ.environ,
 302                               stdout=self.logger.logTxtFile,
 303                               stderr=subprocess.STDOUT)
 304
 305         res_check=self.check_install()
 306         if res_check > 0 :
 307             self.log_command("Error in sat check install - some files are not installed!")
 308         self.put_txt_log_in_appli_log_dir("makeinstall")
 309
 310         res+=res_check
 311         if res == 0:
 312             return res
 313         else:
 314             return 1
 315
 316     def check_install(self):
 317         res=0
 318         if "check_install" in self.product_info:
 319             self.log_command("Check installation of files")
 320             for f in self.product_info.check_install:
 321                 complete_path=os.path.join(self.product_info.install_dir, f)
 322                 self.log_command("    -> check %s" % complete_path)
 323                 if os.path.isfile(complete_path) == False :
 324                     res+=1
 325                     self.logger.write("Error, sat check install failed for file %s\n" % complete_path, 1)
 326                     self.log_command("Error, sat check install failed for file %s" % complete_path)
 327         return res
 328
 329     ##
 330     # Runs 'make_check'.
 331     def check(self, command=""):
 332         if src.architecture.is_windows():
 333             cmd = 'msbuild RUN_TESTS.vcxproj'
 334         else :
 335             if self.product_info.build_source=="autotools" :
 336                 cmd = 'make check'
 337             else:
 338                 cmd = 'make test'
 339
 340         if command:
 341             cmd = command
 342
 343         self.log_command(cmd)
 344         self.log_command("For more detailed logs, see test logs in %s" % self.build_dir)
 345
 346         res = subprocess.call(cmd,
 347                               shell=True,
 348                               cwd=str(self.build_dir),
 349                               env=self.launch_environ.environ.environ,
 350                               stdout=self.logger.logTxtFile,
 351                               stderr=subprocess.STDOUT)
 352
 353         self.put_txt_log_in_appli_log_dir("makecheck")
 354         if res == 0:
 355             return res
 356         else:
 357             return 1
 358
 359     ##
 360     # Performs a default build for this module.
 361     def do_default_build(self,
 362                          build_conf_options="",
 363                          configure_options="",
 364                          show_warning=True):
 365         use_autotools = False
 366         if 'use_autotools' in self.product_info:
 367             uc = self.product_info.use_autotools
 368             if uc in ['always', 'yes']:
 369                 use_autotools = True
 370             elif uc == 'option':
 371                 use_autotools = self.options.autotools
 372
 373
 374         self.use_autotools = use_autotools
 375
 376         use_ctest = False
 377         if 'use_ctest' in self.product_info:
 378             uc = self.product_info.use_ctest
 379             if uc in ['always', 'yes']:
 380                 use_ctest = True
 381             elif uc == 'option':
 382                 use_ctest = self.options.ctest
 383
 384         self.use_ctest = use_ctest
 385
 386         if show_warning:
 387             cmd = ""
 388             if use_autotools: cmd = "(autotools)"
 389             if use_ctest: cmd = "(ctest)"
 390
 391             self.log("\n", 4, False)
 392             self.log("%(module)s: Run default compilation method %(cmd)s\n" % \
 393                 { "module": self.module, "cmd": cmd }, 4)
 394
 395         if use_autotools:
 396             if not self.prepare(): return self.get_result()
 397             if not self.build_configure(
 398                                    build_conf_options): return self.get_result()
 399             if not self.configure(configure_options): return self.get_result()
 400             if not self.make(): return self.get_result()
 401             if not self.install(): return self.get_result()
 402             if not self.clean(): return self.get_result()
 403
 404         else: # CMake
 405             if self.config.VARS.dist_name=='Win':
 406                 if not self.wprepare(): return self.get_result()
 407                 if not self.cmake(): return self.get_result()
 408                 if not self.wmake(): return self.get_result()
 409                 if not self.install(): return self.get_result()
 410                 if not self.clean(): return self.get_result()
 411             else :
 412                 if not self.prepare(): return self.get_result()
 413                 if not self.cmake(): return self.get_result()
 414                 if not self.make(): return self.get_result()
 415                 if not self.install(): return self.get_result()
 416                 if not self.clean(): return self.get_result()
 417
 418         return self.get_result()
 419
 420     ##
 421     # Performs a build with a script.
 422     def do_python_script_build(self, script, nb_proc):
 423         # script found
 424         self.logger.write(_("Compile %(product)s using script %(script)s\n") % \
 425             { 'product': self.product_info.name,
 426              'script': src.printcolors.printcLabel(script) }, 4)
 427         try:
 428             import imp
 429             product = self.product_info.name
 430             pymodule = imp.load_source(product + "_compile_script", script)
 431             self.nb_proc = nb_proc
 432             retcode = pymodule.compil(self.config, self, self.logger)
 433         except:
 434             __, exceptionValue, exceptionTraceback = sys.exc_info()
 435             self.logger.write(str(exceptionValue), 1)
 436             import traceback
 437             traceback.print_tb(exceptionTraceback)
 438             traceback.print_exc()
 439             retcode = 1
 440         finally:
 441             self.put_txt_log_in_appli_log_dir("script")
 442
 443         return retcode
 444
 445     def complete_environment(self, make_options):
 446         assert self.build_environ is not None
 447         # pass additional variables to environment
 448         # (may be used by the build script)
 449         self.build_environ.set("SOURCE_DIR", str(self.source_dir))
 450         self.build_environ.set("INSTALL_DIR", str(self.install_dir))
 451         self.build_environ.set("PRODUCT_INSTALL", str(self.install_dir))
 452         self.build_environ.set("BUILD_DIR", str(self.build_dir))
 453         self.build_environ.set("PRODUCT_BUILD", str(self.build_dir))
 454         self.build_environ.set("MAKE_OPTIONS", make_options)
 455         self.build_environ.set("DIST_NAME", self.config.VARS.dist_name)
 456         self.build_environ.set("DIST_VERSION", self.config.VARS.dist_version)
 457         self.build_environ.set("DIST", self.config.VARS.dist)
 458         self.build_environ.set("VERSION", self.product_info.version)
 459         # if product is in hpc mode, set SAT_HPC to 1
 460         # in order for the compilation script to take it into account
 461         if src.product.product_is_hpc(self.product_info):
 462             self.build_environ.set("SAT_HPC", "1")
 463
 464     def do_batch_script_build(self, script, nb_proc):
 465
 466         if src.architecture.is_windows():
 467             make_options = "/maxcpucount:%s" % nb_proc
 468         else :
 469             make_options = "-j%s" % nb_proc
 470
 471         self.log_command("  " + _("Run build script %s\n") % script)
 472         self.complete_environment(make_options)
 473
 474         res = subprocess.call(script,
 475                               shell=True,
 476                               stdout=self.logger.logTxtFile,
 477                               stderr=subprocess.STDOUT,
 478                               cwd=str(self.build_dir),
 479                               env=self.build_environ.environ.environ)
 480
 481         res_check=self.check_install()
 482         if res_check > 0 :
 483             self.log_command("Error in sat check install - some files are not installed!")
 484
 485         self.put_txt_log_in_appli_log_dir("script")
 486         res += res_check
 487         if res == 0:
 488             return res
 489         else:
 490             return 1
 491
 492     def do_script_build(self, script, number_of_proc=0):
 493         # define make options (may not be used by the script)
 494         if number_of_proc==0:
 495             nb_proc = src.get_cfg_param(self.product_info,"nb_proc", 0)
 496             if nb_proc == 0:
 497                 nb_proc = self.config.VARS.nb_proc
 498         else:
 499             nb_proc = min(number_of_proc, self.config.VARS.nb_proc)
 500
 501         extension = script.split('.')[-1]
 502         if extension in ["bat","sh"]:
 503             return self.do_batch_script_build(script, nb_proc)
 504         if extension == "py":
 505             return self.do_python_script_build(script, nb_proc)
 506
 507         msg = _("The script %s must have .sh, .bat or .py extension." % script)
 508         raise src.SatException(msg)
 509
 510     def put_txt_log_in_appli_log_dir(self, file_name):
 511         '''Put the txt log (that contain the system logs, like make command
 512            output) in the directory <APPLICATION DIR>/LOGS/<product_name>/
 513
 514         :param file_name Str: the name of the file to write
 515         '''
 516         if self.logger.logTxtFile == sys.__stdout__:
 517             return
 518         dir_where_to_put = os.path.join(self.config.APPLICATION.workdir,
 519                                         "LOGS",
 520                                         self.product_info.name)
 521         file_path = os.path.join(dir_where_to_put, file_name)
 522         src.ensure_path_exists(dir_where_to_put)
 523         # write the logTxtFile copy it to the destination, and then recreate
 524         # it as it was
 525         self.logger.logTxtFile.close()
 526         shutil.move(self.logger.txtFilePath, file_path)
 527         self.logger.logTxtFile = open(str(self.logger.txtFilePath), 'w')
 528         self.logger.logTxtFile.write(open(file_path, "r").read())
 529