src/compilation.py

   1 #!/usr/bin/env python
   2 #-*- coding:utf-8 -*-
   3 #  Copyright (C) 2010-2013  CEA/DEN
   4 #
   5 #  This library is free software; you can redistribute it and/or
   6 #  modify it under the terms of the GNU Lesser General Public
   7 #  License as published by the Free Software Foundation; either
   8 #  version 2.1 of the License.
   9 #
  10 #  This library is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 #  Lesser General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU Lesser General Public
  16 #  License along with this library; if not, write to the Free Software
  17 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18
  19 import os
  20 import subprocess
  21 import sys
  22
  23 import src
  24
  25 C_COMPILE_ENV_LIST = ["CC",
  26                       "CXX",
  27                       "F77",
  28                       "CFLAGS",
  29                       "CXXFLAGS",
  30                       "LIBS",
  31                       "LDFLAGS"]
  32
  33 class CompilationResult:
  34     def __init__(self):
  35         self.prepare = False
  36         self.buildconfigure = False
  37         self.configure = False
  38         self.cmake = False
  39         self.make = False
  40         self.install = False
  41         self.check = False
  42         self.check_tried = False
  43         self.ignored = False
  44         self.reason = ""
  45
  46     def isOK(self):
  47         if self.ignored:
  48             return False
  49
  50         return self.prepare \
  51             and self.buildconfigure \
  52             and self.configure \
  53             and self.cmake \
  54             and self.make \
  55             and self.install \
  56             and self.check
  57
  58     def setAllFail(self):
  59         self.prepare = False
  60         self.buildconfigure = False
  61         self.configure = False
  62         self.cmake = False
  63         self.make = False
  64         self.install = False
  65         self.check = False
  66
  67     def setIgnored(self, reason=None):
  68         self.ignored = True
  69         if reason:
  70             self.reason = reason
  71         else:
  72             self.reason = _("ignored")
  73
  74     def getErrorText(self):
  75         if self.ignored or len(self.reason):
  76             return self.reason
  77
  78         if not self.prepare: return "PREPARE BUILD"
  79         if not self.buildconfigure: return "BUILD CONFIGURE"
  80         if not self.configure: return "CONFIGURE"
  81         if not self.cmake: return "CMAKE"
  82         if not self.make: return "MAKE"
  83         if not self.install: return "INSTALL"
  84         if not self.check: return "CHECK"
  85
  86         return ""
  87
  88 class Builder:
  89     """Class to handle all construction steps, like cmake, configure, make, ...
  90     """
  91     def __init__(self, config, logger, options, product_info, debug_mode=False, check_src=True):
  92         self.config = config
  93         self.logger = logger
  94         self.options = options
  95         self.product_info = product_info
  96         self.build_dir = src.Path(self.product_info.build_dir)
  97         self.source_dir = src.Path(self.product_info.source_dir)
  98         self.source_dir = src.Path(self.product_info.install_dir)
  99         self.header = ""
 100         self.debug_mode = debug_mode
 101
 102         if not self.source_dir.exists() and check_src:
 103             raise src.SatException(_("No sources found for product %(product)s in %(source_dir)s" % \
 104                 { "product": self.product, "source_dir": self.source_dir } ))
 105
 106         # check that required modules exist
 107         for dep in self.product_info.depend:
 108             assert dep in self.config.TOOLS.src.product_info, "UNDEFINED product: %s" % dep
 109             dep_info = self.config.TOOLS.src.product_info[dep]
 110             if 'install_dir' in dep_info and not os.path.exists(dep_info.install_dir):
 111                 raise src.SatException(_("Module %s is required") % dep)
 112
 113         self.results = CompilationResult()
 114
 115     ##
 116     # Shortcut method to log in both log files.
 117     def log(self, text, level, showInfo=True):
 118         self.logger.write(text, level, showInfo)
 119         self.logger.logTxtFile.write(src.printcolors.cleancolor(text))
 120
 121     ##
 122     # Shortcut method to log a command.
 123     def log_command(self, command):
 124         self.log("> %s\n" % command, 5)
 125
 126     def log_result(self, res):
 127         if res == 0:
 128             self.logger.write("%s\n" % src.printcolors.printc(src.OK_STATUS), 5)
 129         else:
 130             self.logger.write("%s, code = %s\n" % (src.printcolors.printc(src.KO_STATUS), res), 5)
 131
 132     ##
 133     # Logs a compilation step (configure, make ...)
 134     def log_step(self, step):
 135         if self.config.USER.output_level == 3:
 136             self.logger.write("\r%s%s" % (self.header, " " * 20), 3)
 137             self.logger.write("\r%s%s" % (self.header, step), 3)
 138         self.log("==== %s \n" % src.printcolors.printcInfo(step), 4)
 139         self.logger.flush()
 140
 141     ##
 142     # Prepares the environment for windows.
 143     # Build two environment: one for building and one for testing (launch).
 144     def wprepare(self):
 145         self.log_step('PREPARE BUILD')
 146
 147         if not self.build_dir.exists():
 148             # create build dir
 149             self.build_dir.make()
 150         elif self.options.clean_all:
 151             self.log('  %s\n' % src.printcolors.printcWarning("CLEAN ALL"), 4)
 152             # clean build dir if clean_all option given
 153             self.log('  clean previous build = %s\n' % str(self.build_dir), 4)
 154             self.build_dir.rm()
 155             self.build_dir.make()
 156
 157         if self.options.clean_all or self.options.clean_install:
 158             if os.path.exists(str(self.install_dir)) and not self.single_dir:
 159                 self.log('  clean previous install = %s\n' % str(self.install_dir), 4)
 160                 self.install_dir.rm()
 161
 162         self.log('  build_dir   = %s\n' % str(self.build_dir), 4)
 163         self.log('  install_dir = %s\n' % str(self.install_dir), 4)
 164         self.log('\n', 4)
 165
 166         environ_info = {}
 167
 168         # add products in depend and opt_depend list recursively
 169         environ_info['products'] = src.product.get_product_dependencies(self.config, self.product_info)
 170
 171         # create build environment
 172         self.build_environ = src.environment.SalomeEnviron(self.config, src.environment.Environ(dict(os.environ)), True)
 173         self.build_environ.silent = (self.config.USER.output_level < 5)
 174         self.build_environ.set_full_environ(self.logger, environ_info)
 175
 176         # create runtime environment
 177         self.launch_environ = src.environment.SalomeEnviron(self.config, src.environment.Environ(dict(os.environ)), False)
 178         self.launch_environ.silent = True # no need to show here
 179         self.launch_environ.set_full_environ(self.logger, environ_info)
 180
 181         for ee in C_COMPILE_ENV_LIST:
 182             vv = self.build_environ.get(ee)
 183             if len(vv) > 0:
 184                 self.log("  %s = %s\n" % (ee, vv), 4, False)
 185
 186         self.results.prepare = True
 187         self.log_result(0)
 188         return self.results.prepare
 189
 190     ##
 191     # Prepares the environment.
 192     # Build two environment: one for building and one for testing (launch).
 193     def prepare(self):
 194         self.log_step('PREPARE BUILD')
 195
 196         if not self.build_dir.exists():
 197             # create build dir
 198             self.build_dir.make()
 199         elif self.options.clean_all:
 200             self.log('  %s\n' % src.printcolors.printcWarning("CLEAN ALL"), 4)
 201             # clean build dir if clean_all option given
 202             self.log('  clean previous build = %s\n' % str(self.build_dir), 4)
 203             self.build_dir.rm()
 204             self.build_dir.make()
 205
 206         if self.options.clean_all or self.options.clean_install:
 207             if os.path.exists(str(self.install_dir)) and not self.single_dir:
 208                 self.log('  clean previous install = %s\n' % str(self.install_dir), 4)
 209                 self.install_dir.rm()
 210
 211         self.log('  build_dir   = %s\n' % str(self.build_dir), 4)
 212         self.log('  install_dir = %s\n' % str(self.install_dir), 4)
 213         self.log('\n', 4)
 214
 215         # set the environment
 216         environ_info = {}
 217
 218         # add products in depend and opt_depend list recursively
 219         environ_info['products'] = src.product.get_product_dependencies(self.config, self.product_info)
 220
 221         # create build environment
 222         self.build_environ = src.environment.SalomeEnviron(self.config, src.environment.Environ(dict(os.environ)), True)
 223         self.build_environ.silent = (self.config.USER.output_level < 5)
 224         self.build_environ.set_full_environ(self.logger, environ_info)
 225
 226         # create runtime environment
 227         self.launch_environ = src.environment.SalomeEnviron(self.config, src.environment.Environ(dict(os.environ)), False)
 228         self.launch_environ.silent = True # no need to show here
 229         self.launch_environ.set_full_environ(self.logger, environ_info)
 230
 231         for ee in C_COMPILE_ENV_LIST:
 232             vv = self.build_environ.get(ee)
 233             if len(vv) > 0:
 234                 self.log("  %s = %s\n" % (ee, vv), 4, False)
 235
 236         self.results.prepare = True
 237         self.log_result(0)
 238         return self.results.prepare
 239
 240     ##
 241     # Runs cmake with the given options.
 242     def cmake(self, options=""):
 243         self.log_step('CMAKE')
 244
 245         # cmake so no (build)configure
 246         self.results.configure = True
 247
 248         cmake_option = options
 249         cmake_option +=' -DCMAKE_VERBOSE_MAKEFILE=ON -DSALOME_CMAKE_DEBUG=ON'
 250         if 'cmake_options' in self.product_info:
 251             cmake_option += " %s " % " ".join(self.product_info.cmake_options.split())
 252
 253         # add debug option
 254         if self.debug_mode:
 255             cmake_option += " -DCMAKE_BUILD_TYPE=Debug"
 256         else :
 257             cmake_option += " -DCMAKE_BUILD_TYPE=Release"
 258
 259         # In case CMAKE_GENERATOR is defined in environment, use it in spite of automatically detect it
 260         if 'cmake_generator' in self.config.APPLICATION:
 261             cmake_option += ' -DCMAKE_GENERATOR=%s' % self.config.PRODUCT.cmake_generator
 262
 263         command = "cmake %s -DCMAKE_INSTALL_PREFIX=%s %s" %(cmake_option, self.install_dir, self.source_dir)
 264
 265         self.log_command(command)
 266         res = subprocess.call(command,
 267                               shell=True,
 268                               cwd=str(self.build_dir),
 269                               env=self.build_environ.environ.environ,
 270                               stdout=self.logger.logTxtFile,
 271                               stderr=subprocess.STDOUT)
 272
 273         self.results.cmake = (res == 0)
 274         self.log_result(res)
 275         return self.results.cmake
 276
 277     ##
 278     # Runs build_configure with the given options.
 279     def build_configure(self, options=""):
 280         skip = src.get_cfg_param(self.product_info, "build_configure", False)
 281         if skip:
 282             self.results.buildconfigure = True
 283             res = 0
 284         else:
 285             self.log_step('BUILD CONFIGURE')
 286
 287             self.results.buildconfigure = False
 288
 289             if 'buildconfigure_options' in self.product_info:
 290                 options += " %s " % self.product_info.buildconfigure_options
 291
 292             command = str('./build_configure')
 293             command = command + " " + options
 294             self.log_command(command)
 295
 296             res = subprocess.call(command,
 297                                   shell=True,
 298                                   cwd=str(self.source_dir),
 299                                   env=self.build_environ.environ.environ,
 300                                   stdout=self.logger.logTxtFile,
 301                                   stderr=subprocess.STDOUT)
 302             self.results.buildconfigure = (res == 0)
 303
 304         self.log_result(res)
 305         return self.results.buildconfigure
 306
 307     ##
 308     # Runs configure with the given options.
 309     def configure(self, options=""):
 310         self.log_step('CONFIGURE')
 311
 312         # configure so no cmake
 313         self.results.cmake = True
 314
 315         if 'configure_options' in self.product_info:
 316             options += " %s " % self.product_info.configure_options
 317
 318         command = "%s/configure --prefix=%s" % (self.source_dir, str(self.install_dir))
 319
 320         command = command + " " + options
 321         self.log_command(command)
 322
 323         res = subprocess.call(command,
 324                               shell=True,
 325                               cwd=str(self.build_dir),
 326                               env=self.build_environ.environ.environ,
 327                               stdout=self.logger.logTxtFile,
 328                               stderr=subprocess.STDOUT)
 329
 330         self.log_result(res)
 331         self.results.configure = (res == 0)
 332         return self.results.configure
 333
 334     def hack_libtool(self):
 335         if not os.path.exists(str(self.build_dir + 'libtool')):
 336             return
 337
 338         lf = open(os.path.join(str(self.build_dir), "libtool"), 'r')
 339         for line in lf.readlines():
 340             if 'hack_libtool' in line:
 341                 return
 342
 343         # fix libtool by replacing CC="<compil>" with hack_libtool function
 344         hack_command='''sed -i "s%^CC=\\"\(.*\)\\"%hack_libtool() { \\n\\
 345 if test \\"\$(echo \$@ | grep -E '\\\\\\-L/usr/lib(/../lib)?(64)? ')\\" == \\\"\\\" \\n\\
 346   then\\n\\
 347     cmd=\\"\\1 \$@\\"\\n\\
 348   else\\n\\
 349     cmd=\\"\\1 \\"\`echo \$@ | sed -r -e 's|(.*)-L/usr/lib(/../lib)?(64)? (.*)|\\\\\\1\\\\\\4 -L/usr/lib\\\\\\3|g'\`\\n\\
 350   fi\\n\\
 351   \$cmd\\n\\
 352 }\\n\\
 353 CC=\\"hack_libtool\\"%g" libtool'''
 354
 355         self.log_command(hack_command)
 356         subprocess.call(hack_command,
 357                         shell=True,
 358                         cwd=str(self.build_dir),
 359                         env=self.build_environ.environ.environ,
 360                         stdout=self.logger.logTxtFile,
 361                         stderr=subprocess.STDOUT)
 362
 363     def get_nb_proc(self):
 364         nbproc = -1
 365         if "nb_proc" in self.product_info:
 366             # nb proc is specified in module definition
 367             nbproc = self.product_info.nb_proc
 368             if self.options.nb_proc and self.options.nb_proc < self.product_info.nb_proc:
 369                 # use command line value only if it is lower than module definition
 370                 nbproc = self.options.nb_proc
 371         else:
 372             # nb proc is not specified in module definition
 373             if self.options.nb_proc:
 374                 nbproc = self.options.nb_proc
 375             else:
 376                 nbproc = self.config.VARS.nb_proc
 377
 378         assert nbproc > 0
 379         return nbproc
 380
 381     ##
 382     # Runs make to build the module.
 383     def make(self):
 384         nbproc = self.get_nb_proc()
 385
 386         hh = 'MAKE -j%s' % str(nbproc)
 387         if self.debug_mode:
 388             hh += " " + src.printcolors.printcWarning("DEBUG")
 389         self.log_step(hh)
 390
 391         # make
 392         command = 'make'
 393         if self.options.makeflags:
 394             command = command + " " + self.options.makeflags
 395         command = command + " -j" + str(nbproc)
 396
 397         self.log_command(command)
 398         res = subprocess.call(command,
 399                               shell=True,
 400                               cwd=str(self.build_dir),
 401                               env=self.build_environ.environ.environ,
 402                               stdout=self.logger.logTxtFile,
 403                               stderr=subprocess.STDOUT)
 404
 405         self.results.make = (res == 0)
 406         self.log_result(res)
 407         return self.results.make
 408
 409     ##
 410     # Runs msbuild to build the module.
 411     def wmake(self):
 412         nbproc = self.get_nb_proc()
 413
 414         hh = 'MSBUILD /m:%s' % str(nbproc)
 415         if self.debug_mode:
 416             hh += " " + src.printcolors.printcWarning("DEBUG")
 417         self.log_step(hh)
 418
 419         # make
 420         command = 'msbuild'
 421         if self.options.makeflags:
 422             command = command + " " + self.options.makeflags
 423         command = command + " /maxcpucount:" + str(nbproc)
 424         if self.debug_mode:
 425             command = command + " /p:Configuration=Debug"
 426         else:
 427             command = command + " /p:Configuration=Release"
 428         command = command + " ALL_BUILD.vcxproj"
 429
 430         self.log_command(command)
 431         res = subprocess.call(command,
 432                               shell=True,
 433                               cwd=str(self.build_dir),
 434                               env=self.build_environ.environ.environ,
 435                               stdout=self.logger.logTxtFile,
 436                               stderr=subprocess.STDOUT)
 437
 438         self.results.make = (res == 0)
 439         self.log_result(res)
 440         return self.results.make
 441
 442     ##
 443     # Runs 'make install'.
 444     def install(self):
 445         self.log_step('INSTALL')
 446         if self.config.VARS.dist_name=="Win":
 447             command = 'msbuild INSTALL.vcxproj'
 448             if self.debug_mode:
 449                 command = command + " /p:Configuration=Debug"
 450             else:
 451                 command = command + " /p:Configuration=Release"
 452         else :
 453             command = 'make install'
 454
 455         self.log_command(command)
 456
 457         res = subprocess.call(command,
 458                               shell=True,
 459                               cwd=str(self.build_dir),
 460                               env=self.build_environ.environ.environ,
 461                               stdout=self.logger.logTxtFile,
 462                               stderr=subprocess.STDOUT)
 463
 464         self.results.install = (res == 0)
 465         self.log_result(res)
 466         return self.results.install
 467
 468     ##
 469     # Runs 'make_check'.
 470     def check(self):
 471         self.log_step('CHECK')
 472         if src.architecture.is_windows():
 473             command = 'msbuild RUN_TESTS.vcxproj'
 474         else :
 475             if self.use_autotools :
 476                 command = 'make check'
 477             else :
 478                 command = 'make test'
 479
 480         self.log_command(command)
 481
 482         self.results.check_tried = True
 483         res = subprocess.call(command,
 484                               shell=True,
 485                               cwd=str(self.build_dir),
 486                               env=self.launch_environ.environ.environ,
 487                               stdout=self.logger.logTxtFile,
 488                               stderr=subprocess.STDOUT)
 489
 490         self.results.check = (res == 0)
 491         self.log_result(res)
 492         return self.results.check
 493
 494     ##
 495     # Cleans the build.
 496     def clean(self):
 497         self.log_step('CLEAN')
 498
 499         if src.get_cfg_param(self.config.PRODUCT, 'clean_build_dir', 'no') == "yes":
 500             if self.results.buildconfigure and self.results.configure \
 501                 and self.results.make and self.results.install and self.results.check:
 502                 self.log(_('Clean BUILD directory\n'), 4)
 503                 self.build_dir.rm()
 504             else:
 505                 self.log(_('No clean: some error during compilation\n'), 5)
 506         else:
 507             self.log(_('No clean: not specified in the config\n'), 5)
 508
 509     def get_result(self):
 510         return self.results
 511
 512     ##
 513     # Performs a default build for this module.
 514     def do_default_build(self, build_conf_options="", configure_options="", show_warning=True):
 515         use_autotools = False
 516         if 'use_autotools' in self.product_info:
 517             uc = self.product_info.use_autotools
 518             if uc in ['always', 'yes']:
 519                 use_autotools = True
 520             elif uc == 'option':
 521                 use_autotools = self.options.autotools
 522
 523
 524         self.use_autotools = use_autotools
 525
 526         use_ctest = False
 527         if 'use_ctest' in self.product_info:
 528             uc = self.product_info.use_ctest
 529             if uc in ['always', 'yes']:
 530                 use_ctest = True
 531             elif uc == 'option':
 532                 use_ctest = self.options.ctest
 533
 534         self.use_ctest = use_ctest
 535
 536         if show_warning:
 537             cmd = ""
 538             if use_autotools: cmd = "(autotools)"
 539             if use_ctest: cmd = "(ctest)"
 540
 541             self.log("\n", 4, False)
 542             self.log("%(module)s: Run default compilation method %(cmd)s\n" % \
 543                 { "module": self.module, "cmd": cmd }, 4)
 544
 545         if use_autotools:
 546             if not self.prepare(): return self.get_result()
 547             if not self.build_configure(build_conf_options): return self.get_result()
 548             if not self.configure(configure_options): return self.get_result()
 549             if not self.make(): return self.get_result()
 550             if not self.install(): return self.get_result()
 551             self.results.check = True
 552             if not self.clean(): return self.get_result()
 553
 554         else: # CMake
 555             if self.config.VARS.dist_name=='Win':
 556                 if not self.wprepare(): return self.get_result()
 557                 self.results.buildconfigure = True
 558                 if not self.cmake(): return self.get_result()
 559                 self.results.ctest = True
 560                 if not self.wmake(): return self.get_result()
 561                 if not self.install(): return self.get_result()
 562                 self.results.check = True
 563                 if not self.clean(): return self.get_result()
 564             else :
 565                 if not self.prepare(): return self.get_result()
 566                 self.results.buildconfigure = True
 567                 if not self.cmake(): return self.get_result()
 568                 self.results.ctest = True
 569                 if not self.make(): return self.get_result()
 570                 if not self.install(): return self.get_result()
 571                 self.results.check = True
 572                 if not self.clean(): return self.get_result()
 573
 574         return self.get_result()
 575
 576     ##
 577     # Performs a build with a script.
 578     def do_script_build(self, script):
 579         retcode = CompilationResult()
 580         retcode.setAllFail()
 581         # script found
 582         self.logger.write(_("Compile %(module)s using script %(script)s\n") % \
 583             { 'module': self.module, 'script': src.printcolors.printcLabel(script) }, 4)
 584         try:
 585             import imp
 586             pymodule = imp.load_source(self.module + "_compile_script", script)
 587             retcode = pymodule.compil(self.config, self, self.logger)
 588         except:
 589             __, exceptionValue, exceptionTraceback = sys.exc_info()
 590             print(exceptionValue)
 591             import traceback
 592             traceback.print_tb(exceptionTraceback)
 593             traceback.print_exc()
 594
 595         return retcode
 596
 597     ##
 598     # Builds the module.
 599     # If a script is specified used it, else use 'default' method.
 600     def run_compile(self, no_compile=False):
 601         retcode = CompilationResult()
 602         retcode.setAllFail()
 603
 604         if no_compile:
 605             if os.path.exists(str(self.install_dir)):
 606                 retcode.setIgnored(_("already installed"))
 607             else:
 608                 retcode.setIgnored(src.printcolors.printcError(_("NOT INSTALLED")))
 609
 610             self.log_file.close()
 611             os.remove(os.path.realpath(self.log_file.name))
 612             return retcode
 613
 614         # check if the module is already installed
 615         if not self.single_dir and os.path.exists(str(self.install_dir)) \
 616             and not self.options.clean_all and not self.options.clean_install:
 617
 618             retcode.setIgnored(_("already installed"))
 619             self.log_file.close()
 620             os.remove(os.path.realpath(self.log_file.name))
 621             return retcode
 622
 623         if 'compile_method' in self.product_info:
 624             if self.product_info.compile_method == "copy":
 625                 self.prepare()
 626                 retcode.prepare = self.results.prepare
 627                 retcode.buildconfigure = True
 628                 retcode.configure = True
 629                 retcode.make = True
 630                 retcode.cmake = True
 631                 retcode.ctest = True
 632
 633                 if not self.source_dir.smartcopy(self.install_dir):
 634                     raise src.SatException(_("Error when copying %s sources to install dir") % self.module)
 635                 retcode.install = True
 636                 retcode.check = True
 637
 638             elif self.product_info.compile_method == "default":
 639                 retcode = self.do_default_build(show_warning=False)
 640
 641
 642             elif os.path.isfile(self.product_info.compile_method):
 643                 retcode = self.do_script_build(self.product_info.compile_method)
 644
 645             else:
 646                 raise src.SatException(_("Unknown compile_method: %s") % self.product_info.compile_method)
 647
 648         else:
 649             script = os.path.join(self.config.VARS.dataDir, 'compil_scripts', 'modules', self.module + '.py')
 650
 651             if not os.path.exists(script):
 652                 # no script use default method
 653                 retcode = self.do_default_build(show_warning=False)
 654             else:
 655                 retcode = self.do_script_build(script)
 656
 657         return retcode