From c39c41b632197f7f4f2f3beed1397b4e35bd158b Mon Sep 17 00:00:00 2001 From: =?utf8?q?C=C3=A9dric=20Aguerre?= Date: Thu, 7 Apr 2016 10:42:14 +0200 Subject: [PATCH] Manage dependencies --- scripts/BuildSalomeMini.py | 178 +++++++++++++++++++++++++---------- scripts/SalomeMiniConfig.cfg | 10 +- 2 files changed, 135 insertions(+), 53 deletions(-) diff --git a/scripts/BuildSalomeMini.py b/scripts/BuildSalomeMini.py index dd01742..b601d89 100755 --- a/scripts/BuildSalomeMini.py +++ b/scripts/BuildSalomeMini.py @@ -111,6 +111,30 @@ compOut="compile_out.log" compErr="compile_err.log" okFlag=".install_ok" +# Manage dependencies +__softwares = ["cmake","python","hdf5","swig","qt","med","medcoupling","paraview","gui","paravis"] +__softwareChanged = {} +for s in __softwares: + __softwareChanged[s] = False + +__dependencies={ + "cmake": [], + "python": [], + "hdf5": ["cmake"], + "swig": ["python"], + "qt": [], + "med": ["hdf5","python"], + "medcoupling": ["python","med"], + "paraview": ["hdf5","python","qt","cmake"], + "gui": [""], # not built, only used to get cmake files from sources + "paravis": ["cmake","paraview","medcoupling","gui"], +} + +def some_dependency_has_changed(software): + for x in __dependencies[software]: + if __softwareChanged[x]: + return True + return False # Manage environment @@ -271,13 +295,20 @@ def extractSrcFromGit(gitLoc,srcDir,pattern): assert(_status==0) else: assert(os.path.exists(os.path.join(dirGitLoc,".git"))) + p=subprocess.Popen(["git","fetch"],cwd=dirGitLoc,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + p.communicate() + pass p=subprocess.Popen(["git","checkout",gitLoc[1]],cwd=dirGitLoc,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - p.communicate() - assert(p.returncode==0) + p_out,p_err=p.communicate() + if p.returncode != 0: + msg = "Error in checkout\n" + p_err + raise Exception(msg) if dirGitLoc=="ParaView": p=subprocess.Popen(["git","submodule","update","--init"],cwd=dirGitLoc,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - p.communicate() - assert(p.returncode==0) + p_out,p_err=p.communicate() + if p.returncode != 0: + msg = "Error in submodule update\n" + p_err + raise Exception(msg) return ret @@ -316,80 +347,128 @@ def computeEnv(envAcc,withDft=True): d["PATH"]=os.environ["PATH"] return d -def compilePart0(envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild): - assert(os.path.isdir(srcDir)) - baseName=os.path.basename(srcDir) - myBuildDir=os.path.realpath(os.path.join(topBuildDir,baseName)) - myInstallDir=os.path.realpath(os.path.join(topInstallDir,baseName)) - if os.path.exists(myBuildDir): - if forceRebuild: - shutil.rmtree(myBuildDir) - else: - if not os.path.exists(os.path.join(myBuildDir,okFlag)): - shutil.rmtree(myBuildDir) - os.mkdir(myBuildDir) - else: - print("For \"%s\" : Do nothing !"%srcDir) - return myBuildDir,myInstallDir,False +def getCurrentVersion(srcDir,pattern): + zz=AutoChdir(srcDir) + srcLoc=eval("%sLoc"%pattern) + if isinstance(srcLoc,str): + s = os.path.splitext(os.path.basename(srcLoc))[0] + if s.endswith(".tar"): + s = s[:-4] + return s + elif isinstance(srcLoc,tuple): + dirGitLoc=os.path.splitext(os.path.basename(srcLoc[0]))[0] + p=subprocess.Popen(["git","rev-parse","--verify","HEAD"],cwd=dirGitLoc,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + p_out,p_err=p.communicate() + if p.returncode != 0: + msg = "Error in getCurrentVersion\n" + p_err + raise Exception(msg) + return p_out.strip() + +def compilePart0(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild): + assert(os.path.isdir(srcDir)) + baseName=os.path.basename(srcDir) + myBuildDir=os.path.realpath(os.path.join(topBuildDir,baseName)) + myInstallDir=os.path.realpath(os.path.join(topInstallDir,baseName)) + if os.path.exists(myBuildDir): + if forceRebuild: + shutil.rmtree(myBuildDir) + os.mkdir(myBuildDir) else: + if not os.path.exists(os.path.join(myBuildDir,okFlag)): + shutil.rmtree(myBuildDir) os.mkdir(myBuildDir) - return myBuildDir,myInstallDir,True + else: + with open(os.path.join(myBuildDir,okFlag)) as f: + built_version = f.readline() + built_version = built_version.strip() + current_version = getCurrentVersion(topSrcDir,pattern) + if current_version != built_version: + print("For \"%s\" : Version has changed !"%srcDir) + print(" Built version: %s"%built_version) + print(" Needed version: %s"%current_version) + shutil.rmtree(myBuildDir) + os.mkdir(myBuildDir) + else: + if some_dependency_has_changed(pattern): + print("For \"%s\" : Some dependencies have changed !"%srcDir) + shutil.rmtree(myBuildDir) + os.mkdir(myBuildDir) + else: + print("For \"%s\" : Do nothing !"%srcDir) + return myBuildDir,myInstallDir,False + else: + os.mkdir(myBuildDir) + pass + __softwareChanged[pattern] = True + return myBuildDir,myInstallDir,True def compilePart1(p,buildDir): a,b=p.communicate() f=file(os.path.join(buildDir,cfgOut),"w") ; f.write(a) ; f.flush() f=file(os.path.join(buildDir,cfgErr),"w") ; f.write(b) ; f.flush() - assert(p.returncode==0) + if p.returncode != 0: + msg = "Error in build step. See details in %s and %s\n"%(os.path.join(buildDir,cfgOut), os.path.join(buildDir,cfgErr)) + raise Exception(msg) -def compilePart2(envAcc,buildDir,splitInstall): +def compilePart2(topSrcDir,pattern,envAcc,buildDir,splitInstall): print("make in %s"%(buildDir)) if not splitInstall: p=subprocess.Popen(["make",paraCompFlag,"install"],cwd=buildDir,env=computeEnv(envAcc),stdout=subprocess.PIPE,stderr=subprocess.PIPE) a,b=p.communicate() f=file(os.path.join(buildDir,compOut),"w") ; f.write(a) ; f.flush() f=file(os.path.join(buildDir,compErr),"w") ; f.write(b) ; f.flush() - assert(p.returncode==0) + if p.returncode != 0: + msg = "Error in build step. See details in %s and %s\n"%(os.path.join(buildDir,compOut), os.path.join(buildDir,compErr)) + raise Exception(msg) else: p=subprocess.Popen(["make",paraCompFlag],cwd=buildDir,env=computeEnv(envAcc),stdout=subprocess.PIPE,stderr=subprocess.PIPE) a0,b0=p.communicate() f=file(os.path.join(buildDir,compOut),"w") ; f.write(a0) ; f.flush() f=file(os.path.join(buildDir,compErr),"w") ; f.write(b0) ; f.flush() - assert(p.returncode==0) + if p.returncode != 0: + msg = "Error in build step. See details in %s and %s\n"%(os.path.join(buildDir,compOut), os.path.join(buildDir,compErr)) + raise Exception(msg) p=subprocess.Popen(["make",paraCompFlag,"install"],cwd=buildDir,env=computeEnv(envAcc),stdout=subprocess.PIPE,stderr=subprocess.PIPE) a1,b1=p.communicate() f=file(os.path.join(buildDir,compOut),"w") ; f.write(a0+a1) ; f.flush() f=file(os.path.join(buildDir,compErr),"w") ; f.write(b0+b1) ; f.flush() - assert(p.returncode==0) + if p.returncode != 0: + msg = "Error in build step. See details in %s and %s\n"%(os.path.join(buildDir,compOut), os.path.join(buildDir,compErr)) + raise Exception(msg) pass - f=file(os.path.join(buildDir,okFlag),"w") ; f.write(str(datetime.datetime.now())) ; f.flush() + current_version = getCurrentVersion(topSrcDir,pattern) + with open(os.path.join(buildDir,okFlag),"w") as f: + f.write(current_version + "\n") + f.write(str(datetime.datetime.now())) + f.flush() pass -def compileOldCommon(envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild,splitInstall): - myBuildDir,myInstallDir,shouldContinue=compilePart0(envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild) +def compileOldCommon(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild,splitInstall): + myBuildDir,myInstallDir,shouldContinue=compilePart0(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild) if not shouldContinue: return myInstallDir myCmd=[os.path.join(srcDir,"configure"),"--prefix=%s"%(myInstallDir)]+addCfgParam print "Command: ", " ".join(myCmd) p=subprocess.Popen(myCmd,cwd=myBuildDir,env=computeEnv(envAcc),stdout=subprocess.PIPE,stderr=subprocess.PIPE) compilePart1(p,myBuildDir) - compilePart2(envAcc,myBuildDir,splitInstall) + compilePart2(topSrcDir,pattern,envAcc,myBuildDir,splitInstall) return myInstallDir -def compileOld(envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild): - return compileOldCommon(envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild,False) +def compileOld(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild): + return compileOldCommon(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild,False) -def compileVeryOld(envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild): - return compileOldCommon(envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild,True) +def compileVeryOld(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild): + return compileOldCommon(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild,True) -def compileNew(envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild): - myBuildDir,myInstallDir,shouldContinue=compilePart0(envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild) +def compileNew(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild): + myBuildDir,myInstallDir,shouldContinue=compilePart0(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,addCfgParam,forceRebuild) if not shouldContinue: return myInstallDir myCmd=["cmake","-DCMAKE_INSTALL_PREFIX=%s"%(myInstallDir)]+addCfgParam+[srcDir] print(" ".join(myCmd)) p=subprocess.Popen(myCmd,cwd=myBuildDir,env=computeEnv(envAcc),stdout=subprocess.PIPE,stderr=subprocess.PIPE) compilePart1(p,myBuildDir) - compilePart2(envAcc,myBuildDir,False) + compilePart2(topSrcDir,pattern,envAcc,myBuildDir,False) return myInstallDir def accumulateEnv(installDir,pattern,envAcc): @@ -402,7 +481,6 @@ def generateInlineEnv(envAcc): d=computeEnv(envAcc) return "env "+" ".join(["%s=%s"%(k,d[k]) for k in d.keys()]) - def generateFileEnv(envAcc): d=computeEnv(envAcc) return "\n".join(["export %s=\"%s\""%(k,d[k]) for k in d.keys()]) @@ -451,6 +529,7 @@ ENDIF() f=file(cmakeToPatch,"w") ; f.writelines(lines) return os.path.abspath(destReal) + ########### FINI LA PARLOTE ! if not os.path.exists(rootDir): @@ -470,46 +549,49 @@ for d in [topSrcDir,topBuildDir,topInstallDir]: pattern="cmake" srcDir=extractSrc(topSrcDir,pattern) -installDir=compileOld(envAcc,srcDir,topBuildDir,topInstallDir,[],False) +installDir=compileOld(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,[],False) accumulateEnv(installDir,pattern,envAcc) # pattern="python" srcDir=extractSrc(topSrcDir,pattern) -installDir=compileOld(envAcc,srcDir,topBuildDir,topInstallDir,["--enable-shared","--enable-unicode=ucs4","-without-pymalloc"],False) +installDir=compileOld(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,["--enable-shared","--enable-unicode=ucs4","-without-pymalloc"],False) accumulateEnv(installDir,pattern,envAcc) # pattern="hdf5" srcDir=extractSrc(topSrcDir,pattern) -installDir=compileNew(envAcc,srcDir,topBuildDir,topInstallDir, +installDir=compileNew(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir, ['-DHDF5_BUILD_HL_LIB=ON','-DHDF5_BUILD_TOOLS=ON','-DBUILD_SHARED_LIBS=ON','-DHDF5_ENABLE_Z_LIB_SUPPORT=ON','-DCMAKE_BUILD_TYPE=Release'],False) accumulateEnv(installDir,pattern,envAcc) # pattern="swig" srcDir=extractSrc(topSrcDir,pattern) -installDir=compileVeryOld(envAcc,srcDir,topBuildDir,topInstallDir,["--without-pcre","--without-octave"],False) +installDir=compileVeryOld(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,["--without-pcre","--without-octave"],False) accumulateEnv(installDir,pattern,envAcc) # pattern="qt" srcDir=extractSrc(topSrcDir,pattern) -installDir=compileVeryOld(envAcc,srcDir,topBuildDir,topInstallDir,["-opensource","-confirm-license","-no-rpath","-qt-xcb"],False) +installDir=compileVeryOld(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,["-opensource","-confirm-license","-no-rpath","-qt-xcb"],False) accumulateEnv(installDir,pattern,envAcc) # pattern="med" srcDir=extractSrc(topSrcDir,pattern) -installDir=compileOld(envAcc,srcDir,topBuildDir,topInstallDir,["--with-hdf5=%s"%(envAcc["HDF5HOME"]),"--enable-production"],False) +installDir=compileOld(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir,["--with-hdf5=%s"%(envAcc["HDF5HOME"]),"--enable-production"],False) accumulateEnv(installDir,pattern,envAcc) # pattern="medcoupling" -srcDir=extractSrc(topSrcDir,pattern) ; medcouplingSrcDir=srcDir -installDir=compileNew(envAcc,srcDir,topBuildDir,topInstallDir, +srcDir=extractSrc(topSrcDir,pattern) +installDir=compileNew(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir, ["-DMEDFILE_ROOT_DIR=%s"%(envAcc["MED3HOME"]),"-DCMAKE_BUILD_TYPE=Debug","-DMEDCOUPLING_BUILD_DOC=OFF","-DMEDCOUPLING_BUILD_TESTS=OFF", - "-DMEDCOUPLING_PARTITIONER_METIS=OFF","-DMEDCOUPLING_ENABLE_RENUMBER=OFF","-DMEDCOUPLING_PARTITIONER_SCOTCH=OFF"],False) + "-DMEDCOUPLING_PARTITIONER_METIS=OFF","-DMEDCOUPLING_ENABLE_RENUMBER=OFF","-DMEDCOUPLING_PARTITIONER_SCOTCH=OFF", + "-DPYTHONINTERP_ROOT_DIR=%s"%envAcc["PYTHONHOME"], + "-DPYTHONLIBS_ROOT_DIR=%s"%envAcc["PYTHONHOME"], +],False) accumulateEnv(installDir,pattern,envAcc) #copyCMakeFromSrcToInstall(srcDir,installDir)#patch # pattern="paraview" srcDir=extractSrc(topSrcDir,pattern) -installDir=compileNew(envAcc,srcDir,topBuildDir,topInstallDir, +installDir=compileNew(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir, ['-DPYTHON_ROOT_DIR=%s'%(envAcc["PYTHONHOME"]),'-DHDF5_ROOT_DIR=%s'%(envAcc["HDF5HOME"]),'-DPYTHON_EXECUTABLE=%s/bin/python%s'%(envAcc["PYTHONHOME"],envAcc["PYTHON_VERSION"]), '-DPYTHON_INCLUDE_DIR=%s'%(envAcc["PYTHON_INCLUDE"]),'-DPYTHON_LIBRARY=%s/lib/libpython%s.so'%(envAcc["PYTHONHOME"],envAcc["PYTHON_VERSION"]), '-DHDF5_LIBRARIES=%s/lib'%(envAcc["HDF5HOME"]), @@ -539,7 +621,7 @@ fakeKernelDir=bidouilleForCMake(topSrcDir,srcDir) pattern="paravis"# for MEDReader srcDir=extractSrc(topSrcDir,pattern) srcDir=patchCMakeLists(topSrcDir,srcDir)# change Reference for source -installDir=compileNew(envAcc,srcDir,topBuildDir,topInstallDir, +installDir=compileNew(topSrcDir,pattern,envAcc,srcDir,topBuildDir,topInstallDir, ["-DMEDCOUPLING_ROOT_DIR=%s"%(envAcc["MEDCOUPLINGHOME"]),"-DKERNEL_ROOT_DIR=%s"%fakeKernelDir, '-DPYTHON_ROOT_DIR=%s'%(envAcc["PYTHONHOME"]),'-DHDF5_ROOT_DIR=%s'%(envAcc["HDF5HOME"]), '-DPYTHON_EXECUTABLE=%s/bin/python%s'%(envAcc["PYTHONHOME"],envAcc["PYTHON_VERSION"]), diff --git a/scripts/SalomeMiniConfig.cfg b/scripts/SalomeMiniConfig.cfg index 5ed4baf..7dbbd72 100644 --- a/scripts/SalomeMiniConfig.cfg +++ b/scripts/SalomeMiniConfig.cfg @@ -17,15 +17,15 @@ swig_url=http://downloads.sourceforge.net/project/swig/swig/swig-2.0.8/swig-2.0. [ParaView] paraview_git_repo=https://github.com/Kitware/ParaView.git -paraview_git_version=v5.0.0 +paraview_git_version=2afbf297 paraview_version=5.0 [SALOME] salome_git_repo=https://git.salome-platform.org/gitpub -medcoupling_version=76601490 -kernel_version=49410ad3 -gui_version=792a0f7d -paravis_version=314918a6 +medcoupling_version=d985c98a +kernel_version=8ede54c6 +gui_version=c3dc40d6 +paravis_version=467172fc [Build options] make_nb_jobs=8 -- 2.39.2