From 956d4dc88e33480efe631b4d8e8c12df98a20478 Mon Sep 17 00:00:00 2001 From: Jean-Philippe ARGAUD Date: Sun, 4 May 2014 16:09:36 +0200 Subject: [PATCH] Adding parallel jacobian calculation --- bin/AdaoCatalogGenerator.py | 3 +- .../daNumerics/ApproximatedDerivatives.py | 150 +++++++++++++++--- src/daEficas/generator_adao.py | 2 + src/daSalome/daYacsSchemaCreator/methods.py | 4 + 4 files changed, 137 insertions(+), 22 deletions(-) diff --git a/bin/AdaoCatalogGenerator.py b/bin/AdaoCatalogGenerator.py index 0db5c04..8cc64c2 100644 --- a/bin/AdaoCatalogGenerator.py +++ b/bin/AdaoCatalogGenerator.py @@ -64,7 +64,8 @@ def F_${data_name}(statut) : return FACT(statut = statut, SCRIPTWITHONEFUNCTION_FILE = SIMP(statut = "o", typ = "FichierNoAbs", validators=(OnlyStr()), fr="En attente d'un nom de fichier script, avec ou sans le chemin complet pour le trouver, contenant en variable interne une seule fonction de calcul nommée DirectOperator", ang="Waiting for a script file name, with or without the full path to find it, containing as internal variable only one function named DirectOperator"), DifferentialIncrement = SIMP(statut="o", typ = "R", val_min=0, val_max=1, defaut=0.01, fr="Incrément de la perturbation dX pour calculer la dérivée, construite en multipliant X par l'incrément en évitant les valeurs nulles", ang="Increment of dX perturbation to calculate the derivative, build multiplying X by the increment avoiding null values"), - CenteredFiniteDifference = SIMP(statut="o", typ = "I", into=(0, 1), defaut=0, fr="Formulation centrée (1) ou décentrée (0) pour la méthode des différences finies", ang="Centered (1) or uncentered (0) formulation for the finite difference method"), + CenteredFiniteDifference = SIMP(statut="o", typ = "I", into=(0, 1), defaut=0, fr="Formulation centrée (1) ou décentrée (0) pour la méthode des différences finies", ang="Centered (1) or uncentered (0) formulation for the finite differences method"), + EnableMultiProcessing = SIMP(statut="f", typ = "I", into=(0, 1), defaut=0, fr="EXPERIMENTAL : Calculs élémentaires effectués en séquentiel (0) ou en parallèle (1) dans la méthode des différences finies", ang="EXPERIMENTAL: Elementary calculations done sequentially (0) or in parallel (1) in the finite differences method"), ), SCRIPTWITHSWITCH_DATA = BLOC ( condition = " FROM in ( 'ScriptWithSwitch', ) ", diff --git a/src/daComposant/daNumerics/ApproximatedDerivatives.py b/src/daComposant/daNumerics/ApproximatedDerivatives.py index 3292f4e..35a38a6 100644 --- a/src/daComposant/daNumerics/ApproximatedDerivatives.py +++ b/src/daComposant/daNumerics/ApproximatedDerivatives.py @@ -31,7 +31,17 @@ from daCore.BasicObjects import Operator # logging.getLogger().setLevel(logging.DEBUG) # ============================================================================== -class FDApproximation: +def ExecuteFunction( (X, funcrepr) ): + __X = numpy.asmatrix(numpy.ravel( X )).T + __sys_path_tmp = sys.path ; sys.path.insert(0,funcrepr["__userFunction__path"]) + __module = __import__(funcrepr["__userFunction__modl"], globals(), locals(), []) + __fonction = getattr(__module,funcrepr["__userFunction__name"]) + sys.path = __sys_path_tmp ; del __sys_path_tmp + __HX = __fonction( __X ) + return numpy.ravel( __HX ) + +# ============================================================================== +class FDApproximation(object): """ Cette classe sert d'interface pour définir les opérateurs approximés. A la création d'un objet, en fournissant une fonction "Function", on obtient un @@ -52,8 +62,52 @@ class FDApproximation: mpEnabled = False, mpWorkers = None, ): - self.__userOperator = Operator( fromMethod = Function ) - self.__userFunction = self.__userOperator.appliedTo + if mpEnabled: + try: + import multiprocessing + self.__mpEnabled = True + except ImportError: + self.__mpEnabled = False + else: + self.__mpEnabled = False + self.__mpWorkers = mpWorkers + logging.debug("FDA Calculs en multiprocessing : %s (nombre de processus : %s)"%(self.__mpEnabled,self.__mpWorkers)) + # + if self.__mpEnabled: + if isinstance(Function,types.FunctionType): + logging.debug("FDA Calculs en multiprocessing : FunctionType") + self.__userFunction__name = Function.__name__ + try: + mod = os.path.join(Function.__globals__['filepath'],Function.__globals__['filename']) + except: + mod = os.path.abspath(Function.__globals__['__file__']) + if not os.path.isfile(mod): + raise ImportError("No user defined function or method found with the name %s"%(mod,)) + self.__userFunction__modl = os.path.basename(mod).replace('.pyc','').replace('.pyo','').replace('.py','') + self.__userFunction__path = os.path.dirname(mod) + del mod + self.__userOperator = Operator( fromMethod = Function ) + self.__userFunction = self.__userOperator.appliedTo # Pour le calcul Direct + elif isinstance(Function,types.MethodType): + logging.debug("FDA Calculs en multiprocessing : MethodType") + self.__userFunction__name = Function.__name__ + try: + mod = os.path.join(Function.__globals__['filepath'],Function.__globals__['filename']) + except: + mod = os.path.abspath(Function.im_func.__globals__['__file__']) + if not os.path.isfile(mod): + raise ImportError("No user defined function or method found with the name %s"%(mod,)) + self.__userFunction__modl = os.path.basename(mod).replace('.pyc','').replace('.pyo','').replace('.py','') + self.__userFunction__path = os.path.dirname(mod) + del mod + self.__userOperator = Operator( fromMethod = Function ) + self.__userFunction = self.__userOperator.appliedTo # Pour le calcul Direct + else: + raise TypeError("User defined function or method has to be provided for finite differences approximation.") + else: + self.__userOperator = Operator( fromMethod = Function ) + self.__userFunction = self.__userOperator.appliedTo + # self.__centeredDF = bool(centeredDF) if avoidingRedundancy: self.__avoidRC = True @@ -161,31 +215,85 @@ class FDApproximation: logging.debug("FDA Calcul Jacobienne (explicite)") if self.__centeredDF: # - _Jacobienne = [] - for i in range( _dX.size ): - _dXi = _dX[i] - _X_plus_dXi = numpy.array( _X.A1, dtype=float ) - _X_plus_dXi[i] = _X[i] + _dXi - _X_moins_dXi = numpy.array( _X.A1, dtype=float ) - _X_moins_dXi[i] = _X[i] - _dXi + if self.__mpEnabled: + funcrepr = { + "__userFunction__path" : self.__userFunction__path, + "__userFunction__modl" : self.__userFunction__modl, + "__userFunction__name" : self.__userFunction__name, + } + _jobs = [] + for i in range( len(_dX) ): + _dXi = _dX[i] + _X_plus_dXi = numpy.array( _X.A1, dtype=float ) + _X_plus_dXi[i] = _X[i] + _dXi + _X_moins_dXi = numpy.array( _X.A1, dtype=float ) + _X_moins_dXi[i] = _X[i] - _dXi + # + _jobs.append( (_X_plus_dXi, funcrepr) ) + _jobs.append( (_X_moins_dXi, funcrepr) ) # - _HX_plus_dXi = self.DirectOperator( _X_plus_dXi ) - _HX_moins_dXi = self.DirectOperator( _X_moins_dXi ) + import multiprocessing + self.__pool = multiprocessing.Pool(self.__mpWorkers) + _HX_plusmoins_dX = self.__pool.map( ExecuteFunction, _jobs ) + self.__pool.close() + self.__pool.join() # - _Jacobienne.append( numpy.ravel( _HX_plus_dXi - _HX_moins_dXi ) / (2.*_dXi) ) + _Jacobienne = [] + for i in range( len(_dX) ): + _Jacobienne.append( numpy.ravel( _HX_plusmoins_dX[2*i] - _HX_plusmoins_dX[2*i+1] ) / (2.*_dX[i]) ) + else: + _Jacobienne = [] + for i in range( _dX.size ): + _dXi = _dX[i] + _X_plus_dXi = numpy.array( _X.A1, dtype=float ) + _X_plus_dXi[i] = _X[i] + _dXi + _X_moins_dXi = numpy.array( _X.A1, dtype=float ) + _X_moins_dXi[i] = _X[i] - _dXi + # + _HX_plus_dXi = self.DirectOperator( _X_plus_dXi ) + _HX_moins_dXi = self.DirectOperator( _X_moins_dXi ) + # + _Jacobienne.append( numpy.ravel( _HX_plus_dXi - _HX_moins_dXi ) / (2.*_dXi) ) # else: # - _Jacobienne = [] - _HX = self.DirectOperator( _X ) - for i in range( _dX.size ): - _dXi = _dX[i] - _X_plus_dXi = numpy.array( _X.A1, dtype=float ) - _X_plus_dXi[i] = _X[i] + _dXi + if self.__mpEnabled: + _HX_plus_dX = [] + funcrepr = { + "__userFunction__path" : self.__userFunction__path, + "__userFunction__modl" : self.__userFunction__modl, + "__userFunction__name" : self.__userFunction__name, + } + _jobs = [] + _jobs.append( (_X.A1, funcrepr) ) + for i in range( len(_dX) ): + _X_plus_dXi = numpy.array( _X.A1, dtype=float ) + _X_plus_dXi[i] = _X[i] + _dX[i] + # + _jobs.append( (_X_plus_dXi, funcrepr) ) + # + import multiprocessing + self.__pool = multiprocessing.Pool(self.__mpWorkers) + _HX_plus_dX = self.__pool.map( ExecuteFunction, _jobs ) + self.__pool.close() + self.__pool.join() # - _HX_plus_dXi = self.DirectOperator( _X_plus_dXi ) + _HX = _HX_plus_dX.pop(0) # - _Jacobienne.append( numpy.ravel(( _HX_plus_dXi - _HX ) / _dXi) ) + _Jacobienne = [] + for i in range( len(_dX) ): + _Jacobienne.append( numpy.ravel(( _HX_plus_dX[i] - _HX ) / _dX[i]) ) + else: + _Jacobienne = [] + _HX = self.DirectOperator( _X ) + for i in range( _dX.size ): + _dXi = _dX[i] + _X_plus_dXi = numpy.array( _X.A1, dtype=float ) + _X_plus_dXi[i] = _X[i] + _dXi + # + _HX_plus_dXi = self.DirectOperator( _X_plus_dXi ) + # + _Jacobienne.append( numpy.ravel(( _HX_plus_dXi - _HX ) / _dXi) ) # # _Jacobienne = numpy.matrix( numpy.vstack( _Jacobienne ) ).T diff --git a/src/daEficas/generator_adao.py b/src/daEficas/generator_adao.py index afbc2f1..713c3fe 100644 --- a/src/daEficas/generator_adao.py +++ b/src/daEficas/generator_adao.py @@ -214,6 +214,8 @@ class AdaoGenerator(PythonGenerator): self.text_da += data_name + "_ScriptWithOneFunction['Script']['Adjoint'] = '" + data + "'\n" self.text_da += data_name + "_ScriptWithOneFunction['DifferentialIncrement'] = " + str(float(self.dictMCVal[search_type + "SCRIPTWITHONEFUNCTION_DATA__DifferentialIncrement"])) + "\n" self.text_da += data_name + "_ScriptWithOneFunction['CenteredFiniteDifference'] = " + str(self.dictMCVal[search_type + "SCRIPTWITHONEFUNCTION_DATA__CenteredFiniteDifference"]) + "\n" + if search_type + "SCRIPTWITHONEFUNCTION_DATA__EnableMultiProcessing" in self.dictMCVal.keys(): + self.text_da += data_name + "_ScriptWithOneFunction['EnableMultiProcessing'] = " + str(self.dictMCVal[search_type + "SCRIPTWITHONEFUNCTION_DATA__EnableMultiProcessing"]) + "\n" self.text_da += data_name + "_config = {}\n" self.text_da += data_name + "_config['Type'] = 'Function'\n" self.text_da += data_name + "_config['From'] = 'ScriptWithOneFunction'\n" diff --git a/src/daSalome/daYacsSchemaCreator/methods.py b/src/daSalome/daYacsSchemaCreator/methods.py index b450414..6aa2a2a 100644 --- a/src/daSalome/daYacsSchemaCreator/methods.py +++ b/src/daSalome/daYacsSchemaCreator/methods.py @@ -679,6 +679,8 @@ def create_yacs_proc(study_config): node_script += """ Function = DirectOperator,\n""" node_script += """ increment = %s,\n"""%str(ScriptWithOneFunction['DifferentialIncrement']) node_script += """ centeredDF = %s,\n"""%str(ScriptWithOneFunction['CenteredFiniteDifference']) + if 'EnableMultiProcessing' in ScriptWithOneFunction.keys(): + node_script += """ mpEnabled = %s,\n"""%str(ScriptWithOneFunction['EnableMultiProcessing']) node_script += """ )\n""" node_script += """#\n""" node_script += """__data = []\n""" @@ -901,6 +903,8 @@ def create_yacs_proc(study_config): node_script += """ Function = DirectOperator,\n""" node_script += """ increment = %s,\n"""%str(ScriptWithOneFunction['DifferentialIncrement']) node_script += """ centeredDF = %s,\n"""%str(ScriptWithOneFunction['CenteredFiniteDifference']) + if 'EnableMultiProcessing' in ScriptWithOneFunction.keys(): + node_script += """ mpEnabled = %s,\n"""%str(ScriptWithOneFunction['EnableMultiProcessing']) node_script += """ )\n""" node_script += """#\n""" node_script += """__data = []\n""" -- 2.39.2