X-Git-Url: http://git.salome-platform.org/gitweb/?a=blobdiff_plain;f=src%2FdaComposant%2FdaCore%2FBasicObjects.py;h=3b6b0d5ae74a4fc8333930c71e85c55c20c96a4c;hb=ba52f8705ad24049d6fa5957da1cbe00d7bcb4d6;hp=aba702c66b5c44ffe56df8bc375f22eeb8801bdf;hpb=018c6093cf0f1f41fb2a99f3c32cfa4c7cdb9d12;p=modules%2Fadao.git diff --git a/src/daComposant/daCore/BasicObjects.py b/src/daComposant/daCore/BasicObjects.py index aba702c..3b6b0d5 100644 --- a/src/daComposant/daCore/BasicObjects.py +++ b/src/daComposant/daCore/BasicObjects.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2008-2021 EDF R&D +# Copyright (C) 2008-2022 EDF R&D # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -125,6 +125,7 @@ class Operator(object): fromMethod = None, fromMatrix = None, avoidingRedundancy = True, + reducingMemoryUse = False, inputAsMultiFunction = False, enableMultiProcess = False, extraArguments = None, @@ -136,8 +137,10 @@ class Operator(object): Arguments : - name : nom d'opérateur - fromMethod : argument de type fonction Python - - fromMatrix : argument adapté au constructeur numpy.matrix + - fromMatrix : argument adapté au constructeur numpy.array/matrix - avoidingRedundancy : booléen évitant (ou pas) les calculs redondants + - reducingMemoryUse : booléen forçant (ou pas) des calculs moins + gourmands en mémoire - inputAsMultiFunction : booléen indiquant une fonction explicitement définie (ou pas) en multi-fonction - extraArguments : arguments supplémentaires passés à la fonction de @@ -145,7 +148,8 @@ class Operator(object): """ self.__name = str(name) self.__NbCallsAsMatrix, self.__NbCallsAsMethod, self.__NbCallsOfCached = 0, 0, 0 - self.__AvoidRC = bool( avoidingRedundancy ) + self.__reduceM = bool( reducingMemoryUse ) + self.__avoidRC = bool( avoidingRedundancy ) self.__inputAsMF = bool( inputAsMultiFunction ) self.__mpEnabled = bool( enableMultiProcess ) self.__extraArgs = extraArguments @@ -159,7 +163,9 @@ class Operator(object): self.__Type = "Method" elif fromMatrix is not None: self.__Method = None - self.__Matrix = numpy.matrix( fromMatrix, numpy.float ) + if isinstance(fromMatrix, str): + fromMatrix = PlatformInfo.strmatrix2liststr( fromMatrix ) + self.__Matrix = numpy.asarray( fromMatrix, dtype=float ) self.__Type = "Matrix" else: self.__Method = None @@ -172,7 +178,7 @@ class Operator(object): def enableAvoidingRedundancy(self): "Active le cache" - if self.__AvoidRC: + if self.__avoidRC: Operator.CM.enable() else: Operator.CM.disable() @@ -207,15 +213,15 @@ class Operator(object): assert len(_xValue) == len(_HValue), "Incompatible number of elements in xValue and HValue" _HxValue = [] for i in range(len(_HValue)): - _HxValue.append( numpy.asmatrix( numpy.ravel( _HValue[i] ) ).T ) - if self.__AvoidRC: + _HxValue.append( _HValue[i] ) + if self.__avoidRC: Operator.CM.storeValueInX(_xValue[i],_HxValue[-1],self.__name) else: _HxValue = [] _xserie = [] _hindex = [] for i, xv in enumerate(_xValue): - if self.__AvoidRC: + if self.__avoidRC: __alreadyCalculated, __HxV = Operator.CM.wasCalculatedIn(xv,self.__name) else: __alreadyCalculated = False @@ -226,8 +232,7 @@ class Operator(object): else: if self.__Matrix is not None: self.__addOneMatrixCall() - _xv = numpy.ravel(xv).reshape((-1,1)) - _hv = self.__Matrix * _xv + _hv = self.__Matrix @ numpy.ravel(xv) else: self.__addOneMethodCall() _xserie.append( xv ) @@ -246,7 +251,7 @@ class Operator(object): _xv = _xserie.pop(0) _hv = _hserie.pop(0) _HxValue[i] = _hv - if self.__AvoidRC: + if self.__avoidRC: Operator.CM.storeValueInX(_xv,_hv,self.__name) # if returnSerieAsArrayMatrix: @@ -275,9 +280,8 @@ class Operator(object): _HxValue = [] for paire in _xuValue: _xValue, _uValue = paire - _xValue = numpy.matrix(numpy.ravel(_xValue)).T self.__addOneMatrixCall() - _HxValue.append( self.__Matrix * _xValue ) + _HxValue.append( self.__Matrix @ numpy.ravel(_xValue) ) else: _xuArgs = [] for paire in _xuValue: @@ -322,9 +326,8 @@ class Operator(object): _HxValue = [] for paire in _nxValue: _xNominal, _xValue = paire - _xValue = numpy.matrix(numpy.ravel(_xValue)).T self.__addOneMatrixCall() - _HxValue.append( self.__Matrix * _xValue ) + _HxValue.append( self.__Matrix @ numpy.ravel(_xValue) ) else: self.__addOneMethodCall( len(_nxValue) ) if self.__extraArgs is None: @@ -350,7 +353,7 @@ class Operator(object): if argsAsSerie: self.__addOneMethodCall( len(ValueForMethodForm) ) for _vfmf in ValueForMethodForm: - mValue.append( numpy.matrix( self.__Method(((_vfmf, None),)) ) ) + mValue.append( self.__Method(((_vfmf, None),)) ) else: self.__addOneMethodCall() mValue = self.__Method(((ValueForMethodForm, None),)) @@ -417,7 +420,7 @@ class FullOperator(object): asDict = None, # Parameters appliedInX = None, extraArguments = None, - avoidRC = True, + performancePrf = None, inputAsMF = False,# Fonction(s) as Multi-Functions scheduledBy = None, toBeChecked = False, @@ -444,6 +447,15 @@ class FullOperator(object): __Parameters["EnableMultiProcessingInEvaluation"] = False if "withIncrement" in __Parameters: # Temporaire __Parameters["DifferentialIncrement"] = __Parameters["withIncrement"] + # Le défaut est équivalent à "ReducedOverallRequirements" + __reduceM, __avoidRC = True, True + if performancePrf is not None: + if performancePrf == "ReducedAmountOfCalculation": + __reduceM, __avoidRC = False, True + elif performancePrf == "ReducedMemoryFootprint": + __reduceM, __avoidRC = True, False + elif performancePrf == "NoSavings": + __reduceM, __avoidRC = False, False # if asScript is not None: __Matrix, __Function = None, None @@ -512,7 +524,8 @@ class FullOperator(object): if "CenteredFiniteDifference" not in __Function: __Function["CenteredFiniteDifference"] = False if "DifferentialIncrement" not in __Function: __Function["DifferentialIncrement"] = 0.01 if "withdX" not in __Function: __Function["withdX"] = None - if "withAvoidingRedundancy" not in __Function: __Function["withAvoidingRedundancy"] = avoidRC + if "withReducingMemoryUse" not in __Function: __Function["withReducingMemoryUse"] = __reduceM + if "withAvoidingRedundancy" not in __Function: __Function["withAvoidingRedundancy"] = __avoidRC if "withToleranceInRedundancy" not in __Function: __Function["withToleranceInRedundancy"] = 1.e-18 if "withLenghtOfRedundancy" not in __Function: __Function["withLenghtOfRedundancy"] = -1 if "NumberOfProcesses" not in __Function: __Function["NumberOfProcesses"] = None @@ -525,6 +538,7 @@ class FullOperator(object): increment = __Function["DifferentialIncrement"], dX = __Function["withdX"], extraArguments = self.__extraArgs, + reducingMemoryUse = __Function["withReducingMemoryUse"], avoidingRedundancy = __Function["withAvoidingRedundancy"], toleranceInRedundancy = __Function["withToleranceInRedundancy"], lenghtOfRedundancy = __Function["withLenghtOfRedundancy"], @@ -532,20 +546,22 @@ class FullOperator(object): mpWorkers = __Function["NumberOfProcesses"], mfEnabled = __Function["withmfEnabled"], ) - self.__FO["Direct"] = Operator( name = self.__name, fromMethod = FDA.DirectOperator, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) - self.__FO["Tangent"] = Operator( name = self.__name+"Tangent", fromMethod = FDA.TangentOperator, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) - self.__FO["Adjoint"] = Operator( name = self.__name+"Adjoint", fromMethod = FDA.AdjointOperator, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) + self.__FO["Direct"] = Operator( name = self.__name, fromMethod = FDA.DirectOperator, reducingMemoryUse = __reduceM, avoidingRedundancy = __avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) + self.__FO["Tangent"] = Operator( name = self.__name+"Tangent", fromMethod = FDA.TangentOperator, reducingMemoryUse = __reduceM, avoidingRedundancy = __avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) + self.__FO["Adjoint"] = Operator( name = self.__name+"Adjoint", fromMethod = FDA.AdjointOperator, reducingMemoryUse = __reduceM, avoidingRedundancy = __avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) elif isinstance(__Function, dict) and \ ("Direct" in __Function) and ("Tangent" in __Function) and ("Adjoint" in __Function) and \ (__Function["Direct"] is not None) and (__Function["Tangent"] is not None) and (__Function["Adjoint"] is not None): - self.__FO["Direct"] = Operator( name = self.__name, fromMethod = __Function["Direct"], avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) - self.__FO["Tangent"] = Operator( name = self.__name+"Tangent", fromMethod = __Function["Tangent"], avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) - self.__FO["Adjoint"] = Operator( name = self.__name+"Adjoint", fromMethod = __Function["Adjoint"], avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) + self.__FO["Direct"] = Operator( name = self.__name, fromMethod = __Function["Direct"], reducingMemoryUse = __reduceM, avoidingRedundancy = __avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) + self.__FO["Tangent"] = Operator( name = self.__name+"Tangent", fromMethod = __Function["Tangent"], reducingMemoryUse = __reduceM, avoidingRedundancy = __avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) + self.__FO["Adjoint"] = Operator( name = self.__name+"Adjoint", fromMethod = __Function["Adjoint"], reducingMemoryUse = __reduceM, avoidingRedundancy = __avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) elif asMatrix is not None: - __matrice = numpy.matrix( __Matrix, numpy.float ) - self.__FO["Direct"] = Operator( name = self.__name, fromMatrix = __matrice, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) - self.__FO["Tangent"] = Operator( name = self.__name+"Tangent", fromMatrix = __matrice, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF ) - self.__FO["Adjoint"] = Operator( name = self.__name+"Adjoint", fromMatrix = __matrice.T, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF ) + if isinstance(__Matrix, str): + __Matrix = PlatformInfo.strmatrix2liststr( __Matrix ) + __matrice = numpy.asarray( __Matrix, dtype=float ) + self.__FO["Direct"] = Operator( name = self.__name, fromMatrix = __matrice, reducingMemoryUse = __reduceM, avoidingRedundancy = __avoidRC, inputAsMultiFunction = inputAsMF, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) + self.__FO["Tangent"] = Operator( name = self.__name+"Tangent", fromMatrix = __matrice, reducingMemoryUse = __reduceM, avoidingRedundancy = __avoidRC, inputAsMultiFunction = inputAsMF ) + self.__FO["Adjoint"] = Operator( name = self.__name+"Adjoint", fromMatrix = __matrice.T, reducingMemoryUse = __reduceM, avoidingRedundancy = __avoidRC, inputAsMultiFunction = inputAsMF ) del __matrice else: raise ValueError("The %s object is improperly defined or undefined, it requires at minima either a matrix, a Direct operator for approximate derivatives or a Tangent/Adjoint operators pair. Please check your operator input."%self.__name) @@ -726,7 +742,12 @@ class Algorithm(object): else: logging.debug("%s %s vector %s is not set, but is not required."%(self._name,argname,symbol)) else: - logging.debug("%s %s vector %s is set, and its size is %i."%(self._name,argname,symbol,numpy.array(argument).size)) + if variable in self.__required_inputs["RequiredInputValues"]["mandatory"]: + logging.debug("%s %s vector %s is required and set, and its size is %i."%(self._name,argname,symbol,numpy.array(argument).size)) + elif variable in self.__required_inputs["RequiredInputValues"]["optional"]: + logging.debug("%s %s vector %s is optional and set, and its size is %i."%(self._name,argname,symbol,numpy.array(argument).size)) + else: + logging.debug("%s %s vector %s is set although neither required nor optional, and its size is %i."%(self._name,argname,symbol,numpy.array(argument).size)) return 0 __test_vvalue( Xb, "Xb", "Background or initial state" ) __test_vvalue( Y, "Y", "Observation" ) @@ -743,7 +764,12 @@ class Algorithm(object): else: logging.debug("%s %s error covariance matrix %s is not set, but is not required."%(self._name,argname,symbol)) else: - logging.debug("%s %s error covariance matrix %s is set."%(self._name,argname,symbol)) + if variable in self.__required_inputs["RequiredInputValues"]["mandatory"]: + logging.debug("%s %s error covariance matrix %s is required and set."%(self._name,argname,symbol)) + elif variable in self.__required_inputs["RequiredInputValues"]["optional"]: + logging.debug("%s %s error covariance matrix %s is optional and set."%(self._name,argname,symbol)) + else: + logging.debug("%s %s error covariance matrix %s is set although neither required nor optional."%(self._name,argname,symbol)) return 0 __test_cvalue( B, "B", "Background" ) __test_cvalue( R, "R", "Observation" ) @@ -760,7 +786,12 @@ class Algorithm(object): else: logging.debug("%s %s operator %s is not set, but is not required."%(self._name,argname,symbol)) else: - logging.debug("%s %s operator %s is set."%(self._name,argname,symbol)) + if variable in self.__required_inputs["RequiredInputValues"]["mandatory"]: + logging.debug("%s %s operator %s is required and set."%(self._name,argname,symbol)) + elif variable in self.__required_inputs["RequiredInputValues"]["optional"]: + logging.debug("%s %s operator %s is optional and set."%(self._name,argname,symbol)) + else: + logging.debug("%s %s operator %s is set although neither required nor optional."%(self._name,argname,symbol)) return 0 __test_ovalue( HO, "HO", "Observation", "H" ) __test_ovalue( EM, "EM", "Evolution", "M" ) @@ -995,7 +1026,10 @@ class Algorithm(object): self._parameters[k] = self.setParameterValue(k) else: pass - logging.debug("%s %s : %s", self._name, self.__required_parameters[k]["message"], self._parameters[k]) + if hasattr(self._parameters[k],"__len__") and len(self._parameters[k]) > 100: + logging.debug("%s %s de longueur %s", self._name, self.__required_parameters[k]["message"], len(self._parameters[k])) + else: + logging.debug("%s %s : %s", self._name, self.__required_parameters[k]["message"], self._parameters[k]) def _setInternalState(self, key=None, value=None, fromDico={}, reset=False): """ @@ -1043,6 +1077,45 @@ class Algorithm(object): else: return __SC +# ============================================================================== +class PartialAlgorithm(object): + """ + Classe pour mimer "Algorithm" du point de vue stockage, mais sans aucune + action avancée comme la vérification . Pour les méthodes reprises ici, + le fonctionnement est identique à celles de la classe "Algorithm". + """ + def __init__(self, name): + self._name = str( name ) + self._parameters = {"StoreSupplementaryCalculations":[]} + # + self.StoredVariables = {} + self.StoredVariables["Analysis"] = Persistence.OneVector(name = "Analysis") + self.StoredVariables["CostFunctionJ"] = Persistence.OneScalar(name = "CostFunctionJ") + self.StoredVariables["CostFunctionJb"] = Persistence.OneScalar(name = "CostFunctionJb") + self.StoredVariables["CostFunctionJo"] = Persistence.OneScalar(name = "CostFunctionJo") + self.StoredVariables["CurrentIterationNumber"] = Persistence.OneIndex(name = "CurrentIterationNumber") + # + self.__canonical_stored_name = {} + for k in self.StoredVariables: + self.__canonical_stored_name[k.lower()] = k + + def _toStore(self, key): + "True if in StoreSupplementaryCalculations, else False" + return key in self._parameters["StoreSupplementaryCalculations"] + + def get(self, key=None): + """ + Renvoie l'une des variables stockées identifiée par la clé, ou le + dictionnaire de l'ensemble des variables disponibles en l'absence de + clé. Ce sont directement les variables sous forme objet qui sont + renvoyées, donc les méthodes d'accès à l'objet individuel sont celles + des classes de persistance. + """ + if key is not None: + return self.StoredVariables[self.__canonical_stored_name[key.lower()]] + else: + return self.StoredVariables + # ============================================================================== class AlgorithmAndParameters(object): """ @@ -1398,9 +1471,9 @@ class AlgorithmAndParameters(object): if self.__B is not None and len(self.__B) > 0 and not( __B_shape[1] == max(__Xb_shape) ): if self.__algorithmName in ["EnsembleBlue",]: asPersistentVector = self.__Xb.reshape((-1,min(__B_shape))) - self.__Xb = Persistence.OneVector("Background", basetype=numpy.matrix) + self.__Xb = Persistence.OneVector("Background") for member in asPersistentVector: - self.__Xb.store( numpy.matrix( numpy.ravel(member), numpy.float ).T ) + self.__Xb.store( numpy.asarray(member, dtype=float) ) __Xb_shape = min(__B_shape) else: raise ValueError("Shape characteristic of a priori errors covariance matrix (B) \"%s\" and background (Xb) \"%s\" are incompatible."%(__B_shape,__Xb_shape)) @@ -1688,16 +1761,22 @@ class State(object): # if __Vector is not None: self.__is_vector = True - self.__V = numpy.matrix( numpy.asmatrix(__Vector).A1, numpy.float ).T + if isinstance(__Vector, str): + __Vector = PlatformInfo.strvect2liststr( __Vector ) + self.__V = numpy.ravel(numpy.asarray( __Vector, dtype=float )).reshape((-1,1)) self.shape = self.__V.shape self.size = self.__V.size elif __Series is not None: self.__is_series = True if isinstance(__Series, (tuple, list, numpy.ndarray, numpy.matrix, str)): - self.__V = Persistence.OneVector(self.__name, basetype=numpy.matrix) - if isinstance(__Series, str): __Series = eval(__Series) + #~ self.__V = Persistence.OneVector(self.__name, basetype=numpy.matrix) + self.__V = Persistence.OneVector(self.__name) + if isinstance(__Series, str): + __Series = PlatformInfo.strmatrix2liststr(__Series) for member in __Series: - self.__V.store( numpy.matrix( numpy.asmatrix(member).A1, numpy.float ).T ) + if isinstance(member, str): + member = PlatformInfo.strvect2liststr( member ) + self.__V.store(numpy.asarray( member, dtype=float )) else: self.__V = __Series if isinstance(self.__V.shape, (tuple, list)): @@ -1792,7 +1871,7 @@ class Covariance(object): # if __Scalar is not None: if isinstance(__Scalar, str): - __Scalar = __Scalar.replace(";"," ").replace(","," ").split() + __Scalar = PlatformInfo.strvect2liststr( __Scalar ) if len(__Scalar) > 0: __Scalar = __Scalar[0] if numpy.array(__Scalar).size != 1: raise ValueError(' The diagonal multiplier given to define a sparse matrix is not a unique scalar value.\n Its actual measured size is %i. Please check your scalar input.'%numpy.array(__Scalar).size) @@ -1802,9 +1881,9 @@ class Covariance(object): self.size = 0 elif __Vector is not None: if isinstance(__Vector, str): - __Vector = __Vector.replace(";"," ").replace(","," ").split() + __Vector = PlatformInfo.strvect2liststr( __Vector ) self.__is_vector = True - self.__C = numpy.abs( numpy.array( numpy.ravel( __Vector ), dtype=float ) ) + self.__C = numpy.abs( numpy.ravel(numpy.asarray( __Vector, dtype=float )) ) self.shape = (self.__C.size,self.__C.size) self.size = self.__C.size**2 elif __Matrix is not None: @@ -1986,14 +2065,14 @@ class Covariance(object): def asfullmatrix(self, msize=None): "Matrice pleine" if self.ismatrix(): - return numpy.asarray(self.__C) + return numpy.asarray(self.__C, dtype=float) elif self.isvector(): - return numpy.asarray( numpy.diag(self.__C), float ) + return numpy.asarray( numpy.diag(self.__C), dtype=float ) elif self.isscalar(): if msize is None: raise ValueError("the size of the %s covariance matrix has to be given in case of definition as a scalar over the diagonal."%(self.__name,)) else: - return numpy.asarray( self.__C * numpy.eye(int(msize)), float ) + return numpy.asarray( self.__C * numpy.eye(int(msize)), dtype=float ) elif self.isobject() and hasattr(self.__C,"asfullmatrix"): return self.__C.asfullmatrix() else: @@ -2149,6 +2228,8 @@ class Covariance(object): raise ValueError("operands could not be broadcast together with shapes %s %s in %s matrix"%(numpy.ravel(other).shape,self.shape,self.__name)) elif self.isscalar() and isinstance(other,numpy.matrix): return other * self.__C + elif self.isscalar() and isinstance(other,float): + return other * self.__C elif self.isobject(): return self.__C.__rmul__(other) else: