X-Git-Url: http://git.salome-platform.org/gitweb/?a=blobdiff_plain;f=src%2FdaComposant%2FdaCore%2FBasicObjects.py;h=76fd292115541ec3f7f48a185d9533f34158f3ee;hb=2201c84e2047882b9bd918b3e62de45711d514d5;hp=14d29dc35bd2fbcd4f741a7ec2997600554dbbfa;hpb=fdfa4718d30e8aee85ade24f8296a9ad108fc6c9;p=modules%2Fadao.git diff --git a/src/daComposant/daCore/BasicObjects.py b/src/daComposant/daCore/BasicObjects.py index 14d29dc..76fd292 100644 --- a/src/daComposant/daCore/BasicObjects.py +++ b/src/daComposant/daCore/BasicObjects.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2008-2020 EDF R&D +# Copyright (C) 2008-2021 EDF R&D # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -30,6 +30,7 @@ import os import sys import logging import copy +import time import numpy from functools import partial from daCore import Persistence, PlatformInfo, Interfaces @@ -45,38 +46,47 @@ class CacheManager(object): lenghtOfRedundancy = -1, ): """ - Les caractéristiques de tolérance peuvent être modifées à la création. + Les caractéristiques de tolérance peuvent être modifiées à la création. """ - self.__tolerBP = float(toleranceInRedundancy) - self.__lenghtOR = int(lenghtOfRedundancy) - self.__initlnOR = self.__lenghtOR + self.__tolerBP = float(toleranceInRedundancy) + self.__lenghtOR = int(lenghtOfRedundancy) + self.__initlnOR = self.__lenghtOR + self.__seenNames = [] + self.__enabled = True self.clearCache() def clearCache(self): "Vide le cache" - self.__listOPCV = [] # Operator Previous Calculated Points, Results, Point Norms + self.__listOPCV = [] # Previous Calculated Points, Results, Point Norms, Operator + self.__seenNames = [] # logging.debug("CM Tolerance de determination des doublons : %.2e", self.__tolerBP) - def wasCalculatedIn(self, xValue ): #, info="" ): + def wasCalculatedIn(self, xValue, oName="" ): #, info="" ): "Vérifie l'existence d'un calcul correspondant à la valeur" __alc = False __HxV = None - for i in range(min(len(self.__listOPCV),self.__lenghtOR)-1,-1,-1): - if not hasattr(xValue, 'size') or (xValue.size != self.__listOPCV[i][0].size): - # logging.debug("CM Différence de la taille %s de X et de celle %s du point %i déjà calculé", xValue.shape,i,self.__listOPCP[i].shape) - continue - if numpy.linalg.norm(numpy.ravel(xValue) - self.__listOPCV[i][0]) < self.__tolerBP * self.__listOPCV[i][2]: - __alc = True - __HxV = self.__listOPCV[i][1] - # logging.debug("CM Cas%s déja calculé, portant le numéro %i", info, i) - break + if self.__enabled: + for i in range(min(len(self.__listOPCV),self.__lenghtOR)-1,-1,-1): + if not hasattr(xValue, 'size') or (str(oName) != self.__listOPCV[i][3]) or (xValue.size != self.__listOPCV[i][0].size): + # logging.debug("CM Différence de la taille %s de X et de celle %s du point %i déjà calculé", xValue.shape,i,self.__listOPCP[i].shape) + pass + elif numpy.linalg.norm(numpy.ravel(xValue) - self.__listOPCV[i][0]) < self.__tolerBP * self.__listOPCV[i][2]: + __alc = True + __HxV = self.__listOPCV[i][1] + # logging.debug("CM Cas%s déja calculé, portant le numéro %i", info, i) + break return __alc, __HxV - def storeValueInX(self, xValue, HxValue ): - "Stocke un calcul correspondant à la valeur" + def storeValueInX(self, xValue, HxValue, oName="" ): + "Stocke pour un opérateur o un calcul Hx correspondant à la valeur x" if self.__lenghtOR < 0: self.__lenghtOR = 2 * xValue.size + 2 self.__initlnOR = self.__lenghtOR + self.__seenNames.append(str(oName)) + if str(oName) not in self.__seenNames: # Etend la liste si nouveau + self.__lenghtOR += 2 * xValue.size + 2 + self.__initlnOR += self.__lenghtOR + self.__seenNames.append(str(oName)) while len(self.__listOPCV) > self.__lenghtOR: # logging.debug("CM Réduction de la liste des cas à %i éléments par suppression du premier", self.__lenghtOR) self.__listOPCV.pop(0) @@ -84,16 +94,19 @@ class CacheManager(object): copy.copy(numpy.ravel(xValue)), copy.copy(HxValue), numpy.linalg.norm(xValue), + str(oName), ) ) def disable(self): "Inactive le cache" self.__initlnOR = self.__lenghtOR self.__lenghtOR = 0 + self.__enabled = False def enable(self): "Active le cache" self.__lenghtOR = self.__initlnOR + self.__enabled = True # ============================================================================== class Operator(object): @@ -106,6 +119,7 @@ class Operator(object): CM = CacheManager() # def __init__(self, + name = "GenericOperator", fromMethod = None, fromMatrix = None, avoidingRedundancy = True, @@ -118,6 +132,7 @@ class Operator(object): deux mots-clé, soit une fonction ou un multi-fonction python, soit une matrice. Arguments : + - name : nom d'opérateur - fromMethod : argument de type fonction Python - fromMatrix : argument adapté au constructeur numpy.matrix - avoidingRedundancy : booléen évitant (ou pas) les calculs redondants @@ -126,6 +141,7 @@ class Operator(object): - extraArguments : arguments supplémentaires passés à la fonction de base et ses dérivées (tuple ou dictionnaire) """ + self.__name = str(name) self.__NbCallsAsMatrix, self.__NbCallsAsMethod, self.__NbCallsOfCached = 0, 0, 0 self.__AvoidRC = bool( avoidingRedundancy ) self.__inputAsMF = bool( inputAsMultiFunction ) @@ -163,7 +179,7 @@ class Operator(object): "Renvoie le type" return self.__Type - def appliedTo(self, xValue, HValue = None, argsAsSerie = False): + def appliedTo(self, xValue, HValue = None, argsAsSerie = False, returnSerieAsArrayMatrix = False): """ Permet de restituer le résultat de l'application de l'opérateur à une série d'arguments xValue. Cette méthode se contente d'appliquer, chaque @@ -187,18 +203,18 @@ class Operator(object): # if _HValue is not None: assert len(_xValue) == len(_HValue), "Incompatible number of elements in xValue and HValue" - HxValue = [] + _HxValue = [] for i in range(len(_HValue)): - HxValue.append( numpy.asmatrix( numpy.ravel( _HValue[i] ) ).T ) + _HxValue.append( numpy.asmatrix( numpy.ravel( _HValue[i] ) ).T ) if self.__AvoidRC: - Operator.CM.storeValueInX(_xValue[i],HxValue[-1]) + Operator.CM.storeValueInX(_xValue[i],_HxValue[-1],self.__name) else: - HxValue = [] + _HxValue = [] _xserie = [] _hindex = [] for i, xv in enumerate(_xValue): if self.__AvoidRC: - __alreadyCalculated, __HxV = Operator.CM.wasCalculatedIn(xv) + __alreadyCalculated, __HxV = Operator.CM.wasCalculatedIn(xv,self.__name) else: __alreadyCalculated = False # @@ -214,7 +230,7 @@ class Operator(object): _xserie.append( xv ) _hindex.append( i ) _hv = None - HxValue.append( _hv ) + _HxValue.append( _hv ) # if len(_xserie)>0 and self.__Matrix is None: if self.__extraArgs is None: @@ -226,14 +242,17 @@ class Operator(object): for i in _hindex: _xv = _xserie.pop(0) _hv = _hserie.pop(0) - HxValue[i] = _hv + _HxValue[i] = _hv if self.__AvoidRC: - Operator.CM.storeValueInX(_xv,_hv) + Operator.CM.storeValueInX(_xv,_hv,self.__name) # - if argsAsSerie: return HxValue - else: return HxValue[-1] + if returnSerieAsArrayMatrix: + _HxValue = numpy.stack([numpy.ravel(_hv) for _hv in _HxValue], axis=1) + # + if argsAsSerie: return _HxValue + else: return _HxValue[-1] - def appliedControledFormTo(self, paires, argsAsSerie = False): + def appliedControledFormTo(self, paires, argsAsSerie = False, returnSerieAsArrayMatrix = False): """ Permet de restituer le résultat de l'application de l'opérateur à des paires (xValue, uValue). Cette méthode se contente d'appliquer, son @@ -250,30 +269,32 @@ class Operator(object): PlatformInfo.isIterable( _xuValue, True, " in Operator.appliedControledFormTo" ) # if self.__Matrix is not None: - HxValue = [] + _HxValue = [] for paire in _xuValue: _xValue, _uValue = paire self.__addOneMatrixCall() - HxValue.append( self.__Matrix * _xValue ) + _HxValue.append( self.__Matrix * _xValue ) else: - HxValue = [] + _xuArgs = [] for paire in _xuValue: - _xuValue = [] _xValue, _uValue = paire if _uValue is not None: - _xuValue.append( paire ) + _xuArgs.append( paire ) else: - _xuValue.append( _xValue ) - self.__addOneMethodCall( len(_xuValue) ) + _xuArgs.append( _xValue ) + self.__addOneMethodCall( len(_xuArgs) ) if self.__extraArgs is None: - HxValue = self.__Method( _xuValue ) # Calcul MF + _HxValue = self.__Method( _xuArgs ) # Calcul MF else: - HxValue = self.__Method( _xuValue, self.__extraArgs ) # Calcul MF + _HxValue = self.__Method( _xuArgs, self.__extraArgs ) # Calcul MF + # + if returnSerieAsArrayMatrix: + _HxValue = numpy.stack([numpy.ravel(_hv) for _hv in _HxValue], axis=1) # - if argsAsSerie: return HxValue - else: return HxValue[-1] + if argsAsSerie: return _HxValue + else: return _HxValue[-1] - def appliedInXTo(self, paires, argsAsSerie = False): + def appliedInXTo(self, paires, argsAsSerie = False, returnSerieAsArrayMatrix = False): """ Permet de restituer le résultat de l'application de l'opérateur à une série d'arguments xValue, sachant que l'opérateur est valable en @@ -294,20 +315,23 @@ class Operator(object): PlatformInfo.isIterable( _nxValue, True, " in Operator.appliedInXTo" ) # if self.__Matrix is not None: - HxValue = [] + _HxValue = [] for paire in _nxValue: _xNominal, _xValue = paire self.__addOneMatrixCall() - HxValue.append( self.__Matrix * _xValue ) + _HxValue.append( self.__Matrix * _xValue ) else: self.__addOneMethodCall( len(_nxValue) ) if self.__extraArgs is None: - HxValue = self.__Method( _nxValue ) # Calcul MF + _HxValue = self.__Method( _nxValue ) # Calcul MF else: - HxValue = self.__Method( _nxValue, self.__extraArgs ) # Calcul MF + _HxValue = self.__Method( _nxValue, self.__extraArgs ) # Calcul MF # - if argsAsSerie: return HxValue - else: return HxValue[-1] + if returnSerieAsArrayMatrix: + _HxValue = numpy.stack([numpy.ravel(_hv) for _hv in _HxValue], axis=1) + # + if argsAsSerie: return _HxValue + else: return _HxValue[-1] def asMatrix(self, ValueForMethodForm = "UnknownVoidValue", argsAsSerie = False): """ @@ -316,7 +340,7 @@ class Operator(object): if self.__Matrix is not None: self.__addOneMatrixCall() mValue = [self.__Matrix,] - elif ValueForMethodForm is not "UnknownVoidValue": # Ne pas utiliser "None" + elif not isinstance(ValueForMethodForm,str) or ValueForMethodForm != "UnknownVoidValue": # Ne pas utiliser "None" mValue = [] if argsAsSerie: self.__addOneMethodCall( len(ValueForMethodForm) ) @@ -490,6 +514,7 @@ class FullOperator(object): if "withmfEnabled" not in __Function: __Function["withmfEnabled"] = inputAsMF from daCore import NumericObjects FDA = NumericObjects.FDApproximation( + name = self.__name, Function = __Function["Direct"], centeredDF = __Function["CenteredFiniteDifference"], increment = __Function["DifferentialIncrement"], @@ -501,20 +526,20 @@ class FullOperator(object): mpWorkers = __Function["NumberOfProcesses"], mfEnabled = __Function["withmfEnabled"], ) - self.__FO["Direct"] = Operator( fromMethod = FDA.DirectOperator, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) - self.__FO["Tangent"] = Operator( fromMethod = FDA.TangentOperator, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) - self.__FO["Adjoint"] = Operator( fromMethod = FDA.AdjointOperator, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) + self.__FO["Direct"] = Operator( name = self.__name, fromMethod = FDA.DirectOperator, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) + self.__FO["Tangent"] = Operator( name = self.__name+"Tangent", fromMethod = FDA.TangentOperator, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) + self.__FO["Adjoint"] = Operator( name = self.__name+"Adjoint", fromMethod = FDA.AdjointOperator, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) elif isinstance(__Function, dict) and \ ("Direct" in __Function) and ("Tangent" in __Function) and ("Adjoint" in __Function) and \ (__Function["Direct"] is not None) and (__Function["Tangent"] is not None) and (__Function["Adjoint"] is not None): - self.__FO["Direct"] = Operator( fromMethod = __Function["Direct"], avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) - self.__FO["Tangent"] = Operator( fromMethod = __Function["Tangent"], avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) - self.__FO["Adjoint"] = Operator( fromMethod = __Function["Adjoint"], avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) + self.__FO["Direct"] = Operator( name = self.__name, fromMethod = __Function["Direct"], avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) + self.__FO["Tangent"] = Operator( name = self.__name+"Tangent", fromMethod = __Function["Tangent"], avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) + self.__FO["Adjoint"] = Operator( name = self.__name+"Adjoint", fromMethod = __Function["Adjoint"], avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) elif asMatrix is not None: __matrice = numpy.matrix( __Matrix, numpy.float ) - self.__FO["Direct"] = Operator( fromMatrix = __matrice, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) - self.__FO["Tangent"] = Operator( fromMatrix = __matrice, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF ) - self.__FO["Adjoint"] = Operator( fromMatrix = __matrice.T, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF ) + self.__FO["Direct"] = Operator( name = self.__name, fromMatrix = __matrice, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) + self.__FO["Tangent"] = Operator( name = self.__name+"Tangent", fromMatrix = __matrice, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF ) + self.__FO["Adjoint"] = Operator( name = self.__name+"Adjoint", fromMatrix = __matrice.T, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF ) del __matrice else: raise ValueError("The %s object is improperly defined or undefined, it requires at minima either a matrix, a Direct operator for approximate derivatives or a Tangent/Adjoint operators pair. Please check your operator input."%self.__name) @@ -575,6 +600,7 @@ class Algorithm(object): - CostFunctionJbAtCurrentOptimum : partie ébauche à l'état optimal courant lors d'itérations - CostFunctionJo : partie observations de la fonction-coût : Jo - CostFunctionJoAtCurrentOptimum : partie observations à l'état optimal courant lors d'itérations + - CurrentIterationNumber : numéro courant d'itération dans les algorithmes itératifs, à partir de 0 - CurrentOptimum : état optimal courant lors d'itérations - CurrentState : état courant lors d'itérations - GradientOfCostFunctionJ : gradient de la fonction-coût globale @@ -590,7 +616,7 @@ class Algorithm(object): - MahalanobisConsistency : indicateur de consistance des covariances - OMA : Observation moins Analyse : Y - Xa - OMB : Observation moins Background : Y - Xb - - PredictedState : état prédit courant lors d'itérations + - ForecastState : état prédit courant lors d'itérations - Residu : dans le cas des algorithmes de vérification - SigmaBck2 : indicateur de correction optimale des erreurs d'ébauche - SigmaObs2 : indicateur de correction optimale des erreurs d'observation @@ -608,7 +634,10 @@ class Algorithm(object): self._name = str( name ) self._parameters = {"StoreSupplementaryCalculations":[]} self.__required_parameters = {} - self.__required_inputs = {"RequiredInputValues":{"mandatory":(), "optional":()}} + self.__required_inputs = { + "RequiredInputValues":{"mandatory":(), "optional":()}, + "ClassificationTags":[], + } self.__variable_names_not_public = {"nextStep":False} # Duplication dans AlgorithmAndParameters self.__canonical_parameter_name = {} # Correspondance "lower"->"correct" self.__canonical_stored_name = {} # Correspondance "lower"->"correct" @@ -626,12 +655,14 @@ class Algorithm(object): self.StoredVariables["CostFunctionJbAtCurrentOptimum"] = Persistence.OneScalar(name = "CostFunctionJbAtCurrentOptimum") self.StoredVariables["CostFunctionJo"] = Persistence.OneScalar(name = "CostFunctionJo") self.StoredVariables["CostFunctionJoAtCurrentOptimum"] = Persistence.OneScalar(name = "CostFunctionJoAtCurrentOptimum") + self.StoredVariables["CurrentIterationNumber"] = Persistence.OneIndex(name = "CurrentIterationNumber") self.StoredVariables["CurrentOptimum"] = Persistence.OneVector(name = "CurrentOptimum") self.StoredVariables["CurrentState"] = Persistence.OneVector(name = "CurrentState") + self.StoredVariables["ForecastState"] = Persistence.OneVector(name = "ForecastState") self.StoredVariables["GradientOfCostFunctionJ"] = Persistence.OneVector(name = "GradientOfCostFunctionJ") self.StoredVariables["GradientOfCostFunctionJb"] = Persistence.OneVector(name = "GradientOfCostFunctionJb") self.StoredVariables["GradientOfCostFunctionJo"] = Persistence.OneVector(name = "GradientOfCostFunctionJo") - self.StoredVariables["IndexOfOptimum"] = Persistence.OneIndex(name = "IndexOfOptimum") + self.StoredVariables["IndexOfOptimum"] = Persistence.OneIndex(name = "IndexOfOptimum") self.StoredVariables["Innovation"] = Persistence.OneVector(name = "Innovation") self.StoredVariables["InnovationAtCurrentAnalysis"] = Persistence.OneVector(name = "InnovationAtCurrentAnalysis") self.StoredVariables["InnovationAtCurrentState"] = Persistence.OneVector(name = "InnovationAtCurrentState") @@ -642,7 +673,6 @@ class Algorithm(object): self.StoredVariables["MahalanobisConsistency"] = Persistence.OneScalar(name = "MahalanobisConsistency") self.StoredVariables["OMA"] = Persistence.OneVector(name = "OMA") self.StoredVariables["OMB"] = Persistence.OneVector(name = "OMB") - self.StoredVariables["PredictedState"] = Persistence.OneVector(name = "PredictedState") self.StoredVariables["Residu"] = Persistence.OneScalar(name = "Residu") self.StoredVariables["SigmaBck2"] = Persistence.OneScalar(name = "SigmaBck2") self.StoredVariables["SigmaObs2"] = Persistence.OneScalar(name = "SigmaObs2") @@ -661,10 +691,11 @@ class Algorithm(object): self.__canonical_parameter_name["algorithm"] = "Algorithm" self.__canonical_parameter_name["storesupplementarycalculations"] = "StoreSupplementaryCalculations" - def _pre_run(self, Parameters, Xb=None, Y=None, R=None, B=None, Q=None ): + def _pre_run(self, Parameters, Xb=None, Y=None, U=None, HO=None, EM=None, CM=None, R=None, B=None, Q=None ): "Pré-calcul" logging.debug("%s Lancement", self._name) logging.debug("%s Taille mémoire utilisée de %.0f Mio"%(self._name, self._m.getUsedMemory("Mio"))) + self._getTimeState(reset=True) # # Mise a jour des paramètres internes avec le contenu de Parameters, en # reprenant les valeurs par défauts pour toutes celles non définies @@ -672,36 +703,57 @@ class Algorithm(object): for k, v in self.__variable_names_not_public.items(): if k not in self._parameters: self.__setParameters( {k:v} ) # - # Corrections et compléments - def __test_vvalue(argument, variable, argname): + # Corrections et compléments des vecteurs + def __test_vvalue(argument, variable, argname, symbol=None): + if symbol is None: symbol = variable if argument is None: if variable in self.__required_inputs["RequiredInputValues"]["mandatory"]: - raise ValueError("%s %s vector %s has to be properly defined!"%(self._name,argname,variable)) + raise ValueError("%s %s vector %s is not set and has to be properly defined!"%(self._name,argname,symbol)) elif variable in self.__required_inputs["RequiredInputValues"]["optional"]: - logging.debug("%s %s vector %s is not set, but is optional."%(self._name,argname,variable)) + logging.debug("%s %s vector %s is not set, but is optional."%(self._name,argname,symbol)) else: - logging.debug("%s %s vector %s is not set, but is not required."%(self._name,argname,variable)) + logging.debug("%s %s vector %s is not set, but is not required."%(self._name,argname,symbol)) else: - logging.debug("%s %s vector %s is set, and its size is %i."%(self._name,argname,variable,numpy.array(argument).size)) + logging.debug("%s %s vector %s is set, and its size is %i."%(self._name,argname,symbol,numpy.array(argument).size)) return 0 __test_vvalue( Xb, "Xb", "Background or initial state" ) __test_vvalue( Y, "Y", "Observation" ) + __test_vvalue( U, "U", "Control" ) # - def __test_cvalue(argument, variable, argname): + # Corrections et compléments des covariances + def __test_cvalue(argument, variable, argname, symbol=None): + if symbol is None: symbol = variable if argument is None: if variable in self.__required_inputs["RequiredInputValues"]["mandatory"]: - raise ValueError("%s %s error covariance matrix %s has to be properly defined!"%(self._name,argname,variable)) + raise ValueError("%s %s error covariance matrix %s is not set and has to be properly defined!"%(self._name,argname,symbol)) elif variable in self.__required_inputs["RequiredInputValues"]["optional"]: - logging.debug("%s %s error covariance matrix %s is not set, but is optional."%(self._name,argname,variable)) + logging.debug("%s %s error covariance matrix %s is not set, but is optional."%(self._name,argname,symbol)) else: - logging.debug("%s %s error covariance matrix %s is not set, but is not required."%(self._name,argname,variable)) + logging.debug("%s %s error covariance matrix %s is not set, but is not required."%(self._name,argname,symbol)) else: - logging.debug("%s %s error covariance matrix %s is set."%(self._name,argname,variable)) + logging.debug("%s %s error covariance matrix %s is set."%(self._name,argname,symbol)) return 0 - __test_cvalue( R, "R", "Observation" ) __test_cvalue( B, "B", "Background" ) + __test_cvalue( R, "R", "Observation" ) __test_cvalue( Q, "Q", "Evolution" ) # + # Corrections et compléments des opérateurs + def __test_ovalue(argument, variable, argname, symbol=None): + if symbol is None: symbol = variable + if argument is None or (isinstance(argument,dict) and len(argument)==0): + if variable in self.__required_inputs["RequiredInputValues"]["mandatory"]: + raise ValueError("%s %s operator %s is not set and has to be properly defined!"%(self._name,argname,symbol)) + elif variable in self.__required_inputs["RequiredInputValues"]["optional"]: + logging.debug("%s %s operator %s is not set, but is optional."%(self._name,argname,symbol)) + else: + logging.debug("%s %s operator %s is not set, but is not required."%(self._name,argname,symbol)) + else: + logging.debug("%s %s operator %s is set."%(self._name,argname,symbol)) + return 0 + __test_ovalue( HO, "HO", "Observation", "H" ) + __test_ovalue( EM, "EM", "Evolution", "M" ) + __test_ovalue( CM, "CM", "Control Model", "C" ) + # if ("Bounds" in self._parameters) and isinstance(self._parameters["Bounds"], (list, tuple)) and (len(self._parameters["Bounds"]) > 0): logging.debug("%s Prise en compte des bornes effectuee"%(self._name,)) else: @@ -737,10 +789,11 @@ class Algorithm(object): _EI = numpy.diag(1./numpy.sqrt(numpy.diag(_A))) _C = numpy.dot(_EI, numpy.dot(_A, _EI)) self.StoredVariables["APosterioriCorrelations"].store( _C ) - if _oH is not None: + if _oH is not None and "Direct" in _oH and "Tangent" in _oH and "Adjoint" in _oH: logging.debug("%s Nombre d'évaluation(s) de l'opérateur d'observation direct/tangent/adjoint.: %i/%i/%i", self._name, _oH["Direct"].nbcalls(0),_oH["Tangent"].nbcalls(0),_oH["Adjoint"].nbcalls(0)) logging.debug("%s Nombre d'appels au cache d'opérateur d'observation direct/tangent/adjoint..: %i/%i/%i", self._name, _oH["Direct"].nbcalls(3),_oH["Tangent"].nbcalls(3),_oH["Adjoint"].nbcalls(3)) logging.debug("%s Taille mémoire utilisée de %.0f Mio", self._name, self._m.getUsedMemory("Mio")) + logging.debug("%s Durées d'utilisation CPU de %.1fs et elapsed de %.1fs", self._name, self._getTimeState()[0], self._getTimeState()[1]) logging.debug("%s Terminé", self._name) return 0 @@ -797,7 +850,7 @@ class Algorithm(object): """ raise NotImplementedError("Mathematical assimilation calculation has not been implemented!") - def defineRequiredParameter(self, name = None, default = None, typecast = None, message = None, minval = None, maxval = None, listval = None): + def defineRequiredParameter(self, name = None, default = None, typecast = None, message = None, minval = None, maxval = None, listval = None, listadv = None): """ Permet de définir dans l'algorithme des paramètres requis et leurs caractéristiques par défaut. @@ -811,6 +864,7 @@ class Algorithm(object): "minval" : minval, "maxval" : maxval, "listval" : listval, + "listadv" : listadv, "message" : message, } self.__canonical_parameter_name[name.lower()] = name @@ -836,6 +890,7 @@ class Algorithm(object): minval = self.__required_parameters[__k]["minval"] maxval = self.__required_parameters[__k]["maxval"] listval = self.__required_parameters[__k]["listval"] + listadv = self.__required_parameters[__k]["listadv"] # if value is None and default is None: __val = None @@ -854,23 +909,39 @@ class Algorithm(object): raise ValueError("The parameter named '%s' of value '%s' can not be less than %s."%(__k, __val, minval)) if maxval is not None and (numpy.array(__val, float) > maxval).any(): raise ValueError("The parameter named '%s' of value '%s' can not be greater than %s."%(__k, __val, maxval)) - if listval is not None: + if listval is not None or listadv is not None: if typecast is list or typecast is tuple or isinstance(__val,list) or isinstance(__val,tuple): for v in __val: - if v not in listval: + if listval is not None and v in listval: continue + elif listadv is not None and v in listadv: continue + else: raise ValueError("The value '%s' is not allowed for the parameter named '%s', it has to be in the list %s."%(v, __k, listval)) - elif __val not in listval: + elif not (listval is not None and __val in listval) and not (listadv is not None and __val in listadv): raise ValueError("The value '%s' is not allowed for the parameter named '%s', it has to be in the list %s."%( __val, __k,listval)) # return __val def requireInputArguments(self, mandatory=(), optional=()): """ - Permet d'imposer des arguments requises en entrée + Permet d'imposer des arguments de calcul requis en entrée. """ self.__required_inputs["RequiredInputValues"]["mandatory"] = tuple( mandatory ) self.__required_inputs["RequiredInputValues"]["optional"] = tuple( optional ) + def getInputArguments(self): + """ + Permet d'obtenir les listes des arguments de calcul requis en entrée. + """ + return self.__required_inputs["RequiredInputValues"]["mandatory"], self.__required_inputs["RequiredInputValues"]["optional"] + + def setAttributes(self, tags=()): + """ + Permet d'adjoindre des attributs comme les tags de classification. + Renvoie la liste actuelle dans tous les cas. + """ + self.__required_inputs["ClassificationTags"].extend( tags ) + return self.__required_inputs["ClassificationTags"] + def __setParameters(self, fromDico={}, reset=False): """ Permet de stocker les paramètres reçus dans le dictionnaire interne. @@ -891,6 +962,33 @@ class Algorithm(object): pass logging.debug("%s %s : %s", self._name, self.__required_parameters[k]["message"], self._parameters[k]) + def _getTimeState(self, reset=False): + """ + Initialise ou restitue le temps de calcul (cpu/elapsed) à la seconde + """ + if reset: + self.__initial_cpu_time = time.process_time() + self.__initial_elapsed_time = time.perf_counter() + return 0., 0. + else: + self.__cpu_time = time.process_time() - self.__initial_cpu_time + self.__elapsed_time = time.perf_counter() - self.__initial_elapsed_time + return self.__cpu_time, self.__elapsed_time + + def _StopOnTimeLimit(self, X=None, withReason=False): + "Stop criteria on time limit: True/False [+ Reason]" + c, e = self._getTimeState() + if "MaximumCpuTime" in self._parameters and c > self._parameters["MaximumCpuTime"]: + __SC, __SR = True, "Reached maximum CPU time (%.1fs > %.1fs)"%(c, self._parameters["MaximumCpuTime"]) + elif "MaximumElapsedTime" in self._parameters and e > self._parameters["MaximumElapsedTime"]: + __SC, __SR = True, "Reached maximum elapsed time (%.1fs > %.1fs)"%(e, self._parameters["MaximumElapsedTime"]) + else: + __SC, __SR = False, "" + if withReason: + return __SC, __SR + else: + return __SC + # ============================================================================== class AlgorithmAndParameters(object): """ @@ -1053,6 +1151,14 @@ class AlgorithmAndParameters(object): "Renvoie la liste des paramètres requis selon l'algorithme" return self.__algorithm.getRequiredParameters(noDetails) + def getAlgorithmInputArguments(self): + "Renvoie la liste des entrées requises selon l'algorithme" + return self.__algorithm.getInputArguments() + + def getAlgorithmAttributes(self): + "Renvoie la liste des attributs selon l'algorithme" + return self.__algorithm.setAttributes() + def setObserver(self, __V, __O, __I, __S): if self.__algorithm is None \ or isinstance(self.__algorithm, dict) \ @@ -1292,7 +1398,7 @@ class RegulationAndParameters(object): self.__P.update( dict(__Dict) ) # if __Algo is not None: - self.__P.update( {"Algorithm":__Algo} ) + self.__P.update( {"Algorithm":str(__Algo)} ) def get(self, key = None): "Vérifie l'existence d'une clé de variable ou de paramètres" @@ -1341,19 +1447,11 @@ class DataObserver(object): else: raise ValueError("setting an observer has to be done over a variable name or a list of variable names.") # - if asString is not None: - __FunctionText = asString - elif (asTemplate is not None) and (asTemplate in Templates.ObserverTemplates): - __FunctionText = Templates.ObserverTemplates[asTemplate] - elif asScript is not None: - __FunctionText = Interfaces.ImportFromScript(asScript).getstring() - else: - __FunctionText = "" - __Function = ObserverF(__FunctionText) - # if asObsObject is not None: self.__O = asObsObject else: + __FunctionText = str(UserScript('Observer', asTemplate, asString, asScript)) + __Function = Observer2Func(__FunctionText) self.__O = __Function.getfunc() # for k in range(len(self.__V)): @@ -1372,6 +1470,89 @@ class DataObserver(object): "x.__str__() <==> str(x)" return str(self.__V)+"\n"+str(self.__O) +# ============================================================================== +class UserScript(object): + """ + Classe générale d'interface de type texte de script utilisateur + """ + def __init__(self, + name = "GenericUserScript", + asTemplate = None, + asString = None, + asScript = None, + ): + """ + """ + self.__name = str(name) + # + if asString is not None: + self.__F = asString + elif self.__name == "UserPostAnalysis" and (asTemplate is not None) and (asTemplate in Templates.UserPostAnalysisTemplates): + self.__F = Templates.UserPostAnalysisTemplates[asTemplate] + elif self.__name == "Observer" and (asTemplate is not None) and (asTemplate in Templates.ObserverTemplates): + self.__F = Templates.ObserverTemplates[asTemplate] + elif asScript is not None: + self.__F = Interfaces.ImportFromScript(asScript).getstring() + else: + self.__F = "" + + def __repr__(self): + "x.__repr__() <==> repr(x)" + return repr(self.__F) + + def __str__(self): + "x.__str__() <==> str(x)" + return str(self.__F) + +# ============================================================================== +class ExternalParameters(object): + """ + Classe générale d'interface de type texte de script utilisateur + """ + def __init__(self, + name = "GenericExternalParameters", + asDict = None, + asScript = None, + ): + """ + """ + self.__name = str(name) + self.__P = {} + # + self.updateParameters( asDict, asScript ) + + def updateParameters(self, + asDict = None, + asScript = None, + ): + "Mise a jour des parametres" + if asDict is None and asScript is not None: + __Dict = Interfaces.ImportFromScript(asScript).getvalue( self.__name, "ExternalParameters" ) + else: + __Dict = asDict + # + if __Dict is not None: + self.__P.update( dict(__Dict) ) + + def get(self, key = None): + if key in self.__P: + return self.__P[key] + else: + return list(self.__P.keys()) + + def keys(self): + return list(self.__P.keys()) + + def pop(self, k, d): + return self.__P.pop(k, d) + + def items(self): + return self.__P.items() + + def __contains__(self, key=None): + "D.__contains__(k) -> True if D has a key k, else False" + return key in self.__P + # ============================================================================== class State(object): """ @@ -1671,6 +1852,30 @@ class Covariance(object): elif self.isobject() and hasattr(self.__C,"choleskyI"): return Covariance(self.__name+"H", asCovObject = self.__C.choleskyI() ) + def sqrtm(self): + "Racine carrée matricielle" + if self.ismatrix(): + import scipy + return Covariance(self.__name+"C", asCovariance = scipy.linalg.sqrtm(self.__C) ) + elif self.isvector(): + return Covariance(self.__name+"C", asEyeByVector = numpy.sqrt( self.__C ) ) + elif self.isscalar(): + return Covariance(self.__name+"C", asEyeByScalar = numpy.sqrt( self.__C ) ) + elif self.isobject() and hasattr(self.__C,"sqrt"): + return Covariance(self.__name+"C", asCovObject = self.__C.sqrt() ) + + def sqrtmI(self): + "Inversion de la racine carrée matricielle" + if self.ismatrix(): + import scipy + return Covariance(self.__name+"H", asCovariance = scipy.linalg.sqrtm(self.__C).I ) + elif self.isvector(): + return Covariance(self.__name+"H", asEyeByVector = 1.0 / numpy.sqrt( self.__C ) ) + elif self.isscalar(): + return Covariance(self.__name+"H", asEyeByScalar = 1.0 / numpy.sqrt( self.__C ) ) + elif self.isobject() and hasattr(self.__C,"sqrtI"): + return Covariance(self.__name+"H", asCovObject = self.__C.sqrtI() ) + def diag(self, msize=None): "Diagonale de la matrice" if self.ismatrix(): @@ -1814,7 +2019,7 @@ class Covariance(object): return self.shape[0] # ============================================================================== -class ObserverF(object): +class Observer2Func(object): """ Creation d'une fonction d'observateur a partir de son texte """