X-Git-Url: http://git.salome-platform.org/gitweb/?a=blobdiff_plain;f=src%2FdaComposant%2FdaCore%2FBasicObjects.py;h=0cc2cb04fd55367082218c5480a1033ae341fe4f;hb=8f27741af02e5f1125f56475f0bb80e2fe709bf9;hp=4404d2effe42152307067551438ba7abf4269df9;hpb=4790fb60acb36159350ee1cda40107e6833ead3f;p=modules%2Fadao.git diff --git a/src/daComposant/daCore/BasicObjects.py b/src/daComposant/daCore/BasicObjects.py index 4404d2e..0cc2cb0 100644 --- a/src/daComposant/daCore/BasicObjects.py +++ b/src/daComposant/daCore/BasicObjects.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2008-2017 EDF R&D +# Copyright (C) 2008-2021 EDF R&D # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -22,16 +22,18 @@ """ Définit les outils généraux élémentaires. - - Ce module est destiné à être appelée par AssimilationStudy. """ __author__ = "Jean-Philippe ARGAUD" __all__ = [] -import os, sys, logging, copy +import os +import sys +import logging +import copy +import time import numpy -from daCore import Persistence -from daCore import PlatformInfo +from functools import partial +from daCore import Persistence, PlatformInfo, Interfaces from daCore import Templates # ============================================================================== @@ -44,38 +46,47 @@ class CacheManager(object): lenghtOfRedundancy = -1, ): """ - Les caractéristiques de tolérance peuvent être modifées à la création. + Les caractéristiques de tolérance peuvent être modifiées à la création. """ - self.__tolerBP = float(toleranceInRedundancy) - self.__lenghtOR = int(lenghtOfRedundancy) - self.__initlnOR = self.__lenghtOR + self.__tolerBP = float(toleranceInRedundancy) + self.__lenghtOR = int(lenghtOfRedundancy) + self.__initlnOR = self.__lenghtOR + self.__seenNames = [] + self.__enabled = True self.clearCache() def clearCache(self): "Vide le cache" - self.__listOPCV = [] # Operator Previous Calculated Points, Results, Point Norms + self.__listOPCV = [] # Previous Calculated Points, Results, Point Norms, Operator + self.__seenNames = [] # logging.debug("CM Tolerance de determination des doublons : %.2e", self.__tolerBP) - def wasCalculatedIn(self, xValue ): #, info="" ): + def wasCalculatedIn(self, xValue, oName="" ): #, info="" ): "Vérifie l'existence d'un calcul correspondant à la valeur" __alc = False __HxV = None - for i in range(min(len(self.__listOPCV),self.__lenghtOR)-1,-1,-1): - if xValue.size != self.__listOPCV[i][0].size: - # logging.debug("CM Différence de la taille %s de X et de celle %s du point %i déjà calculé", xValue.shape,i,self.__listOPCP[i].shape) - continue - if numpy.linalg.norm(numpy.ravel(xValue) - self.__listOPCV[i][0]) < self.__tolerBP * self.__listOPCV[i][2]: - __alc = True - __HxV = self.__listOPCV[i][1] - # logging.debug("CM Cas%s déja calculé, portant le numéro %i", info, i) - break + if self.__enabled: + for i in range(min(len(self.__listOPCV),self.__lenghtOR)-1,-1,-1): + if not hasattr(xValue, 'size') or (str(oName) != self.__listOPCV[i][3]) or (xValue.size != self.__listOPCV[i][0].size): + # logging.debug("CM Différence de la taille %s de X et de celle %s du point %i déjà calculé", xValue.shape,i,self.__listOPCP[i].shape) + pass + elif numpy.linalg.norm(numpy.ravel(xValue) - self.__listOPCV[i][0]) < self.__tolerBP * self.__listOPCV[i][2]: + __alc = True + __HxV = self.__listOPCV[i][1] + # logging.debug("CM Cas%s déja calculé, portant le numéro %i", info, i) + break return __alc, __HxV - def storeValueInX(self, xValue, HxValue ): - "Stocke un calcul correspondant à la valeur" + def storeValueInX(self, xValue, HxValue, oName="" ): + "Stocke pour un opérateur o un calcul Hx correspondant à la valeur x" if self.__lenghtOR < 0: self.__lenghtOR = 2 * xValue.size + 2 self.__initlnOR = self.__lenghtOR + self.__seenNames.append(str(oName)) + if str(oName) not in self.__seenNames: # Etend la liste si nouveau + self.__lenghtOR += 2 * xValue.size + 2 + self.__initlnOR += self.__lenghtOR + self.__seenNames.append(str(oName)) while len(self.__listOPCV) > self.__lenghtOR: # logging.debug("CM Réduction de la liste des cas à %i éléments par suppression du premier", self.__lenghtOR) self.__listOPCV.pop(0) @@ -83,16 +94,19 @@ class CacheManager(object): copy.copy(numpy.ravel(xValue)), copy.copy(HxValue), numpy.linalg.norm(xValue), + str(oName), ) ) def disable(self): "Inactive le cache" self.__initlnOR = self.__lenghtOR self.__lenghtOR = 0 + self.__enabled = False def enable(self): "Active le cache" self.__lenghtOR = self.__initlnOR + self.__enabled = True # ============================================================================== class Operator(object): @@ -104,19 +118,41 @@ class Operator(object): NbCallsOfCached = 0 CM = CacheManager() # - def __init__(self, fromMethod=None, fromMatrix=None, avoidingRedundancy = True): + def __init__(self, + name = "GenericOperator", + fromMethod = None, + fromMatrix = None, + avoidingRedundancy = True, + inputAsMultiFunction = False, + enableMultiProcess = False, + extraArguments = None, + ): """ - On construit un objet de ce type en fournissant à l'aide de l'un des - deux mots-clé, soit une fonction python, soit une matrice. + On construit un objet de ce type en fournissant, à l'aide de l'un des + deux mots-clé, soit une fonction ou un multi-fonction python, soit une + matrice. Arguments : + - name : nom d'opérateur - fromMethod : argument de type fonction Python - fromMatrix : argument adapté au constructeur numpy.matrix - - avoidingRedundancy : évite ou pas les calculs redondants + - avoidingRedundancy : booléen évitant (ou pas) les calculs redondants + - inputAsMultiFunction : booléen indiquant une fonction explicitement + définie (ou pas) en multi-fonction + - extraArguments : arguments supplémentaires passés à la fonction de + base et ses dérivées (tuple ou dictionnaire) """ + self.__name = str(name) self.__NbCallsAsMatrix, self.__NbCallsAsMethod, self.__NbCallsOfCached = 0, 0, 0 - self.__AvoidRC = bool( avoidingRedundancy ) - if fromMethod is not None: - self.__Method = fromMethod + self.__AvoidRC = bool( avoidingRedundancy ) + self.__inputAsMF = bool( inputAsMultiFunction ) + self.__mpEnabled = bool( enableMultiProcess ) + self.__extraArgs = extraArguments + if fromMethod is not None and self.__inputAsMF: + self.__Method = fromMethod # logtimer(fromMethod) + self.__Matrix = None + self.__Type = "Method" + elif fromMethod is not None and not self.__inputAsMF: + self.__Method = partial( MultiFonction, _sFunction=fromMethod, _mpEnabled=self.__mpEnabled) self.__Matrix = None self.__Type = "Method" elif fromMatrix is not None: @@ -143,95 +179,184 @@ class Operator(object): "Renvoie le type" return self.__Type - def appliedTo(self, xValue, HValue = None): + def appliedTo(self, xValue, HValue = None, argsAsSerie = False, returnSerieAsArrayMatrix = False): """ - Permet de restituer le résultat de l'application de l'opérateur à un - argument xValue. Cette méthode se contente d'appliquer, son argument - devant a priori être du bon type. + Permet de restituer le résultat de l'application de l'opérateur à une + série d'arguments xValue. Cette méthode se contente d'appliquer, chaque + argument devant a priori être du bon type. Arguments : - - xValue : argument adapté pour appliquer l'opérateur + - les arguments par série sont : + - xValue : argument adapté pour appliquer l'opérateur + - HValue : valeur précalculée de l'opérateur en ce point + - argsAsSerie : indique si les arguments sont une mono ou multi-valeur """ - if HValue is not None: - HxValue = numpy.asmatrix( numpy.ravel( HValue ) ).T - if self.__AvoidRC: - Operator.CM.storeValueInX(xValue,HxValue) + if argsAsSerie: + _xValue = xValue + _HValue = HValue else: - if self.__AvoidRC: - __alreadyCalculated, __HxV = Operator.CM.wasCalculatedIn(xValue) + _xValue = (xValue,) + if HValue is not None: + _HValue = (HValue,) else: - __alreadyCalculated = False + _HValue = HValue + PlatformInfo.isIterable( _xValue, True, " in Operator.appliedTo" ) + # + if _HValue is not None: + assert len(_xValue) == len(_HValue), "Incompatible number of elements in xValue and HValue" + _HxValue = [] + for i in range(len(_HValue)): + _HxValue.append( numpy.asmatrix( numpy.ravel( _HValue[i] ) ).T ) + if self.__AvoidRC: + Operator.CM.storeValueInX(_xValue[i],_HxValue[-1],self.__name) + else: + _HxValue = [] + _xserie = [] + _hindex = [] + for i, xv in enumerate(_xValue): + if self.__AvoidRC: + __alreadyCalculated, __HxV = Operator.CM.wasCalculatedIn(xv,self.__name) + else: + __alreadyCalculated = False + # + if __alreadyCalculated: + self.__addOneCacheCall() + _hv = __HxV + else: + if self.__Matrix is not None: + self.__addOneMatrixCall() + _xv = numpy.matrix(numpy.ravel(xv)).T + _hv = self.__Matrix * _xv + else: + self.__addOneMethodCall() + _xserie.append( xv ) + _hindex.append( i ) + _hv = None + _HxValue.append( _hv ) # - if __alreadyCalculated: - self.__addOneCacheCall() - HxValue = __HxV - else: - if self.__Matrix is not None: - self.__addOneMatrixCall() - HxValue = self.__Matrix * xValue + if len(_xserie)>0 and self.__Matrix is None: + if self.__extraArgs is None: + _hserie = self.__Method( _xserie ) # Calcul MF else: - self.__addOneMethodCall() - HxValue = self.__Method( xValue ) - if self.__AvoidRC: - Operator.CM.storeValueInX(xValue,HxValue) - # - return HxValue - - def appliedControledFormTo(self, paire ): + _hserie = self.__Method( _xserie, self.__extraArgs ) # Calcul MF + if not hasattr(_hserie, "pop"): + raise TypeError("The user input multi-function doesn't seem to return sequence results, behaving like a mono-function. It has to be checked.") + for i in _hindex: + _xv = _xserie.pop(0) + _hv = _hserie.pop(0) + _HxValue[i] = _hv + if self.__AvoidRC: + Operator.CM.storeValueInX(_xv,_hv,self.__name) + # + if returnSerieAsArrayMatrix: + _HxValue = numpy.stack([numpy.ravel(_hv) for _hv in _HxValue], axis=1) + # + if argsAsSerie: return _HxValue + else: return _HxValue[-1] + + def appliedControledFormTo(self, paires, argsAsSerie = False, returnSerieAsArrayMatrix = False): """ - Permet de restituer le résultat de l'application de l'opérateur à une - paire (xValue, uValue). Cette méthode se contente d'appliquer, son + Permet de restituer le résultat de l'application de l'opérateur à des + paires (xValue, uValue). Cette méthode se contente d'appliquer, son argument devant a priori être du bon type. Si la uValue est None, on suppose que l'opérateur ne s'applique qu'à xValue. Arguments : - - xValue : argument X adapté pour appliquer l'opérateur - - uValue : argument U adapté pour appliquer l'opérateur + - paires : les arguments par paire sont : + - xValue : argument X adapté pour appliquer l'opérateur + - uValue : argument U adapté pour appliquer l'opérateur + - argsAsSerie : indique si l'argument est une mono ou multi-valeur """ - assert len(paire) == 2, "Incorrect number of arguments" - xValue, uValue = paire + if argsAsSerie: _xuValue = paires + else: _xuValue = (paires,) + PlatformInfo.isIterable( _xuValue, True, " in Operator.appliedControledFormTo" ) + # if self.__Matrix is not None: - self.__addOneMatrixCall() - return self.__Matrix * xValue - elif uValue is not None: - self.__addOneMethodCall() - return self.__Method( (xValue, uValue) ) + _HxValue = [] + for paire in _xuValue: + _xValue, _uValue = paire + _xValue = numpy.matrix(numpy.ravel(_xValue)).T + self.__addOneMatrixCall() + _HxValue.append( self.__Matrix * _xValue ) else: - self.__addOneMethodCall() - return self.__Method( xValue ) + _xuArgs = [] + for paire in _xuValue: + _xValue, _uValue = paire + if _uValue is not None: + _xuArgs.append( paire ) + else: + _xuArgs.append( _xValue ) + self.__addOneMethodCall( len(_xuArgs) ) + if self.__extraArgs is None: + _HxValue = self.__Method( _xuArgs ) # Calcul MF + else: + _HxValue = self.__Method( _xuArgs, self.__extraArgs ) # Calcul MF + # + if returnSerieAsArrayMatrix: + _HxValue = numpy.stack([numpy.ravel(_hv) for _hv in _HxValue], axis=1) + # + if argsAsSerie: return _HxValue + else: return _HxValue[-1] - def appliedInXTo(self, paire ): + def appliedInXTo(self, paires, argsAsSerie = False, returnSerieAsArrayMatrix = False): """ - Permet de restituer le résultat de l'application de l'opérateur à un - argument xValue, sachant que l'opérateur est valable en xNominal. - Cette méthode se contente d'appliquer, son argument devant a priori - être du bon type. Si l'opérateur est linéaire car c'est une matrice, - alors il est valable en tout point nominal et il n'est pas nécessaire - d'utiliser xNominal. - Arguments : une liste contenant - - xNominal : argument permettant de donner le point où l'opérateur - est construit pour etre ensuite appliqué - - xValue : argument adapté pour appliquer l'opérateur + Permet de restituer le résultat de l'application de l'opérateur à une + série d'arguments xValue, sachant que l'opérateur est valable en + xNominal. Cette méthode se contente d'appliquer, son argument devant a + priori être du bon type. Si l'opérateur est linéaire car c'est une + matrice, alors il est valable en tout point nominal et xNominal peut + être quelconque. Il n'y a qu'une seule paire par défaut, et argsAsSerie + permet d'indiquer que l'argument est multi-paires. + Arguments : + - paires : les arguments par paire sont : + - xNominal : série d'arguments permettant de donner le point où + l'opérateur est construit pour être ensuite appliqué + - xValue : série d'arguments adaptés pour appliquer l'opérateur + - argsAsSerie : indique si l'argument est une mono ou multi-valeur """ - assert len(paire) == 2, "Incorrect number of arguments" - xNominal, xValue = paire + if argsAsSerie: _nxValue = paires + else: _nxValue = (paires,) + PlatformInfo.isIterable( _nxValue, True, " in Operator.appliedInXTo" ) + # if self.__Matrix is not None: - self.__addOneMatrixCall() - return self.__Matrix * xValue + _HxValue = [] + for paire in _nxValue: + _xNominal, _xValue = paire + _xValue = numpy.matrix(numpy.ravel(_xValue)).T + self.__addOneMatrixCall() + _HxValue.append( self.__Matrix * _xValue ) else: - self.__addOneMethodCall() - return self.__Method( (xNominal, xValue) ) + self.__addOneMethodCall( len(_nxValue) ) + if self.__extraArgs is None: + _HxValue = self.__Method( _nxValue ) # Calcul MF + else: + _HxValue = self.__Method( _nxValue, self.__extraArgs ) # Calcul MF + # + if returnSerieAsArrayMatrix: + _HxValue = numpy.stack([numpy.ravel(_hv) for _hv in _HxValue], axis=1) + # + if argsAsSerie: return _HxValue + else: return _HxValue[-1] - def asMatrix(self, ValueForMethodForm = "UnknownVoidValue"): + def asMatrix(self, ValueForMethodForm = "UnknownVoidValue", argsAsSerie = False): """ Permet de renvoyer l'opérateur sous la forme d'une matrice """ if self.__Matrix is not None: self.__addOneMatrixCall() - return self.__Matrix - elif ValueForMethodForm is not "UnknownVoidValue": # Ne pas utiliser "None" - self.__addOneMethodCall() - return numpy.matrix( self.__Method( (ValueForMethodForm, None) ) ) + mValue = [self.__Matrix,] + elif not isinstance(ValueForMethodForm,str) or ValueForMethodForm != "UnknownVoidValue": # Ne pas utiliser "None" + mValue = [] + if argsAsSerie: + self.__addOneMethodCall( len(ValueForMethodForm) ) + for _vfmf in ValueForMethodForm: + mValue.append( numpy.matrix( self.__Method(((_vfmf, None),)) ) ) + else: + self.__addOneMethodCall() + mValue = self.__Method(((ValueForMethodForm, None),)) else: raise ValueError("Matrix form of the operator defined as a function/method requires to give an operating point.") + # + if argsAsSerie: return mValue + else: return mValue[-1] def shape(self): """ @@ -265,10 +390,10 @@ class Operator(object): self.__NbCallsAsMatrix += 1 # Decompte local Operator.NbCallsAsMatrix += 1 # Decompte global - def __addOneMethodCall(self): + def __addOneMethodCall(self, nb = 1): "Comptabilise un appel" - self.__NbCallsAsMethod += 1 # Decompte local - Operator.NbCallsAsMethod += 1 # Decompte global + self.__NbCallsAsMethod += nb # Decompte local + Operator.NbCallsAsMethod += nb # Decompte global def __addOneCacheCall(self): "Comptabilise un appel" @@ -284,46 +409,53 @@ class FullOperator(object): def __init__(self, name = "GenericFullOperator", asMatrix = None, - asOneFunction = None, # Fonction - asThreeFunctions = None, # Dictionnaire de fonctions - asScript = None, + asOneFunction = None, # 1 Fonction + asThreeFunctions = None, # 3 Fonctions in a dictionary + asScript = None, # 1 or 3 Fonction(s) by script asDict = None, # Parameters appliedInX = None, + extraArguments = None, avoidRC = True, + inputAsMF = False,# Fonction(s) as Multi-Functions scheduledBy = None, toBeChecked = False, ): "" - self.__name = str(name) - self.__check = bool(toBeChecked) + self.__name = str(name) + self.__check = bool(toBeChecked) + self.__extraArgs = extraArguments # - self.__FO = {} + self.__FO = {} # __Parameters = {} if (asDict is not None) and isinstance(asDict, dict): __Parameters.update( asDict ) - if "DifferentialIncrement" in asDict: - __Parameters["withIncrement"] = asDict["DifferentialIncrement"] - if "CenteredFiniteDifference" in asDict: - __Parameters["withCenteredDF"] = asDict["CenteredFiniteDifference"] - if "EnableMultiProcessing" in asDict: - __Parameters["withmpEnabled"] = asDict["EnableMultiProcessing"] - if "NumberOfProcesses" in asDict: - __Parameters["withmpWorkers"] = asDict["NumberOfProcesses"] + # Priorité à EnableMultiProcessingInDerivatives=True + if "EnableMultiProcessing" in __Parameters and __Parameters["EnableMultiProcessing"]: + __Parameters["EnableMultiProcessingInDerivatives"] = True + __Parameters["EnableMultiProcessingInEvaluation"] = False + if "EnableMultiProcessingInDerivatives" not in __Parameters: + __Parameters["EnableMultiProcessingInDerivatives"] = False + if __Parameters["EnableMultiProcessingInDerivatives"]: + __Parameters["EnableMultiProcessingInEvaluation"] = False + if "EnableMultiProcessingInEvaluation" not in __Parameters: + __Parameters["EnableMultiProcessingInEvaluation"] = False + if "withIncrement" in __Parameters: # Temporaire + __Parameters["DifferentialIncrement"] = __Parameters["withIncrement"] # if asScript is not None: __Matrix, __Function = None, None if asMatrix: - __Matrix = ImportFromScript(asScript).getvalue( self.__name ) + __Matrix = Interfaces.ImportFromScript(asScript).getvalue( self.__name ) elif asOneFunction: - __Function = { "Direct":ImportFromScript(asScript).getvalue( "DirectOperator" ) } + __Function = { "Direct":Interfaces.ImportFromScript(asScript).getvalue( "DirectOperator" ) } __Function.update({"useApproximatedDerivatives":True}) __Function.update(__Parameters) elif asThreeFunctions: __Function = { - "Direct" :ImportFromScript(asScript).getvalue( "DirectOperator" ), - "Tangent":ImportFromScript(asScript).getvalue( "TangentOperator" ), - "Adjoint":ImportFromScript(asScript).getvalue( "AdjointOperator" ), + "Direct" :Interfaces.ImportFromScript(asScript).getvalue( "DirectOperator" ), + "Tangent":Interfaces.ImportFromScript(asScript).getvalue( "TangentOperator" ), + "Adjoint":Interfaces.ImportFromScript(asScript).getvalue( "AdjointOperator" ), } __Function.update(__Parameters) else: @@ -375,48 +507,48 @@ class FullOperator(object): if isinstance(__Function, dict) and \ ("useApproximatedDerivatives" in __Function) and bool(__Function["useApproximatedDerivatives"]) and \ ("Direct" in __Function) and (__Function["Direct"] is not None): - if "withCenteredDF" not in __Function: __Function["withCenteredDF"] = False - if "withIncrement" not in __Function: __Function["withIncrement"] = 0.01 - if "withdX" not in __Function: __Function["withdX"] = None - if "withAvoidingRedundancy" not in __Function: __Function["withAvoidingRedundancy"] = True - if "withToleranceInRedundancy" not in __Function: __Function["withToleranceInRedundancy"] = 1.e-18 - if "withLenghtOfRedundancy" not in __Function: __Function["withLenghtOfRedundancy"] = -1 - if "withmpEnabled" not in __Function: __Function["withmpEnabled"] = False - if "withmpWorkers" not in __Function: __Function["withmpWorkers"] = None - from daNumerics.ApproximatedDerivatives import FDApproximation - FDA = FDApproximation( + if "CenteredFiniteDifference" not in __Function: __Function["CenteredFiniteDifference"] = False + if "DifferentialIncrement" not in __Function: __Function["DifferentialIncrement"] = 0.01 + if "withdX" not in __Function: __Function["withdX"] = None + if "withAvoidingRedundancy" not in __Function: __Function["withAvoidingRedundancy"] = avoidRC + if "withToleranceInRedundancy" not in __Function: __Function["withToleranceInRedundancy"] = 1.e-18 + if "withLenghtOfRedundancy" not in __Function: __Function["withLenghtOfRedundancy"] = -1 + if "NumberOfProcesses" not in __Function: __Function["NumberOfProcesses"] = None + if "withmfEnabled" not in __Function: __Function["withmfEnabled"] = inputAsMF + from daCore import NumericObjects + FDA = NumericObjects.FDApproximation( + name = self.__name, Function = __Function["Direct"], - centeredDF = __Function["withCenteredDF"], - increment = __Function["withIncrement"], + centeredDF = __Function["CenteredFiniteDifference"], + increment = __Function["DifferentialIncrement"], dX = __Function["withdX"], avoidingRedundancy = __Function["withAvoidingRedundancy"], toleranceInRedundancy = __Function["withToleranceInRedundancy"], lenghtOfRedundancy = __Function["withLenghtOfRedundancy"], - mpEnabled = __Function["withmpEnabled"], - mpWorkers = __Function["withmpWorkers"], + mpEnabled = __Function["EnableMultiProcessingInDerivatives"], + mpWorkers = __Function["NumberOfProcesses"], + mfEnabled = __Function["withmfEnabled"], ) - self.__FO["Direct"] = Operator( fromMethod = FDA.DirectOperator, avoidingRedundancy = avoidRC ) - self.__FO["Tangent"] = Operator( fromMethod = FDA.TangentOperator, avoidingRedundancy = avoidRC ) - self.__FO["Adjoint"] = Operator( fromMethod = FDA.AdjointOperator, avoidingRedundancy = avoidRC ) + self.__FO["Direct"] = Operator( name = self.__name, fromMethod = FDA.DirectOperator, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) + self.__FO["Tangent"] = Operator( name = self.__name+"Tangent", fromMethod = FDA.TangentOperator, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) + self.__FO["Adjoint"] = Operator( name = self.__name+"Adjoint", fromMethod = FDA.AdjointOperator, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) elif isinstance(__Function, dict) and \ ("Direct" in __Function) and ("Tangent" in __Function) and ("Adjoint" in __Function) and \ (__Function["Direct"] is not None) and (__Function["Tangent"] is not None) and (__Function["Adjoint"] is not None): - self.__FO["Direct"] = Operator( fromMethod = __Function["Direct"], avoidingRedundancy = avoidRC ) - self.__FO["Tangent"] = Operator( fromMethod = __Function["Tangent"], avoidingRedundancy = avoidRC ) - self.__FO["Adjoint"] = Operator( fromMethod = __Function["Adjoint"], avoidingRedundancy = avoidRC ) + self.__FO["Direct"] = Operator( name = self.__name, fromMethod = __Function["Direct"], avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) + self.__FO["Tangent"] = Operator( name = self.__name+"Tangent", fromMethod = __Function["Tangent"], avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) + self.__FO["Adjoint"] = Operator( name = self.__name+"Adjoint", fromMethod = __Function["Adjoint"], avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, extraArguments = self.__extraArgs ) elif asMatrix is not None: - __matrice = numpy.matrix( asMatrix, numpy.float ) - self.__FO["Direct"] = Operator( fromMatrix = __matrice, avoidingRedundancy = avoidRC ) - self.__FO["Tangent"] = Operator( fromMatrix = __matrice, avoidingRedundancy = avoidRC ) - self.__FO["Adjoint"] = Operator( fromMatrix = __matrice.T, avoidingRedundancy = avoidRC ) + __matrice = numpy.matrix( __Matrix, numpy.float ) + self.__FO["Direct"] = Operator( name = self.__name, fromMatrix = __matrice, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF, enableMultiProcess = __Parameters["EnableMultiProcessingInEvaluation"] ) + self.__FO["Tangent"] = Operator( name = self.__name+"Tangent", fromMatrix = __matrice, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF ) + self.__FO["Adjoint"] = Operator( name = self.__name+"Adjoint", fromMatrix = __matrice.T, avoidingRedundancy = avoidRC, inputAsMultiFunction = inputAsMF ) del __matrice else: - raise ValueError("Improperly defined observation operator, it requires at minima either a matrix, a Direct for approximate derivatives or a Tangent/Adjoint pair.") + raise ValueError("The %s object is improperly defined or undefined, it requires at minima either a matrix, a Direct operator for approximate derivatives or a Tangent/Adjoint operators pair. Please check your operator input."%self.__name) # if __appliedInX is not None: self.__FO["AppliedInX"] = {} - if type(__appliedInX) is not dict: - raise ValueError("Error: observation operator defined by \"AppliedInX\" need a dictionary as argument.") for key in list(__appliedInX.keys()): if type( __appliedInX[key] ) is type( numpy.matrix([]) ): # Pour le cas où l'on a une vraie matrice @@ -434,11 +566,11 @@ class FullOperator(object): def __repr__(self): "x.__repr__() <==> repr(x)" - return repr(self.__V) + return repr(self.__FO) def __str__(self): "x.__str__() <==> str(x)" - return str(self.__V) + return str(self.__FO) # ============================================================================== class Algorithm(object): @@ -459,30 +591,43 @@ class Algorithm(object): interne à l'objet, mais auquel on accède par la méthode "get". Les variables prévues sont : - - CostFunctionJ : fonction-cout globale, somme des deux parties suivantes - - CostFunctionJb : partie ébauche ou background de la fonction-cout - - CostFunctionJo : partie observations de la fonction-cout - - GradientOfCostFunctionJ : gradient de la fonction-cout globale - - GradientOfCostFunctionJb : gradient de la partie ébauche de la fonction-cout - - GradientOfCostFunctionJo : gradient de la partie observations de la fonction-cout + - APosterioriCorrelations : matrice de corrélations de la matrice A + - APosterioriCovariance : matrice de covariances a posteriori : A + - APosterioriStandardDeviations : vecteur des écart-types de la matrice A + - APosterioriVariances : vecteur des variances de la matrice A + - Analysis : vecteur d'analyse : Xa + - BMA : Background moins Analysis : Xa - Xb + - CostFunctionJ : fonction-coût globale, somme des deux parties suivantes Jb et Jo + - CostFunctionJAtCurrentOptimum : fonction-coût globale à l'état optimal courant lors d'itérations + - CostFunctionJb : partie ébauche ou background de la fonction-coût : Jb + - CostFunctionJbAtCurrentOptimum : partie ébauche à l'état optimal courant lors d'itérations + - CostFunctionJo : partie observations de la fonction-coût : Jo + - CostFunctionJoAtCurrentOptimum : partie observations à l'état optimal courant lors d'itérations + - CurrentIterationNumber : numéro courant d'itération dans les algorithmes itératifs, à partir de 0 + - CurrentOptimum : état optimal courant lors d'itérations - CurrentState : état courant lors d'itérations - - Analysis : l'analyse Xa - - SimulatedObservationAtBackground : l'état observé H(Xb) à l'ébauche - - SimulatedObservationAtCurrentState : l'état observé H(X) à l'état courant - - SimulatedObservationAtOptimum : l'état observé H(Xa) à l'optimum + - GradientOfCostFunctionJ : gradient de la fonction-coût globale + - GradientOfCostFunctionJb : gradient de la partie ébauche de la fonction-coût + - GradientOfCostFunctionJo : gradient de la partie observations de la fonction-coût + - IndexOfOptimum : index de l'état optimal courant lors d'itérations - Innovation : l'innovation : d = Y - H(X) - InnovationAtCurrentState : l'innovation à l'état courant : dn = Y - H(Xn) - - SigmaObs2 : indicateur de correction optimale des erreurs d'observation - - SigmaBck2 : indicateur de correction optimale des erreurs d'ébauche + - JacobianMatrixAtBackground : matrice jacobienne à l'état d'ébauche + - JacobianMatrixAtCurrentState : matrice jacobienne à l'état courant + - JacobianMatrixAtOptimum : matrice jacobienne à l'optimum + - KalmanGainAtOptimum : gain de Kalman à l'optimum - MahalanobisConsistency : indicateur de consistance des covariances - - OMA : Observation moins Analysis : Y - Xa + - OMA : Observation moins Analyse : Y - Xa - OMB : Observation moins Background : Y - Xb - - AMB : Analysis moins Background : Xa - Xb - - APosterioriCovariance : matrice A - - APosterioriVariances : variances de la matrice A - - APosterioriStandardDeviations : écart-types de la matrice A - - APosterioriCorrelations : correlations de la matrice A + - ForecastState : état prédit courant lors d'itérations - Residu : dans le cas des algorithmes de vérification + - SigmaBck2 : indicateur de correction optimale des erreurs d'ébauche + - SigmaObs2 : indicateur de correction optimale des erreurs d'observation + - SimulatedObservationAtBackground : l'état observé H(Xb) à l'ébauche + - SimulatedObservationAtCurrentOptimum : l'état observé H(X) à l'état optimal courant + - SimulatedObservationAtCurrentState : l'état observé H(X) à l'état courant + - SimulatedObservationAtOptimum : l'état observé H(Xa) à l'optimum + - SimulationQuantiles : états observés H(X) pour les quantiles demandés On peut rajouter des variables à stocker dans l'initialisation de l'algorithme élémentaire qui va hériter de cette classe """ @@ -492,64 +637,126 @@ class Algorithm(object): self._name = str( name ) self._parameters = {"StoreSupplementaryCalculations":[]} self.__required_parameters = {} - self.__required_inputs = {"RequiredInputValues":{"mandatory":(), "optional":()}} + self.__required_inputs = { + "RequiredInputValues":{"mandatory":(), "optional":()}, + "ClassificationTags":[], + } + self.__variable_names_not_public = {"nextStep":False} # Duplication dans AlgorithmAndParameters + self.__canonical_parameter_name = {} # Correspondance "lower"->"correct" + self.__canonical_stored_name = {} # Correspondance "lower"->"correct" # self.StoredVariables = {} + self.StoredVariables["APosterioriCorrelations"] = Persistence.OneMatrix(name = "APosterioriCorrelations") + self.StoredVariables["APosterioriCovariance"] = Persistence.OneMatrix(name = "APosterioriCovariance") + self.StoredVariables["APosterioriStandardDeviations"] = Persistence.OneVector(name = "APosterioriStandardDeviations") + self.StoredVariables["APosterioriVariances"] = Persistence.OneVector(name = "APosterioriVariances") + self.StoredVariables["Analysis"] = Persistence.OneVector(name = "Analysis") + self.StoredVariables["BMA"] = Persistence.OneVector(name = "BMA") self.StoredVariables["CostFunctionJ"] = Persistence.OneScalar(name = "CostFunctionJ") - self.StoredVariables["CostFunctionJb"] = Persistence.OneScalar(name = "CostFunctionJb") - self.StoredVariables["CostFunctionJo"] = Persistence.OneScalar(name = "CostFunctionJo") self.StoredVariables["CostFunctionJAtCurrentOptimum"] = Persistence.OneScalar(name = "CostFunctionJAtCurrentOptimum") + self.StoredVariables["CostFunctionJb"] = Persistence.OneScalar(name = "CostFunctionJb") self.StoredVariables["CostFunctionJbAtCurrentOptimum"] = Persistence.OneScalar(name = "CostFunctionJbAtCurrentOptimum") + self.StoredVariables["CostFunctionJo"] = Persistence.OneScalar(name = "CostFunctionJo") self.StoredVariables["CostFunctionJoAtCurrentOptimum"] = Persistence.OneScalar(name = "CostFunctionJoAtCurrentOptimum") + self.StoredVariables["CurrentIterationNumber"] = Persistence.OneIndex(name = "CurrentIterationNumber") + self.StoredVariables["CurrentOptimum"] = Persistence.OneVector(name = "CurrentOptimum") + self.StoredVariables["CurrentState"] = Persistence.OneVector(name = "CurrentState") + self.StoredVariables["ForecastState"] = Persistence.OneVector(name = "ForecastState") self.StoredVariables["GradientOfCostFunctionJ"] = Persistence.OneVector(name = "GradientOfCostFunctionJ") self.StoredVariables["GradientOfCostFunctionJb"] = Persistence.OneVector(name = "GradientOfCostFunctionJb") self.StoredVariables["GradientOfCostFunctionJo"] = Persistence.OneVector(name = "GradientOfCostFunctionJo") - self.StoredVariables["CurrentState"] = Persistence.OneVector(name = "CurrentState") - self.StoredVariables["Analysis"] = Persistence.OneVector(name = "Analysis") - self.StoredVariables["IndexOfOptimum"] = Persistence.OneIndex(name = "IndexOfOptimum") - self.StoredVariables["CurrentOptimum"] = Persistence.OneVector(name = "CurrentOptimum") - self.StoredVariables["SimulatedObservationAtBackground"] = Persistence.OneVector(name = "SimulatedObservationAtBackground") - self.StoredVariables["SimulatedObservationAtCurrentState"] = Persistence.OneVector(name = "SimulatedObservationAtCurrentState") - self.StoredVariables["SimulatedObservationAtOptimum"] = Persistence.OneVector(name = "SimulatedObservationAtOptimum") - self.StoredVariables["SimulatedObservationAtCurrentOptimum"] = Persistence.OneVector(name = "SimulatedObservationAtCurrentOptimum") + self.StoredVariables["IndexOfOptimum"] = Persistence.OneIndex(name = "IndexOfOptimum") self.StoredVariables["Innovation"] = Persistence.OneVector(name = "Innovation") + self.StoredVariables["InnovationAtCurrentAnalysis"] = Persistence.OneVector(name = "InnovationAtCurrentAnalysis") self.StoredVariables["InnovationAtCurrentState"] = Persistence.OneVector(name = "InnovationAtCurrentState") - self.StoredVariables["SigmaObs2"] = Persistence.OneScalar(name = "SigmaObs2") - self.StoredVariables["SigmaBck2"] = Persistence.OneScalar(name = "SigmaBck2") + self.StoredVariables["JacobianMatrixAtBackground"] = Persistence.OneMatrix(name = "JacobianMatrixAtBackground") + self.StoredVariables["JacobianMatrixAtCurrentState"] = Persistence.OneMatrix(name = "JacobianMatrixAtCurrentState") + self.StoredVariables["JacobianMatrixAtOptimum"] = Persistence.OneMatrix(name = "JacobianMatrixAtOptimum") + self.StoredVariables["KalmanGainAtOptimum"] = Persistence.OneMatrix(name = "KalmanGainAtOptimum") self.StoredVariables["MahalanobisConsistency"] = Persistence.OneScalar(name = "MahalanobisConsistency") self.StoredVariables["OMA"] = Persistence.OneVector(name = "OMA") self.StoredVariables["OMB"] = Persistence.OneVector(name = "OMB") - self.StoredVariables["BMA"] = Persistence.OneVector(name = "BMA") - self.StoredVariables["APosterioriCovariance"] = Persistence.OneMatrix(name = "APosterioriCovariance") - self.StoredVariables["APosterioriVariances"] = Persistence.OneVector(name = "APosterioriVariances") - self.StoredVariables["APosterioriStandardDeviations"] = Persistence.OneVector(name = "APosterioriStandardDeviations") - self.StoredVariables["APosterioriCorrelations"] = Persistence.OneMatrix(name = "APosterioriCorrelations") - self.StoredVariables["SimulationQuantiles"] = Persistence.OneMatrix(name = "SimulationQuantiles") self.StoredVariables["Residu"] = Persistence.OneScalar(name = "Residu") + self.StoredVariables["SigmaBck2"] = Persistence.OneScalar(name = "SigmaBck2") + self.StoredVariables["SigmaObs2"] = Persistence.OneScalar(name = "SigmaObs2") + self.StoredVariables["SimulatedObservationAtBackground"] = Persistence.OneVector(name = "SimulatedObservationAtBackground") + self.StoredVariables["SimulatedObservationAtCurrentAnalysis"]= Persistence.OneVector(name = "SimulatedObservationAtCurrentAnalysis") + self.StoredVariables["SimulatedObservationAtCurrentOptimum"] = Persistence.OneVector(name = "SimulatedObservationAtCurrentOptimum") + self.StoredVariables["SimulatedObservationAtCurrentState"] = Persistence.OneVector(name = "SimulatedObservationAtCurrentState") + self.StoredVariables["SimulatedObservationAtOptimum"] = Persistence.OneVector(name = "SimulatedObservationAtOptimum") + self.StoredVariables["SimulationQuantiles"] = Persistence.OneMatrix(name = "SimulationQuantiles") + # + for k in self.StoredVariables: + self.__canonical_stored_name[k.lower()] = k + # + for k, v in self.__variable_names_not_public.items(): + self.__canonical_parameter_name[k.lower()] = k + self.__canonical_parameter_name["algorithm"] = "Algorithm" + self.__canonical_parameter_name["storesupplementarycalculations"] = "StoreSupplementaryCalculations" - def _pre_run(self, Parameters, R=None, B=None, Q=None ): + def _pre_run(self, Parameters, Xb=None, Y=None, U=None, HO=None, EM=None, CM=None, R=None, B=None, Q=None ): "Pré-calcul" logging.debug("%s Lancement", self._name) - logging.debug("%s Taille mémoire utilisée de %.0f Mio", self._name, self._m.getUsedMemory("Mio")) - # - # Mise a jour de self._parameters avec Parameters - self.__setParameters(Parameters) - # - # Corrections et complements - def __test_cvalue( argument, variable, argname): + logging.debug("%s Taille mémoire utilisée de %.0f Mio"%(self._name, self._m.getUsedMemory("Mio"))) + self._getTimeState(reset=True) + # + # Mise a jour des paramètres internes avec le contenu de Parameters, en + # reprenant les valeurs par défauts pour toutes celles non définies + self.__setParameters(Parameters, reset=True) + for k, v in self.__variable_names_not_public.items(): + if k not in self._parameters: self.__setParameters( {k:v} ) + # + # Corrections et compléments des vecteurs + def __test_vvalue(argument, variable, argname, symbol=None): + if symbol is None: symbol = variable if argument is None: if variable in self.__required_inputs["RequiredInputValues"]["mandatory"]: - raise ValueError("%s %s error covariance matrix %s has to be properly defined!"%(self._name,argname,variable)) + raise ValueError("%s %s vector %s is not set and has to be properly defined!"%(self._name,argname,symbol)) elif variable in self.__required_inputs["RequiredInputValues"]["optional"]: - logging.debug("%s %s error covariance matrix %s is not set, but is optional."%(self._name,argname,variable)) + logging.debug("%s %s vector %s is not set, but is optional."%(self._name,argname,symbol)) else: - logging.debug("%s %s error covariance matrix %s is not set, but is not required."%(self._name,argname,variable)) + logging.debug("%s %s vector %s is not set, but is not required."%(self._name,argname,symbol)) else: - logging.debug("%s %s error covariance matrix %s is set."%(self._name,argname,variable)) - __test_cvalue( R, "R", "Observation" ) + logging.debug("%s %s vector %s is set, and its size is %i."%(self._name,argname,symbol,numpy.array(argument).size)) + return 0 + __test_vvalue( Xb, "Xb", "Background or initial state" ) + __test_vvalue( Y, "Y", "Observation" ) + __test_vvalue( U, "U", "Control" ) + # + # Corrections et compléments des covariances + def __test_cvalue(argument, variable, argname, symbol=None): + if symbol is None: symbol = variable + if argument is None: + if variable in self.__required_inputs["RequiredInputValues"]["mandatory"]: + raise ValueError("%s %s error covariance matrix %s is not set and has to be properly defined!"%(self._name,argname,symbol)) + elif variable in self.__required_inputs["RequiredInputValues"]["optional"]: + logging.debug("%s %s error covariance matrix %s is not set, but is optional."%(self._name,argname,symbol)) + else: + logging.debug("%s %s error covariance matrix %s is not set, but is not required."%(self._name,argname,symbol)) + else: + logging.debug("%s %s error covariance matrix %s is set."%(self._name,argname,symbol)) + return 0 __test_cvalue( B, "B", "Background" ) + __test_cvalue( R, "R", "Observation" ) __test_cvalue( Q, "Q", "Evolution" ) # + # Corrections et compléments des opérateurs + def __test_ovalue(argument, variable, argname, symbol=None): + if symbol is None: symbol = variable + if argument is None or (isinstance(argument,dict) and len(argument)==0): + if variable in self.__required_inputs["RequiredInputValues"]["mandatory"]: + raise ValueError("%s %s operator %s is not set and has to be properly defined!"%(self._name,argname,symbol)) + elif variable in self.__required_inputs["RequiredInputValues"]["optional"]: + logging.debug("%s %s operator %s is not set, but is optional."%(self._name,argname,symbol)) + else: + logging.debug("%s %s operator %s is not set, but is not required."%(self._name,argname,symbol)) + else: + logging.debug("%s %s operator %s is set."%(self._name,argname,symbol)) + return 0 + __test_ovalue( HO, "HO", "Observation", "H" ) + __test_ovalue( EM, "EM", "Evolution", "M" ) + __test_ovalue( CM, "CM", "Control Model", "C" ) + # if ("Bounds" in self._parameters) and isinstance(self._parameters["Bounds"], (list, tuple)) and (len(self._parameters["Bounds"]) > 0): logging.debug("%s Prise en compte des bornes effectuee"%(self._name,)) else: @@ -585,13 +792,18 @@ class Algorithm(object): _EI = numpy.diag(1./numpy.sqrt(numpy.diag(_A))) _C = numpy.dot(_EI, numpy.dot(_A, _EI)) self.StoredVariables["APosterioriCorrelations"].store( _C ) - if _oH is not None: + if _oH is not None and "Direct" in _oH and "Tangent" in _oH and "Adjoint" in _oH: logging.debug("%s Nombre d'évaluation(s) de l'opérateur d'observation direct/tangent/adjoint.: %i/%i/%i", self._name, _oH["Direct"].nbcalls(0),_oH["Tangent"].nbcalls(0),_oH["Adjoint"].nbcalls(0)) logging.debug("%s Nombre d'appels au cache d'opérateur d'observation direct/tangent/adjoint..: %i/%i/%i", self._name, _oH["Direct"].nbcalls(3),_oH["Tangent"].nbcalls(3),_oH["Adjoint"].nbcalls(3)) logging.debug("%s Taille mémoire utilisée de %.0f Mio", self._name, self._m.getUsedMemory("Mio")) + logging.debug("%s Durées d'utilisation CPU de %.1fs et elapsed de %.1fs", self._name, self._getTimeState()[0], self._getTimeState()[1]) logging.debug("%s Terminé", self._name) return 0 + def _toStore(self, key): + "True if in StoreSupplementaryCalculations, else False" + return key in self._parameters["StoreSupplementaryCalculations"] + def get(self, key=None): """ Renvoie l'une des variables stockées identifiée par la clé, ou le @@ -601,13 +813,16 @@ class Algorithm(object): des classes de persistance. """ if key is not None: - return self.StoredVariables[key] + return self.StoredVariables[self.__canonical_stored_name[key.lower()]] else: return self.StoredVariables def __contains__(self, key=None): "D.__contains__(k) -> True if D has a key k, else False" - return key in self.StoredVariables + if key is None or key.lower() not in self.__canonical_stored_name: + return False + else: + return self.__canonical_stored_name[key.lower()] in self.StoredVariables def keys(self): "D.keys() -> list of D's keys" @@ -618,8 +833,8 @@ class Algorithm(object): def pop(self, k, d): "D.pop(k[,d]) -> v, remove specified key and return the corresponding value" - if hasattr(self, "StoredVariables"): - return self.StoredVariables.pop(k, d) + if hasattr(self, "StoredVariables") and k.lower() in self.__canonical_stored_name: + return self.StoredVariables.pop(self.__canonical_stored_name[k.lower()], d) else: try: msg = "'%s'"%k @@ -638,7 +853,7 @@ class Algorithm(object): """ raise NotImplementedError("Mathematical assimilation calculation has not been implemented!") - def defineRequiredParameter(self, name = None, default = None, typecast = None, message = None, minval = None, maxval = None, listval = None): + def defineRequiredParameter(self, name = None, default = None, typecast = None, message = None, minval = None, maxval = None, listval = None, listadv = None): """ Permet de définir dans l'algorithme des paramètres requis et leurs caractéristiques par défaut. @@ -652,8 +867,10 @@ class Algorithm(object): "minval" : minval, "maxval" : maxval, "listval" : listval, + "listadv" : listadv, "message" : message, } + self.__canonical_parameter_name[name.lower()] = name logging.debug("%s %s (valeur par défaut = %s)", self._name, message, self.setParameterValue(name)) def getRequiredParameters(self, noDetails=True): @@ -670,11 +887,13 @@ class Algorithm(object): """ Renvoie la valeur d'un paramètre requis de manière contrôlée """ - default = self.__required_parameters[name]["default"] - typecast = self.__required_parameters[name]["typecast"] - minval = self.__required_parameters[name]["minval"] - maxval = self.__required_parameters[name]["maxval"] - listval = self.__required_parameters[name]["listval"] + __k = self.__canonical_parameter_name[name.lower()] + default = self.__required_parameters[__k]["default"] + typecast = self.__required_parameters[__k]["typecast"] + minval = self.__required_parameters[__k]["minval"] + maxval = self.__required_parameters[__k]["maxval"] + listval = self.__required_parameters[__k]["listval"] + listadv = self.__required_parameters[__k]["listadv"] # if value is None and default is None: __val = None @@ -683,169 +902,95 @@ class Algorithm(object): else: __val = typecast( default ) else: if typecast is None: __val = value - else: __val = typecast( value ) + else: + try: + __val = typecast( value ) + except: + raise ValueError("The value '%s' for the parameter named '%s' can not be correctly evaluated with type '%s'."%(value, __k, typecast)) # if minval is not None and (numpy.array(__val, float) < minval).any(): - raise ValueError("The parameter named \"%s\" of value \"%s\" can not be less than %s."%(name, __val, minval)) + raise ValueError("The parameter named '%s' of value '%s' can not be less than %s."%(__k, __val, minval)) if maxval is not None and (numpy.array(__val, float) > maxval).any(): - raise ValueError("The parameter named \"%s\" of value \"%s\" can not be greater than %s."%(name, __val, maxval)) - if listval is not None: + raise ValueError("The parameter named '%s' of value '%s' can not be greater than %s."%(__k, __val, maxval)) + if listval is not None or listadv is not None: if typecast is list or typecast is tuple or isinstance(__val,list) or isinstance(__val,tuple): for v in __val: - if v not in listval: - raise ValueError("The value \"%s\" of the parameter named \"%s\" is not allowed, it has to be in the list %s."%(v, name, listval)) - elif __val not in listval: - raise ValueError("The value \"%s\" of the parameter named \"%s\" is not allowed, it has to be in the list %s."%( __val, name,listval)) + if listval is not None and v in listval: continue + elif listadv is not None and v in listadv: continue + else: + raise ValueError("The value '%s' is not allowed for the parameter named '%s', it has to be in the list %s."%(v, __k, listval)) + elif not (listval is not None and __val in listval) and not (listadv is not None and __val in listadv): + raise ValueError("The value '%s' is not allowed for the parameter named '%s', it has to be in the list %s."%( __val, __k,listval)) + # return __val def requireInputArguments(self, mandatory=(), optional=()): """ - Permet d'imposer des arguments requises en entrée + Permet d'imposer des arguments de calcul requis en entrée. """ self.__required_inputs["RequiredInputValues"]["mandatory"] = tuple( mandatory ) self.__required_inputs["RequiredInputValues"]["optional"] = tuple( optional ) - def __setParameters(self, fromDico={}): + def getInputArguments(self): """ - Permet de stocker les paramètres reçus dans le dictionnaire interne. + Permet d'obtenir les listes des arguments de calcul requis en entrée. """ - self._parameters.update( fromDico ) - for k in self.__required_parameters.keys(): - if k in fromDico.keys(): - self._parameters[k] = self.setParameterValue(k,fromDico[k]) - else: - self._parameters[k] = self.setParameterValue(k) - logging.debug("%s %s : %s", self._name, self.__required_parameters[k]["message"], self._parameters[k]) - -# ============================================================================== -class Diagnostic(object): - """ - Classe générale d'interface de type diagnostic - - Ce template s'utilise de la manière suivante : il sert de classe "patron" en - même temps que l'une des classes de persistance, comme "OneScalar" par - exemple. - - Une classe élémentaire de diagnostic doit implémenter ses deux méthodes, la - méthode "_formula" pour écrire explicitement et proprement la formule pour - l'écriture mathématique du calcul du diagnostic (méthode interne non - publique), et "calculate" pour activer la précédente tout en ayant vérifié - et préparé les données, et pour stocker les résultats à chaque pas (méthode - externe d'activation). - """ - def __init__(self, name = "", parameters = {}): - "Initialisation" - self.name = str(name) - self.parameters = dict( parameters ) + return self.__required_inputs["RequiredInputValues"]["mandatory"], self.__required_inputs["RequiredInputValues"]["optional"] - def _formula(self, *args): + def setAttributes(self, tags=()): """ - Doit implémenter l'opération élémentaire de diagnostic sous sa forme - mathématique la plus naturelle possible. + Permet d'adjoindre des attributs comme les tags de classification. + Renvoie la liste actuelle dans tous les cas. """ - raise NotImplementedError("Diagnostic mathematical formula has not been implemented!") + self.__required_inputs["ClassificationTags"].extend( tags ) + return self.__required_inputs["ClassificationTags"] - def calculate(self, *args): + def __setParameters(self, fromDico={}, reset=False): """ - Active la formule de calcul avec les arguments correctement rangés + Permet de stocker les paramètres reçus dans le dictionnaire interne. """ - raise NotImplementedError("Diagnostic activation method has not been implemented!") + self._parameters.update( fromDico ) + __inverse_fromDico_keys = {} + for k in fromDico.keys(): + if k.lower() in self.__canonical_parameter_name: + __inverse_fromDico_keys[self.__canonical_parameter_name[k.lower()]] = k + #~ __inverse_fromDico_keys = dict([(self.__canonical_parameter_name[k.lower()],k) for k in fromDico.keys()]) + __canonic_fromDico_keys = __inverse_fromDico_keys.keys() + for k in self.__required_parameters.keys(): + if k in __canonic_fromDico_keys: + self._parameters[k] = self.setParameterValue(k,fromDico[__inverse_fromDico_keys[k]]) + elif reset: + self._parameters[k] = self.setParameterValue(k) + else: + pass + logging.debug("%s %s : %s", self._name, self.__required_parameters[k]["message"], self._parameters[k]) -# ============================================================================== -class DiagnosticAndParameters(object): - """ - Classe générale d'interface d'interface de type diagnostic - """ - def __init__(self, - name = "GenericDiagnostic", - asDiagnostic = None, - asIdentifier = None, - asDict = None, - asScript = None, - asUnit = None, - asBaseType = None, - asExistingDiags = None, - ): + def _getTimeState(self, reset=False): """ + Initialise ou restitue le temps de calcul (cpu/elapsed) à la seconde """ - self.__name = str(name) - self.__D = None - self.__I = None - self.__P = {} - self.__U = "" - self.__B = None - self.__E = tuple(asExistingDiags) - self.__TheDiag = None - # - if asScript is not None: - __Diag = ImportFromScript(asScript).getvalue( "Diagnostic" ) - __Iden = ImportFromScript(asScript).getvalue( "Identifier" ) - __Dict = ImportFromScript(asScript).getvalue( self.__name, "Parameters" ) - __Unit = ImportFromScript(asScript).getvalue( "Unit" ) - __Base = ImportFromScript(asScript).getvalue( "BaseType" ) + if reset: + self.__initial_cpu_time = time.process_time() + self.__initial_elapsed_time = time.perf_counter() + return 0., 0. else: - __Diag = asDiagnostic - __Iden = asIdentifier - __Dict = asDict - __Unit = asUnit - __Base = asBaseType - # - if __Diag is not None: - self.__D = str(__Diag) - if __Iden is not None: - self.__I = str(__Iden) + self.__cpu_time = time.process_time() - self.__initial_cpu_time + self.__elapsed_time = time.perf_counter() - self.__initial_elapsed_time + return self.__cpu_time, self.__elapsed_time + + def _StopOnTimeLimit(self, X=None, withReason=False): + "Stop criteria on time limit: True/False [+ Reason]" + c, e = self._getTimeState() + if "MaximumCpuTime" in self._parameters and c > self._parameters["MaximumCpuTime"]: + __SC, __SR = True, "Reached maximum CPU time (%.1fs > %.1fs)"%(c, self._parameters["MaximumCpuTime"]) + elif "MaximumElapsedTime" in self._parameters and e > self._parameters["MaximumElapsedTime"]: + __SC, __SR = True, "Reached maximum elapsed time (%.1fs > %.1fs)"%(e, self._parameters["MaximumElapsedTime"]) else: - self.__I = str(__Diag) - if __Dict is not None: - self.__P.update( dict(__Dict) ) - if __Unit is None or __Unit == "None": - self.__U = "" - if __Base is None or __Base == "None": - self.__B = None - # - self.__setDiagnostic( self.__D, self.__I, self.__U, self.__B, self.__P, self.__E ) - - def get(self): - "Renvoie l'objet" - return self.__TheDiag - - def __setDiagnostic(self, __choice = None, __name = "", __unit = "", __basetype = None, __parameters = {}, __existings = () ): - """ - Permet de sélectionner un diagnostic a effectuer - """ - if __choice is None: - raise ValueError("Error: diagnostic choice has to be given") - __daDirectory = "daDiagnostics" - # - # Recherche explicitement le fichier complet - # ------------------------------------------ - __module_path = None - for directory in sys.path: - if os.path.isfile(os.path.join(directory, __daDirectory, str(__choice)+'.py')): - __module_path = os.path.abspath(os.path.join(directory, __daDirectory)) - if __module_path is None: - raise ImportError("No diagnostic module named \"%s\" was found in a \"%s\" subdirectory\n The search path is %s"%(__choice, __daDirectory, sys.path)) - # - # Importe le fichier complet comme un module - # ------------------------------------------ - try: - __sys_path_tmp = sys.path ; sys.path.insert(0,__module_path) - self.__diagnosticFile = __import__(str(__choice), globals(), locals(), []) - sys.path = __sys_path_tmp ; del __sys_path_tmp - except ImportError as e: - raise ImportError("The module named \"%s\" was found, but is incorrect at the import stage.\n The import error message is: %s"%(__choice,e)) - # - # Instancie un objet du type élémentaire du fichier - # ------------------------------------------------- - if __name in __existings: - raise ValueError("A default input with the same name \"%s\" already exists."%str(__name)) + __SC, __SR = False, "" + if withReason: + return __SC, __SR else: - self.__TheDiag = self.__diagnosticFile.ElementaryDiagnostic( - name = __name, - unit = __unit, - basetype = __basetype, - parameters = __parameters ) - return 0 + return __SC # ============================================================================== class AlgorithmAndParameters(object): @@ -870,8 +1015,8 @@ class AlgorithmAndParameters(object): # self.updateParameters( asDict, asScript ) # - if asScript is not None: - __Algo = ImportFromScript(asScript).getvalue( "Algorithm" ) + if asAlgorithm is None and asScript is not None: + __Algo = Interfaces.ImportFromScript(asScript).getvalue( "Algorithm" ) else: __Algo = asAlgorithm # @@ -880,14 +1025,16 @@ class AlgorithmAndParameters(object): self.__P.update( {"Algorithm":self.__A} ) # self.__setAlgorithm( self.__A ) + # + self.__variable_names_not_public = {"nextStep":False} # Duplication dans Algorithm def updateParameters(self, asDict = None, asScript = None, ): "Mise a jour des parametres" - if asScript is not None: - __Dict = ImportFromScript(asScript).getvalue( self.__name, "Parameters" ) + if asDict is None and asScript is not None: + __Dict = Interfaces.ImportFromScript(asScript).getvalue( self.__name, "Parameters" ) else: __Dict = asDict # @@ -899,7 +1046,7 @@ class AlgorithmAndParameters(object): Operator.CM.clearCache() # if not isinstance(asDictAO, dict): - raise ValueError("The objects for algorithm calculation has to be given as a dictionnary, and is not") + raise ValueError("The objects for algorithm calculation have to be given together as a dictionnary, and they are not") if hasattr(asDictAO["Background"],"getO"): self.__Xb = asDictAO["Background"].getO() elif hasattr(asDictAO["CheckingPoint"],"getO"): self.__Xb = asDictAO["CheckingPoint"].getO() else: self.__Xb = None @@ -908,7 +1055,7 @@ class AlgorithmAndParameters(object): if hasattr(asDictAO["ControlInput"],"getO"): self.__U = asDictAO["ControlInput"].getO() else: self.__U = asDictAO["ControlInput"] if hasattr(asDictAO["ObservationOperator"],"getO"): self.__HO = asDictAO["ObservationOperator"].getO() - else: self.__HO = asDictAO["ObservationOperator"] + else: self.__HO = asDictAO["ObservationOperator"] if hasattr(asDictAO["EvolutionModel"],"getO"): self.__EM = asDictAO["EvolutionModel"].getO() else: self.__EM = asDictAO["EvolutionModel"] if hasattr(asDictAO["ControlModel"],"getO"): self.__CM = asDictAO["ControlModel"].getO() @@ -936,67 +1083,55 @@ class AlgorithmAndParameters(object): def executeYACSScheme(self, FileName=None): "Permet de lancer le calcul d'assimilation" if FileName is None or not os.path.exists(FileName): - raise ValueError("a existing DIC Python file name has to be given for YACS execution.\n") - if not os.environ.has_key("ADAO_ROOT_DIR"): - raise ImportError("Unable to get ADAO_ROOT_DIR environnement variable. Please launch SALOME to add ADAO_ROOT_DIR to your environnement.\n") - # - __converterExe = os.path.join(os.environ["ADAO_ROOT_DIR"], "bin/salome", "AdaoYacsSchemaCreator.py") - __inputFile = os.path.abspath(FileName) - __outputFile = __inputFile[:__inputFile.rfind(".")] + '.xml' - # - __args = ["python", __converterExe, __inputFile, __outputFile] - import subprocess - __p = subprocess.Popen(__args) - (__stdoutdata, __stderrdata) = __p.communicate() - if not os.path.exists(__outputFile): - __msg = "An error occured during the execution of the ADAO YACS Schema\n" - __msg += "Creator applied on the input file:\n" - __msg += " %s\n"%__outputFile - __msg += "If SALOME GUI is launched by command line, see errors\n" - __msg += "details in your terminal.\n" - raise ValueError(__msg) - # + raise ValueError("a YACS file name has to be given for YACS execution.\n") + else: + __file = os.path.abspath(FileName) + logging.debug("The YACS file name is \"%s\"."%__file) + if not PlatformInfo.has_salome or \ + not PlatformInfo.has_yacs or \ + not PlatformInfo.has_adao: + raise ImportError("\n\n"+\ + "Unable to get SALOME, YACS or ADAO environnement variables.\n"+\ + "Please load the right environnement before trying to use it.\n") + # + import pilot + import SALOMERuntime + import loader + SALOMERuntime.RuntimeSALOME_setRuntime() + + r = pilot.getRuntime() + xmlLoader = loader.YACSLoader() + xmlLoader.registerProcCataLoader() try: - import pilot - import SALOMERuntime - import loader - SALOMERuntime.RuntimeSALOME_setRuntime() - - r = pilot.getRuntime() - xmlLoader = loader.YACSLoader() - xmlLoader.registerProcCataLoader() - try: - catalogAd = r.loadCatalog("proc", __outputFile) - except: - pass + catalogAd = r.loadCatalog("proc", __file) r.addCatalog(catalogAd) - - try: - p = xmlLoader.load(__outputFile) - except IOError as ex: - print("IO exception: %s"%(ex,)) - - logger = p.getLogger("parser") - if not logger.isEmpty(): - print("The imported file has errors :") - print(logger.getStr()) - - if not p.isValid(): - print("Le schéma n'est pas valide et ne peut pas être exécuté") - print(p.getErrorReport()) - - info=pilot.LinkInfo(pilot.LinkInfo.ALL_DONT_STOP) - p.checkConsistency(info) - if info.areWarningsOrErrors(): - print("Le schéma n'est pas cohérent et ne peut pas être exécuté") - print(info.getGlobalRepr()) - - e = pilot.ExecutorSwig() - e.RunW(p) - if p.getEffectiveState() != pilot.DONE: - print(p.getErrorReport()) except: - raise ValueError("execution error of YACS scheme") + pass + + try: + p = xmlLoader.load(__file) + except IOError as ex: + print("The YACS XML schema file can not be loaded: %s"%(ex,)) + + logger = p.getLogger("parser") + if not logger.isEmpty(): + print("The imported YACS XML schema has errors on parsing:") + print(logger.getStr()) + + if not p.isValid(): + print("The YACS XML schema is not valid and will not be executed:") + print(p.getErrorReport()) + + info=pilot.LinkInfo(pilot.LinkInfo.ALL_DONT_STOP) + p.checkConsistency(info) + if info.areWarningsOrErrors(): + print("The YACS XML schema is not coherent and will not be executed:") + print(info.getGlobalRepr()) + + e = pilot.ExecutorSwig() + e.RunW(p) + if p.getEffectiveState() != pilot.DONE: + print(p.getErrorReport()) # return 0 @@ -1007,7 +1142,9 @@ class AlgorithmAndParameters(object): elif key in self.__P: return self.__P[key] else: - return self.__P + allvariables = self.__P + for k in self.__variable_names_not_public: allvariables.pop(k, None) + return allvariables def pop(self, k, d): "Necessaire pour le pickling" @@ -1017,6 +1154,14 @@ class AlgorithmAndParameters(object): "Renvoie la liste des paramètres requis selon l'algorithme" return self.__algorithm.getRequiredParameters(noDetails) + def getAlgorithmInputArguments(self): + "Renvoie la liste des entrées requises selon l'algorithme" + return self.__algorithm.getInputArguments() + + def getAlgorithmAttributes(self): + "Renvoie la liste des attributs selon l'algorithme" + return self.__algorithm.setAttributes() + def setObserver(self, __V, __O, __I, __S): if self.__algorithm is None \ or isinstance(self.__algorithm, dict) \ @@ -1054,7 +1199,10 @@ class AlgorithmAndParameters(object): return self.__algorithm.StoredVariables[ __V ].hasDataObserver() def keys(self): - return list(self.__algorithm.keys()) + list(self.__P.keys()) + __allvariables = list(self.__algorithm.keys()) + list(self.__P.keys()) + for k in self.__variable_names_not_public: + if k in __allvariables: __allvariables.remove(k) + return __allvariables def __contains__(self, key=None): "D.__contains__(k) -> True if D has a key k, else False" @@ -1072,8 +1220,7 @@ class AlgorithmAndParameters(object): """ Permet de sélectionner l'algorithme à utiliser pour mener à bien l'étude d'assimilation. L'argument est un champ caractère se rapportant au nom - d'un fichier contenu dans "../daAlgorithms" et réalisant l'opération - d'assimilation sur les arguments fixes. + d'un algorithme réalisant l'opération sur les arguments fixes. """ if choice is None: raise ValueError("Error: algorithm choice has to be given") @@ -1088,13 +1235,15 @@ class AlgorithmAndParameters(object): if os.path.isfile(os.path.join(directory, daDirectory, str(choice)+'.py')): module_path = os.path.abspath(os.path.join(directory, daDirectory)) if module_path is None: - raise ImportError("No algorithm module named \"%s\" was found in a \"%s\" subdirectory\n The search path is %s"%(choice, daDirectory, sys.path)) + raise ImportError("No algorithm module named \"%s\" has been found in the search path.\n The search path is %s"%(choice, sys.path)) # # Importe le fichier complet comme un module # ------------------------------------------ try: sys_path_tmp = sys.path ; sys.path.insert(0,module_path) self.__algorithmFile = __import__(str(choice), globals(), locals(), []) + if not hasattr(self.__algorithmFile, "ElementaryAlgorithm"): + raise ImportError("this module does not define a valid elementary algorithm.") self.__algorithmName = str(choice) sys.path = sys_path_tmp ; del sys_path_tmp except ImportError as e: @@ -1186,13 +1335,13 @@ class AlgorithmAndParameters(object): if not( min(__EM_shape) == max(__EM_shape) ): raise ValueError("Shape characteristic of evolution operator (EM) is incorrect: \"%s\"."%(__EM_shape,)) # - if len(self.__HO) > 0 and not(type(self.__HO) is type({})) and not( __HO_shape[1] == max(__Xb_shape) ): + if len(self.__HO) > 0 and not isinstance(self.__HO, dict) and not( __HO_shape[1] == max(__Xb_shape) ): raise ValueError("Shape characteristic of observation operator (H) \"%s\" and state (X) \"%s\" are incompatible."%(__HO_shape,__Xb_shape)) - if len(self.__HO) > 0 and not(type(self.__HO) is type({})) and not( __HO_shape[0] == max(__Y_shape) ): + if len(self.__HO) > 0 and not isinstance(self.__HO, dict) and not( __HO_shape[0] == max(__Y_shape) ): raise ValueError("Shape characteristic of observation operator (H) \"%s\" and observation (Y) \"%s\" are incompatible."%(__HO_shape,__Y_shape)) - if len(self.__HO) > 0 and not(type(self.__HO) is type({})) and len(self.__B) > 0 and not( __HO_shape[1] == __B_shape[0] ): + if len(self.__HO) > 0 and not isinstance(self.__HO, dict) and len(self.__B) > 0 and not( __HO_shape[1] == __B_shape[0] ): raise ValueError("Shape characteristic of observation operator (H) \"%s\" and a priori errors covariance matrix (B) \"%s\" are incompatible."%(__HO_shape,__B_shape)) - if len(self.__HO) > 0 and not(type(self.__HO) is type({})) and len(self.__R) > 0 and not( __HO_shape[0] == __R_shape[1] ): + if len(self.__HO) > 0 and not isinstance(self.__HO, dict) and len(self.__R) > 0 and not( __HO_shape[0] == __R_shape[1] ): raise ValueError("Shape characteristic of observation operator (H) \"%s\" and observation errors covariance matrix (R) \"%s\" are incompatible."%(__HO_shape,__R_shape)) # if self.__B is not None and len(self.__B) > 0 and not( __B_shape[1] == max(__Xb_shape) ): @@ -1208,10 +1357,10 @@ class AlgorithmAndParameters(object): if self.__R is not None and len(self.__R) > 0 and not( __R_shape[1] == max(__Y_shape) ): raise ValueError("Shape characteristic of observation errors covariance matrix (R) \"%s\" and observation (Y) \"%s\" are incompatible."%(__R_shape,__Y_shape)) # - if self.__EM is not None and len(self.__EM) > 0 and not(type(self.__EM) is type({})) and not( __EM_shape[1] == max(__Xb_shape) ): + if self.__EM is not None and len(self.__EM) > 0 and not isinstance(self.__EM, dict) and not( __EM_shape[1] == max(__Xb_shape) ): raise ValueError("Shape characteristic of evolution model (EM) \"%s\" and state (X) \"%s\" are incompatible."%(__EM_shape,__Xb_shape)) # - if self.__CM is not None and len(self.__CM) > 0 and not(type(self.__CM) is type({})) and not( __CM_shape[1] == max(__U_shape) ): + if self.__CM is not None and len(self.__CM) > 0 and not isinstance(self.__CM, dict) and not( __CM_shape[1] == max(__U_shape) ): raise ValueError("Shape characteristic of control model (CM) \"%s\" and control (U) \"%s\" are incompatible."%(__CM_shape,__U_shape)) # if ("Bounds" in self.__P) \ @@ -1222,6 +1371,45 @@ class AlgorithmAndParameters(object): # return 1 +# ============================================================================== +class RegulationAndParameters(object): + """ + Classe générale d'interface d'action pour la régulation et ses paramètres + """ + def __init__(self, + name = "GenericRegulation", + asAlgorithm = None, + asDict = None, + asScript = None, + ): + """ + """ + self.__name = str(name) + self.__P = {} + # + if asAlgorithm is None and asScript is not None: + __Algo = Interfaces.ImportFromScript(asScript).getvalue( "Algorithm" ) + else: + __Algo = asAlgorithm + # + if asDict is None and asScript is not None: + __Dict = Interfaces.ImportFromScript(asScript).getvalue( self.__name, "Parameters" ) + else: + __Dict = asDict + # + if __Dict is not None: + self.__P.update( dict(__Dict) ) + # + if __Algo is not None: + self.__P.update( {"Algorithm":str(__Algo)} ) + + def get(self, key = None): + "Vérifie l'existence d'une clé de variable ou de paramètres" + if key in self.__P: + return self.__P[key] + else: + return self.__P + # ============================================================================== class DataObserver(object): """ @@ -1262,19 +1450,11 @@ class DataObserver(object): else: raise ValueError("setting an observer has to be done over a variable name or a list of variable names.") # - if asString is not None: - __FunctionText = asString - elif (asTemplate is not None) and (asTemplate in Templates.ObserverTemplates): - __FunctionText = Templates.ObserverTemplates[asTemplate] - elif asScript is not None: - __FunctionText = ImportFromScript(asScript).getstring() - else: - __FunctionText = "" - __Function = ObserverF(__FunctionText) - # if asObsObject is not None: self.__O = asObsObject else: + __FunctionText = str(UserScript('Observer', asTemplate, asString, asScript)) + __Function = Observer2Func(__FunctionText) self.__O = __Function.getfunc() # for k in range(len(self.__V)): @@ -1293,6 +1473,89 @@ class DataObserver(object): "x.__str__() <==> str(x)" return str(self.__V)+"\n"+str(self.__O) +# ============================================================================== +class UserScript(object): + """ + Classe générale d'interface de type texte de script utilisateur + """ + def __init__(self, + name = "GenericUserScript", + asTemplate = None, + asString = None, + asScript = None, + ): + """ + """ + self.__name = str(name) + # + if asString is not None: + self.__F = asString + elif self.__name == "UserPostAnalysis" and (asTemplate is not None) and (asTemplate in Templates.UserPostAnalysisTemplates): + self.__F = Templates.UserPostAnalysisTemplates[asTemplate] + elif self.__name == "Observer" and (asTemplate is not None) and (asTemplate in Templates.ObserverTemplates): + self.__F = Templates.ObserverTemplates[asTemplate] + elif asScript is not None: + self.__F = Interfaces.ImportFromScript(asScript).getstring() + else: + self.__F = "" + + def __repr__(self): + "x.__repr__() <==> repr(x)" + return repr(self.__F) + + def __str__(self): + "x.__str__() <==> str(x)" + return str(self.__F) + +# ============================================================================== +class ExternalParameters(object): + """ + Classe générale d'interface de type texte de script utilisateur + """ + def __init__(self, + name = "GenericExternalParameters", + asDict = None, + asScript = None, + ): + """ + """ + self.__name = str(name) + self.__P = {} + # + self.updateParameters( asDict, asScript ) + + def updateParameters(self, + asDict = None, + asScript = None, + ): + "Mise a jour des parametres" + if asDict is None and asScript is not None: + __Dict = Interfaces.ImportFromScript(asScript).getvalue( self.__name, "ExternalParameters" ) + else: + __Dict = asDict + # + if __Dict is not None: + self.__P.update( dict(__Dict) ) + + def get(self, key = None): + if key in self.__P: + return self.__P[key] + else: + return list(self.__P.keys()) + + def keys(self): + return list(self.__P.keys()) + + def pop(self, k, d): + return self.__P.pop(k, d) + + def items(self): + return self.__P.items() + + def __contains__(self, key=None): + "D.__contains__(k) -> True if D has a key k, else False" + return key in self.__P + # ============================================================================== class State(object): """ @@ -1303,6 +1566,9 @@ class State(object): asVector = None, asPersistentVector = None, asScript = None, + asDataFile = None, + colNames = None, + colMajor = False, scheduledBy = None, toBeChecked = False, ): @@ -1317,6 +1583,14 @@ class State(object): nommée "name", la variable est de type "asVector" (par défaut) ou "asPersistentVector" selon que l'une de ces variables est placée à "True". + - asDataFile : si un ou plusieurs fichiers valides sont donnés + contenant des valeurs en colonnes, elles-mêmes nommées "colNames" + (s'il n'y a pas de nom de colonne indiquée, on cherche une colonne + nommée "name"), on récupère les colonnes et on les range ligne après + ligne (colMajor=False, par défaut) ou colonne après colonne + (colMajor=True). La variable résultante est de type "asVector" (par + défaut) ou "asPersistentVector" selon que l'une de ces variables est + placée à "True". """ self.__name = str(name) self.__check = bool(toBeChecked) @@ -1329,9 +1603,29 @@ class State(object): if asScript is not None: __Vector, __Series = None, None if asPersistentVector: - __Series = ImportFromScript(asScript).getvalue( self.__name ) + __Series = Interfaces.ImportFromScript(asScript).getvalue( self.__name ) + else: + __Vector = Interfaces.ImportFromScript(asScript).getvalue( self.__name ) + elif asDataFile is not None: + __Vector, __Series = None, None + if asPersistentVector: + if colNames is not None: + __Series = Interfaces.ImportFromFile(asDataFile).getvalue( colNames )[1] + else: + __Series = Interfaces.ImportFromFile(asDataFile).getvalue( [self.__name,] )[1] + if bool(colMajor) and not Interfaces.ImportFromFile(asDataFile).getformat() == "application/numpy.npz": + __Series = numpy.transpose(__Series) + elif not bool(colMajor) and Interfaces.ImportFromFile(asDataFile).getformat() == "application/numpy.npz": + __Series = numpy.transpose(__Series) else: - __Vector = ImportFromScript(asScript).getvalue( self.__name ) + if colNames is not None: + __Vector = Interfaces.ImportFromFile(asDataFile).getvalue( colNames )[1] + else: + __Vector = Interfaces.ImportFromFile(asDataFile).getvalue( [self.__name,] )[1] + if bool(colMajor): + __Vector = numpy.ravel(__Vector, order = "F") + else: + __Vector = numpy.ravel(__Vector, order = "C") else: __Vector, __Series = asVector, asPersistentVector # @@ -1342,14 +1636,14 @@ class State(object): self.size = self.__V.size elif __Series is not None: self.__is_series = True - if type(__Series) in (tuple, list, numpy.ndarray, numpy.matrix): + if isinstance(__Series, (tuple, list, numpy.ndarray, numpy.matrix, str)): self.__V = Persistence.OneVector(self.__name, basetype=numpy.matrix) + if isinstance(__Series, str): __Series = eval(__Series) for member in __Series: self.__V.store( numpy.matrix( numpy.asmatrix(member).A1, numpy.float ).T ) - import sys ; sys.stdout.flush() else: self.__V = __Series - if type(self.__V.shape) in (tuple, list): + if isinstance(self.__V.shape, (tuple, list)): self.shape = self.__V.shape else: self.shape = self.__V.shape() @@ -1357,7 +1651,7 @@ class State(object): self.shape = (self.shape[0],1) self.size = self.shape[0] * self.shape[1] else: - raise ValueError("The %s object is improperly defined, it requires at minima either a vector, a list/tuple of vectors or a persistent object. Please check your vector input."%self.__name) + raise ValueError("The %s object is improperly defined or undefined, it requires at minima either a vector, a list/tuple of vectors or a persistent object. Please check your vector input."%self.__name) # if scheduledBy is not None: self.__T = scheduledBy @@ -1429,13 +1723,13 @@ class Covariance(object): if asScript is not None: __Matrix, __Scalar, __Vector, __Object = None, None, None, None if asEyeByScalar: - __Scalar = ImportFromScript(asScript).getvalue( self.__name ) + __Scalar = Interfaces.ImportFromScript(asScript).getvalue( self.__name ) elif asEyeByVector: - __Vector = ImportFromScript(asScript).getvalue( self.__name ) + __Vector = Interfaces.ImportFromScript(asScript).getvalue( self.__name ) elif asCovObject: - __Object = ImportFromScript(asScript).getvalue( self.__name ) + __Object = Interfaces.ImportFromScript(asScript).getvalue( self.__name ) else: - __Matrix = ImportFromScript(asScript).getvalue( self.__name ) + __Matrix = Interfaces.ImportFromScript(asScript).getvalue( self.__name ) else: __Matrix, __Scalar, __Vector, __Object = asCovariance, asEyeByScalar, asEyeByVector, asCovObject # @@ -1478,6 +1772,8 @@ class Covariance(object): def __validate(self): "Validation" + if self.__C is None: + raise UnboundLocalError("%s covariance matrix value has not been set!"%(self.__name,)) if self.ismatrix() and min(self.shape) != max(self.shape): raise ValueError("The given matrix for %s is not a square one, its shape is %s. Please check your matrix input."%(self.__name,self.shape)) if self.isobject() and min(self.shape) != max(self.shape): @@ -1559,6 +1855,30 @@ class Covariance(object): elif self.isobject() and hasattr(self.__C,"choleskyI"): return Covariance(self.__name+"H", asCovObject = self.__C.choleskyI() ) + def sqrtm(self): + "Racine carrée matricielle" + if self.ismatrix(): + import scipy + return Covariance(self.__name+"C", asCovariance = scipy.linalg.sqrtm(self.__C) ) + elif self.isvector(): + return Covariance(self.__name+"C", asEyeByVector = numpy.sqrt( self.__C ) ) + elif self.isscalar(): + return Covariance(self.__name+"C", asEyeByScalar = numpy.sqrt( self.__C ) ) + elif self.isobject() and hasattr(self.__C,"sqrt"): + return Covariance(self.__name+"C", asCovObject = self.__C.sqrt() ) + + def sqrtmI(self): + "Inversion de la racine carrée matricielle" + if self.ismatrix(): + import scipy + return Covariance(self.__name+"H", asCovariance = scipy.linalg.sqrtm(self.__C).I ) + elif self.isvector(): + return Covariance(self.__name+"H", asEyeByVector = 1.0 / numpy.sqrt( self.__C ) ) + elif self.isscalar(): + return Covariance(self.__name+"H", asEyeByScalar = 1.0 / numpy.sqrt( self.__C ) ) + elif self.isobject() and hasattr(self.__C,"sqrtI"): + return Covariance(self.__name+"H", asCovObject = self.__C.sqrtI() ) + def diag(self, msize=None): "Diagonale de la matrice" if self.ismatrix(): @@ -1644,21 +1964,16 @@ class Covariance(object): def __mul__(self, other): "x.__mul__(y) <==> x*y" - if self.ismatrix() and isinstance(other,numpy.matrix): + if self.ismatrix() and isinstance(other, (int, numpy.matrix, float)): return self.__C * other - elif self.ismatrix() and (isinstance(other,numpy.ndarray) \ - or isinstance(other,list) \ - or isinstance(other,tuple)): + elif self.ismatrix() and isinstance(other, (list, numpy.ndarray, tuple)): if numpy.ravel(other).size == self.shape[1]: # Vecteur return self.__C * numpy.asmatrix(numpy.ravel(other)).T elif numpy.asmatrix(other).shape[0] == self.shape[1]: # Matrice return self.__C * numpy.asmatrix(other) else: raise ValueError("operands could not be broadcast together with shapes %s %s in %s matrix"%(self.shape,numpy.asmatrix(other).shape,self.__name)) - elif self.isvector() and (isinstance(other,numpy.matrix) \ - or isinstance(other,numpy.ndarray) \ - or isinstance(other,list) \ - or isinstance(other,tuple)): + elif self.isvector() and isinstance(other, (list, numpy.matrix, numpy.ndarray, tuple)): if numpy.ravel(other).size == self.shape[1]: # Vecteur return numpy.asmatrix(self.__C * numpy.ravel(other)).T elif numpy.asmatrix(other).shape[0] == self.shape[1]: # Matrice @@ -1667,9 +1982,7 @@ class Covariance(object): raise ValueError("operands could not be broadcast together with shapes %s %s in %s matrix"%(self.shape,numpy.ravel(other).shape,self.__name)) elif self.isscalar() and isinstance(other,numpy.matrix): return self.__C * other - elif self.isscalar() and (isinstance(other,numpy.ndarray) \ - or isinstance(other,list) \ - or isinstance(other,tuple)): + elif self.isscalar() and isinstance(other, (list, numpy.ndarray, tuple)): if len(numpy.asarray(other).shape) == 1 or numpy.asarray(other).shape[1] == 1 or numpy.asarray(other).shape[0] == 1: return self.__C * numpy.asmatrix(numpy.ravel(other)).T else: @@ -1681,15 +1994,22 @@ class Covariance(object): def __rmul__(self, other): "x.__rmul__(y) <==> y*x" - if self.ismatrix() and isinstance(other,numpy.matrix): + if self.ismatrix() and isinstance(other, (int, numpy.matrix, float)): return other * self.__C + elif self.ismatrix() and isinstance(other, (list, numpy.ndarray, tuple)): + if numpy.ravel(other).size == self.shape[1]: # Vecteur + return numpy.asmatrix(numpy.ravel(other)) * self.__C + elif numpy.asmatrix(other).shape[0] == self.shape[1]: # Matrice + return numpy.asmatrix(other) * self.__C + else: + raise ValueError("operands could not be broadcast together with shapes %s %s in %s matrix"%(numpy.asmatrix(other).shape,self.shape,self.__name)) elif self.isvector() and isinstance(other,numpy.matrix): if numpy.ravel(other).size == self.shape[0]: # Vecteur return numpy.asmatrix(numpy.ravel(other) * self.__C) elif numpy.asmatrix(other).shape[1] == self.shape[0]: # Matrice return numpy.asmatrix(numpy.array(other) * self.__C) else: - raise ValueError("operands could not be broadcast together with shapes %s %s in %s matrix"%(self.shape,numpy.ravel(other).shape,self.__name)) + raise ValueError("operands could not be broadcast together with shapes %s %s in %s matrix"%(numpy.ravel(other).shape,self.shape,self.__name)) elif self.isscalar() and isinstance(other,numpy.matrix): return other * self.__C elif self.isobject(): @@ -1702,7 +2022,7 @@ class Covariance(object): return self.shape[0] # ============================================================================== -class ObserverF(object): +class Observer2Func(object): """ Creation d'une fonction d'observateur a partir de son texte """ @@ -1716,35 +2036,120 @@ class ObserverF(object): return self.func # ============================================================================== -class ImportFromScript(object): +class CaseLogger(object): """ - Obtention d'une variable nommee depuis un fichier script importe + Conservation des commandes de creation d'un cas """ - def __init__(self, __filename=None): - "Verifie l'existence et importe le script" - self.__filename = __filename.rstrip(".py") - if self.__filename is None: - raise ValueError("The name of the file containing the variable to be imported has to be specified.") - if not os.path.isfile(str(self.__filename)+".py"): - raise ValueError("The file containing the variable to be imported doesn't seem to exist. The given file name is:\n \"%s\""%self.__filename) - self.__scriptfile = __import__(self.__filename, globals(), locals(), []) - self.__scriptstring = open(self.__filename+".py",'r').read() - def getvalue(self, __varname=None, __synonym=None ): - "Renvoie la variable demandee" - if __varname is None: - raise ValueError("The name of the variable to be imported has to be specified.") - if not hasattr(self.__scriptfile, __varname): - if __synonym is None: - raise ValueError("The imported script file \"%s\" doesn't contain the specified variable \"%s\"."%(str(self.__filename)+".py",__varname)) - elif not hasattr(self.__scriptfile, __synonym): - raise ValueError("The imported script file \"%s\" doesn't contain the specified variable \"%s\"."%(str(self.__filename)+".py",__synonym)) - else: - return getattr(self.__scriptfile, __synonym) + def __init__(self, __name="", __objname="case", __addViewers=None, __addLoaders=None): + self.__name = str(__name) + self.__objname = str(__objname) + self.__logSerie = [] + self.__switchoff = False + self.__viewers = { + "TUI" :Interfaces._TUIViewer, + "SCD" :Interfaces._SCDViewer, + "YACS":Interfaces._YACSViewer, + } + self.__loaders = { + "TUI" :Interfaces._TUIViewer, + "COM" :Interfaces._COMViewer, + } + if __addViewers is not None: + self.__viewers.update(dict(__addViewers)) + if __addLoaders is not None: + self.__loaders.update(dict(__addLoaders)) + + def register(self, __command=None, __keys=None, __local=None, __pre=None, __switchoff=False): + "Enregistrement d'une commande individuelle" + if __command is not None and __keys is not None and __local is not None and not self.__switchoff: + if "self" in __keys: __keys.remove("self") + self.__logSerie.append( (str(__command), __keys, __local, __pre, __switchoff) ) + if __switchoff: + self.__switchoff = True + if not __switchoff: + self.__switchoff = False + + def dump(self, __filename=None, __format="TUI", __upa=""): + "Restitution normalisée des commandes" + if __format in self.__viewers: + __formater = self.__viewers[__format](self.__name, self.__objname, self.__logSerie) + else: + raise ValueError("Dumping as \"%s\" is not available"%__format) + return __formater.dump(__filename, __upa) + + def load(self, __filename=None, __content=None, __object=None, __format="TUI"): + "Chargement normalisé des commandes" + if __format in self.__loaders: + __formater = self.__loaders[__format]() + else: + raise ValueError("Loading as \"%s\" is not available"%__format) + return __formater.load(__filename, __content, __object) + +# ============================================================================== +def MultiFonction( + __xserie, + _extraArguments = None, + _sFunction = lambda x: x, + _mpEnabled = False, + _mpWorkers = None, + ): + """ + Pour une liste ordonnée de vecteurs en entrée, renvoie en sortie la liste + correspondante de valeurs de la fonction en argument + """ + # Vérifications et définitions initiales + # logging.debug("MULTF Internal multifonction calculations begin with function %s"%(_sFunction.__name__,)) + if not PlatformInfo.isIterable( __xserie ): + raise TypeError("MultiFonction not iterable unkown input type: %s"%(type(__xserie),)) + if _mpEnabled: + if (_mpWorkers is None) or (_mpWorkers is not None and _mpWorkers < 1): + __mpWorkers = None else: - return getattr(self.__scriptfile, __varname) - def getstring(self): - "Renvoie le script complet" - return self.__scriptstring + __mpWorkers = int(_mpWorkers) + try: + import multiprocessing + __mpEnabled = True + except ImportError: + __mpEnabled = False + else: + __mpEnabled = False + __mpWorkers = None + # + # Calculs effectifs + if __mpEnabled: + _jobs = [] + if _extraArguments is None: + _jobs = __xserie + elif _extraArguments is not None and isinstance(_extraArguments, (list, tuple, map)): + for __xvalue in __xserie: + _jobs.append( [__xvalue, ] + list(_extraArguments) ) + else: + raise TypeError("MultiFonction extra arguments unkown input type: %s"%(type(_extraArguments),)) + # logging.debug("MULTF Internal multiprocessing calculations begin : evaluation of %i point(s)"%(len(_jobs),)) + import multiprocessing + with multiprocessing.Pool(__mpWorkers) as pool: + __multiHX = pool.map( _sFunction, _jobs ) + pool.close() + pool.join() + # logging.debug("MULTF Internal multiprocessing calculation end") + else: + # logging.debug("MULTF Internal monoprocessing calculation begin") + __multiHX = [] + if _extraArguments is None: + for __xvalue in __xserie: + __multiHX.append( _sFunction( __xvalue ) ) + elif _extraArguments is not None and isinstance(_extraArguments, (list, tuple, map)): + for __xvalue in __xserie: + __multiHX.append( _sFunction( __xvalue, *_extraArguments ) ) + elif _extraArguments is not None and isinstance(_extraArguments, dict): + for __xvalue in __xserie: + __multiHX.append( _sFunction( __xvalue, **_extraArguments ) ) + else: + raise TypeError("MultiFonction extra arguments unkown input type: %s"%(type(_extraArguments),)) + # logging.debug("MULTF Internal monoprocessing calculation end") + # + # logging.debug("MULTF Internal multifonction calculations end") + return __multiHX # ============================================================================== def CostFunction3D(_x, @@ -1863,4 +2268,4 @@ def CostFunction3D(_x, # ============================================================================== if __name__ == "__main__": - print('\n AUTODIAGNOSTIC \n') + print('\n AUTODIAGNOSTIC\n')