From: Jean-Philippe ARGAUD Date: Mon, 31 Jan 2011 14:22:00 +0000 (+0100) Subject: Mise à jour d'avancement de la doc, avec un premier jet d'une introduction méthodologique X-Git-Tag: V6_4_0rc3~93 X-Git-Url: http://git.salome-platform.org/gitweb/?a=commitdiff_plain;h=14f3554c7b07bb06399bcdecbe309fa82a36f921;p=modules%2Fadao.git Mise à jour d'avancement de la doc, avec un premier jet d'une introduction méthodologique --- diff --git a/doc/advanced.rst b/doc/advanced.rst index 5f48310..8dd33bb 100644 --- a/doc/advanced.rst +++ b/doc/advanced.rst @@ -1,3 +1,5 @@ +.. _section_advanced: + ================================================================================ Advanced usage of ADAO ================================================================================ @@ -32,7 +34,7 @@ Running an ADAO calculation scheme in YACS in console mode This section describes how to execute in console mode a YACS calculation scheme, obtained using the ADAO "Export to YACS" function. It uses the standard YACS -console mode, wich will be briefly recall here (see YACS documentation for more +console mode, which will be briefly recall here (see YACS documentation for more information). The way to do that is as follows: diff --git a/doc/conf.py b/doc/conf.py index dc6a04c..f5e458b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -22,7 +22,7 @@ import sys, os # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [] +extensions = ["sphinx.ext.pngmath"] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/doc/examples.rst b/doc/examples.rst index e040bd6..5ac3096 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -1,3 +1,5 @@ +.. _section_examples: + ================================================================================ Examples on using the ADAO module ================================================================================ @@ -59,7 +61,7 @@ Using the GUI to build the ADAO case First, you have to activate the ADAO module by choosing the appropriate module button or menu of SALOME, and you will see: - .. _adao_activate: + .. _adao_activate2: .. image:: images/adao_activate.png :align: center .. centered:: diff --git a/doc/index.rst b/doc/index.rst index 6816059..0385fcf 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -3,36 +3,37 @@ ADAO module documentation ================================================================================ -The ADAO module provides **data assimilation** features [#]_ in SALOME context. -It is based on an extensible generic underlying data assimilation library, and -on usage on other SALOME modules, namely YACS and EFICAS. +The ADAO module provides **data assimilation** features in SALOME context. It is +based on usage on other SALOME modules, namely YACS and EFICAS, and on an +extensible generic underlying data assimilation library. Briefly stated, Data Assimilation is a methodological framework to compute the -optimal estimate of the inaccesible true value of a system state over time. It +optimal estimate of the inaccessible true value of a system state over time. It uses information coming from experimental measurements or observations, and from numerical *a priori* models, including information about their errors. Parts of the framework are also known as *parameter estimation*, *inverse problems*, -*bayesian estimation*, *optimal interpolation*, etc. - -The documentation of this module is divided in 4 parts, the first one being an -introduction. The second part describes how to use the module ADAO. The third -part focuses on advanced usages of the module, how to get more information, or -how to use it without the graphical user interface (GUI). The last part gives -examples on ADAO usage. - -Users interested in quick use of the module can jump to the last part, but a -valuable use of the module requires to read and come back regularly the second -part. +*bayesian estimation*, *optimal interpolation*, etc. More details can be found +in the section :ref:`section_theory`. + +The documentation of this module is divided in 5 parts, the first one being an +introduction. The second part briefly introduces data assimilation and concepts. +The third part describes how to use the module ADAO. The fourth part focuses on +advanced usages of the module, how to get more information, or how to use it +without the graphical user interface (GUI). The last part gives examples on ADAO +usage. Users interested in quick use of the module can jump to the section +:ref:`section_examples`, but a valuable use of the module requires to read and +come back regularly the section :ref:`section_using`. In all this documentation, we use standard notations of data assimilation. -Vectors are written horizontally or vertically without difference. Matrices are -written either normally, either with a condensed notation, consisting in the -use of a "``;``" to separate the rows in a continuous line. +Moreover, vectors are written horizontally or vertically without difference. +Matrices are written either normally, either with a condensed notation, +consisting in the use of a "``;``" to separate the rows in a continuous line. .. toctree:: :maxdepth: 2 intro + theory using advanced examples @@ -42,8 +43,3 @@ Indices and tables * :ref:`genindex` * :ref:`search` - -.. _ECMWF: http://www.ecmwf.int/newsevents/training/rcourse_notes/DATA_ASSIMILATION/ASSIM_CONCEPTS/Assim_concepts.html -.. _Wikipedia/Data_assimilation: http://en.wikipedia.org/wiki/Data_assimilation - -.. [#] If you are not familiar with data assimilation concepts in general, an introductory on-line training course can be found at ECMWF_, along with other materials coming from geosciences applications. See also for example `Wikipedia/Data_assimilation`_. Note that data assimilation is not restricted to meteorology or geosciences, but is widely used in other scientific domains. diff --git a/doc/ressources/ADAO.png b/doc/ressources/ADAO.png index ff778d9..6b465d7 100644 Binary files a/doc/ressources/ADAO.png and b/doc/ressources/ADAO.png differ diff --git a/doc/ressources/ADAO_small.png b/doc/ressources/ADAO_small.png index 0f7bbbd..7faf412 100644 Binary files a/doc/ressources/ADAO_small.png and b/doc/ressources/ADAO_small.png differ diff --git a/doc/theory.rst b/doc/theory.rst new file mode 100644 index 0000000..bb465cb --- /dev/null +++ b/doc/theory.rst @@ -0,0 +1,204 @@ +.. _section_theory: + +================================================================================ +A brief introduction to Data Assimilation +================================================================================ + +**Data Assimilation** is a general framework for computing the optimal estimate +of the true state of a system, over time if necessary. It uses values obtained +both from observations and *a priori* models, including information about their +errors. + +In other words, data assimilation merges measurement data, the observations, +with *a priori* physical and mathematical knowledge, embedded in numerical +models, to obtain the best possible estimate of the true state and of its +stochastic properties. Note that this true state can not be reached, but can +only be estimated. Moreover, despite the fact that used information are +stochastic by nature, data assimilation provides deterministic techniques in +order to realize the estimation. + +Two main types of applications exist in data assimilation being covered by the +same formalism: **parameters identification** and **fields reconstruction**. +Before introducing the `Simple description of the data assimilation framework`_ +in a next section, we describe briefly these two types. At the end, some +references allow `Going further in the data assimilation framework`_. + +Fields reconstruction or measures interpolation +----------------------------------------------- + +Fields reconstruction consists in finding, from a restricted set of real +measures, the physical field which is the most *consistent* with these measures. + +This consistency is to understand in terms of interpolation, that is to say that +the field, we want to reconstruct using data assimilation on measures, has to +fit at best the measures, while remaining constrained by the overall +calculation. The calculation is thus an *a priori* estimation of the field that +we seek to identify. + +If the system evolves in time, the reconstruction has to be established on every +time step, as a whole. The interpolation process in this case is more +complicated since it is temporal, not only in terms of instantaneous values of +the field. + +A simple example of fields reconstruction comes from of meteorology, in which we +look for value of variables such as temperature or pressure in all points of the +spatial domain. We have instantaneous measurements of these quantities at +certain points, but also a history set of these measures. Moreover, these +variables are constrained by evolution equations for the state of the +atmosphere, which indicates for example that the pressure at a point can not +take any value independently of the value at this same point in previous time. +We must therefore make the reconstruction of a field at any point in space, in +order "consistent" with the evolution equations and measures of the previous +time steps. + +Parameters identification or calibration +---------------------------------------- + +The identification of parameters by data assimilation is a form of calibration +which uses both the measurement and an *a priori* estimation (called the +"*background*") of the state that one seeks to identify, as well as a +characterization of their errors. From this point of view, it uses all available +information on the physical system (even if assumptions about errors are +relatively restrictive) to find the "*optimal*" estimation from the true state. +We note, in terms of optimization, that the background realizes a mathematical +regularization of the main problem of identification. + +In practice, the two gaps "*calculation-background*" and +"*calculation-measures*" are added to build the calibration correction of +parameters or initial conditions. The addition of these two gaps requires a +relative weight, which is chosen to reflect the trust we give to each piece of +information. This confidence is measured by the covariance of the errors on the +background and on the observations. Thus the stochastic aspect of information, +measured or *a priori*, is essential for building the calibration error +function. + +Simple description of the data assimilation framework +----------------------------------------------------- + +We can write these features in a simple manner. By default, all variables are +vectors, as there are several parameters to readjust. + +According to standard notations in data assimilation, we note +:math:`\mathbf{x}^a` the optimal unknown parameters that is to be determined by +calibration, :math:`\mathbf{y}^o` the observations (or experimental +measurements) that we must compare the simulation outputs, :math:`\mathbf{x}^b` +the background (*a priori* values, or regularization values) of searched +parameters, :math:`\mathbf{x}^t` unknown ideals parameters that would give as +output exactly the observations (assuming that the errors are zero and the model +exact). + +In the simplest case, static, the steps of simulation and of observation can be +combined into a single operator noted :math:`H` (linear or nonlinear), which +transforms the input parameters :math:`\mathbf{x}` to results :math:`\mathbf{y}` +to be compared to observations :math:`\mathbf{y}^o`. Moreover, we use the +linearized operator :math:`\mathbf{H}` to represent the effect of the full +operator :math:`H` around a linearization point (and we omit thereafter to +mention :math:`H` even if it is possible to keep it). In reality, we have already +indicated that the stochastic nature of variables is essential, coming from the +fact that model, background and observations are incorrect. We therefore +introduce errors of observations additively, in the form of a random vector +:math:`\mathbf{\epsilon}^o` such that: + +.. math:: \mathbf{y}^o = \mathbf{H} \mathbf{x}^t + \mathbf{\epsilon}^o + +The errors represented here are not only those from observation, but also from +the simulation. We can always consider that these errors are of zero mean. We +can then define a matrix :math:`\mathbf{R}` of the observation error covariance +by: + +.. math:: \mathbf{R} = E[\mathbf{\epsilon}^o.{\mathbf{\epsilon}^o}^T] + +The background can also be written as a function of the true value, by +introducing the error vector :math:`\mathbf{\epsilon}^b`: + +.. math:: \mathbf{x}^b = \mathbf{x}^t + \mathbf{\epsilon}^b + +where errors are also assumed of zero mean, in the same manner as for +observations. We define the :math:`\mathbf{B}` matrix of background error +covariances by: + +.. math:: \mathbf{B} = E[\mathbf{\epsilon}^b.{\mathbf{\epsilon}^b}^T] + +The optimal estimation of the true parameters :math:`\mathbf{x}^t`, given the +background :math:`\mathbf{x}^b` and the observations :math:`\mathbf{y}^o`, is +then the "*analysis*" :math:`\mathbf{x}^a` and comes from the minimisation of an +error function (in variational assimilation) or from the filtering correction (in +assimilation by filtering). + +In **variational assimilation**, one classically attempts to minimize the +following function :math:`J`: + +.. math:: J(\mathbf{x})=(\mathbf{x}-\mathbf{x}^b)^T.\mathbf{B}^{-1}.(\mathbf{x}-\mathbf{x}^b)+(\mathbf{y}^o-\mathbf{H}.\mathbf{x})^T.\mathbf{R}^{-1}.(\mathbf{y}^o-\mathbf{H}.\mathbf{x}) + +which is usually designed as the "*3D-VAR*" function. Since covariance matrices +are proportional to the variances of errors, their presence in both terms of the +function :math:`J` can effectively weight the differences by confidence in the +background or observations. The parameters :math:`\mathbf{x}` realizing the +minimum of this function therefore constitute the analysis :math:`\mathbf{x}^a`. +It is at this level that we have to use the full panoply of function +minimization methods otherwise known in optimization. Depending on the size of +the parameters vector :math:`\mathbf{x}` to identify and ot the availability of +gradient and Hessian of :math:`J`, it is appropriate to adapt the chosen +optimization method (gradient, Newton, quasi-Newton ...). + +In **assimilation by filtering**, in this simple case usually referred to as +"*BLUE*"(for "*Best Linear Unbiased Estimator*"), the :math:`\mathbf{x}^a` +analysis is given as a correction of the background :math:`\mathbf{x}^b` by a +term proportional to the difference between observations :math:`\mathbf{y}^o` +and calculations :math:`\mathbf{H}\mathbf{x}^b`: + +.. math:: \mathbf{x}^a = \mathbf{x}^b + \mathbf{K}(\mathbf{y}^o - \mathbf{H}\mathbf{x}^b) + +where :math:`\mathbf{K}` is the Kalman gain matrix, which is expressed using +covariance matrices in the following form: + +.. math:: \mathbf{K} = \mathbf{B}\mathbf{H}^T(\mathbf{H}\mathbf{B}\mathbf{H}^T+\mathbf{R})^{-1} + +The advantage of filtering is to explicitly calculate the gain, to produce then +the *a posteriori* covariance analysis matrix. + +In this simple static case, we can show, under the assumption of Gaussian error +distributions, that the two *variational* and *filtering* approaches are +equivalent. + +It is indicated here that these methods of "*3D-VAR*" and "*BLUE*" may be +extended to dynamic problems, called respectively "*4D-VAR*" and "*Kalman +filter*". They can take account of the evolution operator to establish an +analysis at the right time steps of the gap between observations and simulations, +and to have, at every moment, the propagation of the background through the +evolution model. Many other variants have been developed to improve the +numerical quality or to take into account computer requirements such as +calculation size and time. + +Going further in the data assimilation framework +++++++++++++++++++++++++++++++++++++++++++++++++ + +To get more information about all the data assimilation techniques, the reader +can consult introductory documents like [Argaud09], on-line training courses or +lectures like [Bouttier99] and [Bocquet04] (along with other materials coming +from geosciences applications), or general documents like [Talagrand97], +[Tarantola87], [Kalnay03] and [WikipediaDA]. + +Note that data assimilation is not restricted to meteorology or geo-sciences, but +is widely used in other scientific domains. There are several fields in science +and technology where the effective use of observed but incomplete data is +crucial. + +Some aspects of data assimilation are also known as *parameter estimation*, +*inverse problems*, *bayesian estimation*, *optimal interpolation*, +*mathematical regularisation*, *data smoothing*, etc. These terms can be used in +bibliographical searches. + +.. [Argaud09] Argaud J.-P., Bouriquet B., Hunt J., *Data Assimilation from Operational and Industrial Applications to Complex Systems*, Mathematics Today, pp.150-152, October 2009 + +.. [Bouttier99] Bouttier B., Courtier P., *Data assimilation concepts and methods*, Meteorological Training Course Lecture Series, ECMWF, 1999, http://www.ecmwf.int/newsevents/training/rcourse_notes/pdf_files/Assim_concepts.pdf + +.. [Bocquet04] Bocquet M., *Introduction aux principes et méthodes de l'assimilation de données en géophysique*, Lecture Notes, 2004-2008, http://cerea.enpc.fr/HomePages/bocquet/assim.pdf + +.. [Tarantola87] Tarantola A., *Inverse Problem: Theory Methods for Data Fitting and Parameter Estimation*, Elsevier, 1987 + +.. [Talagrand97] Talagrand O., *Assimilation of Observations, an Introduction*, Journal of the Meteorological Society of Japan, 75(1B), pp. 191-209, 1997 + +.. [Kalnay03] Kalnay E., *Atmospheric Modeling, Data Assimilation and Predictability*, Cambridge University Press, 2003 + +.. [WikipediaDA] Wikipedia/Data_assimilation: http://en.wikipedia.org/wiki/Data_assimilation diff --git a/doc/using.rst b/doc/using.rst index 9d8c4a3..71b23d6 100644 --- a/doc/using.rst +++ b/doc/using.rst @@ -1,3 +1,5 @@ +.. _section_using: + ================================================================================ Using the module ADAO ================================================================================ @@ -5,7 +7,7 @@ Using the module ADAO This section presents the usage of the ADAO module in SALOME. Logical procedure to build an ADAO test case --------------------------------------------------------------------------------- +-------------------------------------------- The construction of an ADAO case follows a simple approach to define the set of input data, either static or dynamic data, and then generates a complete block @@ -26,16 +28,16 @@ data assimilation study. These data can be in SALOME or not. Each step will be detailed later in the next section. Detailed procedure to build an ADAO test case --------------------------------------------------------------------------------- +--------------------------------------------- Activate the ADAO module and use the editor GUI -++++++++++++++++++++++++++++++++++++++++ ++++++++++++++++++++++++++++++++++++++++++++++++ As always for a module, it has to be activated by choosing the appropriate module button (or menu) in the toolbar of SALOME. A popup appears, allowing to choose between creating a new study, or opening an already existing one: - .. _adao_activate: + .. _adao_activate1: .. image:: images/adao_activate.png :align: center .. centered:: @@ -64,7 +66,7 @@ You will be prompted for a location in your file tree and a name, that will be completed by a "*.comm*" extension used for JDC EFICAS files. Build and modify the ADAO case and save it -++++++++++++++++++++++++++++++++++++++++ +++++++++++++++++++++++++++++++++++++++++++ To build a case using EFICAS, you have to go through a series of steps for selecting a keyword and then filling in its value. The structured editor @@ -79,16 +81,16 @@ incorrect keywords are identified by a visual error red flag. At the end, you have to save your ADAO case. Export the ADAO case as a YACS scheme -++++++++++++++++++++++++++++++++++++++++ ++++++++++++++++++++++++++++++++++++++ Modify and supplement the YACS scheme and save it -++++++++++++++++++++++++++++++++++++++++ ++++++++++++++++++++++++++++++++++++++++++++++++++ Execute the YACS case and obtain the results -++++++++++++++++++++++++++++++++++++++++ +++++++++++++++++++++++++++++++++++++++++++++ @@ -96,7 +98,7 @@ Execute the YACS case and obtain the results Reference description of the commands and keywords available throught the GUI --------------------------------------------------------------------------------- +----------------------------------------------------------------------------- --TODO-- diff --git a/src/daComposant/daCore/AssimilationStudy.py b/src/daComposant/daCore/AssimilationStudy.py index 5587dd5..8a69181 100644 --- a/src/daComposant/daCore/AssimilationStudy.py +++ b/src/daComposant/daCore/AssimilationStudy.py @@ -648,8 +648,8 @@ class AssimilationStudy: Scheduler = None, ): """ - Permet d'associer un observer à une variable nommée gérée en interne, - activable selon des règles définies dans le Scheduler. + Permet d'associer un observer à une ou des variables nommées gérées en + interne, activable selon des règles définies dans le Scheduler. """ # if type( self.__algorithm ) is dict: @@ -676,6 +676,36 @@ class AssimilationStudy: HookParameters = HookParameters, ) + def removeDataObserver(self, + VariableName = None, + HookFunction = None, + ): + """ + Permet de retirer un observer à une ou des variable nommée. + """ + # + if type( self.__algorithm ) is dict: + raise ValueError("No observer can be removed before choosing an algorithm.") + # + # Vérification du nom de variable et typage + # ----------------------------------------- + if type( VariableName ) is str: + VariableNames = [VariableName,] + elif type( VariableName ) is list: + VariableNames = map( str, VariableName ) + else: + raise ValueError("The observer requires a name or a list of names of variables.") + # + # Association interne de l'observer à la variable + # ----------------------------------------------- + for n in VariableNames: + if not self.__algorithm.has_key( n ): + raise ValueError("An observer requires to be removed on a variable named %s which does not exist."%n) + else: + self.__algorithm.StoredVariables[ n ].removeDataObserver( + HookFunction = HookFunction, + ) + def prepare_to_pickle(self): self.__algorithmFile = None self.__diagnosticFile = None diff --git a/src/daComposant/daCore/Persistence.py b/src/daComposant/daCore/Persistence.py index 79dae99..b8d100d 100644 --- a/src/daComposant/daCore/Persistence.py +++ b/src/daComposant/daCore/Persistence.py @@ -538,6 +538,32 @@ class Persistence: # ----------------------------------------------- self.__dataobservers.append( [HookFunction, HookParameters, Schedulers] ) + def removeDataObserver(self, + HookFunction = None, + ): + """ + Méthode de suppression d'un observer sur la variable. + + On peut donner dans HookFunction la meme fonction que lors de la + définition, ou un simple string qui est le nom de la fonction. + + """ + if hasattr(HookFunction,"func_name"): + name = str( HookFunction.func_name ) + elif type(HookFunction) is str: + name = str( HookFunction ) + else: + name = None + # + i = -1 + index_to_remove = [] + for [hf, hp, hs] in self.__dataobservers: + i = i + 1 + if name is hf.func_name: index_to_remove.append( i ) + index_to_remove.reverse() + for i in index_to_remove: + self.__dataobservers.pop( i ) + # ============================================================================== class OneScalar(Persistence): """ @@ -1005,21 +1031,35 @@ if __name__ == "__main__": print " ---> Mise en oeuvre de l'observer" print " var =",var.valueserie(-1) print " info =",info + def obs_bis(var=None,info=None): + print " ---> Mise en oeuvre de l'observer" + print " var =",var.valueserie(-1) + print " info =",info OBJET_DE_TEST = Persistence("My object", unit="", basetype=list) D = OBJET_DE_TEST D.setDataObserver( HookFunction = obs, Scheduler = [2, 4], - HookParameters = "Second observer", + HookParameters = "Premier observer", ) D.setDataObserver( HookFunction = obs, Scheduler = xrange(1,3), + HookParameters = "Second observer", + ) + D.setDataObserver( + HookFunction = obs_bis, + Scheduler = range(1,3)+range(7,9), HookParameters = "Troisième observer", ) for i in range(5): - # print - print "Action de 2 observers sur la variable observée, étape :",i + print "Action de 3 observers sur la variable observée, étape :",i + D.store( [i, i, i] ) + D.removeDataObserver( + HookFunction = obs, + ) + for i in range(5,10): + print "Action d'un seul observer sur la variable observée, étape :",i D.store( [i, i, i] ) del OBJET_DE_TEST print