From a04ccaff4f19a6d7b7069603824a532f7f1a9a34 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Andr=C3=A9=20Ribes?= Date: Wed, 7 Jul 2010 16:56:32 +0200 Subject: [PATCH] Debut generation catalogue --- bin/AdaoCalatogGenerator.py | 250 ++++++++++++++++++ bin/Makefile.am | 10 +- src/daEficas/ADAO_Cata_V0.py | 3 +- .../daYacsSchemaCreator/infos_daComposant.py | 44 ++- src/daSalome/daYacsSchemaCreator/methods.py | 4 +- .../daSalome/test000_Blue_AnalysisFile.py.in | 2 +- src/tests/daSalome/test_aster_zzzz159a.py.in | 2 +- .../daSalome/test_aster_zzzz159a_LBFGSB.py.in | 2 +- 8 files changed, 302 insertions(+), 15 deletions(-) create mode 100644 bin/AdaoCalatogGenerator.py diff --git a/bin/AdaoCalatogGenerator.py b/bin/AdaoCalatogGenerator.py new file mode 100644 index 0000000..7c27456 --- /dev/null +++ b/bin/AdaoCalatogGenerator.py @@ -0,0 +1,250 @@ +#!/usr/bin/python +#-*- coding:utf-8 -*- +# Copyright (C) 2008-2009 EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# -- +# Author : André RIBES (EDF R&D) +# -- + +import logging +import traceback +import sys +import string +import StringIO + +logging.basicConfig(level=logging.DEBUG) + +#----------- Templates Part ---------------# +begin_catalog_file = """ +# -*- coding: utf-8 -*- + +# -------------------------------------------------------- +# generated by AdaoCatalogGenerator at ${date} +# -------------------------------------------------------- + +import Accas +from Accas import * + +JdC = JDC_CATA (code = 'ADAO', + execmodul = None, + regles = ( AU_MOINS_UN ('ASSIM_STUDY')), + ) +""" + +String_data_bloc = """ + STRING_DATA = BLOC ( condition = " FROM in ( 'String', ) ", + + STRING = SIMP(statut = "o", typ = "TXM"), + ), +""" + +Script_data_bloc = """ + FILE_DATA = BLOC ( condition = " FROM in ( 'File', ) ", + + FILE = SIMP(statut = "o", typ = "Fichier"), + ), +""" + +Dict_data_bloc = """ + FILE_DATA = BLOC ( condition = " FROM in ( 'File', ) ", + + FILE = SIMP(statut = "o", typ = "Fichier"), + ), +""" + +FunctionDict_data_bloc = """ + FILE_DATA = BLOC ( condition = " FROM in ( 'File', ) ", + + FILE = SIMP(statut = "o", typ = "Fichier"), + ), +""" + +data_method = """ +def F_${data_name}(statut) : return FACT(statut = statut, + FROM = SIMP(statut = "o", typ = "TXM", into=(${data_into})), +${data_bloc} + ) +""" + +assim_data_method = """ +def F_${assim_name}(statut) : return FACT(statut=statut, + regles = ( UN_PARMI (${choices})), +${decl_choices} + ) +""" + +assim_data_choice = """ + ${choice_name} = F_${choice_name}("f"), +""" + +assim_opt_choice = """ + ${choice_name} = F_${choice_name}("f"), +""" + +assim_algo = """ + ${name} = FACT(regles = ( ENSEMBLE ("Background", "BackgroundError", + "Observation", "ObservationError", + "ObservationOperator")), + Background = F_Background("o"), + BackgroundError = F_BackgroundError("o"), + Observation = F_Observation("o"), + ObservationError = F_ObservationError("o"), + ObservationOperator = F_ObservationOperator("o"), +${decl_opts} + ), +""" +assim_study = """ +ASSIM_STUDY = PROC(nom="ASSIM_STUDY", + op=None, + repetable = "n", + STUDY_NAME = SIMP(statut="o", typ = "TXM"), + ALGORITHM = FACT(statut='o', + regles = ( UN_PARMI (${algos}),), +${decl_algos} + ), + ) +""" + +begin_catalog_file = string.Template(begin_catalog_file) +data_method = string.Template(data_method) +assim_data_method = string.Template(assim_data_method) +assim_data_choice = string.Template(assim_data_choice) +assim_opt_choice = string.Template(assim_opt_choice) +assim_algo = string.Template(assim_algo) +assim_study = string.Template(assim_study) + +#----------- End of Templates Part ---------------# + + + +#----------- Begin generation script -----------# +print "-- Starting AdaoCalatogGenerator.py --" + +try: + import daEficas + import daYacsSchemaCreator + import daCore.AssimilationStudy +except: + logging.fatal("Import of ADAO python modules failed !" + + "\n add ADAO python installation directory in your PYTHONPATH") + traceback.print_exc() + sys.exit(1) + +def check_args(args): + logging.debug("Arguments are :" + str(args)) + if len(args) != 2: + logging.fatal("Bad number of arguments: you have to provide two arguments (%d given)" % (len(args))) + sys.exit(1) + +# Parse arguments +from optparse import OptionParser +usage = "usage: %prog [options] catalog_path catalog_name" +version="%prog 0.1" +my_parser = OptionParser(usage=usage, version=version) +(options, args) = my_parser.parse_args() +check_args(args) + +catalog_path = args[0] +catalog_name = args[1] + +# Generates into a string +mem_file = StringIO.StringIO() + +# Start file +from time import strftime +mem_file.write(begin_catalog_file.substitute(date=strftime("%Y-%m-%d %H:%M:%S"))) + +# Step 1: Check basic data input types +import daYacsSchemaCreator.infos_daComposant as infos +for basic_type in infos.BasicDataInputs: + logging.debug('A basic data input type is found: ' + basic_type) + if basic_type + '_data_bloc' not in locals().keys(): + logging.fatal("Basic data input type not found: " + basic_type) + sys.exit(1) + +# Step 2: Add data input dict +for data_input_name in infos.DataTypeDict.keys(): + logging.debug('A data input is found: ' + data_input_name) + data_name = data_input_name + data_into = "" + data_bloc = "" + + for basic_type in infos.DataTypeDict[data_input_name]: + data_into += "\"" + basic_type + "\", " + data_bloc += locals()[basic_type + '_data_bloc'] + "\n" + + mem_file.write(data_method.substitute(data_name = data_name, + data_into = data_into, + data_bloc = data_bloc)) + +# Step 3: Add assimilation algorithm data input +for assim_data_input_name in infos.AssimDataDict.keys(): + logging.debug("An assimilation algorithm data input is found: " + assim_data_input_name) + assim_name = assim_data_input_name + choices = "" + decl_choices = "" + decl_opts = "" + for choice in infos.AssimDataDict[assim_data_input_name]: + choices += "\"" + choice + "\", " + decl_choices += assim_data_choice.substitute(choice_name = choice) + + mem_file.write(assim_data_method.substitute(assim_name = assim_name, + choices = choices, + decl_choices = decl_choices)) + +# Step 4: Add optional nodes +opt_names = [] +for opt_name in infos.OptDict.keys(): + logging.debug("An optional node is found: " + opt_name) + data_name = opt_name + data_into = "" + data_bloc = "" + + for choice in infos.OptDict[opt_name]: + data_into += "\"" + choice + "\", " + data_bloc += locals()[choice + '_data_bloc'] + "\n" + + mem_file.write(data_method.substitute(data_name = data_name, + data_into = data_into, + data_bloc = data_bloc)) + + opt_names.append(opt_name) + +# Final step: Add algorithm and assim_study +algos = "" +decl_algos = "" +decl_opts = "" +for opt_name in opt_names: + decl_opts += assim_opt_choice.substitute(choice_name = opt_name) + +assim_study_object = daCore.AssimilationStudy.AssimilationStudy() +algos_list = assim_study_object.get_available_algorithms() +for algo_name in algos_list: + logging.debug("An assimilation algorithm is found: " + algo_name) + if algo_name == "3DVAR": + algo_name = "ThreeDVAR" + algos += "\"" + algo_name + "\", " + decl_algos += assim_algo.substitute(name = algo_name, decl_opts=decl_opts) + "\n" + +mem_file.write(assim_study.substitute(algos=algos, + decl_algos=decl_algos)) +# Write file +final_file = open(catalog_path + "/" + catalog_name, "wr") +final_file.write(mem_file.getvalue()) +mem_file.close() +final_file.close() + diff --git a/bin/Makefile.am b/bin/Makefile.am index fd3cce8..fd2c567 100644 --- a/bin/Makefile.am +++ b/bin/Makefile.am @@ -1,5 +1,11 @@ include $(top_srcdir)/adm_local/make_common_starter.am -bin_SCRIPTS = AdaoYacsSchemaCreator.py qtEficas_adao_study.py +bin_SCRIPTS = AdaoYacsSchemaCreator.py qtEficas_adao_study.py AdaoCalatogGenerator.py -EXTRA_DIST = AdaoYacsSchemaCreator.py qtEficas_adao_study.py.in +EXTRA_DIST = AdaoYacsSchemaCreator.py qtEficas_adao_study.py.in AdaoCalatogGenerator.py + +install-data-hook: + @export PYTHONPATH=${salomepythondir} && python $(top_srcdir)/bin/AdaoCalatogGenerator.py ${salomepythondir}/daEficas CatalogTest.py + +uninstall-hook: + @rm ${salomepythondir}/daEficas/CatalogTest.py diff --git a/src/daEficas/ADAO_Cata_V0.py b/src/daEficas/ADAO_Cata_V0.py index 4d969c7..807ac4a 100644 --- a/src/daEficas/ADAO_Cata_V0.py +++ b/src/daEficas/ADAO_Cata_V0.py @@ -44,7 +44,8 @@ def F_OBSERVATION_ERROR(statut) : return FACT(statut=statut, def F_OBSERVATION_OPERATOR(statut) : return FACT(statut=statut, regles = ( UN_PARMI ("MATRIX", "FUNCTION")), - MATRIX = F_MATRIX("o"), + MATRIX = F_MATRIX("f"), + FUNCTION = F_VECTOR("f"), ) def F_ANALYSIS(statut) : return FACT(statut = statut, diff --git a/src/daSalome/daYacsSchemaCreator/infos_daComposant.py b/src/daSalome/daYacsSchemaCreator/infos_daComposant.py index 76cb379..6ee5ad2 100644 --- a/src/daSalome/daYacsSchemaCreator/infos_daComposant.py +++ b/src/daSalome/daYacsSchemaCreator/infos_daComposant.py @@ -23,27 +23,31 @@ # -- Infos pour le parser -- AnalysisData = {} -AnalysisFromList = ["String", "File"] +AnalysisFromList = ["String", "Script"] # -- Infos from daCore -- +#AssimData = ["Background", "BackgroundError", +# "Observation", "ObservationError", "ObservationOperator", "ObservationOperatorAppliedToX", +# "EvolutionModel", "EvolutionError", "AlgorithmParameters"] + AssimData = ["Background", "BackgroundError", - "Observation", "ObservationError", "ObservationOperator", "ObservationOperatorAppliedToX", + "Observation", "ObservationError", "ObservationOperator", "EvolutionModel", "EvolutionError", "AlgorithmParameters"] + AssimType = {} AssimType["Background"] = ["Vector"] AssimType["BackgroundError"] = ["Matrix"] AssimType["Observation"] = ["Vector"] AssimType["ObservationError"] = ["Matrix"] -AssimType["ObservationOperator"] = ["Matrix", "Function"] -AssimType["ObservationOperatorAppliedToX"] = ["List"] +AssimType["ObservationOperator"] = ["Matrix", "FunctionDict"] AssimType["AlgorithmParameters"] = ["Dict"] +#AssimType["ObservationOperatorAppliedToX"] = ["List"] FromNumpyList = {} FromNumpyList["Vector"] = ["String", "Script"] FromNumpyList["Matrix"] = ["String", "Script"] -FromNumpyList["Function"] = ["Dict"] -FromNumpyList["List"] = ["List"] +FromNumpyList["Function"] = ["FunctionDict"] FromNumpyList["Dict"] = ["Script"] # -- Infos from daAlgorithms -- @@ -56,6 +60,32 @@ AlgoDataRequirements["Blue"] = ["Background", "BackgroundError", AlgoDataRequirements["3DVAR"] = ["Background", "BackgroundError", "Observation", "ObservationOperator", "ObservationError"] AlgoType = {} -#AlgoType["Blue"] = "Direct" AlgoType["Blue"] = "Optim" AlgoType["3DVAR"] = "Optim" +AlgoType["EnsembleBlue"] = "Optim" +AlgoType["Kalman"] = "Optim" +AlgoType["LinearLeastSquares"] = "Optim" +#AlgoType["Blue"] = "Direct" + +# Basic data types +BasicDataInputs = ["String", "Script", "FunctionDict"] + +# Data input dict +DataTypeDict = {} +DataTypeDict["Vector"] = ["String", "Script"] +DataTypeDict["Matrix"] = ["String", "Script"] +DataTypeDict["Function"] = ["FunctionDict"] +DataTypeDict["Dict"] = ["Script"] + +# Assimilation data input +AssimDataDict = {} +AssimDataDict["Background"] = ["Vector"] +AssimDataDict["BackgroundError"] = ["Matrix"] +AssimDataDict["Observation"] = ["Vector"] +AssimDataDict["ObservationError"] = ["Matrix"] +AssimDataDict["ObservationOperator"] = ["Matrix", "Function"] +AssimDataDict["AlgorithmParameters"] = ["Dict"] + +# Assimilation optional nodes +OptDict = {} +OptDict["Analysis"] = ["String", "Script"] diff --git a/src/daSalome/daYacsSchemaCreator/methods.py b/src/daSalome/daYacsSchemaCreator/methods.py index 661d0dd..56babb6 100644 --- a/src/daSalome/daYacsSchemaCreator/methods.py +++ b/src/daSalome/daYacsSchemaCreator/methods.py @@ -157,7 +157,7 @@ def create_yacs_proc(study_config): back_node.edAddInputPort("init_data", t_pyobj) proc.edAddDFLink(init_node.getOutputPort("init_data"), back_node.getInputPort("init_data")) - if data_config["Type"] == "Function" and data_config["From"] == "Dict" and key == "ObservationOperator": + if data_config["Type"] == "Function" and data_config["From"] == "FunctionDict" and key == "ObservationOperator": FunctionDict = data_config["Data"] for FunctionName in FunctionDict["Function"]: port_name = "ObservationOperator" + FunctionName @@ -187,7 +187,7 @@ def create_yacs_proc(study_config): # Check if we have a python script for OptimizerLoopNode data_config = study_config["ObservationOperator"] - if data_config["Type"] == "Function" and data_config["From"] == "Dict": + if data_config["Type"] == "Function" and data_config["From"] == "FunctionDict": # Get script FunctionDict = data_config["Data"] script_filename = "" diff --git a/src/tests/daSalome/test000_Blue_AnalysisFile.py.in b/src/tests/daSalome/test000_Blue_AnalysisFile.py.in index 42dae9e..e73f772 100644 --- a/src/tests/daSalome/test000_Blue_AnalysisFile.py.in +++ b/src/tests/daSalome/test000_Blue_AnalysisFile.py.in @@ -35,6 +35,6 @@ study_config["ObservationOperator"] = ObservationOperator_config Analysis_config = {} Analysis_config["Data"] = "@prefix@/tests/daSalome/test000_Blue_AnalysisCode.py" -Analysis_config["From"] = "File" +Analysis_config["From"] = "Script" study_config["Analysis"] = Analysis_config diff --git a/src/tests/daSalome/test_aster_zzzz159a.py.in b/src/tests/daSalome/test_aster_zzzz159a.py.in index 6b6109b..3aa8f8d 100644 --- a/src/tests/daSalome/test_aster_zzzz159a.py.in +++ b/src/tests/daSalome/test_aster_zzzz159a.py.in @@ -50,7 +50,7 @@ FunctionDict["Script"]["Adjoint"] = "@prefix@/tests/daSalome/test_aster_zzzz159a ObservationOperator_config = {} ObservationOperator_config["Data"] = FunctionDict ObservationOperator_config["Type"] = "Function" -ObservationOperator_config["From"] = "Dict" +ObservationOperator_config["From"] = "FunctionDict" study_config["ObservationOperator"] = ObservationOperator_config Analysis_config = {} diff --git a/src/tests/daSalome/test_aster_zzzz159a_LBFGSB.py.in b/src/tests/daSalome/test_aster_zzzz159a_LBFGSB.py.in index 04bfd0e..8e58ed1 100644 --- a/src/tests/daSalome/test_aster_zzzz159a_LBFGSB.py.in +++ b/src/tests/daSalome/test_aster_zzzz159a_LBFGSB.py.in @@ -50,7 +50,7 @@ FunctionDict["Script"]["Adjoint"] = "@prefix@/tests/daSalome/test_aster_zzzz159a ObservationOperator_config = {} ObservationOperator_config["Data"] = FunctionDict ObservationOperator_config["Type"] = "Function" -ObservationOperator_config["From"] = "Dict" +ObservationOperator_config["From"] = "FunctionDict" study_config["ObservationOperator"] = ObservationOperator_config Analysis_config = {} -- 2.39.2