src/daComposant/daNumerics/pst4mod/modelica_calibration/case.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # Copyright (C) 2016-2024 EDF R&D
   4 #
   5 # This library is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU Lesser General Public
   7 # License as published by the Free Software Foundation; either
   8 # version 2.1 of the License.
   9 #
  10 # This library is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 # Lesser General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU Lesser General Public
  16 # License along with this library; if not, write to the Free Software
  17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18 #
  19 # Author: Jean-Philippe Argaud, jean-philippe.argaud@edf.fr, EDF R&D
  20 # Author: Luis Corona Mesa-Molles, luis.corona-mesa-molles@edf.fr, EDF R&D
  21
  22 __all__ = ["Calibration", "name", "version", "year"]
  23
  24 # ==============================================================================
  25
  26 name     = "Modelica and Dymola Calibration Tools"
  27 version  = "1.0.9.13.0"  # "x.x"+"adao version"
  28 year     = "2021"
  29
  30 # ==============================================================================
  31 # Default configuration
  32 # ---------------------
  33 import os, sys, io, shutil, time, logging, subprocess, copy, csv  # noqa: E402
  34 import tempfile, warnings, numpy, scipy, pandas  # noqa: E402
  35 from datetime import datetime  # noqa: E402
  36
  37 try:
  38     import adao
  39     from adao import adaoBuilder
  40     from daCore.Interfaces import ImportFromFile, ImportScalarLinesFromFile
  41 except ImportError:
  42     raise ImportError("ADAO module not found, please install it first.")
  43
  44 try:
  45     from buildingspy.io.outputfile import Reader
  46 except ImportError:
  47     raise ImportError("buildingspy module not found, please install it first.")
  48
  49 try:
  50     from pst4mod.modelica_libraries.automatic_simulation import Automatic_Simulation
  51     from pst4mod.modelica_libraries.around_simulation import Around_Simulation
  52     from pst4mod.modelica_libraries import write_in_dsin
  53     from pst4mod.modelica_libraries.change_working_point import Dict_Var_To_Fix
  54     from pst4mod.modelica_libraries.change_working_point import Working_Point_Modification
  55 except ImportError:
  56     raise ImportError("modelica_libraries module not found, please install it first.")
  57
  58 try:
  59     from fmpy import simulate_fmu
  60     from fmpy.fmi1 import FMICallException
  61 except ImportError:
  62     __msg = "fmpy library not found, it will not be possible for you to simulate Functional Mock-up Units (FMUs) as models. Add it to Python installation it if necessary."
  63     warnings.warn(__msg, ImportWarning, stacklevel=50)
  64 except Exception:
  65     pass
  66
  67 _configuration = {
  68     "DirectoryModels"   : "Models",
  69     "DirectoryMeasures" : "Measures",
  70     "DirectoryMethods"  : "Methods",
  71     "DirectoryResults"  : "Results",
  72     "Launcher"          : "configuration.py",
  73 }
  74
  75 # ==============================================================================
  76 class Calibration(object):
  77     "ADAO case based parameters inverse estimation tools"
  78     def __init__(self, Name = "Calibration case", SaveStdoutIn = None, Verbose = False):
  79         """
  80         Name           : name as a string
  81         SaveAllStdoutIn: filename to be used for stdout stream
  82         Verbose        : verbose output
  83         """
  84         self.__name     = str(Name)
  85         self.__measures = {}
  86         self.__model    = {}
  87         self.__link     = {}
  88         self.__method   = {}
  89         self.__result   = {}
  90         self._setConfigurationDefaults()
  91         self.__verbose  = bool(Verbose)
  92         self.__stdoutid = sys.stdout
  93         if SaveStdoutIn is not None:
  94             sys.stdout = open(SaveStdoutIn, "w")
  95         if self.__verbose:
  96             __msg = "[VERBOSE] %s"%self.__name
  97             print("")
  98             print("  %s"%__msg)
  99             print("  %s"%("-" * len(__msg),))
 100             print("")
 101             VersionsLogicielles()
 102
 103     def _setConfigurationDefaults(self, Configuration = None):
 104         """
 105         Impose case directory and files structure configuration defaults based
 106         on argument dictionary
 107         """
 108         if Configuration is None or type(Configuration) is not dict:
 109             Configuration = _configuration
 110         for __k, __v in Configuration.items():
 111             setattr(self, __k, __v)
 112
 113     def _setModelTmpDir(self, __model_name = "dsin.txt", __model_format = "DYMOSIM", __model_dest = "dsin.txt", __before_tmp = None, __suffix=None):
 114         if __model_name is None:
 115             raise ValueError('Model file or directory has to be set and can not be None')
 116         elif os.path.isfile(__model_name):
 117             __model_nam = os.path.basename(__model_name)
 118             __model_dir = os.path.abspath(os.path.dirname(__model_name))
 119             __model_dst = __model_dest
 120         elif os.path.isdir(__model_name):
 121             __model_nam = "dsin.txt"
 122             __model_dir = os.path.abspath(__model_name)
 123             __model_dst = __model_dest
 124         else:
 125             raise ValueError('Model file or directory not found using %s'%str(__model_name))
 126         #
 127         if __before_tmp is not None:  # Mettre "../.." si nécessaire
 128             __mtmp = os.path.join(__model_dir, __before_tmp, "tmp")
 129         else:
 130             __mtmp = os.path.join(__model_dir, "tmp")
 131         if not os.path.exists(__mtmp):
 132             os.mkdir(__mtmp)
 133         __prefix = time.strftime('%Y%m%d_%Hh%Mm%Ss_tmp_', time.localtime())
 134         if __suffix is not None:
 135             __prefix = __prefix + __suffix + "_"
 136         __ltmp = tempfile.mkdtemp( prefix=__prefix, dir=__mtmp )
 137         #
 138         for sim, pmf, dst in (
 139             (__model_nam, __model_format, __model_dst),
 140             ("dymosim.exe", "DYMOSIM", "dymosim.exe"),
 141             ("pythonsim.exe", "PYSIM", "pythonsim.exe"),
 142             ):
 143             # Recherche un fichier de données ou un simulateur dans cet ordre
 144
 145             if os.path.isfile(os.path.join(__model_dir, sim)) and (pmf.upper() in [__model_format, "GUESS"]):
 146                 shutil.copy(
 147                     os.path.join(__model_dir, sim),
 148                     os.path.join(__ltmp, dst)
 149                 )
 150
 151                 # _secure_copy_textfile(
 152                 #         os.path.join(__model_dir, sim),
 153                 #         os.path.join(__ltmp, dst)
 154                 #         )
 155
 156                 break
 157         return __ltmp
 158
 159     def _setModelTmpDir_simple(self, __model_name="dsin.txt", __model_format="DYMOSIM", __model_dest = "dsin.txt"):
 160         if __model_name is None:
 161             raise ValueError('Model file or directory has to be set and can not be None')
 162         elif os.path.isfile(__model_name):
 163             if __model_format == "OPENMODELICA": #same structure as in  the directory case
 164                 __model_dir = os.path.abspath(os.path.dirname(__model_name))
 165
 166                 __model_nam_1 = os.path.basename(__model_name)
 167                 __model_nam_2 = __model_nam_1[:-9]+'.exe'
 168                 __model_nam = [__model_nam_1, __model_nam_2] #the first found model is kept
 169
 170                 __model_nam_3 = __model_nam_1[:-9]+'_info.json' #check if this file exists as well
 171                 if os.path.exists(os.path.join(__model_dir,__model_nam_3)):
 172                     __model_nam.append(__model_nam_3)                      #get the three files necessar for simulation
 173                 __model_dst = __model_nam #the same file name is kept
 174
 175             else: #cas classique
 176                 __model_nam = os.path.basename(__model_name)
 177                 __model_dir = os.path.abspath(os.path.dirname(__model_name))
 178                 __model_dst = os.path.basename(__model_dest)
 179         elif os.path.isdir(__model_name):
 180             if __model_format == "DYMOSIM":
 181                 __model_nam = "dsin.txt"
 182                 __model_dir = os.path.abspath(__model_name)
 183                 __model_dst = os.path.basename(__model_dest)
 184
 185             elif __model_format == "FMI" or __model_format == "FMU":
 186                 __model_dir = os.path.abspath(__model_name)
 187                 __model_nam = [files for files in os.listdir(__model_dir) if files[-4:] =='.fmu'][0] #the first found model is kept
 188                 __model_dst = __model_nam #the same file name is kept
 189
 190             elif __model_format == "OPENMODELICA":
 191                 __model_dir = os.path.abspath(__model_name)
 192                 __model_nam_1 = [files for files in os.listdir(__model_dir) if files[-4:] =='.xml'][0] #the first found model is kept, .xml extension is considered
 193                 __model_nam_2 = __model_nam_1[:-9]+'.exe'
 194                 __model_nam = [__model_nam_1, __model_nam_2] #the first found model is kept
 195
 196                 __model_nam_3 = __model_nam_1[:-9]+'_info.json' #check if this file exists as well
 197                 if os.path.exists(os.path.join(__model_dir,__model_nam_3)):
 198                     __model_nam.append(__model_nam_3)                      #get the three files necessar for simulation
 199                 __model_dst = __model_nam #the same file name is kept
 200
 201             else :
 202                 raise ValueError('If the model is not provided in DYMOSIM, OpenModelica or FMI format, the name of the model must refer to a file name (in a relative way with respect to the configuration file)')
 203         else:
 204             raise ValueError('Model file or directory not found using %s'%str(__model_name))
 205         #
 206         __mtmp = os.path.join(__model_dir, "tmp")
 207         if not os.path.exists(__mtmp):
 208             os.mkdir(__mtmp)
 209         __ltmp = __mtmp
 210         #
 211         if type(__model_nam) == list: #it means that it is an OpenModelica model
 212             for sim_element in __model_nam:
 213                 shutil.copy(
 214                     os.path.join(__model_dir, sim_element),
 215                     os.path.join(__ltmp, sim_element) #the same name is kept for the files
 216                     )
 217         else:
 218             for sim, pmf, dst in (
 219                 (__model_nam,__model_format,__model_dst),
 220                 ("dsin.txt","DYMOSIM","dsin.txt"),
 221                 ("pythonsim.exe","PYSIM","pythonsim.exe"),
 222                 ):
 223                 # Recherche un fichier de données ou un simulateur dans cet ordre
 224                 if os.path.isfile(os.path.join(__model_dir, sim)) and (pmf.upper() in [__model_format, "GUESS"]):
 225
 226                     shutil.copy(
 227                         os.path.join(__model_dir, sim),
 228                         os.path.join(__ltmp, dst)
 229                         )
 230                     # _secure_copy_textfile(
 231                     #         os.path.join(__model_dir, sim),
 232                     #         os.path.join(__ltmp, dst)
 233                     #         )
 234
 235                     break
 236         return __ltmp
 237
 238     def _get_Name_ModelTmpDir_simple(self, __model_name="dsin.txt"):
 239         if __model_name is None:
 240             raise ValueError('Model file or directory has to be set and can not be None')
 241         elif os.path.isfile(__model_name):
 242             __model_dir = os.path.abspath(os.path.dirname(__model_name))
 243         elif os.path.isdir(__model_name):
 244             __model_dir = os.path.abspath(__model_name)
 245         else:
 246             raise ValueError('Model file or directory not found using %s'%str(__model_name))
 247         #
 248         __mtmp = os.path.join(__model_dir, "tmp")
 249         __ltmp = __mtmp
 250         return __ltmp
 251
 252     def _setModelTmpDir_REF_CALCULATION(self, __model_name="dsin.txt", __model_format="DYMOSIM", __suffix=None, __model_dest = "dsin.txt"):
 253         if __model_name is None:
 254             raise ValueError('Model file or directory has to be set and can not be None')
 255         elif os.path.isfile(__model_name):
 256             __model_dir = os.path.abspath(os.path.dirname(__model_name))
 257             __model_nam = os.path.basename(__model_name)
 258         elif os.path.isdir(__model_name):
 259             __model_dir = os.path.abspath(__model_name)
 260             __model_nam = "dsin.txt"
 261         else:
 262             raise ValueError('Model file or directory not found using %s'%str(__model_name))
 263         if __suffix is None:
 264             raise ValueError('Observation files must be named')
 265         #
 266         __mtmp = os.path.join(__model_dir, "tmp")
 267         if not os.path.exists(__mtmp):
 268             os.mkdir(__mtmp)
 269         for sim, pmf in ((__model_nam,__model_format), ("dymosim.exe","DYMOSIM"), ("pythonsim.exe","PYSIM")):
 270             # Recherche un simulateur dans cet ordre
 271             if os.path.isfile(os.path.join(__model_dir, sim)) and (pmf.upper() in [__model_format, "GUESS"]):
 272                 __pref = time.strftime('%Y%m%d_%Hh%Mm%Ss_REF_CALCULATION_',time.localtime())
 273                 if __suffix is not None:
 274                     __pref = __pref+__suffix+"_"
 275                 __ltmp = tempfile.mkdtemp( prefix=__pref, dir=__mtmp )
 276                 shutil.copy(
 277                     os.path.join(__model_dir, sim),
 278                     os.path.join(__ltmp, __model_dest)
 279                     )
 280                 # _secure_copy_textfile(
 281                 #         os.path.join(__model_dir, sim),
 282                 #         os.path.join(__ltmp, __model_dest)
 283                 #         )
 284
 285                 __ok = True
 286                 break
 287             else:
 288                 __ok = False
 289         if not __ok:
 290             raise ValueError("Model simulator not found using \"%s\" path and \"%s\" format"%(__model_name,__model_format))
 291         return __ltmp
 292
 293     def describeMeasures(self, SourceName, Variables, Format="Guess"):
 294         "To store description of measures"
 295         self.__measures["SourceName"] = str(SourceName) # File or data name
 296         self.__measures["Variables"]  = tuple(Variables) # Name of variables
 297         self.__measures["Format"]     = str(Format) # File or data format
 298         if self.__verbose: print("  [VERBOSE] Measures described")
 299         return self.__measures
 300
 301     def describeMultiMeasures(self, SourceNames, Variables, Format="Guess"):
 302         "To store description of measures"
 303         self.__measures["SourceName"] = tuple(SourceNames) # List of file or data name
 304         self.__measures["Variables"]  = tuple(Variables) # Name of variables
 305         self.__measures["Format"]     = str(Format) # File or data format
 306         if self.__verbose: print("  [VERBOSE] Measures described")
 307         return self.__measures
 308
 309     def describeModel(self, SourceName, VariablesToCalibrate, OutputVariables, Format="Guess"):
 310         "To store description of model"
 311         self.__model["SourceName"]           = str(SourceName) # Model command file
 312         self.__model["VariablesToCalibrate"] = tuple(VariablesToCalibrate) # Name of variables
 313         self.__model["OutputVariables"]      = tuple(OutputVariables) # Name of variables
 314         self.__model["Format"]               = str(Format) # File or model format
 315         if os.path.isfile(SourceName):
 316             self.__model["Verbose_printing_name"] = os.path.basename(SourceName)
 317         elif os.path.isdir(SourceName):
 318             if self.__model["Format"] == "DYMOSIM":
 319                 self.__model["Verbose_printing_name"] = "dsin.txt + dymosim.exe in " + os.path.basename(SourceName)
 320             elif self.__model["Format"] == "FMI" or self.__model["Format"] == "FMU":
 321                 self.__model["Verbose_printing_name"] = [files for files in os.listdir(os.path.abspath(SourceName)) if files[-4:] =='.fmu'][0]
 322             elif self.__model["Format"] == "OPENMODELICA":
 323                 xml_file_name = [files for files in os.listdir(os.path.abspath(SourceName)) if files[-4:] =='.xml'][0]
 324                 exe_file_name = xml_file_name[:-9] + '.exe'
 325                 json_file_name = xml_file_name[:-9] + '_info.json'
 326                 if os.path.exists(os.path.join(os.path.abspath(SourceName),json_file_name)):
 327                     self.__model["Verbose_printing_name"] = xml_file_name + ' ' + exe_file_name + ' ' + json_file_name
 328                 else:
 329                     self.__model["Verbose_printing_name"] = xml_file_name + ' ' + exe_file_name + ' '
 330             else:
 331                 print("  [VERBOSE] Model described: ", "/!\\ Unknown ModelFormat /!\\ ")
 332                 exit()
 333         if self.__verbose: print("  [VERBOSE] Model described: ", self.__model["Verbose_printing_name"] )
 334         return self.__model
 335
 336     def describeLink(self, MeasuresNames, ModelName, LinkName, LinkVariable, Format="Guess"):
 337         "To store description of link between measures and model input data"
 338         self.__link["SourceName"]    = str(LinkName) # File or link name
 339         self.__link["MeasuresNames"] = tuple(MeasuresNames) # List of file or name
 340         self.__link["ModelName"]     = tuple(ModelName) # File or model name
 341         self.__link["Variable"]      = str(LinkVariable) # Name of variable names column
 342         self.__link["Format"]        = str(Format) # File or model format
 343         if self.__verbose: print("  [VERBOSE] Link between model and multiple measures described")
 344         return self.__link
 345
 346     def describeMethod(self, SourceName, MethodFormat="PY", BackgroundName=None, BackgroundFormat="Guess"):
 347         "To store description of calibration method"
 348         self.__method["SourceName"]       = str(SourceName) # File or method name
 349         self.__method["Format"]           = str(MethodFormat) # File or model format
 350         self.__method["BackgroundName"]   = str(BackgroundName) # File or background name
 351         self.__method["BackgroundFormat"] = str(BackgroundFormat) # File or model format
 352         if self.__verbose: print("  [VERBOSE] Optimization method settings described")
 353         return self.__method
 354
 355     def describeResult(self, ResultName, Level=None, Format="TXT"):
 356         "To store description of results"
 357         self.__result["SourceName"] = str(ResultName) # File or method name
 358         self.__result["Level"]      = str(Level) # Level = "Final", "IntermediaryFinal"
 359         self.__result["Format"]     = str(Format) # File or model format
 360         if self.__verbose: print("  [VERBOSE] Results settings described")
 361         return self.__result
 362
 363     def calibrate(self,
 364             ModelName            = None, # Model command file
 365             ModelFormat          = "Guess",
 366             DataName             = None, # Measures
 367             DataFormat           = "Guess",
 368             BackgroundName       = None, # Background
 369             BackgroundFormat     = "Guess",
 370             MethodName           = None, # Assimilation
 371             MethodFormat         = "Guess",
 372             VariablesToCalibrate = None,
 373             OutputVariables      = None,
 374             ResultName           = "results.txt", # Results
 375             ResultFormat         = "Guess",
 376             ResultLevel          = None,
 377             Verbose              = False,
 378             ):
 379         """
 380         General method to set and realize a calibration (optimal estimation
 381         task) of model parameters using measures data.
 382         """
 383         if Verbose:
 384             self.__verbose = True
 385             print("  [VERBOSE] Verbose mode activated")
 386         else:
 387             self.__verbose = False
 388         if DataName is not None:
 389             # On force l'utilisateur à nommer dans son fichier de mesures
 390             # les variables avec le même nom que des sorties dans le modèle
 391             self.describeMeasures(DataName, OutputVariables, DataFormat)
 392         if ModelName is not None:
 393             self.describeModel(ModelName, VariablesToCalibrate, OutputVariables, ModelFormat)
 394         if MethodName is not None:
 395             self.describeMethod(MethodName, MethodFormat, BackgroundName, BackgroundFormat)
 396         if ResultName is not None:
 397             self.describeResult(ResultName, ResultLevel, ResultFormat)
 398         #
 399         ONames, Observations = _readMeasures(
 400             self.__measures["SourceName"],
 401             self.__measures["Variables"],
 402             self.__measures["Format"],
 403             )
 404         Algo, Params, CovB, CovR = _readMethod(
 405             self.__method["SourceName"],
 406             self.__method["Format"],
 407             )
 408         __bgfile = None
 409         if self.__method["BackgroundFormat"] == "DSIN":  __bgfile = self.__model["SourceName"]
 410         if self.__method["BackgroundFormat"] == "ADAO":  __bgfile = self.__method["SourceName"]
 411         if self.__method["BackgroundFormat"] == "USER":  __bgfile = self.__method["BackgroundName"]
 412         if self.__method["BackgroundFormat"] == "Guess":
 413             if self.__method["BackgroundName"] is not None: __bgfile = self.__method["BackgroundName"]
 414             if self.__method["SourceName"] is not None:     __bgfile = self.__method["SourceName"]
 415             if self.__model["SourceName"] is not None:      __bgfile = self.__model["SourceName"]
 416         BNames, Background, Bounds = _readBackground(
 417             __bgfile,
 418             self.__model["VariablesToCalibrate"],
 419             self.__method["BackgroundFormat"],
 420             )
 421         if "Bounds" not in Params: # On force la priorité aux bornes utilisateur
 422             Params["Bounds"] = Bounds
 423         if self.__verbose:
 424             print("  [VERBOSE] Measures read")
 425             print("  [VERBOSE] Optimization information read")
 426             if "MaximumNumberOfSteps" in Params:
 427                 print("  [VERBOSE] Maximum possible number of iteration:",Params['MaximumNumberOfSteps'])
 428             print("  [VERBOSE] Background read:",Background)
 429             __v = Params['Bounds']
 430             if isinstance(__v, (numpy.ndarray, numpy.matrix, list, tuple)):
 431                 __v = numpy.array(__v).astype('float')
 432                 __v = numpy.where(numpy.isnan(__v), None, __v)
 433                 __v = __v.tolist()
 434             print("  [VERBOSE] Bounds read:",__v)
 435         #
 436         if "DifferentialIncrement" not in Params:
 437             Params["DifferentialIncrement"] = 0.001
 438         if "StoreSupplementaryCalculations" not in Params:
 439             Params["StoreSupplementaryCalculations"] = ["SimulatedObservationAtOptimum",]
 440         if "StoreSupplementaryCalculations" in Params and \
 441             "SimulatedObservationAtOptimum" not in Params["StoreSupplementaryCalculations"]:
 442             Params["StoreSupplementaryCalculations"].append("SimulatedObservationAtOptimum")
 443         if self.__verbose and "StoreSupplementaryCalculations" in Params and \
 444             "CurrentOptimum" not in Params["StoreSupplementaryCalculations"]:
 445             Params["StoreSupplementaryCalculations"].append("CurrentOptimum")
 446         if self.__verbose and "StoreSupplementaryCalculations" in Params and \
 447             "CostFunctionJAtCurrentOptimum" not in Params["StoreSupplementaryCalculations"]:
 448             Params["StoreSupplementaryCalculations"].append("CostFunctionJAtCurrentOptimum")
 449         #
 450         def exedymosim( x_values_matrix ):
 451             "Appel du modèle et restitution des résultats"
 452             from buildingspy.io.outputfile import Reader
 453             from pst4mod.modelica_libraries.automatic_simulation import Automatic_Simulation
 454             #
 455             # x_values = list(numpy.ravel(x_values_matrix) * Background)
 456             x_values = list(numpy.ravel(x_values_matrix))
 457             dict_inputs=dict(zip(VariablesToCalibrate,x_values))
 458             # if Verbose: print("  Simulation for %s"%numpy.ravel(x_values))
 459             #
 460             simudir = self._setModelTmpDir(ModelName, ModelFormat)
 461             auto_simul = Automatic_Simulation(
 462                 simu_dir=simudir,
 463                 dict_inputs=dict_inputs,
 464                 logfile=True,
 465                 timeout=60,
 466                 without_modelicares=True) #linux=true is removed
 467             auto_simul.single_simulation()
 468             reader = Reader(os.path.join(simudir,'dsres.mat'),'dymola')
 469             y_values = [reader.values(y_name)[1][-1] for y_name in OutputVariables]
 470             y_values_matrix = numpy.asarray(y_values)
 471             if not Verbose:
 472                 shutil.rmtree(simudir, ignore_errors=True)
 473             return y_values_matrix
 474         #
 475         def exepython( x_values_matrix ):
 476             "Appel du modèle et restitution des résultats"
 477             # x_values = list(numpy.ravel(x_values_matrix))
 478             # dict_inputs=dict(zip(VariablesToCalibrate,x_values))
 479             #
 480             simudir = self._setModelTmpDir(ModelName, ModelFormat)
 481             auto_simul = Python_Simulation(
 482                 simu_dir=simudir,
 483                 val_inputs=x_values_matrix,
 484                 logfile=True,
 485                 timeout=60,
 486                 verbose=self.__verbose)
 487             y_values_matrix = auto_simul.single_simulation()
 488             if not Verbose:
 489                 shutil.rmtree(simudir, ignore_errors=True)
 490             return y_values_matrix
 491         #
 492         if self.__model["Format"].upper() in ["DYMOSIM", "GUESS"]:
 493             simulateur = exedymosim
 494         elif self.__model["Format"].upper() == "PYSIM":
 495             simulateur = exepython
 496         else:
 497             raise ValueError("Unknown model format \"%s\""%self.__model["Format"].upper())
 498         #
 499         __adaocase = adaoBuilder.New()
 500         __adaocase.set( 'AlgorithmParameters',  Algorithm=Algo, Parameters=Params )
 501         __adaocase.set( 'Background',           Vector=Background)
 502         __adaocase.set( 'Observation',          Vector=Observations )
 503         if type(CovB) is float:
 504             __adaocase.set( 'BackgroundError',  ScalarSparseMatrix=CovB )
 505         else:
 506             __adaocase.set( 'BackgroundError',  Matrix=CovB )
 507         if type(CovR) is float:
 508             __adaocase.set( 'ObservationError', ScalarSparseMatrix=CovR )
 509         else:
 510             __adaocase.set( 'ObservationError', Matrix=CovR )
 511         __adaocase.set( 'ObservationOperator',  OneFunction=simulateur, Parameters = {"DifferentialIncrement":Params["DifferentialIncrement"]} )
 512         if self.__verbose:
 513             __adaocase.set( 'Observer', Variable="CostFunctionJAtCurrentOptimum", String="print('\\n  -----------------------------------\\n  [VERBOSE] Current iteration: %i'%(len(var),))" )
 514             __adaocase.set( 'Observer', Variable="CostFunctionJAtCurrentOptimum", Template="ValuePrinter", Info="\n  [VERBOSE] Current best cost value:" )
 515             __adaocase.set( 'Observer', Variable="CurrentOptimum",                Template="ValuePrinter", Info="\n  [VERBOSE] Current optimal state..:" )
 516         __adaocase.execute()
 517         #
 518         __resultats = dict(
 519             InitialParameters = numpy.asarray(Background),
 520             OptimalParameters = __adaocase.get("Analysis")[-1], # * Background,
 521             OptimalSimulation = __adaocase.get("SimulatedObservationAtOptimum")[-1],
 522             Measures          = Observations,
 523             NamesOfParameters = BNames,
 524             NamesOfMeasures   = ONames,
 525             )
 526         if self.__verbose: print("")
 527         del __adaocase
 528         #
 529         _saveResults(
 530             __resultats,
 531             self.__result["SourceName"],
 532             self.__result["Level"],
 533             self.__result["Format"],
 534             self.__verbose,
 535             )
 536         #
 537         return __resultats
 538
 539
 540
 541     def calibrateMultiObs(self,
 542             ModelName            = None, # Model command file
 543             ModelFormat          = "Guess",
 544             DataNames            = None, # Multiple measures
 545             DataFormat           = "Guess",
 546             LinkName             = None, # CL with data names
 547             LinkVariable         = "Variable", # Name of variable names column
 548             LinkFormat           = "Guess",
 549             BackgroundName       = None, # Background
 550             BackgroundFormat     = "Guess",
 551             MethodName           = None, # Assimilation
 552             MethodFormat         = "Guess",
 553             VariablesToCalibrate = None,
 554             OutputVariables      = None,
 555             ResultName           = "results.txt", # Results
 556             ResultFormat         = "Guess",
 557             ResultLevel          = None,
 558             Verbose              = False,
 559             IntermediateInformation = False,
 560             ComplexModel         = False,
 561             FMUInput             = None,
 562             NeedInitVariables         = False,
 563             KeepCalculationFolders = False,
 564             Linux = False,
 565             CreateOptimaldsin = False,
 566             InitialSimulation = False,
 567             VerifyFunction = False,
 568             GlobalSensitivityCheck = False,
 569             ParamSensitivityCheck = False,
 570             CreateCSVoutputResults = False,
 571             AdaptObservationError = False,
 572             ResultsSummary = False,
 573             AdvancedDebugModel = False,
 574             TimeoutModelExecution = 300,
 575             ModelStabilityCheck = False,
 576             ModelStabilityCheckingPoints = 10
 577             ):
 578         """
 579         General method to set and realize a calibration (optimal estimation
 580         task) of model parameters using multiple measures data, with an
 581         explicit link between measures and data for the model.
 582         """
 583         if Verbose:
 584             self.__verbose = True
 585             print("  [VERBOSE] Verbose mode activated")
 586         if IntermediateInformation:
 587             if self.__verbose:
 588                 self.__IntermediateInformation = True
 589                 print("  [VERBOSE] IntermediateInformation provided")
 590             else:
 591                 self.__IntermediateInformation = False
 592                 print("  [VERBOSE] IntermediateInformation requires Verbose mode activated")
 593         else:
 594             self.__IntermediateInformation = False
 595
 596         if TimeoutModelExecution < 0:
 597             raise ValueError("TimeoutModelExecution must be positive")
 598
 599         if DataNames is not None:
 600             # On force l'utilisateur à nommer dans son fichier de mesures
 601             # les variables avec le même nom que des sorties dans le modèle
 602             self.describeMultiMeasures(DataNames, OutputVariables, DataFormat)
 603         #
 604         if self.__verbose:
 605             print("  [VERBOSE] Timeoout for model execution is:", TimeoutModelExecution, "seconds")
 606         #
 607         if ModelFormat is not None: #included sot that the model format is allways in upper format
 608             ModelFormat = ModelFormat.upper()
 609             if ModelFormat == "DYMOLA": #to handle situations in which name Dymola is indicated
 610                 ModelFormat = "DYMOSIM"
 611         if ModelName is not None:
 612             self.describeModel(ModelName, VariablesToCalibrate, OutputVariables, ModelFormat)
 613         if LinkName is not None:
 614             # On force l'utilisateur à utiliser les fichiers de mesures requis
 615             # et non pas tous ceux présents dans le Link
 616             self.describeLink(DataNames, ModelName, LinkName, LinkVariable, LinkFormat)
 617         if MethodName is not None:
 618             self.describeMethod(MethodName, MethodFormat, BackgroundName, BackgroundFormat)
 619         if ResultName is not None:
 620             self.describeResult(ResultName, ResultLevel, ResultFormat)
 621         #
 622         #handling of the new option for complex models while keeping the previous key word in the code
 623         if ComplexModel:
 624             print(" The option ComplexModel  is deprecated and has to be replaced by NeedInitVariables. Please update your code.")
 625             NeedInitVariables = ComplexModel
 626         ComplexModel = NeedInitVariables
 627        #
 628
 629         ONames, Observations = _readMultiMeasures(
 630             self.__measures["SourceName"],
 631             self.__measures["Variables"],
 632             self.__measures["Format"],
 633             )
 634
 635
 636         Algo, Params, CovB, CovR = _readMethod(
 637             self.__method["SourceName"],
 638             self.__method["Format"],
 639             )
 640         __bgfile = None
 641         if self.__method["BackgroundFormat"] == "DSIN":  __bgfile = self.__model["SourceName"]
 642         if self.__method["BackgroundFormat"] == "ADAO":  __bgfile = self.__method["SourceName"]
 643         if self.__method["BackgroundFormat"] == "USER":  __bgfile = self.__method["BackgroundName"]
 644         if self.__method["BackgroundFormat"] == "Guess":
 645             if self.__method["BackgroundName"] is not None: __bgfile = self.__method["BackgroundName"]
 646             if self.__method["SourceName"] is not None:     __bgfile = self.__method["SourceName"]
 647             if self.__model["SourceName"] is not None:      __bgfile = self.__model["SourceName"]
 648         BNames, Background, Bounds = _readBackground(
 649             __bgfile,
 650             self.__model["VariablesToCalibrate"],
 651             self.__method["BackgroundFormat"],
 652             )
 653         if "Bounds" not in Params: # On force la priorité aux bornes utilisateur
 654             Params["Bounds"] = Bounds
 655         LNames, LColumns, LVariablesToChange = _readLink(
 656             self.__link["SourceName"],
 657             self.__link["MeasuresNames"],
 658             self.__link["Variable"],
 659             self.__link["Format"],
 660             )
 661         if self.__verbose:
 662             print("  [VERBOSE] Measures read")
 663             print("  [VERBOSE] Links read")
 664             print("  [VERBOSE] Optimization information read")
 665             print("  [VERBOSE] Background read:",Background)
 666             __v = Params['Bounds']
 667             if isinstance(__v, (numpy.ndarray, numpy.matrix, list, tuple)):
 668                 __v = numpy.array(__v).astype('float')
 669                 __v = numpy.where(numpy.isnan(__v), None, __v)
 670                 __v = __v.tolist()
 671             print("  [VERBOSE] Bounds read:",__v)
 672         #
 673         if "DifferentialIncrement" not in Params:
 674             Params["DifferentialIncrement"] = 0.001
 675         if "EnableMultiProcessing" not in Params:
 676             Params["EnableMultiProcessing"] = 0
 677         if "NumberOfProcesses" not in Params:
 678             Params["NumberOfProcesses"] = 0
 679         if "StoreSupplementaryCalculations" not in Params:
 680             Params["StoreSupplementaryCalculations"] = ["SimulatedObservationAtOptimum",]
 681         if "StoreSupplementaryCalculations" in Params and \
 682             "SimulatedObservationAtOptimum" not in Params["StoreSupplementaryCalculations"]:
 683             Params["StoreSupplementaryCalculations"].append("SimulatedObservationAtOptimum")
 684         if self.__verbose and "StoreSupplementaryCalculations" in Params and \
 685             "CurrentOptimum" not in Params["StoreSupplementaryCalculations"]:
 686             Params["StoreSupplementaryCalculations"].append("CurrentOptimum")
 687         if self.__verbose and "StoreSupplementaryCalculations" in Params and \
 688             "CostFunctionJAtCurrentOptimum" not in Params["StoreSupplementaryCalculations"]:
 689             Params["StoreSupplementaryCalculations"].append("CostFunctionJAtCurrentOptimum")
 690
 691         dict_dsin_path_ref ={} #creation of the dict_dsin_ref (for storing the path of the dsfinal file for each dataset
 692         #
 693
 694         VariablesToCalibrate = list(BNames)
 695
 696         def exedymosimMultiobs_REF_CALCULATION( x_values_matrix, dict_dsin_paths = dict_dsin_path_ref): #2ème argument à ne pas modifier
 697             "Appel du modèle et restitution des résultats"
 698
 699             x_values = list(numpy.ravel(x_values_matrix))
 700             x_inputs = dict(zip(VariablesToCalibrate,x_values))
 701             dict_inputs_for_CWP = {}
 702
 703 #            multi_y_values_matrix = []
 704             for etat in range(len(LNames)):
 705                 simudir = self._setModelTmpDir_REF_CALCULATION(ModelName, ModelFormat, LNames[etat])
 706
 707                 try: #to handle situations in which there is only one CL (boundary conditions) file
 708                     var_int = numpy.ravel(LColumns[:,etat])
 709                 except Exception:
 710                     var_int = numpy.ravel(LColumns)
 711                 dict_inputs = dict(zip(LVariablesToChange, var_int))
 712
 713                 dict_inputs.update( x_inputs )
 714
 715                 auto_simul = Automatic_Simulation(
 716                     simu_dir=simudir,
 717                     dymosim_path = os.path.join(simudir, os.pardir, os.pardir) ,
 718                     dsres_path = os.path.join(simudir, str("dsres" +LNames[etat]+ ".mat")),
 719                     dict_inputs=dict_inputs,
 720                     logfile=True,
 721                     timeout=TimeoutModelExecution,
 722                     without_modelicares=True,
 723                     linux = Linux)
 724                 auto_simul.single_simulation()
 725
 726
 727
 728                 if AdvancedDebugModel:
 729                     dslog_file = os.path.join(simudir, 'dslog.txt')
 730                     debug_model_file = os.path.join(simudir, os.pardir, 'log_debug_model.txt')
 731                     try:
 732                         shutil.copy(dslog_file,debug_model_file)
 733                     except Exception:
 734                         pass
 735
 736                 if auto_simul.success_code == 2 :
 737                     raise ValueError("The simulation falis after initialization, please check the log file and your model in order to be sure that the whole simulation can be performed (specially for dynamic models)")
 738
 739                 if auto_simul.success_code == 3 :
 740                     raise ValueError("The simulation did not reach the final time, probably due to a lack of time, please increase the time for model execution by modifying the TimeoutModelExecution value in configuration.py")
 741
 742                 if auto_simul.success_code == 0:
 743
 744                     path_around_simu_no_CWP = os.path.join(simudir, os.pardir, os.pardir)
 745                     around_simu_no_CWP = Around_Simulation(dymola_version='2012',
 746                                       curdir=path_around_simu_no_CWP,
 747                                       source_list_iter_var = 'from_dymtranslog',
 748                                       copy_from_dym_trans_log= os.path.join(path_around_simu_no_CWP,'ini.txt'),
 749                                       mat = os.path.join(simudir, str("dsres" +LNames[etat]+ ".mat")),
 750                                       iter_var_values_options={'source':'from_mat', 'moment':'initial'},
 751                                       verbose = False)
 752                     around_simu_no_CWP.set_list_iter_var()
 753                     around_simu_no_CWP.set_dict_iter_var()
 754
 755                     writer_no_CWP = write_in_dsin.Write_in_dsin(dict_inputs =around_simu_no_CWP.dict_iter_var,filedir=simudir,dsin_name='dsin.txt',old_file_name='old_dsin.txt',new_file_name=str("dsin_updated_" +LNames[etat]+ ".txt"))
 756                     writer_no_CWP.write_in_dsin
 757                     writer_no_CWP.write_in_dsin()
 758
 759                     dict_dsin_paths[LNames[etat]] = os.path.join(simudir,str("dsin_updated_" +LNames[etat]+ ".txt"))
 760
 761                 else:
 762                     from pst4mod.modelica_libraries.change_working_point import Dict_Var_To_Fix
 763                     from pst4mod.modelica_libraries.change_working_point import Working_Point_Modification
 764
 765                     path_around_simu = os.path.join(simudir,os.path.pardir,os.path.pardir)
 766
 767                     if not(os.path.exists(os.path.join(path_around_simu, str("ref" + ".mat")))):
 768                                         auto_simul_ref = Automatic_Simulation(
 769                                                                         simu_dir=path_around_simu,
 770                                                                         dymosim_path = os.path.join(simudir, os.pardir, os.pardir) ,
 771                                                                         dict_inputs={},
 772                                                                         logfile=True,
 773                                                                         dsres_path = os.path.join(path_around_simu, str("ref" + ".mat")),
 774                                                                         timeout=TimeoutModelExecution,
 775                                                                         without_modelicares=True,
 776                                                                         linux = Linux)
 777                                         auto_simul_ref.single_simulation()
 778
 779                                         if not(auto_simul_ref.success_code == 0):
 780                                             raise ValueError("A working dsin.txt file must be provided, provide a dsin.txt that makes it possible for the initial simulation to converge")
 781
 782                                         temporary_files_to_remove = [os.path.join(path_around_simu,'dsfinal.txt'),
 783                                                                      os.path.join(path_around_simu,'dsin_old.txt'),
 784                                                                      os.path.join(path_around_simu,'dslog.txt'),
 785                                                                      os.path.join(path_around_simu,'status'),
 786                                                                      os.path.join(path_around_simu,'success')
 787                                                                      ]
 788                                         for path_to_remove in temporary_files_to_remove:
 789                                             if os.path.exists(path_to_remove):
 790                                                 os.remove(path_to_remove)
 791
 792                     around_simu = Around_Simulation(dymola_version='2012',
 793                                       curdir=path_around_simu,
 794                                       source_list_iter_var = 'from_dymtranslog',
 795                                       copy_from_dym_trans_log= os.path.join(path_around_simu,'ini.txt'),
 796                                       mat = os.path.join(path_around_simu, str("ref" + ".mat")),
 797                                       iter_var_values_options={'source':'from_mat'},
 798                                       verbose = False)
 799                     around_simu.set_list_iter_var()
 800                     around_simu.set_dict_iter_var()
 801
 802                     reader_CWP = Reader(os.path.join(path_around_simu, str("ref" + ".mat")),'dymola')
 803                     x_values_intemediary = [reader_CWP.values(x_name)[1][-1] for x_name in VariablesToCalibrate]
 804                     x_values_from_ref_calculation = numpy.asarray(x_values_intemediary)
 805                     x_inputs_from_ref_calculation = dict(zip(VariablesToCalibrate,x_values_from_ref_calculation))
 806
 807                     cl_values_intemediary = [reader_CWP.values(cl_name)[1][-1] for cl_name in LVariablesToChange]
 808                     cl_values_from_ref_calculation = numpy.asarray(cl_values_intemediary)
 809                     cl_inputs_from_ref_calculation = dict(zip(LVariablesToChange,cl_values_from_ref_calculation))
 810
 811
 812                     for var_calib in x_inputs_from_ref_calculation.keys():
 813                         dict_inputs_for_CWP[var_calib] = (x_inputs_from_ref_calculation[var_calib], x_inputs[var_calib])
 814
 815                     try: #to handle situations in which there is only one CL (boundary conditions) file
 816                         var_int = numpy.ravel(LColumns[:,etat])
 817                     except Exception:
 818                         var_int = numpy.ravel(LColumns)
 819
 820                     dict_inputs = dict(zip(LVariablesToChange, var_int))
 821
 822                     for var_cl in cl_inputs_from_ref_calculation.keys():
 823                         dict_inputs_for_CWP[var_cl] = (cl_inputs_from_ref_calculation[var_cl], dict_inputs[var_cl])
 824
 825                     dict_var_to_fix2 = Dict_Var_To_Fix(option='automatic',
 826                                           dict_auto_var_to_fix=dict_inputs_for_CWP)
 827
 828                     dict_var_to_fix2.set_dict_var_to_fix()
 829
 830                     if Verbose:
 831                         LOG_FILENAME = os.path.join(simudir,'change_working_point.log')
 832                         for handler in logging.root.handlers[:]:
 833                             logging.root.removeHandler(handler)
 834                         logging.basicConfig(filename=LOG_FILENAME,level=logging.DEBUG,filemode='w')
 835
 836                     working_point_modif = Working_Point_Modification(main_dir = simudir,
 837                                  simu_material_dir = simudir,
 838                                  dymosim_path = os.path.join(simudir, os.pardir, os.pardir),
 839                                  simu_dir = 'SIMUS',
 840                                  store_res_dir = 'RES',
 841                                  dict_var_to_fix = dict_var_to_fix2.dict_var_to_fix,
 842                                  around_simulation0 = around_simu,
 843                                  var_to_follow_path= os.path.join(simudir,'var_to_follow.csv'),
 844                                  gen_scripts_ini= False,
 845                                  nit_max= 1000000000000000,
 846                                  min_step_val = 0.00000000000005,
 847                                  timeout = TimeoutModelExecution,
 848                                  linux = Linux)
 849
 850                     working_point_modif.create_working_directory()
 851                     working_point_modif.working_point_modification(skip_reference_simulation=True)
 852
 853                     if AdvancedDebugModel:
 854                         dslog_file = os.path.join(simudir, 'SIMUS', 'dslog.txt')
 855                         debug_model_file = os.path.join(simudir, os.pardir, 'log_debug_model.txt')
 856                         try:
 857                             shutil.copy(dslog_file,debug_model_file)
 858                         except Exception:
 859                             pass
 860
 861                     if not(os.path.exists(os.path.join(simudir, 'RES','1.0.mat'))):
 862                         raise ValueError(str("Simulation with Background values does not converge automatically, try to give a dsin.txt file that makes it possible to run a simulation with boundary conditions of " + LNames[etat]))
 863
 864                     shutil.copy(
 865                         os.path.join(simudir, 'RES','1.0.mat'),
 866                         os.path.join(simudir, str("dsres" +LNames[etat]+ ".mat"))
 867                         )
 868
 869                     path_around_simu_after_CWP = os.path.join(simudir,os.path.pardir,os.path.pardir)
 870                     around_simu_after_CWP = Around_Simulation(dymola_version='2012',
 871                                       curdir=path_around_simu_after_CWP,
 872                                       source_list_iter_var = 'from_dymtranslog',
 873                                       copy_from_dym_trans_log= os.path.join(path_around_simu_after_CWP,'ini.txt'),
 874                                       mat = os.path.join(simudir, 'RES','1.0.mat'),  #Le fichier .mat qui vient d'être créé et pour lequel on atteint les valeurs cibles de la variable
 875                                       iter_var_values_options={'source':'from_mat', 'moment':'initial'},
 876                                       verbose = False)
 877                     around_simu_after_CWP.set_list_iter_var()
 878                     around_simu_after_CWP.set_dict_iter_var()
 879
 880                     writer_after_CWP = write_in_dsin.Write_in_dsin(dict_inputs =around_simu_after_CWP.dict_iter_var,filedir=os.path.join(simudir, 'SIMUS'),dsin_name='dsin.txt',old_file_name='old_dsin.txt',new_file_name='dsin_updated.txt')
 881                     writer_after_CWP.write_in_dsin
 882                     writer_after_CWP.write_in_dsin()
 883
 884                     shutil.copy(
 885                         os.path.join(simudir, 'SIMUS','dsin_updated.txt'),
 886                         os.path.join(simudir, str("dsin_updated_" +LNames[etat]+ ".txt"))
 887                         )
 888                     dict_dsin_paths[LNames[etat]] = os.path.join(simudir,str("dsin_updated_" +LNames[etat]+ ".txt"))
 889
 890                 os.rename(
 891                 os.path.join(os.path.dirname(dict_dsin_paths[LNames[etat]]), str("dsres" +LNames[etat]+ ".mat")),
 892                 os.path.join(os.path.dirname(dict_dsin_paths[LNames[etat]]), str("REF_for_CWP" + ".mat"))
 893                         )
 894
 895             return
 896
 897
 898
 899         def preparation_exedymosimMultiobs_simple():
 900             from pst4mod.modelica_libraries import write_in_dsin
 901
 902             for etat in range(len(LNames)):
 903                 simudir = self._setModelTmpDir_simple(ModelName, ModelFormat, str("dsin_" +LNames[etat]+ ".txt"))
 904                 #
 905                 try: #to handle situations in which there is only one CL (boundary conditions) file
 906                     var_int = numpy.ravel(LColumns[:,etat])
 907                 except Exception:
 908                     var_int = numpy.ravel(LColumns)
 909                 dict_inputs = dict(zip(LVariablesToChange, var_int))
 910                 writer_no_CWP = write_in_dsin.Write_in_dsin(dict_inputs = dict_inputs, filedir=simudir, dsin_name=str("dsin_" +LNames[etat]+ ".txt"), old_file_name='old_dsin.txt',new_file_name = str("dsin_" +LNames[etat]+ ".txt"))
 911                 writer_no_CWP.write_in_dsin
 912                 writer_no_CWP.write_in_dsin()
 913             return
 914
 915         def preparation_exefmu_om_Multiobs():
 916             self._setModelTmpDir_simple(ModelName, ModelFormat, ModelName)
 917
 918         def exepythonMultiobs( x_values_matrix ):
 919             "Appel du modèle et restitution des résultats"
 920             #
 921             x_values = list(numpy.ravel(x_values_matrix))
 922             x_inputs = dict(zip(VariablesToCalibrate,x_values))
 923             #
 924             multi_y_values_matrix = []
 925             for etat in range(len(LNames)):
 926                 simudir = self._setModelTmpDir(ModelName, ModelFormat)
 927                 #
 928                 dict_inputs = dict(zip(LVariablesToChange, numpy.ravel(LColumns[:,etat])))
 929                 dict_inputs.update( x_inputs )
 930                 #
 931                 auto_simul = Python_Simulation(
 932                     simu_dir=simudir,
 933                     val_inputs=x_values_matrix,
 934                     logfile=True,
 935                     timeout=TimeoutModelExecution,
 936                     verbose=self.__verbose)
 937                 y_values_matrix = auto_simul.single_simulation()
 938                 multi_y_values_matrix.append( y_values_matrix )
 939             if not Verbose:
 940                 shutil.rmtree(simudir, ignore_errors=True)
 941             y_values_matrix = numpy.ravel(numpy.array(multi_y_values_matrix))
 942             return y_values_matrix
 943         #
 944
 945
 946         #Test for output variables - not repetition  - This test must be before the redefinition of OutputVariables
 947         OutputVariables_set = set(OutputVariables)
 948         if len(OutputVariables) != len(OutputVariables_set):
 949             diff_number_values = len(OutputVariables) - len(OutputVariables_set)
 950             raise ValueError("There are " + str(diff_number_values) + " repeated output variables in the definition of OutputVariables, please remove repeated names.")
 951
 952         #Handle several times same obs -START
 953         OutputVariables_new_tmp = []
 954
 955         whole_obs_in_fileobs = readObsnamesfile(self.__measures["SourceName"])
 956
 957         for outputname in OutputVariables:
 958             if outputname not in whole_obs_in_fileobs:
 959                 print("  [VERBOSE] /!\\ ", outputname," is not defined in the measurements file")
 960         for obsname in whole_obs_in_fileobs:
 961             if obsname in OutputVariables:
 962                 OutputVariables_new_tmp.append(obsname)
 963
 964         OutputVariables_new = []
 965         for obs_name in OutputVariables: #to get the same order as for measures
 966             count_obs_name = OutputVariables_new_tmp.count(obs_name)
 967             for i in range(count_obs_name):
 968                 OutputVariables_new.append(obs_name)
 969         OutputVariables = OutputVariables_new
 970
 971         ONames = OutputVariables
 972         #Handle several times same obs - END
 973
 974
 975         ODates,OHours = readMultiDatesHours(self.__measures["SourceName"])
 976
 977         List_Multideltatime =[]
 978
 979         for etat in range(len(ODates)):
 980             time_initial = datetime.strptime(ODates[etat][0]+ ' ' + OHours[etat][0], '%d/%m/%Y %H:%M:%S')
 981             list_deltatime = []
 982
 983             if len(ODates[etat])>1:
 984                 list_deltatime.append(0)
 985                 for index_tmp in range(len(ODates[etat])-1): #the first date is considered as reference
 986                     time_index_tmp = datetime.strptime(ODates[etat][index_tmp+1] + ' ' + OHours[etat][index_tmp+1], '%d/%m/%Y %H:%M:%S')
 987                     list_deltatime.append((time_index_tmp-time_initial).total_seconds())
 988
 989                     if (time_index_tmp-time_initial).total_seconds() < 0:
 990                          raise ValueError('The initial date is not chronological for state %s'%str(etat))
 991
 992             List_Multideltatime.append(list_deltatime)
 993
 994
 995         #Bricolage Debut
 996         Observations_new =[]
 997         number_observations_by_cl = [] #to be used in the test function, number of measurements by boundary condition (i.e. number of lines in the measurements file)
 998
 999         for etat in range(len(ODates)): #correspond au nombre de conditions aux limites
1000             Obs_etat = Observations[etat].tolist()
1001             list_obs_etat = []
1002             for obs in range(len(list(ONames))):
1003                 if len(ODates[etat])==1: #cas classique, statique
1004                     list_obs_etat = Obs_etat
1005                     if not(isinstance(list_obs_etat, list)):
1006                         list_obs_etat = [list_obs_etat]
1007
1008                 else:
1009                     for time_etat in range(len(Obs_etat)):
1010                         if not(isinstance(Obs_etat[time_etat], list)): #une seule grandeur observée
1011                             list_obs_etat.append(Obs_etat[time_etat])
1012                         else:
1013                             list_obs_etat.append(Obs_etat[time_etat][obs])
1014
1015             #EXTRA for verify tests - START
1016             if len(ODates[etat])==1:
1017                 number_observations_by_cl.append(1) #in this case only one line in the measurements file, this is just to be used as input of the test functions
1018             else:
1019                 number_observations_by_cl.append(len(Obs_etat)) #we get the number of lines of the measurements file, this is just to be used as input of the test functions
1020             #EXTRA for verify tests - END
1021
1022             Observations_new = Observations_new + list_obs_etat
1023         Observations = Observations_new
1024         #Bricolage Fin
1025
1026
1027
1028
1029         def prev_adao_version_TOP_LEVEL_exedymosimMultiobs( x_values_matrix):
1030             y_values_matrix = TOP_LEVEL_exedymosimMultiobs( x_values_matrix=x_values_matrix, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables, LNames = LNames, ModelFormat = ModelFormat, KeepCalculationFolders = KeepCalculationFolders, Verbose = Verbose, dict_dsin_paths = dict_dsin_path_ref, Linux = Linux, List_Multideltatime = List_Multideltatime, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution)
1031             return y_values_matrix
1032
1033         def prev_adao_version_TOPLEVEL_exedymosimMultiobs_simple(x_values_matrix):
1034             y_values_matrix = TOPLEVEL_exedymosimMultiobs_simple(x_values_matrix = x_values_matrix, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables, LNames = LNames, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName), KeepCalculationFolders = KeepCalculationFolders, Linux = Linux, List_Multideltatime = List_Multideltatime, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution)
1035             return y_values_matrix
1036
1037         if self.__model["Format"].upper() in ["DYMOSIM", "GUESS"]:
1038
1039             if adao.version[:5] < '9.7.0' and adao.version[5]=='.': #to handle differences in the way directoperator is mangaged after modifications brought to direct operator
1040                 if ComplexModel:
1041                     simulateur = prev_adao_version_TOP_LEVEL_exedymosimMultiobs
1042                     exedymosimMultiobs_REF_CALCULATION(Background)
1043                 else:
1044                     simulateur = prev_adao_version_TOPLEVEL_exedymosimMultiobs_simple
1045                     preparation_exedymosimMultiobs_simple()
1046
1047             else:
1048                 if ComplexModel:
1049                     simulateur = TOP_LEVEL_exedymosimMultiobs #to handle parallel computing
1050                     exedymosimMultiobs_REF_CALCULATION(Background)
1051                 else:
1052                     simulateur = TOPLEVEL_exedymosimMultiobs_simple #to handle parallel computing
1053                     preparation_exedymosimMultiobs_simple()
1054
1055         elif self.__model["Format"].upper() == "PYSIM":
1056             simulateur = exepythonMultiobs
1057         elif self.__model["Format"].upper() == "FMI" or self.__model["Format"].upper() == "FMU":
1058             simulateur = TOP_LEVEL_exefmuMultiobs
1059             preparation_exefmu_om_Multiobs()
1060         elif self.__model["Format"].upper() == "OPENMODELICA":
1061             preparation_exefmu_om_Multiobs()
1062             simulateur = TOP_LEVEL_exeOpenModelicaMultiobs
1063
1064         else:
1065             raise ValueError("Unknown model format \"%s\""%self.__model["Format"].upper())
1066
1067         def verify_function(x_values, model_format):
1068
1069
1070             simulation_results_all = []
1071             list_simulation_time = []
1072             for i in list(range(3)): #we consider only three iterations
1073                 start_time_verify = time.time()
1074                 if model_format in ["DYMOSIM"]:
1075                     if ComplexModel:
1076                         simulation_results = prev_adao_version_TOP_LEVEL_exedymosimMultiobs(x_values)
1077                     else:
1078                         simulation_results = prev_adao_version_TOPLEVEL_exedymosimMultiobs_simple(x_values)
1079                 elif model_format in ["FMI","FMU"]:
1080                     simulation_results = TOP_LEVEL_exefmuMultiobs(x_values, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution, FMUInput = FMUInput)
1081                 elif model_format in ["OPENMODELICA"]:
1082                     simulation_results = TOP_LEVEL_exeOpenModelicaMultiobs(x_values, KeepCalculationFolders = KeepCalculationFolders, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, Linux = Linux, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution)
1083                 else:
1084                     raise NotImplementedError("Not yet implemented for current model format: ", model_format)
1085                 end_time_verify = time.time()
1086
1087                 list_simulation_time.append(round(end_time_verify - start_time_verify,3))
1088                 simulation_results_all.append(simulation_results)
1089
1090
1091             elapsed_time = round(sum(list_simulation_time)/len(list_simulation_time),3)
1092                 #not necessary, already in good format
1093             # numpy.set_printoptions(precision=2,linewidth=5000, threshold = 10000)
1094             # reshaped_simulation_results_all = numpy.array(simulation_results_all).reshape((len(OutputVariables),-1)).transpose()
1095
1096             # numpy.set_printoptions(precision=2,linewidth=5000)
1097
1098             list_check_output = []
1099             list_false_output = []
1100             simulation_results_all_one_boundary_condition =[]
1101
1102             for simulation_output in simulation_results_all: #iteration over the three simulations performed
1103                 # THIS DOES NOT WORK IF OBSERVATIONS FILES ARE OF DIFFERENT SIZE# simultation_ouput_reshaped = simulation_output.reshape((len(LNames),-1))[0]    #consideration of only one set of boundary conditions (otherwise outputs could be modified by boundary condition modifications)
1104                 total_number_measurements_first_simulation = number_observations_by_cl[0]*len(OutputVariables)
1105                 first_simulation_results = simulation_output[:total_number_measurements_first_simulation]
1106                 simulation_results_all_one_boundary_condition.append(first_simulation_results)
1107
1108             for index_line_time_simulation in range(len(simulation_results_all_one_boundary_condition[0].reshape(len(OutputVariables),-1).transpose())): #iteration over each time results individually
1109                 for i in range(len(OutputVariables)):
1110                     if (simulation_results_all_one_boundary_condition[0].reshape(len(OutputVariables),-1).transpose()[index_line_time_simulation][i] == simulation_results_all_one_boundary_condition[1].reshape(len(OutputVariables),-1).transpose()[index_line_time_simulation][i]) & (simulation_results_all_one_boundary_condition[0].reshape(len(OutputVariables),-1).transpose()[index_line_time_simulation][i] == simulation_results_all_one_boundary_condition[2].reshape(len(OutputVariables),-1).transpose()[index_line_time_simulation][i]):
1111                         list_check_output.append("True")
1112
1113                     else:
1114                         list_check_output.append("False")
1115                         if OutputVariables[i] in list_false_output: #problematic variable already included
1116                             pass
1117                         else:
1118                                 list_false_output.append(OutputVariables[i])
1119
1120             if "False" in list_check_output:
1121                 verify_function_result = "False"
1122             else:
1123                 verify_function_result = "True"
1124             return verify_function_result, list_false_output, elapsed_time
1125
1126         def verify_function_sensitivity(x_values, increment, model_format):
1127
1128             if model_format in ["DYMOSIM"]:
1129                 if ComplexModel:
1130                     simulation_results = prev_adao_version_TOP_LEVEL_exedymosimMultiobs(x_values)
1131                 else:
1132                     simulation_results = prev_adao_version_TOPLEVEL_exedymosimMultiobs_simple(x_values)
1133             elif model_format in ["FMI","FMU"]:
1134                 simulation_results = TOP_LEVEL_exefmuMultiobs(x_values, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution, FMUInput = FMUInput)
1135             elif model_format in ["OPENMODELICA"]:
1136                 simulation_results = TOP_LEVEL_exeOpenModelicaMultiobs(x_values, KeepCalculationFolders = KeepCalculationFolders, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, Linux = Linux, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution)
1137             else:
1138                 raise NotImplementedError("Not yet implemented for current model format: ", model_format)
1139
1140
1141             x_values_modif = [x*(1+increment) for x in x_values]
1142
1143             if model_format in ["DYMOSIM"]:
1144                 if ComplexModel:
1145                     simulation_results_modif = prev_adao_version_TOP_LEVEL_exedymosimMultiobs(x_values_modif)
1146                 else:
1147                     simulation_results_modif = prev_adao_version_TOPLEVEL_exedymosimMultiobs_simple(x_values_modif)
1148             elif model_format in ["FMI","FMU"]:
1149                 simulation_results_modif = TOP_LEVEL_exefmuMultiobs(x_values_modif, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution, FMUInput = FMUInput)
1150             elif model_format in ["OPENMODELICA"]:
1151                 simulation_results_modif = TOP_LEVEL_exeOpenModelicaMultiobs(x_values_modif, KeepCalculationFolders = KeepCalculationFolders, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, Linux = Linux, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution)
1152             else:
1153                 raise NotImplementedError("Not yet implemented for current model format: ", model_format)
1154
1155             list_check_output = ["False" for i in OutputVariables] #everything initialized as not good and then if something changes (at least one point), then it is OK
1156             list_not_modified_variables =[]
1157
1158             total_number_measurements_first_simulation = number_observations_by_cl[0]*len(OutputVariables)
1159
1160             # THIS DOES NOT WORK IF OBSERVATIONS FILES ARE OF DIFFERENT SIZE# simulation_results_one_boundary_condition = simulation_results.reshape((len(LNames),-1))[0]
1161             # THIS DOES NOT WORK IF OBSERVATIONS FILES ARE OF DIFFERENT SIZE# simulation_results_modif_one_boundary_condition = simulation_results_modif.reshape((len(LNames),-1))[0]
1162             simulation_results_one_boundary_condition = simulation_results[:total_number_measurements_first_simulation]
1163             simulation_results_modif_one_boundary_condition = simulation_results_modif[:total_number_measurements_first_simulation]
1164
1165             for index_line_time_simulation in range(len(simulation_results_one_boundary_condition.reshape(len(OutputVariables),-1).transpose())):
1166
1167                 for i in range(len(OutputVariables)):
1168                     if simulation_results_one_boundary_condition.reshape(len(OutputVariables),-1).transpose()[index_line_time_simulation][i] != simulation_results_modif_one_boundary_condition.reshape(len(OutputVariables),-1).transpose()[index_line_time_simulation][i] :
1169                         list_check_output[i] = "True"
1170
1171
1172             for i in range(len(OutputVariables)):
1173                 if list_check_output[i] == "False": #it means the variable has not changed
1174                     list_not_modified_variables.append(OutputVariables[i])
1175
1176             if "False" in list_check_output:
1177                 verify_function_sensitivity_result = "False"
1178             else:
1179                 verify_function_sensitivity_result = "True"
1180
1181             return verify_function_sensitivity_result, list_not_modified_variables
1182
1183         def verify_function_sensitivity_parameter(x_values, increment, model_format):
1184
1185             if model_format in ["DYMOSIM"]:
1186                 if ComplexModel:
1187                     simulation_results_ref = prev_adao_version_TOP_LEVEL_exedymosimMultiobs(x_values)
1188                 else:
1189                     simulation_results_ref = prev_adao_version_TOPLEVEL_exedymosimMultiobs_simple(x_values)
1190             elif model_format in ["FMI","FMU"]:
1191                 simulation_results_ref = TOP_LEVEL_exefmuMultiobs(x_values, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution, FMUInput = FMUInput)
1192             elif model_format in ["OPENMODELICA"]:
1193                 simulation_results_ref = TOP_LEVEL_exeOpenModelicaMultiobs(x_values, KeepCalculationFolders = KeepCalculationFolders, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, Linux = Linux, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution)
1194             else:
1195                 raise NotImplementedError("Not yet implemented for current model format: ", model_format)
1196
1197             list_param_with_no_impact = []
1198             Check_params_impact = "True"
1199
1200             x_values_modif = [x*(1+increment) for x in x_values]
1201
1202
1203             for param_index in range(len(x_values)):   #the goal is to modify each param individually and check if it is possible to observe variations
1204                 x_values_param = list(x_values) #avoid memory surprises
1205                 x_values_param[param_index] = x_values_modif[param_index]
1206                 if model_format in ["DYMOSIM"]:
1207                     if ComplexModel:
1208                         simulation_results_param = prev_adao_version_TOP_LEVEL_exedymosimMultiobs(x_values_param)
1209                     else:
1210                         simulation_results_param = prev_adao_version_TOPLEVEL_exedymosimMultiobs_simple(x_values_param)
1211                 elif model_format in ["FMI","FMU"]:
1212                     simulation_results_param = TOP_LEVEL_exefmuMultiobs(x_values_param, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution, FMUInput = FMUInput)
1213                 elif model_format in ["OPENMODELICA"]:
1214                     simulation_results_param = TOP_LEVEL_exeOpenModelicaMultiobs(x_values_param, KeepCalculationFolders = KeepCalculationFolders, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, Linux = Linux, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution)
1215                 else:
1216                     raise NotImplementedError("Not yet implemented for current model format: ", model_format)
1217
1218                 if numpy.array_equal(simulation_results_param,simulation_results_ref): #there is not impact on the whole simulation
1219                     list_param_with_no_impact.append(VariablesToCalibrate[param_index])
1220                     Check_params_impact = "False"
1221
1222             return Check_params_impact, list_param_with_no_impact
1223
1224
1225         def check_model_stability(x_values_names, background_values, bounds, number_of_tests, model_format):
1226
1227             dict_check_model_stability = {}
1228             for index_x in range(len(x_values_names)):
1229                 x_bg_tested = x_values_names[index_x]
1230
1231                 print("  [VERBOSE] Model stability check is being performed for the following variable to be calibrated: ", x_bg_tested)
1232
1233                 if bounds[index_x][0] == None or bounds[index_x][0] == "None":
1234                     bounds_inf_x_bg_tested = background_values[index_x] - abs(background_values[index_x]*100)
1235                 else:
1236                     bounds_inf_x_bg_tested = bounds[index_x][0]
1237
1238                 if bounds[index_x][1] == None or bounds[index_x][1] == "None":
1239                     bounds_sup_x_bg_tested = background_values[index_x] + abs(background_values[index_x]*100)
1240                 else:
1241                     bounds_sup_x_bg_tested = bounds[index_x][1]
1242
1243                 #avoid problems with values to be tested hen the bounds are strictly equal to 0
1244                 if bounds_inf_x_bg_tested == 0: #which means that bounds_sup_x_bg_tested > 0
1245                     bounds_inf_x_bg_tested = min(1e-9,bounds_sup_x_bg_tested*1e-9)
1246                 if bounds_sup_x_bg_tested == 0: #which means that bounds_inf_x_bg_tested < 0
1247                     bounds_sup_x_bg_tested = max(-1e-9,bounds_inf_x_bg_tested*1e-9)
1248
1249                 list_x_bg_tested = []
1250
1251                 if bounds_inf_x_bg_tested < 0:
1252                     if bounds_sup_x_bg_tested<0:
1253                         list_x_bg_tested = list(numpy.geomspace(bounds_inf_x_bg_tested,bounds_sup_x_bg_tested,int(number_of_tests)))
1254                     else:
1255                         list_x_bg_tested = list(numpy.geomspace(bounds_inf_x_bg_tested,bounds_inf_x_bg_tested*1e-4,int(number_of_tests/2)))
1256                         list_x_bg_tested = list_x_bg_tested + list((numpy.geomspace(bounds_sup_x_bg_tested*1e-4,bounds_sup_x_bg_tested,int(number_of_tests/2))))
1257                 else: #both bounds are >0
1258                     list_x_bg_tested = list(numpy.geomspace(bounds_inf_x_bg_tested,bounds_sup_x_bg_tested,int(number_of_tests)))
1259
1260                 #for a linear partition, we consider a log repartition better
1261                 # for index_partition in range(number_of_tests):
1262                 #     list_x_bg_tested.append(bounds_inf_x_bg_tested + index_partition*(bounds_sup_x_bg_tested-bounds_inf_x_bg_tested)/(number_of_tests-1))
1263
1264                 list_failed_model_evaluation = []
1265                 for x_value_bg_tested in list_x_bg_tested:
1266
1267                     x_whole_values_with_bg = copy.deepcopy(background_values) #avoid modifiction of backgroundvalues
1268                     x_whole_values_with_bg[index_x] = x_value_bg_tested
1269                     try:
1270                         if model_format in ["DYMOSIM"]:
1271                             if ComplexModel:
1272                                 simulation_results = prev_adao_version_TOP_LEVEL_exedymosimMultiobs(x_whole_values_with_bg)
1273                             else:
1274                                 simulation_results = prev_adao_version_TOPLEVEL_exedymosimMultiobs_simple(x_whole_values_with_bg)
1275                         elif model_format in ["FMI","FMU"]:
1276                             simulation_results = TOP_LEVEL_exefmuMultiobs(x_whole_values_with_bg, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution, FMUInput = FMUInput)
1277                         elif model_format in ["OPENMODELICA"]:
1278                             simulation_results = TOP_LEVEL_exeOpenModelicaMultiobs(x_whole_values_with_bg, KeepCalculationFolders = KeepCalculationFolders, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, Linux = Linux, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution)
1279                         else:
1280                             raise NotImplementedError("Not yet implemented for current model format: ", model_format)
1281                     except Exception:
1282                         list_failed_model_evaluation.append(x_value_bg_tested)
1283
1284                 dict_check_model_stability[x_bg_tested] = list_failed_model_evaluation
1285             return dict_check_model_stability
1286
1287         if VerifyFunction:
1288             check_function, list_not_same_model_outputs, elapsed_time_for_simu = verify_function(x_values = Background, model_format = self.__model["Format"].upper())
1289             if check_function == "True":
1290                 print("  [VERBOSE] Results of function checking: ", "OK for all model outputs, ","simulation time (including all boundary conditions) is ",elapsed_time_for_simu, " seconds" )
1291             else:
1292                 print("  [VERBOSE] Results of function checking: ", "NOT OK for all model outputs")
1293                 print("  [VERBOSE] The following model outputs are not the same when repeating a simulation: ", list_not_same_model_outputs )
1294
1295         if GlobalSensitivityCheck:
1296             check_sensitivity, list_same_model_outputs = verify_function_sensitivity(x_values = Background, increment = Params["DifferentialIncrement"], model_format = self.__model["Format"].upper())
1297             if VerifyFunction == False:
1298                 print("  [VERBOSE] /!\\ Activate VerifyFunction option and check that the result of the check is OK to get reliable results from GlobalSensitivityCheck option")
1299             if check_sensitivity == "True":
1300                 print("  [VERBOSE] Results of function global sensitivity analysis: ", "All model outputs vary when background values are modified")
1301             else:
1302                 print("  [VERBOSE] Results of function global sensitivity analysis: ", "Some model outputs DO NOT vary when background values are modified")
1303                 print("  [VERBOSE] The following model outputs DO NOT vary when background values are modified: ", list_same_model_outputs )
1304
1305         if ParamSensitivityCheck:
1306             check_function, list_params_without_impact = verify_function_sensitivity_parameter(x_values = Background, increment = Params["DifferentialIncrement"], model_format = self.__model["Format"].upper())
1307             if VerifyFunction == False:
1308                 print("  [VERBOSE] /!\\ Activate VerifyFunction option and check that the result of the check is OK to get reliable results from ParamSensitivityCheck option")
1309             if check_function == "True":
1310                 print("  [VERBOSE] Results of parameter sensitivity checking: ", "OK, all parameters have an impact on model outputs")
1311             else:
1312                 print("  [VERBOSE] Results of parameter sensitivity checking: ", "NOT OK, some parameters do not have an impact on model outputs")
1313                 print("  [VERBOSE] The following parameters do not have an impact on model outputs: ", list_params_without_impact )
1314
1315         if ModelStabilityCheck:
1316             print("  [VERBOSE] Model stability check is being performed, this check might take several minutes depending on the model and number of variables to calibrate (you can reduce the value of ModelStabilityCheckingPoints option to reduce checking time, by default its value is 10)")
1317             check_stability = check_model_stability(x_values_names = VariablesToCalibrate, background_values = Background, bounds = Params['Bounds'], number_of_tests = ModelStabilityCheckingPoints, model_format = self.__model["Format"].upper())
1318             if check_stability == dict(zip(VariablesToCalibrate,[list() for i in VariablesToCalibrate])):
1319                 print("  [VERBOSE] Results of Model stability check: ", "The model simulates correctly when modifying individually the values of the variables to calibrate within the range in which their optimal value should be found")
1320             else:
1321                 print("  [VERBOSE] Results of Model stability check: ", "The model FAILS to converge with the following values of variables to calibrate", check_stability)
1322                 print("  [VERBOSE] The data assimilation procedure might not succeed if the model fails to converge, please revise the model and/or the range in which the optimal value of the variables to calibrate should be found")
1323
1324         #
1325
1326         __adaocase = adaoBuilder.New()
1327         __adaocase.set( 'AlgorithmParameters',  Algorithm=Algo, Parameters=Params )
1328         __adaocase.set( 'Background',           Vector=Background)
1329         __adaocase.set( 'Observation',          Vector=Observations )
1330         if type(CovB) is float:
1331             __adaocase.set( 'BackgroundError',  ScalarSparseMatrix=CovB )
1332         else:
1333             __adaocase.set( 'BackgroundError',  Matrix=CovB )
1334
1335         if AdaptObservationError:
1336             Square_Observations = [x*x for x  in Observations]
1337
1338             # if (type(CovR) is float and CovR!= 1.0 ):# by default we still consider the square, better not to modify it
1339             if type(CovR) is float:
1340                 ObsError = CovR*numpy.diag(Square_Observations)
1341                 __adaocase.set( 'ObservationError', Matrix = ObsError )
1342             else:
1343                 ObsError = 1e-15*numpy.diag(Square_Observations) #arbitary low value to eventually handle pressures in Pa
1344                 __adaocase.set( 'ObservationError', Matrix = ObsError )
1345             print("  [VERBOSE] AdaptObservationError was set to True, the following ObservationError matrix has been considered: ")
1346             print(ObsError)
1347         else: #classical situation
1348             if type(CovR) is float:
1349                 __adaocase.set( 'ObservationError', ScalarSparseMatrix=CovR )
1350             else:
1351                 __adaocase.set( 'ObservationError', Matrix=CovR )
1352
1353         if Params["EnableMultiProcessing"]==1 :
1354             ParallelComputationGradient = True
1355         else:
1356             ParallelComputationGradient = False
1357
1358
1359         if self.__model["Format"].upper() in ["DYMOSIM"]:
1360             if ParallelComputationGradient:
1361
1362                 if adao.version[:5] < '9.7.0' and adao.version[5]=='.':
1363                     raise ValueError("ADAO version must be at least 9.7.0 to support parallel computation of the gradient, current version is %s"%str(adao.version[:5]))
1364
1365                 if ComplexModel:
1366                       __adaocase.set( 'ObservationOperator',  OneFunction=simulateur,
1367                                Parameters = {"DifferentialIncrement":Params["DifferentialIncrement"], "EnableMultiProcessing":Params["EnableMultiProcessing"], "NumberOfProcesses" : Params["NumberOfProcesses"]},
1368                                ExtraArguments = {'VariablesToCalibrate':VariablesToCalibrate, 'OutputVariables' : OutputVariables, 'LNames': LNames, 'ModelFormat': ModelFormat, 'KeepCalculationFolders': KeepCalculationFolders, 'Verbose' : Verbose, 'dict_dsin_paths' : dict_dsin_path_ref, 'Linux': Linux, 'List_Multideltatime' : List_Multideltatime, 'AdvancedDebugModel':AdvancedDebugModel, 'TimeoutModelExecution':TimeoutModelExecution   },
1369                                )
1370                 else:
1371                     __adaocase.set( 'ObservationOperator',  OneFunction=simulateur,
1372                                Parameters = {"DifferentialIncrement":Params["DifferentialIncrement"], "EnableMultiProcessing":Params["EnableMultiProcessing"], "NumberOfProcesses" : Params["NumberOfProcesses"]},
1373                                ExtraArguments = {'VariablesToCalibrate':VariablesToCalibrate, 'OutputVariables' : OutputVariables, 'LNames': LNames, 'ref_simudir': self._get_Name_ModelTmpDir_simple(ModelName), 'KeepCalculationFolders': KeepCalculationFolders, 'Linux': Linux, 'List_Multideltatime' : List_Multideltatime, 'AdvancedDebugModel':AdvancedDebugModel, 'TimeoutModelExecution':TimeoutModelExecution  },
1374                                )
1375
1376             else:
1377                 if adao.version[:5] < '9.7.0' and adao.version[5]=='.':
1378                     __adaocase.set( 'ObservationOperator',  OneFunction=simulateur, Parameters = {"DifferentialIncrement":Params["DifferentialIncrement"]})
1379
1380                 else:
1381                     if ComplexModel:
1382                           __adaocase.set( 'ObservationOperator',  OneFunction=simulateur,
1383                                    Parameters = {"DifferentialIncrement":Params["DifferentialIncrement"]},
1384                                    ExtraArguments = {'VariablesToCalibrate':VariablesToCalibrate, 'OutputVariables' : OutputVariables, 'LNames': LNames, 'ModelFormat': ModelFormat, 'KeepCalculationFolders': KeepCalculationFolders, 'Verbose' : Verbose, 'dict_dsin_paths' : dict_dsin_path_ref, 'Linux': Linux, 'List_Multideltatime' : List_Multideltatime, 'AdvancedDebugModel':AdvancedDebugModel, 'TimeoutModelExecution':TimeoutModelExecution  },
1385                                    )
1386                     else:
1387                         __adaocase.set( 'ObservationOperator',  OneFunction=simulateur,
1388                                    Parameters = {"DifferentialIncrement":Params["DifferentialIncrement"]},
1389                                    ExtraArguments = {'VariablesToCalibrate':VariablesToCalibrate, 'OutputVariables' : OutputVariables, 'LNames': LNames, 'ref_simudir': self._get_Name_ModelTmpDir_simple(ModelName), 'KeepCalculationFolders': KeepCalculationFolders, 'Linux': Linux, 'List_Multideltatime' : List_Multideltatime, 'AdvancedDebugModel':AdvancedDebugModel, 'TimeoutModelExecution':TimeoutModelExecution  })
1390
1391         elif self.__model["Format"].upper() in ["OPENMODELICA"]: #OPENMODELICA (different from FMI since there are different keywords for the function: Linux and KeepCalculationFolders)
1392
1393             if ComplexModel:
1394                 print("ComplexModel option is only valid for DYMOSIM format (it has no effect for the current calculation)")
1395
1396             if ParallelComputationGradient:
1397                 if adao.version[:5] < '9.7.0' and adao.version[5]=='.':
1398                     raise ValueError("ADAO version must be at least 9.7.0 to support parallel computation of the gradient, current version is %s"%str(adao.version[:5]))
1399                 else:
1400                     __adaocase.set( 'ObservationOperator',  OneFunction=simulateur,
1401                                Parameters = {"DifferentialIncrement":Params["DifferentialIncrement"]},
1402                                ExtraArguments = {'VariablesToCalibrate':VariablesToCalibrate, 'OutputVariables' : OutputVariables, 'LNames': LNames, 'LColumns':LColumns, 'LVariablesToChange':LVariablesToChange, 'ref_simudir': self._get_Name_ModelTmpDir_simple(ModelName), 'ModelName': ModelName, 'List_Multideltatime' : List_Multideltatime, 'Linux': Linux, 'KeepCalculationFolders': KeepCalculationFolders, 'AdvancedDebugModel':AdvancedDebugModel, 'TimeoutModelExecution':TimeoutModelExecution})
1403
1404             else:
1405                 if adao.version[:5] < '9.7.0' and adao.version[5]=='.':
1406                     raise ValueError("ADAO version must be at least 9.7.0 to support other formats than DYMOSIM")
1407                 else:
1408                     __adaocase.set( 'ObservationOperator',  OneFunction=simulateur,
1409                                Parameters = {"DifferentialIncrement":Params["DifferentialIncrement"]},
1410                                ExtraArguments = {'VariablesToCalibrate':VariablesToCalibrate, 'OutputVariables' : OutputVariables, 'LNames': LNames, 'LColumns':LColumns, 'LVariablesToChange':LVariablesToChange, 'ref_simudir': self._get_Name_ModelTmpDir_simple(ModelName), 'ModelName': ModelName, 'List_Multideltatime' : List_Multideltatime, 'Linux': Linux, 'KeepCalculationFolders': KeepCalculationFolders, 'AdvancedDebugModel':AdvancedDebugModel, 'TimeoutModelExecution':TimeoutModelExecution})
1411
1412         else: #FMI or GUESS format
1413
1414             if ComplexModel:
1415                 print("ComplexModel option is only valid for DYMOSIM format (it has no effect for the current calculation)")
1416
1417             if ParallelComputationGradient:
1418                 if adao.version[:5] < '9.7.0' and adao.version[5]=='.':
1419                     raise ValueError("ADAO version must be at least 9.7.0 to support parallel computation of the gradient, current version is %s"%str(adao.version[:5]))
1420                 else:
1421                     __adaocase.set( 'ObservationOperator',  OneFunction=simulateur,
1422                                Parameters = {"DifferentialIncrement":Params["DifferentialIncrement"]},
1423                                ExtraArguments = {'VariablesToCalibrate':VariablesToCalibrate, 'OutputVariables' : OutputVariables, 'LNames': LNames, 'LColumns':LColumns, 'LVariablesToChange':LVariablesToChange, 'ref_simudir': self._get_Name_ModelTmpDir_simple(ModelName), 'ModelName': ModelName, 'List_Multideltatime' : List_Multideltatime, 'AdvancedDebugModel':AdvancedDebugModel, 'TimeoutModelExecution':TimeoutModelExecution, 'FMUInput':FMUInput})
1424
1425             else:
1426                 if adao.version[:5] < '9.7.0' and adao.version[5]=='.':
1427                     raise ValueError("ADAO version must be at least 9.7.0 to support other formats than DYMOSIM")
1428                 else:
1429                     __adaocase.set( 'ObservationOperator',  OneFunction=simulateur,
1430                                Parameters = {"DifferentialIncrement":Params["DifferentialIncrement"]},
1431                                ExtraArguments = {'VariablesToCalibrate':VariablesToCalibrate, 'OutputVariables' : OutputVariables, 'LNames': LNames, 'LColumns':LColumns, 'LVariablesToChange':LVariablesToChange, 'ref_simudir': self._get_Name_ModelTmpDir_simple(ModelName), 'ModelName': ModelName, 'List_Multideltatime' : List_Multideltatime, 'AdvancedDebugModel':AdvancedDebugModel, 'TimeoutModelExecution':TimeoutModelExecution, 'FMUInput':FMUInput})
1432
1433
1434         #
1435         if self.__verbose:
1436             __adaocase.set( 'Observer', Variable="CostFunctionJAtCurrentOptimum", String="print('\\n  -----------------------------------\\n  [VERBOSE] Current iteration number: %i'%len(var))" )
1437             __adaocase.set( 'Observer', Variable="CostFunctionJAtCurrentOptimum",  Template="ValuePrinter", Info="\n  [VERBOSE] Current optimal cost value.:" )
1438             __adaocase.set( 'Observer', Variable="CurrentOptimum",                 Template="ValuePrinter", Info="\n  [VERBOSE] Current optimal state......:" )        #
1439
1440         if self.__IntermediateInformation:
1441             __adaocase.set( 'Observer', Variable="CostFunctionJ",  Template="ValuePrinter", Info="\n  [VERBOSE] Current cost value.:" )
1442             __adaocase.set( 'Observer', Variable="CurrentState",  Template="ValuePrinter", Info="\n  [VERBOSE] Current state......:" )
1443
1444         __adaocase.execute()
1445         #
1446
1447         if InitialSimulation: #the prev_adao_version_TOP_LEVEL_XXX is already defined with proper arguments (boundary conditions etc.)
1448
1449             if self.__model["Format"].upper() in ["DYMOSIM"]:
1450                 if ComplexModel:
1451                     initialsimulation_results = prev_adao_version_TOP_LEVEL_exedymosimMultiobs(Background)
1452                 else:
1453                     initialsimulation_results = prev_adao_version_TOPLEVEL_exedymosimMultiobs_simple(Background)
1454             elif self.__model["Format"].upper() in ["FMI", "FMU"]:
1455                 initialsimulation_results = TOP_LEVEL_exefmuMultiobs(Background, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution, FMUInput = FMUInput)
1456
1457             elif self.__model["Format"].upper() in ["OPENMODELICA"]:
1458                 initialsimulation_results = TOP_LEVEL_exeOpenModelicaMultiobs(Background, KeepCalculationFolders = KeepCalculationFolders, VariablesToCalibrate=VariablesToCalibrate, OutputVariables=OutputVariables,  LNames = LNames, LColumns = LColumns, LVariablesToChange=LVariablesToChange, ref_simudir = self._get_Name_ModelTmpDir_simple(ModelName),  ModelName = ModelName, List_Multideltatime = List_Multideltatime, Linux = Linux, AdvancedDebugModel = AdvancedDebugModel,  TimeoutModelExecution = TimeoutModelExecution )
1459
1460             __resultats = dict(
1461                 InitialParameters = numpy.asarray(Background),
1462                 OptimalParameters = __adaocase.get("Analysis")[-1], # * Background,
1463                 InitialSimulation = numpy.asarray(initialsimulation_results), #the simulation results are given for backgroung values, can be used to evaluate the overall performance of the calibration procedure
1464                 OptimalSimulation = __adaocase.get("SimulatedObservationAtOptimum")[-1],
1465                 Measures          = Observations,
1466                 NamesOfParameters = BNames,
1467                 NamesOfMeasures   = ONames,
1468                 )
1469         else:
1470             __resultats = dict(
1471                 InitialParameters = numpy.asarray(Background),
1472                 OptimalParameters = __adaocase.get("Analysis")[-1], # * Background,
1473                 OptimalSimulation = __adaocase.get("SimulatedObservationAtOptimum")[-1],
1474                 Measures          = Observations,
1475                 NamesOfParameters = BNames,
1476                 NamesOfMeasures   = ONames,
1477                 )
1478
1479         if "APosterioriCovariance" in Params["StoreSupplementaryCalculations"]:
1480             __resultats["APosterioriCovariance"] = __adaocase.get("APosterioriCovariance")[-1]
1481
1482         if ResultsSummary:
1483             if InitialSimulation:
1484                 measures_input = __resultats ["Measures"]
1485                 initial_simu = __resultats ["InitialSimulation"]
1486                 optimal_simu = __resultats ["OptimalSimulation"]
1487
1488                 diff_measures_initial_rmse = numpy.array([(measures_input[i]-initial_simu[i])**2 for i in range(len(measures_input))])
1489                 diff_measures_initial_reldiff = numpy.array([abs((measures_input[i]-initial_simu[i])/measures_input[i]) for i in range(len(measures_input)) if measures_input[i] !=0])
1490
1491                 diff_measures_optimal_rmse = numpy.array([(measures_input[i]-optimal_simu[i])**2 for i in range(len(measures_input))])
1492                 diff_measures_optimal_reldiff = numpy.array([abs((measures_input[i]-optimal_simu[i])/measures_input[i]) for i in range(len(measures_input)) if measures_input[i] !=0])
1493
1494                 diff_measures_initial_rmse_res = str(numpy.format_float_scientific(numpy.sqrt(diff_measures_initial_rmse.mean()), precision = 3))
1495                 diff_measures_initial_reldiff_res = str(numpy.round(diff_measures_initial_reldiff.mean()*100,decimals =2))
1496
1497                 diff_measures_optimal_rmse_res = str(numpy.format_float_scientific(numpy.sqrt(diff_measures_optimal_rmse.mean()), precision = 3))
1498                 diff_measures_optimal_reldiff_res = str(numpy.round(diff_measures_optimal_reldiff.mean()*100,decimals =2))
1499
1500                 __resultats["ResultsSummary_RMSE"] = ["Average_RMSE_INITIAL = " + diff_measures_initial_rmse_res, "Average_RMSE_OPTIMAL = " + diff_measures_optimal_rmse_res ]
1501                 __resultats["ResultsSummary_RelativeDifference"] = ["Average_RelativeDifference_INITIAL = " +  diff_measures_initial_reldiff_res + "%", "Average_RelativeDifference_OPTIMAL = " + diff_measures_optimal_reldiff_res + "%" ]
1502             else:
1503                 measures_input = __resultats ["Measures"]
1504                 optimal_simu = __resultats ["OptimalSimulation"]
1505
1506                 diff_measures_optimal_rmse = numpy.array([(measures_input[i]-optimal_simu[i])**2 for i in range(len(measures_input))])
1507                 diff_measures_optimal_reldiff = numpy.array([abs((measures_input[i]-optimal_simu[i])/measures_input[i]) for i in range(len(measures_input)) if measures_input[i] !=0])
1508
1509
1510                 diff_measures_optimal_rmse_res = str(numpy.format_float_scientific(numpy.sqrt(diff_measures_optimal_rmse.mean()), precision = 3))
1511                 diff_measures_optimal_reldiff_res = str(numpy.round(diff_measures_optimal_reldiff.mean()*100,decimals =2))
1512
1513                 __resultats["ResultsSummary_RMSE"] = ["Average_RMSE_OPTIMAL = " + diff_measures_optimal_rmse_res ]
1514                 __resultats["ResultsSummary_RelativeDifference"] = ["Average_RelativeDifference_OPTIMAL = " + diff_measures_optimal_reldiff_res + "%" ]
1515
1516         #
1517         _saveResults(
1518             __resultats,
1519             self.__result["SourceName"],
1520             self.__result["Level"],
1521             self.__result["Format"],
1522             self.__verbose,
1523             )
1524         #
1525
1526         if CreateOptimaldsin:
1527              if self.__model["Format"].upper() in ["DYMOSIM"]:
1528                  for etat in range(len(LNames)):
1529                      dict_optimal_params = dict(zip(VariablesToCalibrate, __adaocase.get("Analysis")[-1])) #dans la boucle, sinon il se vide à chaque fois que l'on fait itère
1530                      if not(ComplexModel):
1531                          dir_for_dsin_opti = os.path.abspath(os.path.join(self._get_Name_ModelTmpDir_simple(ModelName),os.pardir))
1532                          simu_dir_opti = os.path.abspath(self._get_Name_ModelTmpDir_simple(ModelName))
1533
1534                          writer_opti = write_in_dsin.Write_in_dsin(dict_inputs =dict_optimal_params,filedir=simu_dir_opti,dsin_name=str("dsin_" +LNames[etat]+ ".txt"),old_file_name='dsin.txt',new_file_name= str("dsin_" +LNames[etat]+ "_optimal.txt"))
1535                          writer_opti.write_in_dsin
1536
1537                          writer_opti.write_in_dsin()
1538
1539                          try:
1540                              os.remove(os.path.join(dir_for_dsin_opti,str("dsin_" +LNames[etat]+ "_optimal.txt")))
1541                          except Exception:
1542                              pass
1543
1544                          shutil.copyfile(
1545                                     os.path.join(simu_dir_opti, str("dsin_" +LNames[etat]+ "_optimal.txt")),
1546                                     os.path.join(dir_for_dsin_opti, str("dsin_" +LNames[etat]+ "_optimal.txt"))
1547                                     )
1548
1549                          auto_simul_opti = Automatic_Simulation(
1550                                                      simu_dir=simu_dir_opti,
1551                                                      dsin_name= str("dsin_" +LNames[etat]+ "_optimal.txt"),
1552                                                      dymosim_path = os.path.join(simu_dir_opti, os.pardir) ,
1553                                                      dsres_path = str("dsres_" +LNames[etat]+ "_opti.mat"),
1554                                                      dict_inputs= {},
1555                                                      logfile=False,
1556                                                      timeout=TimeoutModelExecution,
1557                                                      without_modelicares=True,
1558                                                      linux=Linux
1559                                                      )
1560                          auto_simul_opti.single_simulation()
1561
1562                          if not(auto_simul_opti.success_code == 0):
1563                              print("The dsin.txt with the optimal values does not converge directly, please update the value of iteration variables using Change_working_Point script (consider both calibrated paramaters and boundary conditions in the procedure)")
1564                      else:
1565                          try: #to handle situations in which there is only one CL (boundary conditions) file
1566                              var_int = numpy.ravel(LColumns[:,etat])
1567                          except Exception:
1568                              var_int = numpy.ravel(LColumns)
1569                          dict_inputs_boundary_conditions_optimal_params = dict(zip(LVariablesToChange, var_int))
1570                          dict_inputs_boundary_conditions_optimal_params.update(dict_optimal_params)
1571
1572                          try:
1573                              os.remove(os.path.join(dir_for_dsin_opti,str("dsin_" +LNames[etat]+ "_optimal.txt")))
1574                          except Exception:
1575                              pass
1576                          dir_for_dsin_opti = os.path.abspath(os.path.join(self._get_Name_ModelTmpDir_simple(ModelName),os.pardir))
1577                          shutil.copyfile(
1578                                     os.path.join(dir_for_dsin_opti, str("dsin.txt")),
1579                                     os.path.join(dir_for_dsin_opti, str("dsin_" +LNames[etat]+ "_optimal.txt"))
1580                                     )
1581                          writer_opti = write_in_dsin.Write_in_dsin(dict_inputs =dict_inputs_boundary_conditions_optimal_params,filedir=dir_for_dsin_opti,dsin_name=str("dsin_" +LNames[etat]+ "_optimal.txt"),old_file_name=str("dsin_" +LNames[etat]+ "_old.txt"),new_file_name= str("dsin_" +LNames[etat]+ "_optimal.txt"))
1582                          writer_opti.write_in_dsin
1583                          writer_opti.write_in_dsin()
1584                          os.remove(os.path.join(dir_for_dsin_opti,str("dsin_" +LNames[etat]+ "_old.txt")))
1585                          print("The dsin.txt with the optimal values has not been tested, updating the values of iteration variables might be necessary: this can be done by using Change_working_Point script (consider both calibrated paramaters and boundary conditions in the procedure)")
1586
1587              else: #FMI, OM or GUESS format
1588                  print("CreateOptimaldsin option is only valid for DYMOSIM format (it has no effect for the current calculation)")
1589
1590         if self.__verbose: print("")
1591         del __adaocase
1592
1593         if not(KeepCalculationFolders):
1594             shutil.rmtree(self._get_Name_ModelTmpDir_simple(ModelName), ignore_errors=True)
1595
1596         if CreateCSVoutputResults:
1597             print("Creation of CSV output results")
1598
1599
1600             optimal_results = __resultats["OptimalSimulation"]
1601             for obs_file_index in range(len(self.__measures["SourceName"])):
1602                 obs_filename = self.__measures["SourceName"][obs_file_index] #for the name of the outputresult
1603
1604                 time_measures_number = len(OHours[obs_file_index]) #get the number of lines in the obs file
1605                 total_measures_number_per_cl = time_measures_number*len(__resultats["NamesOfMeasures"]) #size of the list to be removed from the whole results file
1606                 optimal_results_cl = optimal_results[:total_measures_number_per_cl] # output results for the current cl
1607                 optimal_results = optimal_results[total_measures_number_per_cl:] # update of the whole results file (we remove te results already treated)
1608
1609                 optimal_results_cl_to_csv = optimal_results_cl.reshape((len(__resultats["NamesOfMeasures"]),-1)).transpose() #reshape to get one line per timedate
1610                 df_optimal_results_cl_to_csv = pandas.DataFrame(optimal_results_cl_to_csv, columns = __resultats["NamesOfMeasures"]) #creation of the df with the names of measures
1611                 df_optimal_results_cl_to_csv.insert(0,"Hour", OHours[obs_file_index]) #add Hour column
1612                 df_optimal_results_cl_to_csv.insert(0,"Date", ODates[obs_file_index]) # add Date column
1613
1614                 optimal_file_results_name_cl_tmp = "Optimal_simulation_tmp.csv"
1615                 df_optimal_results_cl_to_csv.to_csv(optimal_file_results_name_cl_tmp, sep = ";", index = False)
1616
1617                 optimal_file_results_name_cl = "Optimal_simulation_"+obs_filename
1618                 with open(optimal_file_results_name_cl_tmp, 'r') as infile:
1619                     readie=csv.reader(infile, delimiter=';')
1620                     with open(optimal_file_results_name_cl, 'wt', newline='') as output:
1621                         outwriter=csv.writer(output, delimiter=';')
1622                         outwriter.writerow(["#"])
1623                         for row in readie:
1624                              outwriter.writerow(row)
1625                 try:
1626                     os.remove(optimal_file_results_name_cl_tmp)
1627                 except Exception:
1628                     pass
1629
1630             if InitialSimulation: #If initial simulation is required as well, the Initial results files are given as well
1631                 initial_results = __resultats["InitialSimulation"]
1632                 for obs_file_index in range(len(self.__measures["SourceName"])):
1633                     obs_filename = self.__measures["SourceName"][obs_file_index] #for the name of the outputresult
1634
1635                     time_measures_number = len(OHours[obs_file_index]) #get the number of lines in the obs file
1636                     total_measures_number_per_cl = time_measures_number*len(__resultats["NamesOfMeasures"]) #size of the list to be removed from the whole results file
1637                     initial_results_cl = initial_results[:total_measures_number_per_cl] # output results for the current cl
1638                     initial_results = initial_results[total_measures_number_per_cl:] # update of the whole results file (we remove te results already treated)
1639
1640                     initial_results_cl_to_csv = initial_results_cl.reshape((len(__resultats["NamesOfMeasures"]),-1)).transpose() #reshape to get one line per timedate
1641                     df_initial_results_cl_to_csv = pandas.DataFrame(initial_results_cl_to_csv, columns = __resultats["NamesOfMeasures"]) #creation of the df with the names of measures
1642                     df_initial_results_cl_to_csv.insert(0,"Hour", OHours[obs_file_index]) #add Hour column
1643                     df_initial_results_cl_to_csv.insert(0,"Date", ODates[obs_file_index]) # add Date column
1644
1645                     initial_file_results_name_cl_tmp = "Initial_simulation_tmp.csv"
1646                     df_initial_results_cl_to_csv.to_csv(initial_file_results_name_cl_tmp, sep = ";", index = False)
1647
1648                     initial_file_results_name_cl = "Initial_simulation_"+obs_filename
1649                     with open(initial_file_results_name_cl_tmp, 'r') as infile:
1650                         readie=csv.reader(infile, delimiter=';')
1651                         with open(initial_file_results_name_cl, 'wt', newline='') as output:
1652                             outwriter=csv.writer(output, delimiter=';')
1653                             outwriter.writerow(["#"])
1654                             for row in readie:
1655                                  outwriter.writerow(row)
1656                     try:
1657                         os.remove(initial_file_results_name_cl_tmp)
1658                     except Exception:
1659                         pass
1660
1661         return __resultats
1662
1663
1664         def verify_gradient(self):
1665             raise NotImplementedError("Not yet implemented")
1666
1667 # ==============================================================================
1668 def _readLink(__filename=None, __colnames=None, __indexname="Variable", __format="Guess"):
1669     """
1670     Read file of link between model and measures
1671
1672     Arguments:
1673         - File name
1674         - Names of the columns to read, known by their variable header
1675         - Name of the column containing the variables to adapt
1676         - Format of the data (CSV, TSV... or can be guessed)
1677     """
1678     #
1679     if __colnames is None:
1680         raise
1681     if __indexname is None: __indexname="Variable"
1682
1683     try: #Solution provisoire pou gérer les cas sans CL précisées par l'utilisateur
1684         with ImportFromFile(__filename, __colnames, __indexname, __format, False) as reading:
1685             colnames, columns, indexname, variablestochange = reading.getvalue()
1686     except Exception:
1687         colnames =__colnames
1688         columns =[]
1689         variablestochange =[]
1690         print("/!\\ Boundary conditions indicated in ", __filename, " are not considered for calculation. Indicate at least two boundary conditions so that they are considered. /!\\ ")
1691     #
1692     variablestochange = tuple([v.strip() for v in variablestochange])
1693     return (colnames, columns, variablestochange)
1694
1695 # ==============================================================================
1696 def _readMultiMeasures(__filenames=None, __colnames=None, __format="Guess"):
1697     """
1698     Read files of measures, using only some named variables by columns
1699
1700     Arguments:
1701         - File name
1702         - Names of the columns to read, known by their variable header
1703         - Format of the data (CSV, TSV... or can be guessed)
1704     """
1705     #
1706     MultiMeasures = []
1707     for __filename in __filenames:
1708         colnames, columns = _readMeasures(__filename, __colnames, __format)
1709         MultiMeasures.append( columns )
1710     #
1711     return (colnames, MultiMeasures)
1712
1713 # ==============================================================================
1714 def _readMeasures(__filename=None, __colnames=None, __format="Guess"):
1715     """
1716     Read files of measures, using only some named variables by columns
1717
1718     Arguments:
1719         - File name
1720         - Names of the columns to read, known by their variable header
1721         - Format of the data (CSV, TSV... or can be guessed)
1722     """
1723     #
1724     with ImportFromFile(__filename, __colnames, None, __format) as reading:
1725         colnames, columns, indexname, index = reading.getvalue()
1726     #
1727     return (colnames, columns)
1728
1729 # ==============================================================================
1730 def _readBackground(__filename="dsin.txt", __varnames=None, __backgroundformat="Guess"):
1731     """
1732     Read background in model definition. The priority is "USER", then "ADAO",
1733     then "DSIN", and the default one correspond to "DSIN".
1734
1735     Arguments:
1736         - File name
1737         - Names of the variables to be found in the model data
1738         - Format of the data (can be guessed)
1739     """
1740     __format = None
1741     if __backgroundformat.upper() in ["GUESS", "DSIN"]:
1742         __format = "DSIN"
1743         if __filename is not None and os.path.isfile(__filename):
1744             __model_dir = os.path.dirname(__filename)
1745             __model_fil = os.path.basename(__filename)
1746         elif  __filename is not None and os.path.isdir(__filename) and os.path.isfile(os.path.join(__filename, "dsin.txt")):
1747             __model_dir = os.path.abspath(__filename)
1748             __model_fil = "dsin.txt"
1749         else:
1750             raise ValueError('No such file or directory: %s'%str(__filename))
1751         __bgfile = os.path.abspath(os.path.join(__model_dir, __model_fil))
1752         if (len(__bgfile)<7) and (__bgfile[-8:].lower() != "dsin.txt" ):
1753             raise ValueError("Model definition file \"dsin.txt\" can not be found.")
1754         if __varnames is None:
1755             raise ValueError("Model variables to be read has to be given")
1756     if __backgroundformat.upper() == "ADAO":
1757         __format = "ADAO"
1758         if __filename is None or not os.path.isfile(__filename):
1759             raise ValueError('No such file or directory: %s'%str(__filename))
1760         __bgfile = os.path.abspath(__filename)
1761         if not( len(__bgfile)>4 ):
1762             raise ValueError("User file name \"%s\" is too short and seems not to be valid."%__bgfile)
1763     if __backgroundformat.upper() == "USER":
1764         __format = "USER"
1765         if __filename is None or not os.path.isfile(__filename):
1766             raise ValueError('No such file or directory: %s'%str(__filename))
1767         __bgfile = os.path.abspath(__filename)
1768         if not( len(__bgfile)>5 ):
1769             raise ValueError("User file name \"%s\" is too short and seems not to be valid."%__bgfile)
1770     if __format not in ["DSIN", "ADAO", "USER"]:
1771         raise ValueError("Background initial values asked format \"%s\" is not valid."%__format)
1772     #
1773     #---------------------------------------------
1774     if __format == "DSIN":
1775         if __varnames is None:
1776             raise ValueError("Names of variables to read has to be given")
1777         #
1778         __names, __background, __bounds = [], [], []
1779         with open(__bgfile, 'r') as fid:
1780             __content = fid.readlines()
1781             for v in tuple(__varnames):
1782                 if _verify_existence_of_variable_in_dsin(v, __content):
1783                     __value = _read_existing_variable_in_dsin(v, __content)
1784                     __names.append(v)
1785                     __background.append(__value[0])
1786                     __bounds.append(__value[1:3])
1787                     # print "%s = "%v, __value
1788         __names = tuple(__names)
1789         __background = numpy.ravel(__background)
1790         __bounds = tuple(__bounds)
1791     #---------------------------------------------
1792     elif __format.upper() == "ADAO":
1793         with open(__bgfile, 'r') as fid:
1794             exec(fid.read())
1795         #
1796         __background = locals().get("Background", None)
1797         if __background is None:
1798             raise ValueError("Background is not defined")
1799         else:
1800             __background = numpy.ravel(__background)
1801         __Params     = locals().get("Parameters", {})
1802         if "Bounds" not in __Params:
1803             raise ValueError("Bounds can not be find in file \"%s\""%str(__filename))
1804         else:
1805             __bounds = tuple(__Params["Bounds"])
1806             if len(__bounds) != len(__background):
1807                 raise ValueError("Initial background length does not match bounds number")
1808         #
1809         if __varnames is None:
1810             __names = ()
1811         else:
1812             __names = tuple(__varnames)
1813             if len(__names) != len(__background):
1814                 raise ValueError("Initial background length does not match names number")
1815     #---------------------------------------------
1816     elif __format.upper() == "USER":
1817         __names, __background, __bounds = ImportScalarLinesFromFile(__bgfile).getvalue(__varnames)
1818     #---------------------------------------------
1819     for i,b in enumerate(__bounds) :
1820         if b[0] is not None and b[1] is not None and not (b[0] < b[1]) :
1821             raise ValueError(f'Inconsistent boundaries values for "{__names[i]}": {b[0]} not < {b[1]}')
1822
1823     return (__names, __background, __bounds)
1824
1825 def _verify_existence_of_variable_in_dsin(__variable, __content, __number = 2):
1826     "Verify if the variable exist in the model file content"
1827     if "".join(__content).count(__variable) >= __number:
1828         return True
1829     else:
1830         return False
1831
1832 def _read_existing_variable_in_dsin(__variable, __content):
1833     "Return the value of the real variable"
1834     __previousline, __currentline = "", ""
1835     internalOrder = 0
1836     initialValuePart = False
1837     for line in __content:
1838         if not initialValuePart and "initialValue" not in line: continue
1839         if not initialValuePart and "initialValue" in line: initialValuePart = True
1840         __previousline = __currentline
1841         __currentline  = line
1842         if __variable in __currentline and "#" in __currentline and "#" not in __previousline:
1843             # Informations ecrites sur deux lignes successives
1844             vini, value, vmin, vmax = __previousline.split()
1845             # vcat, vtype, diez, vnam = __currentline.split()
1846             value = float(value)
1847             if float(vmin) >= float(vmax):
1848                 vmin, vmax = None, None
1849             else:
1850                 vmin, vmax = float(vmin), float(vmax)
1851             return (value, vmin, vmax)
1852         elif __variable in __currentline and "#" in __currentline and "#" in __previousline:
1853             # Informations ecrites sur une seule ligne
1854             vini, value, vmin, vmax, reste = __previousline.split(maxsplit=5)
1855             value = float(value)
1856             if float(vmin) >= float(vmax):
1857                 vmin, vmax = None, None
1858             else:
1859                 vmin, vmax = float(vmin), float(vmax)
1860             return (value, vmin, vmax)
1861     #
1862     raise ValueError("Value of variable %s not found in the file content"%__variable)
1863
1864 # ==============================================================================
1865 def _readMethod(__filename="parameters.py", __format="ADAO"):
1866     """
1867     Read data assimilation method parameters
1868
1869     Arguments:
1870         - File name
1871         - Format of the data (can be guessed)
1872     """
1873     if not os.path.isfile(__filename):
1874         raise ValueError('No such file or directory: %s'%str(__filename))
1875     #
1876     #---------------------------------------------
1877     if __format.upper() in ["GUESS", "ADAO"]:
1878         with open(__filename, 'r') as fid:
1879             exec(fid.read())
1880         __Algo   = locals().get("Algorithm", "3DVAR")
1881         __Params = locals().get("Parameters", {})
1882         __CovB   = locals().get("BackgroundError", 1.)
1883         __CovR   = locals().get("ObservationError", 1.)
1884         # __Xb     = locals().get("Background", __background)
1885         # if not isinstance(__Params, dict): __Params = {"Bounds":__bounds}
1886         # if "Bounds" not in __Params:       __Params["Bounds"] = __bounds
1887     #---------------------------------------------
1888     else:
1889         __Algo, __Params, __CovB, __CovR = "3DVAR", {}, 1., 1.
1890     #---------------------------------------------
1891     # if __Xb is None:
1892     #     raise ValueError("Background is not defined")
1893     # else:
1894     #     __Xb = numpy.ravel(__Xb)
1895     #
1896     return (__Algo, __Params, __CovB, __CovR)
1897
1898 # ==============================================================================
1899 def _saveResults(__resultats, __filename=None, __level=None, __format="Guess", __verbose=False):
1900     """
1901     Save results relatively to the level required
1902
1903     Arguments!
1904         - File name
1905         - Level of saving : final output only or intermediary with final output
1906         - Format of the data
1907     """
1908     if __filename is None:
1909         raise ValueError('A file to save results has to be named, please specify one')
1910     if __format.upper() == "GUESS":
1911         if __filename.split(".")[-1].lower() == "txt":
1912             __format = "TXT"
1913         elif __filename.split(".")[-1].lower() == "py":
1914             __format = "PY"
1915         else:
1916             raise ValueError("Can not guess the file format of \"%s\", please specify the good one"%__filename)
1917     else:
1918         __format     = str(__format).upper()
1919     if __format not in ["TXT", "PY"]:
1920         raise ValueError("Result file format \"%s\" is not valid."%__format)
1921     if __format == "PY" and os.path.splitext(__filename)[1] != ".py":
1922         __filename = os.path.splitext(__filename)[0] + ".py"
1923     #
1924     #---------------------------------------------
1925     if __format.upper() == "TXT":
1926         output = ["# Final results",]
1927         keys = list(__resultats.keys())
1928         keys.sort()
1929         for k in keys:
1930             __v = __resultats[k]
1931             if isinstance(__v, numpy.matrix):  # no1
1932                 __v = __v.astype('float').tolist()
1933             elif isinstance(__v, numpy.ndarray):  # no2
1934                 __v = tuple(__v.astype('float').tolist())
1935             else:
1936                 __v = tuple(__v)
1937             output.append("%22s = %s"%(k,__v))
1938         output.append("")
1939         with open(__filename, 'w') as fid:
1940             fid.write( "\n".join(output) )
1941             fid.flush()
1942     #---------------------------------------------
1943     elif __format.upper() == "PY":
1944         output = ["# Final results",]
1945         keys = list(__resultats.keys())
1946         keys.sort()
1947         for k in keys:
1948             __v = __resultats[k]
1949             if isinstance(__v, numpy.matrix):  # no1
1950                 __v = __v.astype('float').tolist()
1951             elif isinstance(__v, numpy.ndarray):  # no2
1952                 __v = tuple(__v.astype('float').tolist())
1953             else:
1954                 __v = tuple(__v)
1955             output.append("%s = %s"%(k,__v))
1956         output.append("")
1957         with open(__filename, 'w') as fid:
1958             fid.write( "\n".join(output) )
1959             fid.flush()
1960     #---------------------------------------------
1961     elif __format.upper() == "CSV":
1962         raise NotImplementedError("Not yet implemented")
1963     #---------------------------------------------
1964     elif __format.upper() == "DICT":
1965         raise NotImplementedError("Not yet implemented")
1966     #---------------------------------------------
1967     if __verbose: # Format TXT
1968         output = ["# Final results",]
1969         keys = list(__resultats.keys())
1970         keys.sort()
1971         for k in keys:
1972             __v = __resultats[k]
1973             if isinstance(__v, numpy.matrix):  # no1
1974                 __v = __v.astype('float').tolist()
1975             elif isinstance(__v, numpy.ndarray):  # no2
1976                 __v = tuple(__v.astype('float').tolist())
1977             else:
1978                 __v = tuple(__v)
1979             output.append("%22s = %s"%(k,__v))
1980         output.append("")
1981         print( "\n".join(output) )
1982     #---------------------------------------------
1983     return True
1984
1985 def _secure_copy_textfile(__original_file, __destination_file):
1986     "Copy the file guranteeing that it is copied"
1987
1988
1989     shutil.copy(
1990                 __original_file,
1991                 __destination_file
1992                 )
1993     shutil.copystat( # commande pour assurer que le fichier soit bien copié
1994         __original_file,
1995         __destination_file
1996         )
1997     os.path.getsize(__destination_file)# commande bis pour assurer que le fichier soit bien copié
1998
1999     def file_len(fname):
2000         count = 0
2001         with open(fname) as f:
2002             for line in f:
2003                 count += 1
2004         return count
2005
2006     while file_len(__original_file)!= file_len(__destination_file):
2007         shutil.copy(
2008             __original_file,
2009             __destination_file
2010             )
2011
2012 # ==============================================================================
2013 class Python_Simulation(object):
2014     def __init__(self, simu_dir=".", val_inputs=(), logfile=True, timeout=60, verbose=True):
2015         self.__simu_dir = os.path.abspath(simu_dir)
2016         self.__inputs = numpy.ravel(val_inputs)
2017         if verbose:
2018             print()
2019             print("  [VERBOSE] Input values %s"%self.__inputs)
2020
2021     def single_simulation(self, __inputs = None):
2022         with open(os.path.join(self.__simu_dir,"pythonsim.exe"), 'r') as fid:
2023             exec(fid.read())
2024         __directoperator = locals().get("DirectOperator", None)
2025         if __inputs is None: __inputs = self.__inputs
2026         return __directoperator(__inputs)
2027
2028 # ==============================================================================
2029 class VersionsLogicielles (object):
2030     "Modules version information"
2031     def __init__(self):
2032         print("  [VERBOSE] System configuration:")
2033         print("      - Python...:",sys.version.split()[0])
2034         print("      - Numpy....:",numpy.version.version)
2035         print("      - Scipy....:",scipy.version.version)
2036         print("      - ADAO.....:",adao.version)
2037         print("")
2038
2039 def TOP_LEVEL_exefmuMultiobs_ref( x_values_matrix , VariablesToCalibrate=None, OutputVariables=None, ref_simudir = None,  ModelName = None ):
2040     "Appel du modèle en format FMU et restitution des résultats"
2041     x_values = list(numpy.ravel(x_values_matrix))
2042     x_inputs=dict(zip(VariablesToCalibrate,x_values))
2043
2044     fmuname = os.path.basename(ModelName) #in case ModelName is given as a path (works also if it is a file name)
2045     fmu = os.path.join(ref_simudir,fmuname)
2046
2047     reader = simulate_fmu(fmu, output = OutputVariables, start_values = x_inputs)
2048     y_values = [reader[y_name][-1] for y_name in OutputVariables]
2049     y_values_matrix = numpy.asarray(y_values)
2050
2051     return y_values_matrix
2052
2053 def TOP_LEVEL_exefmuMultiobs( x_values_matrix , VariablesToCalibrate=None, OutputVariables=None,  LNames = None, LColumns = None, LVariablesToChange=None, ref_simudir = None,  ModelName = None, List_Multideltatime = None, AdvancedDebugModel = None, TimeoutModelExecution = None, FMUInput = None):
2054     "Appel du modèle en format FMU et restitution des résultats"
2055
2056     if VariablesToCalibrate is None:
2057         raise ValueError("VariablesToCalibrate")
2058
2059     if OutputVariables is None:
2060         raise ValueError("OutputVariables")
2061
2062     if LNames is None:
2063         raise ValueError("LNames")
2064
2065     if LColumns is None:
2066         raise ValueError("LColumns")
2067
2068     if LVariablesToChange is None:
2069         raise ValueError("LVariablesToChange")
2070
2071     if ref_simudir is None:
2072         raise ValueError("ref_simudir")
2073
2074     if List_Multideltatime is None:
2075         raise ValueError("Problem defining simulation output results")
2076
2077     if AdvancedDebugModel is None:
2078         raise ValueError("AdvancedDebugModel")
2079
2080     if TimeoutModelExecution is None:
2081         raise ValueError("TimeoutModelExecution")
2082
2083     x_values = list(numpy.ravel(x_values_matrix))
2084     x_inputs=dict(zip(VariablesToCalibrate,x_values))
2085
2086     if os.path.isfile(ModelName):
2087         fmuname = os.path.basename(ModelName)
2088     elif os.path.isdir(ModelName):
2089         fmuname = [files for files in os.listdir(os.path.abspath(ModelName)) if files[-4:] =='.fmu'][0]
2090
2091     fmu = os.path.join(ref_simudir,fmuname)
2092
2093     multi_y_values_matrix = []
2094
2095     for etat in range(len(LNames)):
2096         y_values =[]
2097
2098         try: #to handle situations in which there is only one CL (boundary conditions) file
2099             var_int = numpy.ravel(LColumns[:,etat])
2100         except Exception:
2101             var_int = numpy.ravel(LColumns)
2102
2103         dict_inputs = dict(zip(LVariablesToChange, var_int))
2104         #
2105         dict_inputs.update(x_inputs)
2106
2107         if FMUInput:
2108             fmu_inputs = FMUInput[LNames[etat]]
2109             timestep = fmu_inputs[1][0] - fmu_inputs[0][0]  #Assuming constant timestep
2110             if AdvancedDebugModel: print(f'The timestep for {LNames[etat]} is {timestep} seconds')
2111         else:
2112             fmu_inputs = None
2113             timestep = None
2114
2115         try:
2116             stoptime_fmu = List_Multideltatime[etat][-1]
2117
2118         except IndexError:
2119             stoptime_fmu = None
2120
2121
2122         if AdvancedDebugModel == True:
2123             old_stdout = sys.stdout
2124             new_stdout = io.StringIO()
2125             sys.stdout = new_stdout
2126
2127             start_time_simulation_fmi = time.time()#timeout manangement since fmpy does not raise an error for this, it just ends the simulations and continues
2128
2129             try :
2130                 reader = simulate_fmu(fmu, output = OutputVariables, start_values = dict_inputs, debug_logging = True, timeout = TimeoutModelExecution, input = fmu_inputs, stop_time= stoptime_fmu, output_interval= timestep)
2131             except FMICallException as e:
2132                 output = new_stdout.getvalue()
2133                 sys.stdout = old_stdout
2134                 print('[ERROR] Failed simulation with the following input:\n',dict_inputs)
2135                 raise e
2136
2137             elapsed_time_simulation_fmi =  time.time() - start_time_simulation_fmi
2138             if elapsed_time_simulation_fmi > TimeoutModelExecution:
2139                 raise TimeoutError("Timeout for simulation reached, please increase it in order to be able to simulate your model and/or check if your model is correct (use TimeoutModelExecution option in configuration.py file)")
2140
2141             output = new_stdout.getvalue()
2142             sys.stdout = old_stdout
2143
2144             dir_run=ref_simudir
2145             log_file=os.path.join(dir_run,
2146                                   'log_debug_model.txt')
2147             try: #try toremove the previous file
2148                 os.remove(log_file)
2149             except Exception:
2150                 pass
2151
2152             f=open(log_file,'a')
2153             for line in output:
2154                 f.write(line)
2155
2156         else:
2157             start_time_simulation_fmi = time.time()
2158
2159             try :
2160                 reader = simulate_fmu(fmu, output = OutputVariables, start_values = dict_inputs, timeout = TimeoutModelExecution, input = fmu_inputs, stop_time= stoptime_fmu, output_interval= timestep)
2161             except FMICallException as e:
2162                 print('[ERROR] Failed simulation with the following input:\n',dict_inputs)
2163                 raise e
2164
2165             elapsed_time_simulation_fmi =  time.time() - start_time_simulation_fmi
2166             if elapsed_time_simulation_fmi > TimeoutModelExecution:
2167                 raise TimeoutError("Timeout for simulation reached, please increase it in order to be able to simulate your model and/or check if your model is correct (use TimeoutModelExecution option in configuration.py file)")
2168
2169         y_whole = [reader[y_name] for y_name in OutputVariables]
2170
2171         for y_ind in range(len(OutputVariables)):
2172             y_ind_whole_values = y_whole[y_ind]
2173             y_ind_whole_time = reader['time'] #standard output of fmu
2174
2175             if len(List_Multideltatime[etat])>1:
2176                 index_y_values = [find_nearest_index(y_ind_whole_time,time)[0] for time in List_Multideltatime[etat]]
2177             else:
2178                 index_y_values = [-1] #we only take the last point if one measure point
2179             y_ind_values = [y_ind_whole_values[i] for i in index_y_values]
2180             y_values = y_values + y_ind_values
2181
2182         y_values_matrix = y_values #pbs in results management
2183         multi_y_values_matrix = multi_y_values_matrix + y_values_matrix
2184
2185     y_values_matrix_def = numpy.ravel(numpy.array(multi_y_values_matrix))
2186     return y_values_matrix_def
2187
2188 def TOP_LEVEL_exeOpenModelicaMultiobs( x_values_matrix , KeepCalculationFolders = None, VariablesToCalibrate=None, OutputVariables=None,  LNames = None, LColumns = None, LVariablesToChange=None, ref_simudir = None,  ModelName = None, List_Multideltatime = None, Linux = None, AdvancedDebugModel = None, TimeoutModelExecution = None):
2189     "Appel du modèle en format OpenModelica et restitution des résultats"
2190
2191     if VariablesToCalibrate is None:
2192         raise ValueError("VariablesToCalibrate")
2193
2194     if OutputVariables is None:
2195         raise ValueError("OutputVariables")
2196
2197     if LNames is None:
2198         raise ValueError("LNames")
2199
2200     if LColumns is None:
2201         raise ValueError("LColumns")
2202
2203     if LVariablesToChange is None:
2204         raise ValueError("LVariablesToChange")
2205
2206     if ref_simudir is None:
2207         raise ValueError("ref_simudir")
2208
2209     if KeepCalculationFolders is None:
2210         raise ValueError("KeepCalculationFolders")
2211
2212     if Linux is None:
2213         raise ValueError("Linux")
2214
2215     if List_Multideltatime is None:
2216         raise ValueError("Problem defining simulation output results")
2217
2218     if AdvancedDebugModel is None:
2219         raise ValueError("AdvancedDebugModel")
2220
2221     if TimeoutModelExecution is None:
2222         raise ValueError("TimeoutModelExecution")
2223
2224     x_values = list(numpy.ravel(x_values_matrix))
2225     x_inputs=dict(zip(VariablesToCalibrate,x_values))
2226
2227     if os.path.isfile(ModelName):
2228         om_xml = os.path.basename(ModelName)
2229     elif os.path.isdir(ModelName):
2230         om_xml = [files for files in os.listdir(os.path.abspath(ModelName)) if files[-4:] =='.xml'][0]
2231
2232     om_xml = os.path.abspath(os.path.join(ref_simudir,om_xml))
2233
2234     multi_y_values_matrix = []
2235
2236     for etat in range(len(LNames)):
2237         y_values =[]
2238
2239         try: #to handle situations in which there is only one CL (boundary conditions) file
2240             var_int = numpy.ravel(LColumns[:,etat])
2241         except Exception:
2242             var_int = numpy.ravel(LColumns)
2243
2244         dict_inputs = dict(zip(LVariablesToChange, var_int))
2245         #
2246
2247         dict_inputs.update(x_inputs)
2248
2249
2250         results_dir = tempfile.mkdtemp( dir=os.path.dirname(om_xml) ) #to handle parallel computation
2251
2252         results_file_name = os.path.join(results_dir,os.path.basename(om_xml)[:-9] + '_' + LNames[etat].replace('.','_') + ".mat")
2253
2254         OM_execution = run_OM_model(om_xml, dict_inputs = dict_inputs, result_file_name = results_file_name , Linux = Linux, AdvancedDebugModel = AdvancedDebugModel, TimeoutModelExecution = TimeoutModelExecution)
2255
2256         try:
2257             reader = Reader(results_file_name,'dymola') #dymola even if it is OpenModelica
2258         except Exception:
2259             raise ValueError("Simulation cannot be performed: reduce the number of parameters to calibrate and/or the range in which their optimal value should be found (or modify and simplify the model to make it easier to simulate)" )
2260
2261         y_whole = [reader.values(y_name) for y_name in OutputVariables]
2262
2263         for y_ind in range(len(OutputVariables)):
2264             y_ind_whole_values = y_whole[y_ind][1]
2265             y_ind_whole_time = y_whole[y_ind][0]
2266
2267             if len(List_Multideltatime[etat])>1:
2268                 index_y_values = [find_nearest_index(y_ind_whole_time,time)[0] for time in List_Multideltatime[etat]]
2269             else:
2270                 index_y_values = [-1] #we only take the last point if one measure point
2271
2272             y_ind_values = [y_ind_whole_values[i] for i in index_y_values]
2273             y_values = y_values + y_ind_values
2274 #        y_values_matrix = numpy.asarray(y_values)
2275         y_values_matrix = y_values #pbs in results management
2276         multi_y_values_matrix = multi_y_values_matrix + y_values_matrix
2277
2278         if not KeepCalculationFolders:
2279             shutil.rmtree(results_dir, ignore_errors=True)
2280
2281     y_values_matrix_def = numpy.ravel(numpy.array(multi_y_values_matrix))
2282
2283     return y_values_matrix_def
2284
2285 def TOPLEVEL_exedymosimMultiobs_simple( x_values_matrix , VariablesToCalibrate=None, OutputVariables=None, LNames = None, ref_simudir = None, KeepCalculationFolders = None, Linux = None, List_Multideltatime = None, AdvancedDebugModel = None, TimeoutModelExecution = None):
2286     "Appel du modèle en format DYMOSIM et restitution des résultats"
2287
2288
2289     if VariablesToCalibrate is None:
2290         raise ValueError("VariablesToCalibrate")
2291
2292     if OutputVariables is None:
2293         raise ValueError("OutputVariables")
2294
2295     if LNames is None:
2296         raise ValueError("LNames")
2297
2298     if ref_simudir is None:
2299         raise ValueError("ref_simudir")
2300
2301     if KeepCalculationFolders is None:
2302         raise ValueError("KeepCalculationFolders")
2303
2304     if Linux is None:
2305         raise ValueError("Linux")
2306
2307     if List_Multideltatime is None:
2308         raise ValueError("Problem defining simulation output results")
2309
2310     if AdvancedDebugModel is None:
2311         raise ValueError("AdvancedDebugModel")
2312
2313     if TimeoutModelExecution is None:
2314         raise ValueError("TimeoutModelExecution")
2315     #
2316     # x_values = list(numpy.ravel(x_values_matrix) * Background)
2317     x_values = list(numpy.ravel(x_values_matrix))
2318     x_inputs=dict(zip(VariablesToCalibrate,x_values))
2319     # if Verbose: print("  Simulation for %s"%numpy.ravel(x_values))
2320     #
2321
2322     multi_y_values_matrix = []
2323
2324     for etat in range(len(LNames)):
2325         y_values =[]
2326
2327         simudir = tempfile.mkdtemp( dir=ref_simudir )
2328         #
2329         dict_inputs ={}
2330         dict_inputs.update( x_inputs )
2331         #
2332         shutil.copy(
2333             os.path.join(ref_simudir, str("dsin_" +LNames[etat]+ ".txt")),
2334             os.path.join(simudir, str("dsin_" +LNames[etat]+ ".txt"))
2335             )
2336         _secure_copy_textfile(
2337                         os.path.join(ref_simudir, str("dsin_" +LNames[etat]+ ".txt")),
2338                         os.path.join(simudir, str("dsin_" +LNames[etat]+ ".txt"))
2339                         )
2340
2341         auto_simul = Automatic_Simulation(
2342                                         simu_dir = simudir,
2343                                         dymosim_path = os.path.join(ref_simudir,os.pardir) ,
2344                                         dsin_name = str("dsin_" +LNames[etat]+ ".txt"),
2345                                         dsres_path = os.path.join(simudir, str("dsres" +LNames[etat]+ ".mat")),
2346                                         dict_inputs = dict_inputs,
2347                                         logfile = True,
2348                                         timeout = TimeoutModelExecution,
2349                                         without_modelicares = True,
2350                                         linux=Linux,
2351                                         )
2352         auto_simul.single_simulation()
2353
2354         if AdvancedDebugModel:
2355             dslog_file = os.path.join(simudir, 'dslog.txt')
2356             debug_model_file = os.path.join(ref_simudir,'log_debug_model.txt')
2357             try:
2358                 shutil.copy(dslog_file,debug_model_file)
2359             except Exception:
2360                 pass
2361
2362         if auto_simul.success_code == 2 :
2363             raise ValueError("The simulation falis after initialization, please check the log file and your model in order to be sure that the whole simulation can be performed (specially for dynamic models)")
2364
2365         if auto_simul.success_code == 3 :
2366             raise ValueError("The simulation did not reach the final time, probably due to a lack of time, please increase the time for model execution by modifying the TimeoutModelExecution value in configuration.py")
2367
2368         if auto_simul.success_code == 0:
2369             pass #means that everything OK, we keep going
2370         else  :
2371             raise ValueError("The NeedInitVariables iteration should be set to True, convergence issues found for %s" % str(LNames[etat])) #means this value is 1--> not correct initialization
2372
2373         reader = Reader(os.path.join(simudir, str("dsres" +LNames[etat]+ ".mat")),'dymola')
2374
2375         y_whole = [reader.values(y_name) for y_name in OutputVariables]
2376
2377         for y_ind in range(len(OutputVariables)):
2378             y_ind_whole_values = y_whole[y_ind][1]
2379             y_ind_whole_time = y_whole[y_ind][0]
2380
2381             if len(List_Multideltatime[etat])>1:
2382                 index_y_values = [find_nearest_index(y_ind_whole_time,time)[0] for time in List_Multideltatime[etat]]
2383             else:
2384                 index_y_values = [-1] #we only take the last point if one measure point
2385
2386             y_ind_values = [y_ind_whole_values[i] for i in index_y_values]
2387             y_values = y_values + y_ind_values
2388 #        y_values_matrix = numpy.asarray(y_values)
2389         y_values_matrix = y_values #pbs in results management
2390         multi_y_values_matrix = multi_y_values_matrix + y_values_matrix
2391
2392         if not KeepCalculationFolders:
2393             shutil.rmtree(simudir, ignore_errors=True)
2394
2395     y_values_matrix_def = numpy.ravel(numpy.array(multi_y_values_matrix))
2396     return y_values_matrix_def
2397
2398 def TOP_LEVEL_exedymosimMultiobs( x_values_matrix , VariablesToCalibrate=None, OutputVariables=None, LNames = None, ModelFormat = None, KeepCalculationFolders = None, Verbose = None, dict_dsin_paths = None, Linux = None, List_Multideltatime = None, AdvancedDebugModel = None, TimeoutModelExecution = None): #2ème argument à ne pas modifier
2399     "Appel du modèle en format DYMOSIM et restitution des résultats"
2400
2401     if VariablesToCalibrate is None:
2402         raise ValueError("VariablesToCalibrate")
2403
2404     if OutputVariables is None:
2405         raise ValueError("OutputVariables")
2406
2407     if LNames is None:
2408         raise ValueError("LNames")
2409
2410     if ModelFormat is None:
2411         raise ValueError("ModelFormat")
2412
2413     if KeepCalculationFolders is None:
2414         raise ValueError("KeepCalculationFolders")
2415
2416     if Verbose is None:
2417         raise ValueError("Verbose")
2418
2419     if dict_dsin_paths is None:
2420         raise ValueError("dict_dsin_paths")
2421
2422     if Linux is None:
2423         raise ValueError("Linux")
2424
2425     if List_Multideltatime is None:
2426         raise ValueError("Problem defining simulation output results")
2427
2428     if AdvancedDebugModel is None:
2429         raise ValueError("AdvancedDebugModel")
2430
2431     if TimeoutModelExecution is None:
2432         raise ValueError("TimeoutModelExecution")
2433 #
2434     # x_values = list(numpy.ravel(x_values_matrix) * Background)
2435     x_values = list(numpy.ravel(x_values_matrix))
2436     x_inputs = dict(zip(VariablesToCalibrate,x_values))
2437     x_inputs_for_CWP = {}
2438     # if Verbose: print("  Simulation for %s"%numpy.ravel(x_values))
2439
2440     multi_y_values_matrix = []
2441
2442
2443
2444     for etat in range(len(LNames)):
2445         y_values =[]
2446
2447         ref_simudir = Calibration._setModelTmpDir( None,dict_dsin_paths[LNames[etat]], ModelFormat,   'dsin.txt', os.path.join(os.path.pardir,os.path.pardir),  LNames[etat])
2448         simudir = ref_simudir
2449         dict_inputs={}
2450         dict_inputs.update(x_inputs)
2451
2452
2453         auto_simul = Automatic_Simulation(
2454             simu_dir=simudir,
2455             dymosim_path = os.path.join(simudir, os.pardir, os.pardir) ,
2456             dsres_path = os.path.join(simudir, str("dsres" +LNames[etat]+ ".mat")),
2457             dict_inputs=dict_inputs,
2458             logfile=True,
2459             timeout=TimeoutModelExecution,
2460             without_modelicares=True,
2461             linux=Linux
2462             )
2463         auto_simul.single_simulation()
2464
2465         if AdvancedDebugModel:
2466             dslog_file = os.path.join(simudir, 'dslog.txt')
2467             debug_model_file = os.path.join(simudir, os.pardir, 'log_debug_model.txt')
2468             try:
2469                 shutil.copy(dslog_file,debug_model_file)
2470             except Exception:
2471                 pass
2472
2473         if auto_simul.success_code == 2 :
2474             raise ValueError("The simulation falis after initialization, please check the log file and your model in order to be sure that the whole simulation can be performed (specially for dynamic models)")
2475
2476         if auto_simul.success_code == 3 :
2477             raise ValueError("The simulation did not reach the final time, probably due to a lack of time, please increase the time for model execution by modifying the TimeoutModelExecution value in configuration.py")
2478
2479
2480         if auto_simul.success_code == 0:
2481             reader = Reader(os.path.join(simudir, str("dsres" +LNames[etat]+ ".mat")),'dymola')
2482
2483         else:
2484
2485             path_around_simu = os.path.join(simudir,os.path.pardir,os.path.pardir)
2486             around_simu = Around_Simulation(dymola_version='2012',
2487                               curdir=path_around_simu,
2488                               source_list_iter_var = 'from_dymtranslog',
2489                               copy_from_dym_trans_log= os.path.join(path_around_simu,'ini.txt'),
2490                               mat = os.path.join(os.path.dirname(dict_dsin_paths[LNames[etat]]), str("REF_for_CWP" + ".mat")),
2491                               iter_var_values_options={'source':'from_mat','moment':'initial'},
2492                               verbose = False)
2493             around_simu.set_list_iter_var()
2494             around_simu.set_dict_iter_var()
2495
2496             reader_CWP = Reader(os.path.join(os.path.dirname(dict_dsin_paths[LNames[etat]]), str("REF_for_CWP" + ".mat")),'dymola')
2497             x_values_intemediary = [reader_CWP.values(x_name)[1][-1] for x_name in VariablesToCalibrate]
2498             x_values_from_last_calculation = numpy.asarray(x_values_intemediary)
2499             x_inputs_from_last_calculation = dict(zip(VariablesToCalibrate,x_values_from_last_calculation))
2500
2501
2502             for var_calib in x_inputs_from_last_calculation.keys():
2503                 x_inputs_for_CWP[var_calib] = (x_inputs_from_last_calculation[var_calib], x_inputs[var_calib])
2504
2505             dict_var_to_fix2 = Dict_Var_To_Fix(option='automatic',
2506                                   dict_auto_var_to_fix=x_inputs_for_CWP)
2507
2508             dict_var_to_fix2.set_dict_var_to_fix()
2509
2510             if Verbose:
2511                 LOG_FILENAME = os.path.join(simudir,'change_working_point.log')
2512                 for handler in logging.root.handlers[:]:
2513                     logging.root.removeHandler(handler)
2514                 logging.basicConfig(filename=LOG_FILENAME,level=logging.DEBUG,filemode='w')
2515
2516             working_point_modif = Working_Point_Modification(main_dir = simudir,
2517                          simu_material_dir = simudir,
2518                          dymosim_path = os.path.join(simudir, os.pardir, os.pardir),
2519                          simu_dir = 'SIMUS',
2520                          store_res_dir = 'RES',
2521                          dict_var_to_fix = dict_var_to_fix2.dict_var_to_fix,
2522                          around_simulation0 = around_simu,
2523                          var_to_follow_path= os.path.join(simudir,'var_to_follow.csv'),
2524                          gen_scripts_ini= False,
2525                          nit_max= 1000000000000000,
2526                          min_step_val = 0.00000000000005,
2527                          timeout = TimeoutModelExecution,
2528                          linux=Linux)
2529
2530             working_point_modif.create_working_directory()
2531             working_point_modif.working_point_modification(skip_reference_simulation=True)
2532
2533             if AdvancedDebugModel:
2534                 dslog_file = os.path.join(simudir,'SIMUS', 'dslog.txt')
2535                 debug_model_file = os.path.join(simudir, os.pardir, 'log_debug_model.txt')
2536                 try:
2537                     shutil.copy(dslog_file,debug_model_file)
2538                 except Exception:
2539                     pass
2540             try:
2541                 reader = Reader(os.path.join(simudir, 'RES','1.0.mat'),'dymola')
2542             except Exception:
2543                 raise ValueError("Simulation cannot be performed: reduce the number of parameters to calibrate and/or the range in which their optimal value should be found (or modify and simplify the model to make it easier to simulate)" )
2544
2545
2546         y_whole = [reader.values(y_name) for y_name in OutputVariables]
2547
2548         for y_ind in range(len(OutputVariables)):
2549             y_ind_whole_values = y_whole[y_ind][1]
2550             y_ind_whole_time = y_whole[y_ind][0]
2551             if len(List_Multideltatime[etat])>1:
2552                 index_y_values = [find_nearest_index(y_ind_whole_time,time)[0] for time in List_Multideltatime[etat]]
2553             else:
2554                 index_y_values = [-1] #we only take the last point if one measure point
2555
2556             y_ind_values = [y_ind_whole_values[i] for i in index_y_values]
2557             y_values = y_values + y_ind_values
2558 #        y_values_matrix = numpy.asarray(y_values)
2559         y_values_matrix = y_values #pbs in results management
2560         multi_y_values_matrix = multi_y_values_matrix + y_values_matrix
2561
2562         if not KeepCalculationFolders:
2563             shutil.rmtree(simudir, ignore_errors=True)
2564     y_values_matrix_def = numpy.ravel(numpy.array(multi_y_values_matrix))
2565     return y_values_matrix_def
2566
2567 def run_OM_model(xml_file, #req arg, file path to _init.xml file
2568                  result_file_name = None, #file path for storing results
2569                  dict_inputs = None, #optionnal argument : for overriding paramters
2570                  Linux = False,
2571                  AdvancedDebugModel = None,
2572                  TimeoutModelExecution = None):
2573
2574     Command=[] #initialize command with list of argument
2575
2576     xml_file = os.path.abspath(xml_file)
2577
2578     #set main command to be runned
2579     if Linux:
2580         OM_binary = xml_file.replace('_init.xml','') # not tested
2581     else: #WIndows
2582         OM_binary = xml_file.replace('_init.xml','.exe')
2583     Command.append(OM_binary)
2584
2585     #Get base path for binaries and input
2586
2587
2588     inputPath='-inputPath='+os.path.dirname(xml_file)
2589     Command.append(inputPath)
2590
2591     #Generate override command
2592     if dict_inputs !=None:
2593         Override='-override='
2594         for elt in dict_inputs.items():
2595             Override=Override+elt[0]+'='+str(elt[1])+','
2596         Override=Override.rstrip(',') #remove last ',' if any
2597         Command.append(Override)
2598
2599     #Generate name of result file
2600     if result_file_name != None:
2601         result='-r='+result_file_name
2602         Command.append(result)
2603     else:
2604         result='-r='+OM_binary+'.mat' #by default result is stored next to _init and bin file
2605         Command.append(result)
2606
2607     result='-w' #by default result is stored next to _init and bin file
2608     Command.append(result)
2609
2610     result='-lv='+'LOG_STATS,LOG_NLS_V' #by default result is stored next to _init and bin file
2611     Command.append(result)
2612
2613     inputPath='-outputPath='+os.path.dirname(xml_file)
2614     Command.append(inputPath)
2615
2616    #launch calculation
2617     proc = subprocess.Popen(Command, #Command in the form of a text list
2618                             stderr=subprocess.STDOUT,
2619                             stdout=subprocess.PIPE,
2620                             universal_newlines=True)
2621     # for line in proc.stdout.readlines():
2622     #     print(line)
2623
2624     if AdvancedDebugModel == True:
2625         dir_run=os.path.join(os.path.dirname(result_file_name),os.pardir)
2626         log_file=os.path.join(dir_run,
2627                               'log_debug_model.txt')
2628
2629         try:
2630             os.remove(log_file)
2631         except Exception:
2632             pass
2633         try:
2634             f=open(log_file,'a')
2635             for line in proc.stdout.readlines():
2636                 f.write(line)
2637             f.close()
2638             proc.communicate(timeout = TimeoutModelExecution)
2639         except subprocess.TimeoutExpired:
2640             raise ValueError("Timeout for simulation reached, please increase it in order to be able to simulate your model and/or check if your model is correct (use TimeoutModelExecution option in configuration.py file)")
2641
2642     else:
2643         try:
2644             proc.communicate(timeout = TimeoutModelExecution)
2645         except Exception:
2646             raise ValueError("Timeout for simulation reached, please increase it in order to be able to simulate your model and/or check if your model is correct (use TimeoutModelExecution option in configuration.py file)")
2647
2648     # proc.communicate()
2649
2650     # r = proc.poll()
2651     # while r !=0:
2652     #     r = proc.poll()
2653
2654     # if proc.returncode !=0:
2655     #     raise ValueError("Simulation not ended")
2656
2657
2658     # sys.stdout.flush()
2659     # sys.stderr.flush()
2660
2661     # dir_run=os.path.dirname(result_file_name)
2662     # log_file=os.path.join(dir_run,
2663     #                       'log.txt')
2664     # try:
2665     #     proc.wait(timeout=50)
2666
2667     #     f=open(log_file,'a')
2668     #     for line in proc.stdout.readlines():
2669     #         f.write(line)
2670
2671     #     f.close()
2672     #     return_code=proc.returncode
2673
2674     # except subprocess.TimeoutExpired:
2675
2676     #     f=open(log_file,'a')
2677     #     for line in proc.stdout.readlines():
2678     #         f.write(line)
2679
2680     #     f.write('Simulation stoppe because of TimeOut')
2681     #     f.close()
2682
2683     #     return_code=2
2684
2685     return #return_code
2686
2687 def readObsnamesfile(__filenames=None):
2688     "Provisionnel pour lire les noms des observations d'un fichier"
2689     for __filename in __filenames:
2690         try:
2691             df = pandas.read_csv(__filename, sep = ";", header =0) #so that repeated names are not modified
2692             obsnames_infile = df.loc[0, :].values.tolist()[2:]
2693         except Exception:
2694             df_tmp = pandas.read_csv(__filename, sep = ";", header =1)
2695             with open(__filename, 'r') as infile:
2696                 readie=csv.reader(infile, delimiter=';')
2697                 with open('tmp_obs_file.csv', 'wt', newline='') as output:
2698                     outwriter=csv.writer(output, delimiter=';')
2699                     list_row = ['#'] + df_tmp.columns
2700                     outwriter.writerow(list_row)
2701                     for row in readie:
2702                          outwriter.writerow(row)
2703
2704             df = pandas.read_csv('tmp_obs_file.csv', sep = ";", header =0) #so that repeated names are not modified
2705             df = df.iloc[1: , :]
2706             obsnames_infile = df.iloc[0, :].values.tolist()[2:]
2707
2708             try:
2709                 os.remove('tmp_obs_file.csv')
2710             except Exception:
2711                 pass
2712     return obsnames_infile
2713
2714 def readDatesHours(__filename=None):
2715     "Provisionnel pour lire les heures et les dates d'une simulation dynamique"
2716     df = pandas.read_csv(__filename, sep = ";", header =1)
2717     dates = numpy.array(df['Date'])
2718     hours = numpy.array(df['Hour'])
2719     return (dates,hours)
2720
2721 def readMultiDatesHours(__filenames=None):
2722     "Provisionnel pour lire les heures et les dates d'une simulation dynamique dans le cas multi-observations"
2723     Multidates =[]
2724     Multihours = []
2725     for __filename in __filenames:
2726         df = pandas.read_csv(__filename, sep = ";", header =1)
2727         dates = numpy.array(df[df.columns[0]])
2728         hours = numpy.array(df[df.columns[1]])
2729         Multidates.append( dates )
2730         Multihours.append( hours )
2731     return (Multidates,Multihours)
2732
2733 def find_nearest_index(array, value):
2734     "Trouver le point de la simulation qui se rapproche le plus aux observations"
2735     array = numpy.asarray(array)
2736     idx = (numpy.abs(array - value)).argmin()
2737     return [idx, array[idx]]
2738 # ==============================================================================
2739 if __name__ == "__main__":
2740     print('\n  AUTODIAGNOSTIC\n  ==============\n')
2741     print("  Module name..............: %s"%name)
2742     print("  Module version...........: %s"%version)
2743     print("  Module year..............: %s"%year)
2744     print("")
2745     print("  Configuration attributes.:")
2746     for __k, __v in _configuration.items():
2747         print("%26s : %s"%(__k,__v))
2748     print("")