1 # -*- coding: utf-8 -*-
3 # Copyright (C) 2008-2020 EDF R&D
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Lesser General Public
7 # License as published by the Free Software Foundation; either
8 # version 2.1 of the License.
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # Lesser General Public License for more details.
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with this library; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 # See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
21 # Author: Jean-Philippe Argaud, jean-philippe.argaud@edf.fr, EDF R&D
24 Normalized interface for ADAO scripting (generic API)
26 __author__ = "Jean-Philippe ARGAUD"
32 from daCore.BasicObjects import State, Covariance, FullOperator, Operator
33 from daCore.BasicObjects import AlgorithmAndParameters, DataObserver
34 from daCore.BasicObjects import RegulationAndParameters, CaseLogger
35 from daCore import PlatformInfo
37 from daCore import ExtendedLogging ; ExtendedLogging.ExtendedLogging() # A importer en premier
40 # ==============================================================================
42 """ ADAO Internal Data Structure Model """
43 def __init__(self, name = "", addViewers=None):
44 self.__name = str(name)
45 self.__directory = None
46 self.__case = CaseLogger(self.__name, "case", addViewers)
48 self.__adaoObject = {}
49 self.__StoredInputs = {}
50 self.__PostAnalysis = []
53 self.__Concepts = [ # Liste exhaustive
54 "AlgorithmParameters",
62 "ObservationOperator",
69 "RegulationParameters",
73 for ename in self.__Concepts:
74 self.__adaoObject[ename] = None
75 for ename in ("ObservationOperator", "EvolutionModel", "ControlModel"):
76 self.__adaoObject[ename] = {}
77 for ename in ("Observer",):
78 self.__adaoObject[ename] = []
79 self.__StoredInputs[ename] = [] # Vide par defaut
80 self.__StoredInputs["Name"] = self.__name
81 self.__StoredInputs["Directory"] = self.__directory
83 # Récupère le chemin du répertoire parent et l'ajoute au path
84 # (Cela complète l'action de la classe PathManagement dans PlatformInfo,
85 # qui est activée dans Persistence)
86 self.__parent = os.path.abspath(os.path.join(os.path.dirname(__file__),".."))
87 sys.path.insert(0, self.__parent)
88 sys.path = PlatformInfo.uniq( sys.path ) # Conserve en unique exemplaire chaque chemin
91 Concept = None, # Premier argument
99 DiagonalSparseMatrix = None,
100 ExtraArguments = None,
102 InputFunctionAsMulti = False,
104 ObjectFunction = None,
108 ScalarSparseMatrix = None,
114 ThreeFunctions = None,
119 "Interface unique de definition de variables d'entrees par argument"
120 self.__case.register("set",dir(),locals(),None,True)
122 if Concept in ("Background", "CheckingPoint", "ControlInput", "Observation"):
123 commande = getattr(self,"set"+Concept)
124 commande(Vector, VectorSerie, Script, DataFile, ColNames, ColMajor, Stored, Scheduler, Checked )
125 elif Concept in ("BackgroundError", "ObservationError", "EvolutionError"):
126 commande = getattr(self,"set"+Concept)
127 commande(Matrix, ScalarSparseMatrix, DiagonalSparseMatrix,
128 Script, Stored, ObjectMatrix, Checked )
129 elif Concept == "AlgorithmParameters":
130 self.setAlgorithmParameters( Algorithm, Parameters, Script )
131 elif Concept == "RegulationParameters":
132 self.setRegulationParameters( Algorithm, Parameters, Script )
133 elif Concept == "Name":
135 elif Concept == "Directory":
136 self.setDirectory(String)
137 elif Concept == "Debug":
139 elif Concept == "NoDebug":
141 elif Concept == "Observer":
142 self.setObserver( Variable, Template, String, Script, Info, ObjectFunction, Scheduler )
143 elif Concept == "ObservationOperator":
144 self.setObservationOperator(
145 Matrix, OneFunction, ThreeFunctions, AppliedInXb,
146 Parameters, Script, ExtraArguments,
147 Stored, AvoidRC, InputFunctionAsMulti, Checked )
148 elif Concept in ("EvolutionModel", "ControlModel"):
149 commande = getattr(self,"set"+Concept)
151 Matrix, OneFunction, ThreeFunctions,
152 Parameters, Script, Scheduler, ExtraArguments,
153 Stored, AvoidRC, InputFunctionAsMulti, Checked )
155 raise ValueError("the variable named '%s' is not allowed."%str(Concept))
156 except Exception as e:
157 if isinstance(e, SyntaxError): msg = "at %s: %s"%(e.offset, e.text)
159 raise ValueError(("during settings, the following error occurs:\n"+\
160 "\n%s %s\n\nSee also the potential messages, "+\
161 "which can show the origin of the above error, "+\
162 "in the launching terminal.")%(str(e),msg))
164 # -----------------------------------------------------------
166 def setBackground(self,
176 "Definition d'un concept de calcul"
177 Concept = "Background"
178 self.__case.register("set"+Concept, dir(), locals())
179 self.__adaoObject[Concept] = State(
182 asPersistentVector = VectorSerie,
183 asScript = self.__with_directory(Script),
184 asDataFile = DataFile,
187 scheduledBy = Scheduler,
188 toBeChecked = Checked,
191 self.__StoredInputs[Concept] = self.__adaoObject[Concept].getO()
194 def setCheckingPoint(self,
204 "Definition d'un concept de calcul"
205 Concept = "CheckingPoint"
206 self.__case.register("set"+Concept, dir(), locals())
207 self.__adaoObject[Concept] = State(
210 asPersistentVector = VectorSerie,
211 asScript = self.__with_directory(Script),
212 asDataFile = DataFile,
215 scheduledBy = Scheduler,
216 toBeChecked = Checked,
219 self.__StoredInputs[Concept] = self.__adaoObject[Concept].getO()
222 def setControlInput(self,
232 "Definition d'un concept de calcul"
233 Concept = "ControlInput"
234 self.__case.register("set"+Concept, dir(), locals())
235 self.__adaoObject[Concept] = State(
238 asPersistentVector = VectorSerie,
239 asScript = self.__with_directory(Script),
240 asDataFile = DataFile,
243 scheduledBy = Scheduler,
244 toBeChecked = Checked,
247 self.__StoredInputs[Concept] = self.__adaoObject[Concept].getO()
250 def setObservation(self,
260 "Definition d'un concept de calcul"
261 Concept = "Observation"
262 self.__case.register("set"+Concept, dir(), locals())
263 self.__adaoObject[Concept] = State(
266 asPersistentVector = VectorSerie,
267 asScript = self.__with_directory(Script),
268 asDataFile = DataFile,
271 scheduledBy = Scheduler,
272 toBeChecked = Checked,
275 self.__StoredInputs[Concept] = self.__adaoObject[Concept].getO()
278 def setBackgroundError(self,
280 ScalarSparseMatrix = None,
281 DiagonalSparseMatrix = None,
286 "Definition d'un concept de calcul"
287 Concept = "BackgroundError"
288 self.__case.register("set"+Concept, dir(), locals())
289 self.__adaoObject[Concept] = Covariance(
291 asCovariance = Matrix,
292 asEyeByScalar = ScalarSparseMatrix,
293 asEyeByVector = DiagonalSparseMatrix,
294 asCovObject = ObjectMatrix,
295 asScript = self.__with_directory(Script),
296 toBeChecked = Checked,
299 self.__StoredInputs[Concept] = self.__adaoObject[Concept].getO()
302 def setObservationError(self,
304 ScalarSparseMatrix = None,
305 DiagonalSparseMatrix = None,
310 "Definition d'un concept de calcul"
311 Concept = "ObservationError"
312 self.__case.register("set"+Concept, dir(), locals())
313 self.__adaoObject[Concept] = Covariance(
315 asCovariance = Matrix,
316 asEyeByScalar = ScalarSparseMatrix,
317 asEyeByVector = DiagonalSparseMatrix,
318 asCovObject = ObjectMatrix,
319 asScript = self.__with_directory(Script),
320 toBeChecked = Checked,
323 self.__StoredInputs[Concept] = self.__adaoObject[Concept].getO()
326 def setEvolutionError(self,
328 ScalarSparseMatrix = None,
329 DiagonalSparseMatrix = None,
334 "Definition d'un concept de calcul"
335 Concept = "EvolutionError"
336 self.__case.register("set"+Concept, dir(), locals())
337 self.__adaoObject[Concept] = Covariance(
339 asCovariance = Matrix,
340 asEyeByScalar = ScalarSparseMatrix,
341 asEyeByVector = DiagonalSparseMatrix,
342 asCovObject = ObjectMatrix,
343 asScript = self.__with_directory(Script),
344 toBeChecked = Checked,
347 self.__StoredInputs[Concept] = self.__adaoObject[Concept].getO()
350 def setObservationOperator(self,
353 ThreeFunctions = None,
357 ExtraArguments = None,
360 InputFunctionAsMulti = False,
362 "Definition d'un concept de calcul"
363 Concept = "ObservationOperator"
364 self.__case.register("set"+Concept, dir(), locals())
365 self.__adaoObject[Concept] = FullOperator(
368 asOneFunction = OneFunction,
369 asThreeFunctions = ThreeFunctions,
370 asScript = self.__with_directory(Script),
372 appliedInX = AppliedInXb,
373 extraArguments = ExtraArguments,
375 inputAsMF = InputFunctionAsMulti,
377 toBeChecked = Checked,
380 self.__StoredInputs[Concept] = self.__adaoObject[Concept].getO()
383 def setEvolutionModel(self,
386 ThreeFunctions = None,
390 ExtraArguments = None,
393 InputFunctionAsMulti = False,
395 "Definition d'un concept de calcul"
396 Concept = "EvolutionModel"
397 self.__case.register("set"+Concept, dir(), locals())
398 self.__adaoObject[Concept] = FullOperator(
401 asOneFunction = OneFunction,
402 asThreeFunctions = ThreeFunctions,
403 asScript = self.__with_directory(Script),
406 extraArguments = ExtraArguments,
408 inputAsMF = InputFunctionAsMulti,
409 scheduledBy = Scheduler,
410 toBeChecked = Checked,
413 self.__StoredInputs[Concept] = self.__adaoObject[Concept].getO()
416 def setControlModel(self,
419 ThreeFunctions = None,
423 ExtraArguments = None,
426 InputFunctionAsMulti = False,
428 "Definition d'un concept de calcul"
429 Concept = "ControlModel"
430 self.__case.register("set"+Concept, dir(), locals())
431 self.__adaoObject[Concept] = FullOperator(
434 asOneFunction = OneFunction,
435 asThreeFunctions = ThreeFunctions,
436 asScript = self.__with_directory(Script),
439 extraArguments = ExtraArguments,
441 inputAsMF = InputFunctionAsMulti,
442 scheduledBy = Scheduler,
443 toBeChecked = Checked,
446 self.__StoredInputs[Concept] = self.__adaoObject[Concept].getO()
449 def setName(self, String=None):
450 "Definition d'un concept de calcul"
451 self.__case.register("setName",dir(),locals())
452 if String is not None:
453 self.__name = str(String)
456 self.__StoredInputs["Name"] = self.__name
458 def setDirectory(self, String=None):
459 "Definition d'un concept de calcul"
460 self.__case.register("setDirectory",dir(),locals())
461 if os.path.isdir(os.path.abspath(str(String))):
462 self.__directory = os.path.abspath(str(String))
464 self.__directory = None
465 self.__StoredInputs["Directory"] = self.__directory
467 def setDebug(self, __level = 10):
468 "NOTSET=0 < DEBUG=10 < INFO=20 < WARNING=30 < ERROR=40 < CRITICAL=50"
469 self.__case.register("setDebug",dir(),locals())
470 log = logging.getLogger()
471 log.setLevel( __level )
472 self.__StoredInputs["Debug"] = __level
473 self.__StoredInputs["NoDebug"] = False
476 def setNoDebug(self):
477 "NOTSET=0 < DEBUG=10 < INFO=20 < WARNING=30 < ERROR=40 < CRITICAL=50"
478 self.__case.register("setNoDebug",dir(),locals())
479 log = logging.getLogger()
480 log.setLevel( logging.WARNING )
481 self.__StoredInputs["Debug"] = logging.WARNING
482 self.__StoredInputs["NoDebug"] = True
485 def setAlgorithmParameters(self,
489 "Definition d'un concept de calcul"
490 Concept = "AlgorithmParameters"
491 self.__case.register("set"+Concept, dir(), locals())
492 self.__adaoObject[Concept] = AlgorithmAndParameters(
494 asAlgorithm = Algorithm,
496 asScript = self.__with_directory(Script),
500 def updateAlgorithmParameters(self,
503 "Mise a jour d'un concept de calcul"
504 if "AlgorithmParameters" not in self.__adaoObject or self.__adaoObject["AlgorithmParameters"] is None:
505 raise ValueError("\n\nNo algorithm registred, set one before updating parameters or executing\n")
506 self.__adaoObject["AlgorithmParameters"].updateParameters(
508 asScript = self.__with_directory(Script),
512 def setRegulationParameters(self,
516 "Definition d'un concept de calcul"
517 Concept = "RegulationParameters"
518 self.__case.register("set"+Concept, dir(), locals())
519 self.__adaoObject[Concept] = RegulationAndParameters(
521 asAlgorithm = Algorithm,
523 asScript = self.__with_directory(Script),
529 def setObserver(self,
535 ObjectFunction = None,
537 "Definition d'un concept de calcul"
539 self.__case.register("set"+Concept, dir(), locals())
540 self.__adaoObject[Concept].append( DataObserver(
542 onVariable = Variable,
543 asTemplate = Template,
545 asScript = self.__with_directory(Script),
546 asObsObject = ObjectFunction,
548 scheduledBy = Scheduler,
549 withAlgo = self.__adaoObject["AlgorithmParameters"]
553 def removeObserver(self,
555 ObjectFunction = None,
557 "Permet de retirer un observer à une ou des variable nommée"
558 if "AlgorithmParameters" not in self.__adaoObject:
559 raise ValueError("No algorithm registred, ask for one before removing observers")
561 # Vérification du nom de variable et typage
562 # -----------------------------------------
563 if isinstance(Variable, str):
564 VariableNames = (Variable,)
565 elif isinstance(Variable, list):
566 VariableNames = tuple(map( str, Variable ))
568 raise ValueError("The observer requires a name or a list of names of variables.")
570 # Association interne de l'observer à la variable
571 # -----------------------------------------------
572 for ename in VariableNames:
573 if ename not in self.__adaoObject["AlgorithmParameters"]:
574 raise ValueError("An observer requires to be removed on a variable named %s which does not exist."%ename)
576 return self.__adaoObject["AlgorithmParameters"].removeObserver( ename, ObjectFunction )
578 # -----------------------------------------------------------
580 def get(self, Concept=None, noDetails=True ):
581 "Recuperation d'une sortie du calcul"
582 if Concept is not None:
584 self.__case.register("get", dir(), locals(), Concept) # Break pickle in Python 2
587 if Concept in self.__StoredInputs:
588 return self.__StoredInputs[Concept]
590 elif self.__adaoObject["AlgorithmParameters"] is not None and Concept == "AlgorithmParameters":
591 return self.__adaoObject["AlgorithmParameters"].get()
593 elif self.__adaoObject["AlgorithmParameters"] is not None and Concept in self.__adaoObject["AlgorithmParameters"]:
594 return self.__adaoObject["AlgorithmParameters"].get( Concept )
596 elif Concept == "AlgorithmRequiredParameters" and self.__adaoObject["AlgorithmParameters"] is not None:
597 return self.__adaoObject["AlgorithmParameters"].getAlgorithmRequiredParameters(noDetails)
599 elif Concept == "AlgorithmRequiredInputs" and self.__adaoObject["AlgorithmParameters"] is not None:
600 return self.__adaoObject["AlgorithmParameters"].getAlgorithmInputArguments()
602 elif Concept == "AlgorithmAttributes" and self.__adaoObject["AlgorithmParameters"] is not None:
603 return self.__adaoObject["AlgorithmParameters"].getAlgorithmAttributes()
606 raise ValueError("The requested key \"%s\" does not exists as an input or a stored variable."%Concept)
609 allvariables.update( {"AlgorithmParameters":self.__adaoObject["AlgorithmParameters"].get()} )
610 # allvariables.update( self.__adaoObject["AlgorithmParameters"].get() )
611 allvariables.update( self.__StoredInputs )
612 allvariables.pop('Observer', None)
615 # -----------------------------------------------------------
617 def get_available_variables(self):
619 Renvoie les variables potentiellement utilisables pour l'étude,
620 initialement stockées comme données d'entrées ou dans les algorithmes,
621 identifiés par les chaînes de caractères. L'algorithme doit avoir été
622 préalablement choisi sinon la méthode renvoie "None".
624 if len(list(self.__adaoObject["AlgorithmParameters"].keys())) == 0 and \
625 len(list(self.__StoredInputs.keys())) == 0:
629 if len(list(self.__adaoObject["AlgorithmParameters"].keys())) > 0:
630 variables.extend(list(self.__adaoObject["AlgorithmParameters"].keys()))
631 if len(list(self.__StoredInputs.keys())) > 0:
632 variables.extend( list(self.__StoredInputs.keys()) )
633 variables.remove('Observer')
637 def get_available_algorithms(self):
639 Renvoie la liste des algorithmes potentiellement utilisables, identifiés
640 par les chaînes de caractères.
643 for directory in sys.path:
644 trypath = os.path.join(directory,"daAlgorithms")
645 if os.path.isdir(trypath):
646 for fname in os.listdir(trypath):
647 if os.path.isfile(os.path.join(trypath,fname)):
648 root, ext = os.path.splitext(fname)
649 if ext != ".py": continue
650 fc = open(os.path.join(trypath,fname)).read()
651 iselal = bool("class ElementaryAlgorithm" in fc)
652 if iselal and ext == '.py' and root != '__init__':
657 def get_algorithms_main_path(self):
659 Renvoie le chemin pour le répertoire principal contenant les algorithmes
663 def add_algorithms_path(self, Path=None):
665 Ajoute au chemin de recherche des algorithmes un répertoire dans lequel
666 se trouve un sous-répertoire "daAlgorithms"
668 if not os.path.isdir(Path):
669 raise ValueError("The given "+Path+" argument must exist as a directory")
670 if not os.path.isdir(os.path.join(Path,"daAlgorithms")):
671 raise ValueError("The given \""+Path+"\" argument must contain a subdirectory named \"daAlgorithms\"")
672 if not os.path.isfile(os.path.join(Path,"daAlgorithms","__init__.py")):
673 raise ValueError("The given \""+Path+"/daAlgorithms\" path must contain a file named \"__init__.py\"")
674 sys.path.insert(0, os.path.abspath(Path))
675 sys.path = PlatformInfo.uniq( sys.path ) # Conserve en unique exemplaire chaque chemin
678 # -----------------------------------------------------------
680 def execute(self, Executor=None, SaveCaseInFile=None, nextStep=False):
681 "Lancement du calcul"
682 self.__case.register("execute",dir(),locals(),None,True)
683 self.updateAlgorithmParameters(Parameters={"nextStep":bool(nextStep)})
684 if not nextStep: Operator.CM.clearCache()
686 if Executor == "YACS": self.__executeYACSScheme( SaveCaseInFile )
687 else: self.__executePythonScheme( SaveCaseInFile )
688 except Exception as e:
689 if isinstance(e, SyntaxError): msg = "at %s: %s"%(e.offset, e.text)
691 raise ValueError(("during execution, the following error occurs:\n"+\
692 "\n%s %s\n\nSee also the potential messages, "+\
693 "which can show the origin of the above error, "+\
694 "in the launching terminal.\n")%(str(e),msg))
697 def __executePythonScheme(self, FileName=None):
698 "Lancement du calcul"
699 self.__case.register("executePythonScheme", dir(), locals())
700 if FileName is not None:
701 self.dump( FileName, "TUI")
702 self.__adaoObject["AlgorithmParameters"].executePythonScheme( self.__adaoObject )
705 def __executeYACSScheme(self, FileName=None):
706 "Lancement du calcul"
707 self.__case.register("executeYACSScheme", dir(), locals())
708 self.dump( FileName, "YACS")
709 self.__adaoObject["AlgorithmParameters"].executeYACSScheme( FileName )
712 # -----------------------------------------------------------
714 def dump(self, FileName=None, Formater="TUI"):
715 "Restitution normalisée des commandes"
716 __Upa = "\n".join(self.__PostAnalysis)
717 return self.__case.dump(FileName, Formater, __Upa)
719 def load(self, FileName=None, Content=None, Object=None, Formater="TUI"):
720 "Chargement normalisé des commandes"
721 __commands = self.__case.load(FileName, Content, Object, Formater)
722 from numpy import array, matrix
723 for __command in __commands:
724 if __command.find("set")>-1 and __command.find("set_")<0:
725 exec("self."+__command)
727 self.__PostAnalysis.append(__command)
731 FileNameFrom=None, ContentFrom=None, ObjectFrom=None, FormaterFrom="TUI",
732 FileNameTo=None, FormaterTo="TUI",
734 "Conversion normalisée des commandes"
736 FileName=FileNameFrom, Content=ContentFrom, Object=ObjectFrom, Formater=FormaterFrom
738 FileName=FileNameTo, Formater=FormaterTo
742 "Effacement du contenu du cas en cours"
743 self.__init__(self.__name)
745 # -----------------------------------------------------------
747 def __with_directory(self, __filename=None):
748 if os.path.exists(str(__filename)):
749 __fullpath = __filename
750 elif os.path.exists(os.path.join(str(self.__directory), str(__filename))):
751 __fullpath = os.path.join(self.__directory, str(__filename))
753 __fullpath = __filename
757 "Clarifie la visibilité des méthodes"
758 return ['set', 'get', 'execute', 'dump', 'load', '__doc__', '__init__', '__module__']
760 def prepare_to_pickle(self):
761 "Retire les variables non pickelisables, avec recopie efficace"
762 if self.__adaoObject['AlgorithmParameters'] is not None:
763 for k in self.__adaoObject['AlgorithmParameters'].keys():
764 if k == "Algorithm": continue
765 if k in self.__StoredInputs:
766 raise ValueError("The key \"%s\" to be transfered for pickling will overwrite an existing one."%(k,))
767 if self.__adaoObject['AlgorithmParameters'].hasObserver( k ):
768 self.__adaoObject['AlgorithmParameters'].removeObserver( k, "", True )
769 self.__StoredInputs[k] = self.__adaoObject['AlgorithmParameters'].pop(k, None)
770 if sys.version_info[0] == 2:
771 del self.__adaoObject # Because it breaks pickle in Python 2. Not required for Python 3
772 del self.__case # Because it breaks pickle in Python 2. Not required for Python 3
773 if sys.version_info.major < 3:
776 return self.__StoredInputs
778 # ==============================================================================
779 if __name__ == "__main__":
780 print('\n AUTODIAGNOSTIC\n')