1 # -*- coding: utf-8 -*-
3 # Copyright (C) 2008-2019 EDF R&D
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Lesser General Public
7 # License as published by the Free Software Foundation; either
8 # version 2.1 of the License.
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # Lesser General Public License for more details.
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with this library; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 # See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
21 # Author: Jean-Philippe Argaud, jean-philippe.argaud@edf.fr, EDF R&D
24 Définit les outils d'interfaces normalisées de cas.
26 __author__ = "Jean-Philippe ARGAUD"
35 from daCore import Persistence
36 from daCore import PlatformInfo
37 from daCore import Templates
39 # ==============================================================================
40 class GenericCaseViewer(object):
42 Gestion des commandes de creation d'une vue de cas
44 def __init__(self, __name="", __objname="case", __content=None, __object=None):
45 "Initialisation et enregistrement de l'entete"
46 self._name = str(__name)
47 self._objname = str(__objname)
49 self._switchoff = False
50 self._numobservers = 2
51 self._content = __content
52 self._object = __object
53 self._missing = """raise ValueError("This case requires beforehand to import or define the variable named <%s>. When corrected, remove this command, correct and uncomment the following one.")\n# """
54 def _append(self, *args):
55 "Transformation de commande individuelle en enregistrement"
56 raise NotImplementedError()
57 def _extract(self, *args):
58 "Transformation d'enregistrement en commande individuelle"
59 raise NotImplementedError()
60 def _finalize(self, __upa=None):
61 "Enregistrement du final"
62 if __upa is not None and len(__upa)>0:
63 self._lineSerie.append("%s.execute()"%(self._objname,))
64 self._lineSerie.append(__upa)
65 def _addLine(self, line=""):
66 "Ajoute un enregistrement individuel"
67 self._lineSerie.append(line)
68 def _get_objname(self):
70 def dump(self, __filename=None, __upa=None):
71 "Restitution normalisée des commandes"
73 __text = "\n".join(self._lineSerie)
75 if __filename is not None:
76 __file = os.path.abspath(__filename)
77 __fid = open(__file,"w")
81 def load(self, __filename=None, __content=None, __object=None):
82 "Chargement normalisé des commandes"
83 if __filename is not None and os.path.exists(__filename):
84 self._content = open(__filename, 'r').read()
85 elif __content is not None and type(__content) is str:
86 self._content = __content
87 elif __object is not None and type(__object) is dict:
88 self._object = copy.deepcopy(__object)
90 pass # use "self._content" from initialization
91 __commands = self._extract(self._content, self._object)
94 class _TUIViewer(GenericCaseViewer):
96 Etablissement des commandes d'un cas ADAO TUI (Cas<->TUI)
98 def __init__(self, __name="", __objname="case", __content=None, __object=None):
99 "Initialisation et enregistrement de l'entete"
100 GenericCaseViewer.__init__(self, __name, __objname, __content, __object)
101 self._addLine("# -*- coding: utf-8 -*-")
102 self._addLine("#\n# Python script using ADAO TUI\n#")
103 self._addLine("from numpy import array, matrix")
104 self._addLine("from adao import adaoBuilder")
105 self._addLine("%s = adaoBuilder.New('%s')"%(self._objname, self._name))
106 if self._content is not None:
107 for command in self._content:
108 self._append(*command)
109 def _append(self, __command=None, __keys=None, __local=None, __pre=None, __switchoff=False):
110 "Transformation d'une commande individuelle en un enregistrement"
111 if __command is not None and __keys is not None and __local is not None:
113 if __pre is not None:
114 __text += "%s = "%__pre
115 __text += "%s.%s( "%(self._objname,str(__command))
116 if "self" in __keys: __keys.remove("self")
117 if __command not in ("set","get") and "Concept" in __keys: __keys.remove("Concept")
120 if __v is None: continue
121 if k == "Checked" and not __v: continue
122 if k == "Stored" and not __v: continue
123 if k == "ColMajor" and not __v: continue
124 if k == "InputFunctionAsMulti" and not __v: continue
125 if k == "AvoidRC" and __v: continue
126 if k == "noDetails": continue
127 if isinstance(__v,Persistence.Persistence): __v = __v.values()
128 if callable(__v): __text = self._missing%__v.__name__+__text
129 if isinstance(__v,dict):
130 for val in __v.values():
131 if callable(val): __text = self._missing%val.__name__+__text
132 numpy.set_printoptions(precision=15,threshold=1000000,linewidth=1000*15)
133 __text += "%s=%s, "%(k,repr(__v))
134 numpy.set_printoptions(precision=8,threshold=1000,linewidth=75)
137 self._addLine(__text)
138 def _extract(self, __multilines="", __object=None):
139 "Transformation un enregistrement en une commande individuelle"
142 __multilines = __multilines.replace("\r\n","\n")
143 for line in __multilines.split("\n"):
144 if "adaoBuilder.New" in line and "=" in line:
145 self._objname = line.split("=")[0].strip()
147 logging.debug("TUI Extracting commands of '%s' object..."%(self._objname,))
151 if self._objname+".set" in line:
152 __commands.append( line.replace(self._objname+".","",1) )
153 logging.debug("TUI Extracted command: %s"%(__commands[-1],))
156 class _COMViewer(GenericCaseViewer):
158 Etablissement des commandes d'un cas COMM (Eficas Native Format/Cas<-COM)
160 def __init__(self, __name="", __objname="case", __content=None, __object=None):
161 "Initialisation et enregistrement de l'entete"
162 GenericCaseViewer.__init__(self, __name, __objname, __content, __object)
163 self._observerIndex = 0
164 self._addLine("# -*- coding: utf-8 -*-")
165 self._addLine("#\n# Python script using ADAO COMM\n#")
166 self._addLine("from numpy import array, matrix")
168 self._addLine("%s = {}"%__objname)
169 if self._content is not None:
170 for command in self._content:
171 self._append(*command)
172 def _extract(self, __multilines=None, __object=None):
173 "Transformation un enregistrement en une commande individuelle"
174 if __multilines is not None:
175 __multilines = __multilines.replace("ASSIMILATION_STUDY","dict")
176 __multilines = __multilines.replace("CHECKING_STUDY", "dict")
177 __multilines = __multilines.replace("_F(", "dict(")
178 __multilines = __multilines.replace(",),);", ",),)")
180 for line in __multilines.split("\n"):
181 if len(line) < 1: continue
182 __fulllines += line + "\n"
183 __multilines = __fulllines
184 self._objname = "case"
186 exec("self._objdata = "+__multilines)
188 if self._objdata is None or not(type(self._objdata) is dict) or not('AlgorithmParameters' in self._objdata):
189 raise ValueError("Impossible to load given content as an ADAO COMM one (no dictionnary or no 'AlgorithmParameters' key found).")
190 # ----------------------------------------------------------------------
191 logging.debug("COMM Extracting commands of '%s' object..."%(self._objname,))
193 __UserPostAnalysis = ""
194 for k,r in self._objdata.items():
196 logging.debug("COMM Extracted command: %s:%s"%(k, r))
197 if __command == "StudyName" and len(str(r))>0:
198 __commands.append( "set( Concept='Name', String='%s')"%(str(r),) )
199 elif __command == "StudyRepertory":
200 __commands.append( "set( Concept='Directory', String='%s')"%(str(r),) )
202 elif __command == "UserPostAnalysis" and type(r) is dict:
204 __UserPostAnalysis = r['STRING']
205 elif 'SCRIPT_FILE' in r and os.path.exists(r['SCRIPT_FILE']):
206 __UserPostAnalysis = open(r['SCRIPT_FILE'],'r').read()
207 elif 'Template' in r and 'ValueTemplate' in r:
209 __UserPostAnalysis = r['ValueTemplate']
211 __UserPostAnalysis = ""
212 __UserPostAnalysis = __UserPostAnalysis.replace("ADD",self._objname)
214 elif __command == "AlgorithmParameters" and type(r) is dict and 'Algorithm' in r:
215 if 'data' in r and r['Parameters'] == 'Dict':
217 if 'STRING' in __from:
218 __parameters = ", Parameters=%s"%(repr(eval(__from['STRING'])),)
219 elif 'SCRIPT_FILE' in __from and os.path.exists(__from['SCRIPT_FILE']):
220 __parameters = ", Script='%s'"%(__from['SCRIPT_FILE'],)
221 else: # if 'Parameters' in r and r['Parameters'] == 'Defaults':
222 __Dict = copy.deepcopy(r)
223 __Dict.pop('Algorithm','')
224 __Dict.pop('Parameters','')
225 if 'SetSeed' in __Dict:__Dict['SetSeed'] = int(__Dict['SetSeed'])
226 if 'BoxBounds' in __Dict and type(__Dict['BoxBounds']) is str:
227 __Dict['BoxBounds'] = eval(__Dict['BoxBounds'])
229 __parameters = ', Parameters=%s'%(repr(__Dict),)
232 __commands.append( "set( Concept='AlgorithmParameters', Algorithm='%s'%s )"%(r['Algorithm'],__parameters) )
234 elif __command == "Observers" and type(r) is dict and 'SELECTION' in r:
235 if type(r['SELECTION']) is str:
236 __selection = (r['SELECTION'],)
238 __selection = tuple(r['SELECTION'])
239 for sk in __selection:
240 __idata = r['%s_data'%sk]
241 if __idata['NodeType'] == 'Template' and 'Template' in __idata:
242 __template = __idata['Template']
243 if 'Info' in __idata:
244 __info = ", Info='%s'"%(__idata['Info'],)
247 __commands.append( "set( Concept='Observer', Variable='%s', Template='%s'%s )"%(sk,__template,__info) )
248 if __idata['NodeType'] == 'String' and 'Value' in __idata:
249 __value =__idata['Value']
250 __commands.append( "set( Concept='Observer', Variable='%s', String='%s' )"%(sk,__value) )
252 # Background, ObservationError, ObservationOperator...
253 elif type(r) is dict:
255 if 'Stored' in r and bool(r['Stored']):
256 __argumentsList.append(['Stored',True])
257 if 'INPUT_TYPE' in r and 'data' in r:
258 # Vector, Matrix, ScalarSparseMatrix, DiagonalSparseMatrix, Function
259 __itype = r['INPUT_TYPE']
261 if 'FROM' in __idata:
262 # String, Script, Template, ScriptWithOneFunction, ScriptWithFunctions
263 __ifrom = __idata['FROM']
264 __idata.pop('FROM','')
265 if __ifrom == 'String' or __ifrom == 'Template':
266 __argumentsList.append([__itype,__idata['STRING']])
267 if __ifrom == 'Script':
268 __argumentsList.append([__itype,True])
269 __argumentsList.append(['Script',__idata['SCRIPT_FILE']])
270 if __ifrom == 'ScriptWithOneFunction':
271 __argumentsList.append(['OneFunction',True])
272 __argumentsList.append(['Script',__idata.pop('SCRIPTWITHONEFUNCTION_FILE')])
274 __argumentsList.append(['Parameters',__idata])
275 if __ifrom == 'ScriptWithFunctions':
276 __argumentsList.append(['ThreeFunctions',True])
277 __argumentsList.append(['Script',__idata.pop('SCRIPTWITHFUNCTIONS_FILE')])
279 __argumentsList.append(['Parameters',__idata])
280 __arguments = ["%s = %s"%(k,repr(v)) for k,v in __argumentsList]
281 __commands.append( "set( Concept='%s', %s )"%(__command, ", ".join(__arguments)))
283 # ----------------------------------------------------------------------
284 __commands.sort() # Pour commencer par 'AlgorithmParameters'
285 __commands.append(__UserPostAnalysis)
288 class _SCDViewer(GenericCaseViewer):
290 Etablissement des commandes d'un cas SCD (Study Config Dictionary/Cas->SCD)
292 def __init__(self, __name="", __objname="case", __content=None, __object=None):
293 "Initialisation et enregistrement de l'entete"
294 GenericCaseViewer.__init__(self, __name, __objname, __content, __object)
295 self._addLine("# -*- coding: utf-8 -*-")
296 self._addLine("#\n# Input for ADAO converter to YACS\n#")
297 self._addLine("from numpy import array, matrix")
299 self._addLine("study_config = {}")
300 self._addLine("study_config['StudyType'] = 'ASSIMILATION_STUDY'")
301 self._addLine("study_config['Name'] = '%s'"%self._name)
302 self._addLine("observers = {}")
303 self._addLine("study_config['Observers'] = observers")
305 self._addLine("inputvariables_config = {}")
306 self._addLine("inputvariables_config['Order'] =['adao_default']")
307 self._addLine("inputvariables_config['adao_default'] = -1")
308 self._addLine("study_config['InputVariables'] = inputvariables_config")
310 self._addLine("outputvariables_config = {}")
311 self._addLine("outputvariables_config['Order'] = ['adao_default']")
312 self._addLine("outputvariables_config['adao_default'] = -1")
313 self._addLine("study_config['OutputVariables'] = outputvariables_config")
314 if __content is not None:
315 for command in __content:
316 self._append(*command)
317 def _append(self, __command=None, __keys=None, __local=None, __pre=None, __switchoff=False):
318 "Transformation d'une commande individuelle en un enregistrement"
319 if __command == "set": __command = __local["Concept"]
320 else: __command = __command.replace("set", "", 1)
323 if __command in (None, 'execute', 'executePythonScheme', 'executeYACSScheme', 'get', 'Name'):
325 elif __command in ['Debug', 'setDebug']:
326 __text = "#\nstudy_config['Debug'] = '1'"
327 elif __command in ['NoDebug', 'setNoDebug']:
328 __text = "#\nstudy_config['Debug'] = '0'"
329 elif __command in ['Observer', 'setObserver']:
330 __obs = __local['Variable']
331 self._numobservers += 1
333 __text += "observers['%s'] = {}\n"%__obs
334 if __local['String'] is not None:
335 __text += "observers['%s']['nodetype'] = '%s'\n"%(__obs, 'String')
336 __text += "observers['%s']['String'] = \"\"\"%s\"\"\"\n"%(__obs, __local['String'])
337 if __local['Script'] is not None:
338 __text += "observers['%s']['nodetype'] = '%s'\n"%(__obs, 'Script')
339 __text += "observers['%s']['Script'] = \"%s\"\n"%(__obs, __local['Script'])
340 if __local['Template'] is not None and __local['Template'] in Templates.ObserverTemplates:
341 __text += "observers['%s']['nodetype'] = '%s'\n"%(__obs, 'String')
342 __text += "observers['%s']['String'] = \"\"\"%s\"\"\"\n"%(__obs, Templates.ObserverTemplates[__local['Template']])
343 if __local['Info'] is not None:
344 __text += "observers['%s']['info'] = \"\"\"%s\"\"\"\n"%(__obs, __local['Info'])
346 __text += "observers['%s']['info'] = \"\"\"%s\"\"\"\n"%(__obs, __obs)
347 __text += "observers['%s']['number'] = %s"%(__obs, self._numobservers)
348 elif __local is not None: # __keys is not None and
349 numpy.set_printoptions(precision=15,threshold=1000000,linewidth=1000*15)
351 __text += "%s_config = {}\n"%__command
352 __local.pop('self','')
354 for __k,__v in __local.items():
355 if __v is None: __to_be_removed.append(__k)
356 for __k in __to_be_removed:
358 for __k,__v in __local.items():
359 if __k == "Concept": continue
360 if __k in ['ScalarSparseMatrix','DiagonalSparseMatrix','Matrix','OneFunction','ThreeFunctions'] and 'Script' in __local: continue
361 if __k == 'Algorithm':
362 __text += "study_config['Algorithm'] = %s\n"%(repr(__v))
363 elif __k == 'Script':
366 __v = "'"+repr(__v)+"'"
367 for __lk in ['ScalarSparseMatrix','DiagonalSparseMatrix','Matrix']:
368 if __lk in __local and __local[__lk]: __k = __lk
369 if __command == "AlgorithmParameters": __k = "Dict"
370 if 'OneFunction' in __local and __local['OneFunction']:
371 __text += "%s_ScriptWithOneFunction = {}\n"%(__command,)
372 __text += "%s_ScriptWithOneFunction['Function'] = ['Direct', 'Tangent', 'Adjoint']\n"%(__command,)
373 __text += "%s_ScriptWithOneFunction['Script'] = {}\n"%(__command,)
374 __text += "%s_ScriptWithOneFunction['Script']['Direct'] = %s\n"%(__command,__v)
375 __text += "%s_ScriptWithOneFunction['Script']['Tangent'] = %s\n"%(__command,__v)
376 __text += "%s_ScriptWithOneFunction['Script']['Adjoint'] = %s\n"%(__command,__v)
377 __text += "%s_ScriptWithOneFunction['DifferentialIncrement'] = 1e-06\n"%(__command,)
378 __text += "%s_ScriptWithOneFunction['CenteredFiniteDifference'] = 0\n"%(__command,)
380 __f = 'ScriptWithOneFunction'
381 __v = '%s_ScriptWithOneFunction'%(__command,)
382 if 'ThreeFunctions' in __local and __local['ThreeFunctions']:
383 __text += "%s_ScriptWithFunctions = {}\n"%(__command,)
384 __text += "%s_ScriptWithFunctions['Function'] = ['Direct', 'Tangent', 'Adjoint']\n"%(__command,)
385 __text += "%s_ScriptWithFunctions['Script'] = {}\n"%(__command,)
386 __text += "%s_ScriptWithFunctions['Script']['Direct'] = %s\n"%(__command,__v)
387 __text += "%s_ScriptWithFunctions['Script']['Tangent'] = %s\n"%(__command,__v)
388 __text += "%s_ScriptWithFunctions['Script']['Adjoint'] = %s\n"%(__command,__v)
390 __f = 'ScriptWithFunctions'
391 __v = '%s_ScriptWithFunctions'%(__command,)
392 __text += "%s_config['Type'] = '%s'\n"%(__command,__k)
393 __text += "%s_config['From'] = '%s'\n"%(__command,__f)
394 __text += "%s_config['Data'] = %s\n"%(__command,__v)
395 __text = __text.replace("''","'")
396 elif __k in ('Stored', 'Checked', 'ColMajor', 'InputFunctionAsMulti'):
398 __text += "%s_config['%s'] = '%s'\n"%(__command,__k,int(bool(__v)))
399 elif __k in ('AvoidRC', 'noDetails'):
401 __text += "%s_config['%s'] = '%s'\n"%(__command,__k,int(bool(__v)))
403 if __k == 'Parameters': __k = "Dict"
404 if isinstance(__v,Persistence.Persistence): __v = __v.values()
405 if callable(__v): __text = self._missing%__v.__name__+__text
406 if isinstance(__v,dict):
407 for val in __v.values():
408 if callable(val): __text = self._missing%val.__name__+__text
409 __text += "%s_config['Type'] = '%s'\n"%(__command,__k)
410 __text += "%s_config['From'] = '%s'\n"%(__command,'String')
411 __text += "%s_config['Data'] = \"\"\"%s\"\"\"\n"%(__command,repr(__v))
412 __text += "study_config['%s'] = %s_config"%(__command,__command)
413 numpy.set_printoptions(precision=8,threshold=1000,linewidth=75)
415 self._switchoff = True
416 if __text is not None: self._addLine(__text)
418 self._switchoff = False
419 def _finalize(self, *__args):
420 self.__loadVariablesByScript()
422 self._addLine("Analysis_config = {}")
423 self._addLine("Analysis_config['From'] = 'String'")
424 self._addLine("Analysis_config['Data'] = \"\"\"import numpy")
425 self._addLine("xa=numpy.ravel(ADD.get('Analysis')[-1])")
426 self._addLine("print('Analysis:',xa)\"\"\"")
427 self._addLine("study_config['UserPostAnalysis'] = Analysis_config")
428 def __loadVariablesByScript(self):
429 __ExecVariables = {} # Necessaire pour recuperer la variable
430 exec("\n".join(self._lineSerie), __ExecVariables)
431 study_config = __ExecVariables['study_config']
432 # Pour Python 3 : self.__hasAlgorithm = bool(study_config['Algorithm'])
433 if 'Algorithm' in study_config:
434 self.__hasAlgorithm = True
436 self.__hasAlgorithm = False
437 if not self.__hasAlgorithm and \
438 "AlgorithmParameters" in study_config and \
439 isinstance(study_config['AlgorithmParameters'], dict) and \
440 "From" in study_config['AlgorithmParameters'] and \
441 "Data" in study_config['AlgorithmParameters'] and \
442 study_config['AlgorithmParameters']['From'] == 'Script':
443 __asScript = study_config['AlgorithmParameters']['Data']
444 __var = ImportFromScript(__asScript).getvalue( "Algorithm" )
445 __text = "#\nstudy_config['Algorithm'] = '%s'"%(__var,)
446 self._addLine(__text)
447 if self.__hasAlgorithm and \
448 "AlgorithmParameters" in study_config and \
449 isinstance(study_config['AlgorithmParameters'], dict) and \
450 "From" not in study_config['AlgorithmParameters'] and \
451 "Data" not in study_config['AlgorithmParameters']:
453 __text += "AlgorithmParameters_config['Type'] = 'Dict'\n"
454 __text += "AlgorithmParameters_config['From'] = 'String'\n"
455 __text += "AlgorithmParameters_config['Data'] = '{}'\n"
456 self._addLine(__text)
459 class _YACSViewer(GenericCaseViewer):
461 Etablissement des commandes d'un cas YACS (Cas->SCD->YACS)
463 def __init__(self, __name="", __objname="case", __content=None, __object=None):
464 "Initialisation et enregistrement de l'entete"
465 GenericCaseViewer.__init__(self, __name, __objname, __content, __object)
466 self.__internalSCD = _SCDViewer(__name, __objname, __content, __object)
467 self._append = self.__internalSCD._append
468 def dump(self, __filename=None, __upa=None):
469 "Restitution normalisée des commandes"
471 if __filename is None:
472 raise ValueError("A file name has to be given for YACS XML output.")
474 __file = os.path.abspath(__filename)
475 if os.path.isfile(__file) or os.path.islink(__file):
478 if not PlatformInfo.has_salome or \
479 not PlatformInfo.has_adao:
481 "Unable to get SALOME or ADAO environnement variables for YACS conversion.\n"+\
482 "Please load the right environnement before trying to use it.")
484 from daYacsSchemaCreator.run import create_schema_from_content
486 self.__internalSCD._finalize(__upa)
487 __SCDdump = self.__internalSCD.dump()
488 create_schema_from_content(__SCDdump, __file)
490 if not os.path.exists(__file):
491 __msg = "An error occured during the ADAO YACS Schema build for\n"
492 __msg += "the target output file:\n"
493 __msg += " %s\n"%__file
494 __msg += "See errors details in your launching terminal log.\n"
495 raise ValueError(__msg)
497 __fid = open(__file,"r")
498 __text = __fid.read()
502 # ==============================================================================
503 class ImportFromScript(object):
505 Obtention d'une variable nommee depuis un fichier script importé
507 __slots__ = ("__basename", "__filenspace", "__filestring")
508 def __init__(self, __filename=None):
509 "Verifie l'existence et importe le script"
510 if __filename is None:
511 raise ValueError("The name of the file, containing the variable to be read, has to be specified.")
512 if not os.path.isfile(__filename):
513 raise ValueError("The file containing the variable to be imported doesn't seem to exist. Please check the file. The given file name is:\n \"%s\""%str(__filename))
514 if os.path.dirname(__filename) != '':
515 sys.path.insert(0, os.path.dirname(__filename))
516 __basename = os.path.basename(__filename).rstrip(".py")
518 __basename = __filename.rstrip(".py")
519 PlatformInfo.checkFileNameImportability( __basename+".py" )
520 self.__basename = __basename
521 self.__filenspace = __import__(__basename, globals(), locals(), [])
522 self.__filestring = open(__filename,'r').read()
523 def getvalue(self, __varname=None, __synonym=None ):
524 "Renvoie la variable demandee par son nom ou son synonyme"
525 if __varname is None:
526 raise ValueError("The name of the variable to be read has to be specified. Please check the content of the file and the syntax.")
527 if not hasattr(self.__filenspace, __varname):
528 if __synonym is None:
529 raise ValueError("The imported script file \"%s\" doesn't contain the mandatory variable \"%s\" to be read. Please check the content of the file and the syntax."%(str(self.__basename)+".py",__varname))
530 elif not hasattr(self.__filenspace, __synonym):
531 raise ValueError("The imported script file \"%s\" doesn't contain the mandatory variable \"%s\" to be read. Please check the content of the file and the syntax."%(str(self.__basename)+".py",__synonym))
533 return getattr(self.__filenspace, __synonym)
535 return getattr(self.__filenspace, __varname)
537 "Renvoie le script complet"
538 return self.__filestring
540 # ==============================================================================
541 class ImportDetector(object):
543 Détection des caractéristiques de fichiers ou objets en entrée
546 "__url", "__usr", "__root", "__end")
547 def __enter__(self): return self
548 def __exit__(self, exc_type, exc_val, exc_tb): return False
550 def __init__(self, __url, UserMime=""):
552 raise ValueError("The name or url of the file object has to be specified.")
554 self.__url = __url.decode()
556 self.__url = str(__url)
557 if UserMime is bytes:
558 self.__usr = UserMime.decode().lower()
560 self.__usr = str(UserMime).lower()
561 (self.__root, self.__end) = os.path.splitext(self.__url)
563 mimetypes.add_type('application/numpy.npy', '.npy')
564 mimetypes.add_type('application/numpy.npz', '.npz')
565 mimetypes.add_type('application/dymola.sdf', '.sdf')
566 if sys.platform.startswith("win"):
567 mimetypes.add_type('text/plain', '.txt')
568 mimetypes.add_type('text/csv', '.csv')
569 mimetypes.add_type('text/tab-separated-values', '.tsv')
573 def is_local_file(self):
574 if os.path.isfile(os.path.realpath(self.__url)):
578 def is_not_local_file(self):
579 if not os.path.isfile(os.path.realpath(self.__url)):
583 def raise_error_if_not_local_file(self):
584 if not os.path.isfile(os.path.realpath(self.__url)):
585 raise ValueError("The name or the url of the file object doesn't seem to exist. The given name is:\n \"%s\""%str(self.__url))
589 # Directory related tests
590 # -----------------------
591 def is_local_dir(self):
592 if os.path.isdir(self.__url):
596 def is_not_local_dir(self):
597 if not os.path.isdir(self.__url):
601 def raise_error_if_not_local_dir(self):
602 if not os.path.isdir(self.__url):
603 raise ValueError("The name or the url of the directory object doesn't seem to exist. The given name is:\n \"%s\""%str(self.__url))
607 # Mime related functions
608 # ------------------------
609 def get_standard_mime(self):
610 (__mtype, __encoding) = mimetypes.guess_type(self.__url, strict=False)
612 def get_user_mime(self):
613 __fake = "fake."+self.__usr.lower()
614 (__mtype, __encoding) = mimetypes.guess_type(__fake, strict=False)
616 def get_comprehensive_mime(self):
617 if self.get_standard_mime() is not None:
618 return self.get_standard_mime()
619 elif self.get_user_mime() is not None:
620 return self.get_user_mime()
624 # Name related functions
625 # ----------------------
626 def get_user_name(self):
628 def get_absolute_name(self):
629 return os.path.abspath(os.path.realpath(self.__url))
630 def get_extension(self):
633 # ==============================================================================
634 class ImportFromFile(object):
636 Obtention de variables disrétisées en 1D, définies par une ou des variables
637 nommées, et sous la forme d'une série de points éventuellement indexés. La
638 lecture d'un fichier au format spécifié (ou intuité) permet de charger ces
640 - des fichiers textes en colonnes de type TXT, CSV, TSV...
641 - des fichiers de données binaires NPY, NPZ, SDF...
642 La lecture du fichier complet ne se fait que si nécessaire, pour assurer la
643 performance tout en disposant de l'interprétation du contenu. Les fichiers
644 textes doivent présenter en première ligne (hors commentaire ou ligne vide)
645 les noms des variables de colonnes. Les commentaires commencent par un "#".
648 "_filename", "_colnames", "_colindex", "_varsline", "_format",
649 "_delimiter", "_skiprows", "__url", "__filestring", "__header",
650 "__allowvoid", "__binaryformats", "__supportedformats")
651 def __enter__(self): return self
652 def __exit__(self, exc_type, exc_val, exc_tb): return False
654 def __init__(self, Filename=None, ColNames=None, ColIndex=None, Format="Guess", AllowVoidNameList=True):
656 Verifie l'existence et les informations de définition du fichier. Les
657 noms de colonnes ou de variables sont ignorées si le format ne permet
660 - Filename : nom du fichier
661 - ColNames : noms de la ou des colonnes/variables à lire
662 - ColIndex : nom unique de la colonne/variable servant d'index
663 - Format : format du fichier et/ou des données inclues
664 - AllowVoidNameList : permet, si la liste de noms est vide, de
665 prendre par défaut toutes les colonnes
667 self.__binaryformats =(
668 "application/numpy.npy",
669 "application/numpy.npz",
670 "application/dymola.sdf",
672 self.__url = ImportDetector( Filename, Format)
673 self.__url.raise_error_if_not_local_file()
674 self._filename = self.__url.get_absolute_name()
675 PlatformInfo.checkFileNameConformity( self._filename )
677 self._format = self.__url.get_comprehensive_mime()
679 self.__header, self._varsline, self._skiprows = self.__getentete()
681 if self._format == "text/csv" or Format.upper() == "CSV":
682 self._format = "text/csv"
683 self.__filestring = "".join(self.__header)
684 if self.__filestring.count(",") > 1:
685 self._delimiter = ","
686 elif self.__filestring.count(";") > 1:
687 self._delimiter = ";"
688 elif self._format == "text/tab-separated-values" or Format.upper() == "TSV":
689 self._format = "text/tab-separated-values"
690 self._delimiter = "\t"
692 self._delimiter = None
694 if ColNames is not None: self._colnames = tuple(ColNames)
695 else: self._colnames = None
697 if ColIndex is not None: self._colindex = str(ColIndex)
698 else: self._colindex = None
700 self.__allowvoid = bool(AllowVoidNameList)
702 def __getentete(self, __nblines = 3):
703 "Lit l'entête du fichier pour trouver la définition des variables"
704 __header, __varsline, __skiprows = [], "", 1
705 if self._format in self.__binaryformats:
708 with open(self._filename,'r') as fid:
709 __line = fid.readline().strip()
710 while "#" in __line or len(__line) < 1:
711 __header.append(__line)
713 __line = fid.readline().strip()
714 __varsline = __line # Première ligne non commentée non vide
715 for i in range(max(0,__nblines)):
716 __header.append(fid.readline())
717 return (__header, __varsline, __skiprows)
719 def __getindices(self, __colnames, __colindex, __delimiter=None ):
720 "Indices de colonnes correspondants à l'index et aux variables"
721 if __delimiter is None:
722 __varserie = self._varsline.strip('#').strip().split()
724 __varserie = self._varsline.strip('#').strip().split(str(__delimiter))
726 if __colnames is not None:
728 __colnames = tuple(__colnames)
730 for i, n in enumerate(__varserie):
731 if v == n: __usecols.append(i)
732 __usecols = tuple(__usecols)
733 if len(__usecols) == 0:
737 raise ValueError("Can not found any column corresponding to the required names %s"%(__colnames,))
741 if __colindex is not None:
743 __colindex = str(__colindex)
744 for i, n in enumerate(__varserie):
745 if __colindex == n: __useindex = i
749 return (__usecols, __useindex)
751 def getsupported(self):
752 self.__supportedformats = {}
753 self.__supportedformats["text/plain"] = True
754 self.__supportedformats["text/csv"] = True
755 self.__supportedformats["text/tab-separated-values"] = True
756 self.__supportedformats["application/numpy.npy"] = True
757 self.__supportedformats["application/numpy.npz"] = True
758 self.__supportedformats["application/dymola.sdf"] = PlatformInfo.has_sdf
759 return self.__supportedformats
761 def getvalue(self, ColNames=None, ColIndex=None ):
762 "Renvoie la ou les variables demandees par la liste de leurs noms"
763 # Uniquement si mise à jour
764 if ColNames is not None: self._colnames = tuple(ColNames)
765 if ColIndex is not None: self._colindex = str(ColIndex)
768 if self._format == "application/numpy.npy":
769 __columns = numpy.load(self._filename)
771 elif self._format == "application/numpy.npz":
773 with numpy.load(self._filename) as __allcolumns:
774 if self._colnames is None:
775 self._colnames = __allcolumns.files
776 for nom in self._colnames:
777 if nom in __allcolumns.files:
778 if __columns is not None:
779 # Attention : toutes les variables doivent avoir la même taille
780 __columns = numpy.vstack((__columns, numpy.reshape(__allcolumns[nom], (1,-1))))
783 __columns = numpy.reshape(__allcolumns[nom], (1,-1))
784 if self._colindex is not None and self._colindex in __allcolumns.files:
785 __index = numpy.array(numpy.reshape(__allcolumns[self._colindex], (1,-1)), dtype=bytes)
786 elif self._format == "text/plain":
787 __usecols, __useindex = self.__getindices(self._colnames, self._colindex)
788 __columns = numpy.loadtxt(self._filename, usecols = __usecols, skiprows=self._skiprows)
789 if __useindex is not None:
790 __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), skiprows=self._skiprows)
792 elif self._format == "application/dymola.sdf" and PlatformInfo.has_sdf:
794 __content = sdf.load(self._filename)
796 if self._colnames is None:
797 self._colnames = [__content.datasets[i].name for i in range(len(__content.datasets))]
798 for nom in self._colnames:
800 if __columns is not None:
801 # Attention : toutes les variables doivent avoir la même taille
802 __columns = numpy.vstack((__columns, numpy.reshape(__content[nom].data, (1,-1))))
805 __columns = numpy.reshape(__content[nom].data, (1,-1))
806 if self._colindex is not None and self._colindex in __content:
807 __index = __content[self._colindex].data
809 elif self._format == "text/csv":
810 __usecols, __useindex = self.__getindices(self._colnames, self._colindex, self._delimiter)
811 __columns = numpy.loadtxt(self._filename, usecols = __usecols, delimiter = self._delimiter, skiprows=self._skiprows)
812 if __useindex is not None:
813 __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), delimiter = self._delimiter, skiprows=self._skiprows)
815 elif self._format == "text/tab-separated-values":
816 __usecols, __useindex = self.__getindices(self._colnames, self._colindex, self._delimiter)
817 __columns = numpy.loadtxt(self._filename, usecols = __usecols, delimiter = self._delimiter, skiprows=self._skiprows)
818 if __useindex is not None:
819 __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), delimiter = self._delimiter, skiprows=self._skiprows)
821 raise ValueError("Unkown file format \"%s\" or no reader available"%self._format)
822 if __columns is None: __columns = ()
826 return value.decode()
829 if __index is not None:
830 __index = tuple([toString(v) for v in __index])
832 return (self._colnames, __columns, self._colindex, __index)
835 "Renvoie le fichier texte complet"
836 if self._format in self.__binaryformats:
839 with open(self._filename,'r') as fid:
845 # ==============================================================================
846 class ImportScalarLinesFromFile(ImportFromFile):
848 Importation de fichier contenant des variables scalaires nommées. Le
849 fichier comporte soit 2, soit 4 colonnes, obligatoirement nommées "Name",
850 "Value", "Minimum", "Maximum" si les noms sont précisés. Sur chaque ligne
851 est indiqué le nom, la valeur, et éventuelement deux bornes min et max (ou
852 None si nécessaire pour une borne).
854 Seule la méthode "getvalue" est changée.
856 def __enter__(self): return self
857 def __exit__(self, exc_type, exc_val, exc_tb): return False
859 def __init__(self, Filename=None, ColNames=None, ColIndex=None, Format="Guess"):
860 ImportFromFile.__init__(self, Filename, ColNames, ColIndex, Format)
861 if self._format not in ["text/plain", "text/csv", "text/tab-separated-values"]:
862 raise ValueError("Unkown file format \"%s\""%self._format)
864 def getvalue(self, VarNames = None, HeaderNames=()):
865 "Renvoie la ou les variables demandees par la liste de leurs noms"
866 if VarNames is not None: __varnames = tuple(VarNames)
867 else: __varnames = None
869 if "Name" in self._varsline and "Value" in self._varsline and "Minimum" in self._varsline and "Maximum" in self._varsline:
870 __ftype = "NamValMinMax"
871 __dtypes = {'names' : ('Name', 'Value', 'Minimum', 'Maximum'),
872 'formats': ('S128', 'g', 'g', 'g')}
873 __usecols = (0, 1, 2, 3)
874 def __replaceNoneN( s ):
875 if s.strip() == b'None': return numpy.NINF
877 def __replaceNoneP( s ):
878 if s.strip() == b'None': return numpy.PINF
880 __converters = {2: __replaceNoneN, 3: __replaceNoneP}
881 elif "Name" in self._varsline and "Value" in self._varsline and ("Minimum" not in self._varsline or "Maximum" not in self._varsline):
883 __dtypes = {'names' : ('Name', 'Value'),
884 'formats': ('S128', 'g')}
887 elif len(HeaderNames)>0 and numpy.all([kw in self._varsline for kw in HeaderNames]):
888 __ftype = "NamLotOfVals"
889 __dtypes = {'names' : HeaderNames,
890 'formats': tuple(['S128',]+['g']*(len(HeaderNames)-1))}
891 __usecols = tuple(range(len(HeaderNames)))
892 def __replaceNone( s ):
893 if s.strip() == b'None': return numpy.NAN
895 __converters = dict()
896 for i in range(1,len(HeaderNames)):
897 __converters[i] = __replaceNone
899 raise ValueError("Can not find names of columns for initial values. Wrong first line is:\n \"%s\""%__firstline)
901 if self._format == "text/plain":
902 __content = numpy.loadtxt(self._filename, dtype = __dtypes, usecols = __usecols, skiprows = self._skiprows, converters = __converters)
903 elif self._format in ["text/csv", "text/tab-separated-values"]:
904 __content = numpy.loadtxt(self._filename, dtype = __dtypes, usecols = __usecols, skiprows = self._skiprows, converters = __converters, delimiter = self._delimiter)
906 raise ValueError("Unkown file format \"%s\""%self._format)
908 __names, __thevalue, __bounds = [], [], []
909 for sub in __content:
910 if len(__usecols) == 4:
912 if numpy.isneginf(mi): mi = None # Réattribue les variables None
913 elif numpy.isnan(mi): mi = None # Réattribue les variables None
914 if numpy.isposinf(ma): ma = None # Réattribue les variables None
915 elif numpy.isnan(ma): ma = None # Réattribue les variables None
916 elif len(__usecols) == 2 and __ftype == "NamVal":
922 for i, v in enumerate(nsub[1:]):
923 if numpy.isnan(v): nsub[i+1] = None
927 if (__varnames is None or na in __varnames) and (na not in __names):
928 # Ne stocke que la premiere occurence d'une variable
930 __thevalue.append(va)
931 __bounds.append((mi,ma))
933 __names = tuple(__names)
934 __thevalue = numpy.array(__thevalue)
935 __bounds = tuple(__bounds)
937 return (__names, __thevalue, __bounds)
939 # ==============================================================================
940 if __name__ == "__main__":
941 print('\n AUTODIAGNOSTIC\n')