1 # -*- coding: utf-8 -*-
3 # Copyright (C) 2008-2018 EDF R&D
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Lesser General Public
7 # License as published by the Free Software Foundation; either
8 # version 2.1 of the License.
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # Lesser General Public License for more details.
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with this library; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 # See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
21 # Author: Jean-Philippe Argaud, jean-philippe.argaud@edf.fr, EDF R&D
24 Définit les outils d'interfaces normalisées de cas.
26 __author__ = "Jean-Philippe ARGAUD"
35 from daCore import Persistence
36 from daCore import PlatformInfo
37 from daCore import Templates
39 # ==============================================================================
40 class GenericCaseViewer(object):
42 Gestion des commandes de creation d'une vue de cas
44 def __init__(self, __name="", __objname="case", __content=None, __object=None):
45 "Initialisation et enregistrement de l'entete"
46 self._name = str(__name)
47 self._objname = str(__objname)
49 self._switchoff = False
50 self._numobservers = 2
51 self._content = __content
52 self._object = __object
53 self._missing = """raise ValueError("This case requires beforehand to import or define the variable named <%s>. When corrected, remove this command, correct and uncomment the following one.")\n# """
54 def _append(self, *args):
55 "Transformation de commande individuelle en enregistrement"
56 raise NotImplementedError()
57 def _extract(self, *args):
58 "Transformation d'enregistrement en commande individuelle"
59 raise NotImplementedError()
60 def _finalize(self, __upa=None):
61 "Enregistrement du final"
62 if __upa is not None and len(__upa)>0:
63 self._lineSerie.append("%s.execute()"%(self._objname,))
64 self._lineSerie.append(__upa)
65 def _addLine(self, line=""):
66 "Ajoute un enregistrement individuel"
67 self._lineSerie.append(line)
68 def _get_objname(self):
70 def dump(self, __filename=None, __upa=None):
71 "Restitution normalisée des commandes"
73 __text = "\n".join(self._lineSerie)
75 if __filename is not None:
76 __file = os.path.abspath(__filename)
77 __fid = open(__file,"w")
81 def load(self, __filename=None, __content=None, __object=None):
82 "Chargement normalisé des commandes"
83 if __filename is not None and os.path.exists(__filename):
84 self._content = open(__filename, 'r').read()
85 elif __content is not None and type(__content) is str:
86 self._content = __content
87 elif __object is not None and type(__object) is dict:
88 self._object = copy.deepcopy(__object)
90 pass # use "self._content" from initialization
91 __commands = self._extract(self._content, self._object)
94 class _TUIViewer(GenericCaseViewer):
96 Etablissement des commandes d'un cas ADAO TUI (Cas<->TUI)
98 def __init__(self, __name="", __objname="case", __content=None, __object=None):
99 "Initialisation et enregistrement de l'entete"
100 GenericCaseViewer.__init__(self, __name, __objname, __content, __object)
101 self._addLine("# -*- coding: utf-8 -*-")
102 self._addLine("#\n# Python script using ADAO TUI\n#")
103 self._addLine("from numpy import array, matrix")
104 self._addLine("import adaoBuilder")
105 self._addLine("%s = adaoBuilder.New('%s')"%(self._objname, self._name))
106 if self._content is not None:
107 for command in self._content:
108 self._append(*command)
109 def _append(self, __command=None, __keys=None, __local=None, __pre=None, __switchoff=False):
110 "Transformation d'une commande individuelle en un enregistrement"
111 if __command is not None and __keys is not None and __local is not None:
113 if __pre is not None:
114 __text += "%s = "%__pre
115 __text += "%s.%s( "%(self._objname,str(__command))
116 if "self" in __keys: __keys.remove("self")
117 if __command not in ("set","get") and "Concept" in __keys: __keys.remove("Concept")
120 if __v is None: continue
121 if k == "Checked" and not __v: continue
122 if k == "Stored" and not __v: continue
123 if k == "ColMajor" and not __v: continue
124 if k == "AvoidRC" and __v: continue
125 if k == "noDetails": continue
126 if isinstance(__v,Persistence.Persistence): __v = __v.values()
127 if callable(__v): __text = self._missing%__v.__name__+__text
128 if isinstance(__v,dict):
129 for val in __v.values():
130 if callable(val): __text = self._missing%val.__name__+__text
131 numpy.set_printoptions(precision=15,threshold=1000000,linewidth=1000*15)
132 __text += "%s=%s, "%(k,repr(__v))
133 numpy.set_printoptions(precision=8,threshold=1000,linewidth=75)
136 self._addLine(__text)
137 def _extract(self, __multilines="", __object=None):
138 "Transformation un enregistrement en une commande individuelle"
141 __multilines = __multilines.replace("\r\n","\n")
142 for line in __multilines.split("\n"):
143 if "adaoBuilder.New" in line and "=" in line:
144 self._objname = line.split("=")[0].strip()
146 logging.debug("TUI Extracting commands of '%s' object..."%(self._objname,))
150 if self._objname+".set" in line:
151 __commands.append( line.replace(self._objname+".","",1) )
152 logging.debug("TUI Extracted command: %s"%(__commands[-1],))
155 class _COMViewer(GenericCaseViewer):
157 Etablissement des commandes d'un cas COMM (Eficas Native Format/Cas<-COM)
159 def __init__(self, __name="", __objname="case", __content=None, __object=None):
160 "Initialisation et enregistrement de l'entete"
161 GenericCaseViewer.__init__(self, __name, __objname, __content, __object)
162 self._observerIndex = 0
163 self._addLine("# -*- coding: utf-8 -*-")
164 self._addLine("#\n# Python script using ADAO COMM\n#")
165 self._addLine("from numpy import array, matrix")
167 self._addLine("%s = {}"%__objname)
168 if self._content is not None:
169 for command in self._content:
170 self._append(*command)
171 def _extract(self, __multilines=None, __object=None):
172 "Transformation un enregistrement en une commande individuelle"
173 if __multilines is not None:
174 __multilines = __multilines.replace("ASSIMILATION_STUDY","dict")
175 __multilines = __multilines.replace("CHECKING_STUDY", "dict")
176 __multilines = __multilines.replace("_F(", "dict(")
177 __multilines = __multilines.replace(",),);", ",),)")
179 for line in __multilines.split("\n"):
180 if len(line) < 1: continue
181 __fulllines += line + "\n"
182 __multilines = __fulllines
183 self._objname = "case"
185 exec("self._objdata = "+__multilines)
187 if self._objdata is None or not(type(self._objdata) is dict) or not('AlgorithmParameters' in self._objdata):
188 raise ValueError("Impossible to load given content as an ADAO COMM one (no dictionnary or no 'AlgorithmParameters' key found).")
189 # ----------------------------------------------------------------------
190 logging.debug("COMM Extracting commands of '%s' object..."%(self._objname,))
192 __UserPostAnalysis = ""
193 for k,r in self._objdata.items():
195 logging.debug("COMM Extracted command: %s:%s"%(k, r))
196 if __command == "StudyName" and len(str(r))>0:
197 __commands.append( "set( Concept='Name', String='%s')"%(str(r),) )
198 elif __command == "StudyRepertory":
199 __commands.append( "set( Concept='Directory', String='%s')"%(str(r),) )
201 elif __command == "UserPostAnalysis" and type(r) is dict:
203 __UserPostAnalysis = r['STRING']
204 elif 'SCRIPT_FILE' in r and os.path.exists(r['SCRIPT_FILE']):
205 __UserPostAnalysis = open(r['SCRIPT_FILE'],'r').read()
206 elif 'Template' in r and 'ValueTemplate' in r:
208 __UserPostAnalysis = r['ValueTemplate']
210 __UserPostAnalysis = ""
211 __UserPostAnalysis = __UserPostAnalysis.replace("ADD",self._objname)
213 elif __command == "AlgorithmParameters" and type(r) is dict and 'Algorithm' in r:
214 if 'data' in r and r['Parameters'] == 'Dict':
216 if 'STRING' in __from:
217 __parameters = ", Parameters=%s"%(repr(eval(__from['STRING'])),)
218 elif 'SCRIPT_FILE' in __from and os.path.exists(__from['SCRIPT_FILE']):
219 __parameters = ", Script='%s'"%(__from['SCRIPT_FILE'],)
220 else: # if 'Parameters' in r and r['Parameters'] == 'Defaults':
221 __Dict = copy.deepcopy(r)
222 __Dict.pop('Algorithm','')
223 __Dict.pop('Parameters','')
224 if 'SetSeed' in __Dict:__Dict['SetSeed'] = int(__Dict['SetSeed'])
225 if 'BoxBounds' in __Dict and type(__Dict['BoxBounds']) is str:
226 __Dict['BoxBounds'] = eval(__Dict['BoxBounds'])
228 __parameters = ', Parameters=%s'%(repr(__Dict),)
231 __commands.append( "set( Concept='AlgorithmParameters', Algorithm='%s'%s )"%(r['Algorithm'],__parameters) )
233 elif __command == "Observers" and type(r) is dict and 'SELECTION' in r:
234 if type(r['SELECTION']) is str:
235 __selection = (r['SELECTION'],)
237 __selection = tuple(r['SELECTION'])
238 for sk in __selection:
239 __idata = r['%s_data'%sk]
240 if __idata['NodeType'] == 'Template' and 'Template' in __idata:
241 __template = __idata['Template']
242 if 'Info' in __idata:
243 __info = ", Info='%s'"%(__idata['Info'],)
246 __commands.append( "set( Concept='Observer', Variable='%s', Template='%s'%s )"%(sk,__template,__info) )
247 if __idata['NodeType'] == 'String' and 'Value' in __idata:
248 __value =__idata['Value']
249 __commands.append( "set( Concept='Observer', Variable='%s', String='%s' )"%(sk,__value) )
251 # Background, ObservationError, ObservationOperator...
252 elif type(r) is dict:
254 if 'Stored' in r and bool(r['Stored']):
255 __argumentsList.append(['Stored',True])
256 if 'INPUT_TYPE' in r and 'data' in r:
257 # Vector, Matrix, ScalarSparseMatrix, DiagonalSparseMatrix, Function
258 __itype = r['INPUT_TYPE']
260 if 'FROM' in __idata:
261 # String, Script, Template, ScriptWithOneFunction, ScriptWithFunctions
262 __ifrom = __idata['FROM']
263 __idata.pop('FROM','')
264 if __ifrom == 'String' or __ifrom == 'Template':
265 __argumentsList.append([__itype,__idata['STRING']])
266 if __ifrom == 'Script':
267 __argumentsList.append([__itype,True])
268 __argumentsList.append(['Script',__idata['SCRIPT_FILE']])
269 if __ifrom == 'ScriptWithOneFunction':
270 __argumentsList.append(['OneFunction',True])
271 __argumentsList.append(['Script',__idata.pop('SCRIPTWITHONEFUNCTION_FILE')])
273 __argumentsList.append(['Parameters',__idata])
274 if __ifrom == 'ScriptWithFunctions':
275 __argumentsList.append(['ThreeFunctions',True])
276 __argumentsList.append(['Script',__idata.pop('SCRIPTWITHFUNCTIONS_FILE')])
278 __argumentsList.append(['Parameters',__idata])
279 __arguments = ["%s = %s"%(k,repr(v)) for k,v in __argumentsList]
280 __commands.append( "set( Concept='%s', %s )"%(__command, ", ".join(__arguments)))
282 # ----------------------------------------------------------------------
283 __commands.sort() # Pour commencer par 'AlgorithmParameters'
284 __commands.append(__UserPostAnalysis)
287 class _SCDViewer(GenericCaseViewer):
289 Etablissement des commandes d'un cas SCD (Study Config Dictionary/Cas->SCD)
291 def __init__(self, __name="", __objname="case", __content=None, __object=None):
292 "Initialisation et enregistrement de l'entete"
293 GenericCaseViewer.__init__(self, __name, __objname, __content, __object)
294 self._addLine("# -*- coding: utf-8 -*-")
295 self._addLine("#\n# Input for ADAO converter to YACS\n#")
296 self._addLine("from numpy import array, matrix")
298 self._addLine("study_config = {}")
299 self._addLine("study_config['StudyType'] = 'ASSIMILATION_STUDY'")
300 self._addLine("study_config['Name'] = '%s'"%self._name)
301 self._addLine("observers = {}")
302 self._addLine("study_config['Observers'] = observers")
304 self._addLine("inputvariables_config = {}")
305 self._addLine("inputvariables_config['Order'] =['adao_default']")
306 self._addLine("inputvariables_config['adao_default'] = -1")
307 self._addLine("study_config['InputVariables'] = inputvariables_config")
309 self._addLine("outputvariables_config = {}")
310 self._addLine("outputvariables_config['Order'] = ['adao_default']")
311 self._addLine("outputvariables_config['adao_default'] = -1")
312 self._addLine("study_config['OutputVariables'] = outputvariables_config")
313 if __content is not None:
314 for command in __content:
315 self._append(*command)
316 def _append(self, __command=None, __keys=None, __local=None, __pre=None, __switchoff=False):
317 "Transformation d'une commande individuelle en un enregistrement"
318 if __command == "set": __command = __local["Concept"]
319 else: __command = __command.replace("set", "", 1)
322 if __command in (None, 'execute', 'executePythonScheme', 'executeYACSScheme', 'get', 'Name'):
324 elif __command in ['Debug', 'setDebug']:
325 __text = "#\nstudy_config['Debug'] = '1'"
326 elif __command in ['NoDebug', 'setNoDebug']:
327 __text = "#\nstudy_config['Debug'] = '0'"
328 elif __command in ['Observer', 'setObserver']:
329 __obs = __local['Variable']
330 self._numobservers += 1
332 __text += "observers['%s'] = {}\n"%__obs
333 if __local['String'] is not None:
334 __text += "observers['%s']['nodetype'] = '%s'\n"%(__obs, 'String')
335 __text += "observers['%s']['String'] = \"\"\"%s\"\"\"\n"%(__obs, __local['String'])
336 if __local['Script'] is not None:
337 __text += "observers['%s']['nodetype'] = '%s'\n"%(__obs, 'Script')
338 __text += "observers['%s']['Script'] = \"%s\"\n"%(__obs, __local['Script'])
339 if __local['Template'] is not None and __local['Template'] in Templates.ObserverTemplates:
340 __text += "observers['%s']['nodetype'] = '%s'\n"%(__obs, 'String')
341 __text += "observers['%s']['String'] = \"\"\"%s\"\"\"\n"%(__obs, Templates.ObserverTemplates[__local['Template']])
342 if __local['Info'] is not None:
343 __text += "observers['%s']['info'] = \"\"\"%s\"\"\"\n"%(__obs, __local['Info'])
345 __text += "observers['%s']['info'] = \"\"\"%s\"\"\"\n"%(__obs, __obs)
346 __text += "observers['%s']['number'] = %s"%(__obs, self._numobservers)
347 elif __local is not None: # __keys is not None and
348 numpy.set_printoptions(precision=15,threshold=1000000,linewidth=1000*15)
350 __text += "%s_config = {}\n"%__command
351 __local.pop('self','')
353 for __k,__v in __local.items():
354 if __v is None: __to_be_removed.append(__k)
355 for __k in __to_be_removed:
357 for __k,__v in __local.items():
358 if __k == "Concept": continue
359 if __k in ['ScalarSparseMatrix','DiagonalSparseMatrix','Matrix','OneFunction','ThreeFunctions'] and 'Script' in __local: continue
360 if __k == 'Algorithm':
361 __text += "study_config['Algorithm'] = %s\n"%(repr(__v))
362 elif __k == 'Script':
365 __v = "'"+repr(__v)+"'"
366 for __lk in ['ScalarSparseMatrix','DiagonalSparseMatrix','Matrix']:
367 if __lk in __local and __local[__lk]: __k = __lk
368 if __command == "AlgorithmParameters": __k = "Dict"
369 if 'OneFunction' in __local and __local['OneFunction']:
370 __text += "%s_ScriptWithOneFunction = {}\n"%(__command,)
371 __text += "%s_ScriptWithOneFunction['Function'] = ['Direct', 'Tangent', 'Adjoint']\n"%(__command,)
372 __text += "%s_ScriptWithOneFunction['Script'] = {}\n"%(__command,)
373 __text += "%s_ScriptWithOneFunction['Script']['Direct'] = %s\n"%(__command,__v)
374 __text += "%s_ScriptWithOneFunction['Script']['Tangent'] = %s\n"%(__command,__v)
375 __text += "%s_ScriptWithOneFunction['Script']['Adjoint'] = %s\n"%(__command,__v)
376 __text += "%s_ScriptWithOneFunction['DifferentialIncrement'] = 1e-06\n"%(__command,)
377 __text += "%s_ScriptWithOneFunction['CenteredFiniteDifference'] = 0\n"%(__command,)
379 __f = 'ScriptWithOneFunction'
380 __v = '%s_ScriptWithOneFunction'%(__command,)
381 if 'ThreeFunctions' in __local and __local['ThreeFunctions']:
382 __text += "%s_ScriptWithFunctions = {}\n"%(__command,)
383 __text += "%s_ScriptWithFunctions['Function'] = ['Direct', 'Tangent', 'Adjoint']\n"%(__command,)
384 __text += "%s_ScriptWithFunctions['Script'] = {}\n"%(__command,)
385 __text += "%s_ScriptWithFunctions['Script']['Direct'] = %s\n"%(__command,__v)
386 __text += "%s_ScriptWithFunctions['Script']['Tangent'] = %s\n"%(__command,__v)
387 __text += "%s_ScriptWithFunctions['Script']['Adjoint'] = %s\n"%(__command,__v)
389 __f = 'ScriptWithFunctions'
390 __v = '%s_ScriptWithFunctions'%(__command,)
391 __text += "%s_config['Type'] = '%s'\n"%(__command,__k)
392 __text += "%s_config['From'] = '%s'\n"%(__command,__f)
393 __text += "%s_config['Data'] = %s\n"%(__command,__v)
394 __text = __text.replace("''","'")
395 elif __k in ('Stored', 'Checked', 'ColMajor'):
397 __text += "%s_config['%s'] = '%s'\n"%(__command,__k,int(bool(__v)))
398 elif __k in ('AvoidRC', 'noDetails'):
400 __text += "%s_config['%s'] = '%s'\n"%(__command,__k,int(bool(__v)))
402 if __k == 'Parameters': __k = "Dict"
403 if isinstance(__v,Persistence.Persistence): __v = __v.values()
404 if callable(__v): __text = self._missing%__v.__name__+__text
405 if isinstance(__v,dict):
406 for val in __v.values():
407 if callable(val): __text = self._missing%val.__name__+__text
408 __text += "%s_config['Type'] = '%s'\n"%(__command,__k)
409 __text += "%s_config['From'] = '%s'\n"%(__command,'String')
410 __text += "%s_config['Data'] = \"\"\"%s\"\"\"\n"%(__command,repr(__v))
411 __text += "study_config['%s'] = %s_config"%(__command,__command)
412 numpy.set_printoptions(precision=8,threshold=1000,linewidth=75)
414 self._switchoff = True
415 if __text is not None: self._addLine(__text)
417 self._switchoff = False
418 def _finalize(self, *__args):
419 self.__loadVariablesByScript()
421 self._addLine("Analysis_config = {}")
422 self._addLine("Analysis_config['From'] = 'String'")
423 self._addLine("Analysis_config['Data'] = \"\"\"import numpy")
424 self._addLine("xa=numpy.ravel(ADD.get('Analysis')[-1])")
425 self._addLine("print 'Analysis:',xa\"\"\"")
426 self._addLine("study_config['UserPostAnalysis'] = Analysis_config")
427 def __loadVariablesByScript(self):
428 __ExecVariables = {} # Necessaire pour recuperer la variable
429 exec("\n".join(self._lineSerie), __ExecVariables)
430 study_config = __ExecVariables['study_config']
431 # Pour Python 3 : self.__hasAlgorithm = bool(study_config['Algorithm'])
432 if 'Algorithm' in study_config:
433 self.__hasAlgorithm = True
435 self.__hasAlgorithm = False
436 if not self.__hasAlgorithm and \
437 "AlgorithmParameters" in study_config and \
438 isinstance(study_config['AlgorithmParameters'], dict) and \
439 "From" in study_config['AlgorithmParameters'] and \
440 "Data" in study_config['AlgorithmParameters'] and \
441 study_config['AlgorithmParameters']['From'] == 'Script':
442 __asScript = study_config['AlgorithmParameters']['Data']
443 __var = ImportFromScript(__asScript).getvalue( "Algorithm" )
444 __text = "#\nstudy_config['Algorithm'] = '%s'"%(__var,)
445 self._addLine(__text)
446 if self.__hasAlgorithm and \
447 "AlgorithmParameters" in study_config and \
448 isinstance(study_config['AlgorithmParameters'], dict) and \
449 "From" not in study_config['AlgorithmParameters'] and \
450 "Data" not in study_config['AlgorithmParameters']:
452 __text += "AlgorithmParameters_config['Type'] = 'Dict'\n"
453 __text += "AlgorithmParameters_config['From'] = 'String'\n"
454 __text += "AlgorithmParameters_config['Data'] = '{}'\n"
455 self._addLine(__text)
458 class _YACSViewer(GenericCaseViewer):
460 Etablissement des commandes d'un cas YACS (Cas->SCD->YACS)
462 def __init__(self, __name="", __objname="case", __content=None, __object=None):
463 "Initialisation et enregistrement de l'entete"
464 GenericCaseViewer.__init__(self, __name, __objname, __content, __object)
465 self.__internalSCD = _SCDViewer(__name, __objname, __content, __object)
466 self._append = self.__internalSCD._append
467 def dump(self, __filename=None, __upa=None):
468 "Restitution normalisée des commandes"
470 if __filename is None:
471 raise ValueError("A file name has to be given for YACS XML output.")
473 __file = os.path.abspath(__filename)
474 if os.path.isfile(__file) or os.path.islink(__file):
477 if not PlatformInfo.has_salome or \
478 not PlatformInfo.has_adao:
480 "Unable to get SALOME or ADAO environnement variables for YACS conversion.\n"+\
481 "Please load the right environnement before trying to use it.")
483 from daYacsSchemaCreator.run import create_schema_from_content
485 self.__internalSCD._finalize(__upa)
486 __SCDdump = self.__internalSCD.dump()
487 create_schema_from_content(__SCDdump, __file)
489 if not os.path.exists(__file):
490 __msg = "An error occured during the ADAO YACS Schema build for\n"
491 __msg += "the target output file:\n"
492 __msg += " %s\n"%__file
493 __msg += "See errors details in your launching terminal log.\n"
494 raise ValueError(__msg)
496 __fid = open(__file,"r")
497 __text = __fid.read()
501 # ==============================================================================
502 class ImportFromScript(object):
504 Obtention d'une variable nommee depuis un fichier script importé
506 __slots__ = ("__basename", "__filenspace", "__filestring")
507 def __init__(self, __filename=None):
508 "Verifie l'existence et importe le script"
509 if __filename is None:
510 raise ValueError("The name of the file, containing the variable to be read, has to be specified.")
511 if not os.path.isfile(__filename):
512 raise ValueError("The file containing the variable to be imported doesn't seem to exist. Please check the file. The given file name is:\n \"%s\""%str(__filename))
513 if os.path.dirname(__filename) != '':
514 sys.path.insert(0, os.path.dirname(__filename))
515 __basename = os.path.basename(__filename).rstrip(".py")
517 __basename = __filename.rstrip(".py")
518 self.__basename = __basename
519 self.__filenspace = __import__(__basename, globals(), locals(), [])
520 self.__filestring = open(__filename,'r').read()
521 def getvalue(self, __varname=None, __synonym=None ):
522 "Renvoie la variable demandee par son nom ou son synonyme"
523 if __varname is None:
524 raise ValueError("The name of the variable to be read has to be specified. Please check the content of the file and the syntax.")
525 if not hasattr(self.__filenspace, __varname):
526 if __synonym is None:
527 raise ValueError("The imported script file \"%s\" doesn't contain the mandatory variable \"%s\" to be read. Please check the content of the file and the syntax."%(str(self.__basename)+".py",__varname))
528 elif not hasattr(self.__filenspace, __synonym):
529 raise ValueError("The imported script file \"%s\" doesn't contain the mandatory variable \"%s\" to be read. Please check the content of the file and the syntax."%(str(self.__basename)+".py",__synonym))
531 return getattr(self.__filenspace, __synonym)
533 return getattr(self.__filenspace, __varname)
535 "Renvoie le script complet"
536 return self.__filestring
538 # ==============================================================================
539 class ImportDetector(object):
541 Détection des caractéristiques de fichiers ou objets en entrée
544 "__url", "__usr", "__root", "__end")
545 def __enter__(self): return self
546 def __exit__(self, exc_type, exc_val, exc_tb): return False
548 def __init__(self, __url, UserMime=""):
550 raise ValueError("The name or url of the file object has to be specified.")
552 self.__url = __url.decode()
554 self.__url = str(__url)
555 if UserMime is bytes:
556 self.__usr = UserMime.decode().lower()
558 self.__usr = str(UserMime).lower()
559 (self.__root, self.__end) = os.path.splitext(self.__url)
561 mimetypes.add_type('application/numpy.npy', '.npy')
562 mimetypes.add_type('application/numpy.npz', '.npz')
563 if sys.platform.startswith("win"):
564 mimetypes.add_type('text/plain', '.txt')
565 mimetypes.add_type('text/csv', '.csv')
566 mimetypes.add_type('text/tab-separated-values', '.tsv')
570 def is_local_file(self):
571 if os.path.isfile(os.path.realpath(self.__url)):
575 def is_not_local_file(self):
576 if not os.path.isfile(os.path.realpath(self.__url)):
580 def raise_error_if_not_local_file(self):
581 if not os.path.isfile(os.path.realpath(self.__url)):
582 raise ValueError("The name or the url of the file object doesn't seem to exist. The given name is:\n \"%s\""%str(self.__url))
585 # Directory related tests
586 # -----------------------
587 def is_local_dir(self):
588 if os.path.isdir(self.__url):
592 def is_not_local_dir(self):
593 if not os.path.isdir(self.__url):
597 def raise_error_if_not_local_dir(self):
598 if not os.path.isdir(self.__url):
599 raise ValueError("The name or the url of the directory object doesn't seem to exist. The given name is:\n \"%s\""%str(self.__url))
602 # Mime related functions
603 # ------------------------
604 def get_standard_mime(self):
605 (__mtype, __encoding) = mimetypes.guess_type(self.__url, strict=False)
607 def get_user_mime(self):
608 __fake = "fake."+self.__usr.lower()
609 (__mtype, __encoding) = mimetypes.guess_type(__fake, strict=False)
611 def get_comprehensive_mime(self):
612 if self.get_standard_mime() is not None:
613 return self.get_standard_mime()
614 elif self.get_user_mime() is not None:
615 return self.get_user_mime()
618 # Name related functions
619 # ----------------------
620 def get_user_name(self):
622 def get_absolute_name(self):
623 return os.path.abspath(os.path.realpath(self.__url))
624 def get_extension(self):
627 # ==============================================================================
628 class ImportFromFile(object):
630 Obtention de variables disrétisées en 1D, définies par une ou des variables
631 nommées, et sous la forme d'une série de points éventuellement indexés. La
632 lecture d'un fichier au format spécifié (ou intuité) permet de charger ces
634 - des fichiers textes en colonnes de type TXT, CSV, TSV...
635 - des fichiers de données binaires NPY, NPZ...
636 La lecture du fichier complet ne se fait que si nécessaire, pour assurer la
637 performance tout en disposant de l'interprétation du contenu. Les fichiers
638 textes doivent présenter en première ligne (hors commentaire ou ligne vide)
639 les noms des variables de colonnes. Les commentaires commencent par un "#".
642 "_filename", "_colnames", "_colindex", "_varsline", "_format",
643 "_delimiter", "_skiprows", "__url", "__filestring", "__header",
645 def __enter__(self): return self
646 def __exit__(self, exc_type, exc_val, exc_tb): return False
648 def __init__(self, Filename=None, ColNames=None, ColIndex=None, Format="Guess", AllowVoidNameList=True):
650 Verifie l'existence et les informations de définition du fichier. Les
651 noms de colonnes ou de variables sont ignorées si le format ne permet
654 - Filename : nom du fichier
655 - ColNames : noms de la ou des colonnes/variables à lire
656 - ColIndex : nom unique de la colonne/variable servant d'index
657 - Format : format du fichier et/ou des données inclues
658 - AllowVoidNameList : permet, si la liste de noms est vide, de
659 prendre par défaut toutes les colonnes
661 self.__url = ImportDetector( Filename, Format)
662 self.__url.raise_error_if_not_local_file()
663 self._filename = self.__url.get_absolute_name()
665 self._format = self.__url.get_comprehensive_mime()
667 self.__header, self._varsline, self._skiprows = self.__getentete()
669 if self._format == "text/csv" or Format.upper() == "CSV":
670 self._format = "text/csv"
671 self.__filestring = "".join(self.__header)
672 if self.__filestring.count(",") > 1:
673 self._delimiter = ","
674 elif self.__filestring.count(";") > 1:
675 self._delimiter = ";"
676 elif self._format == "text/tab-separated-values" or Format.upper() == "TSV":
677 self._format = "text/tab-separated-values"
678 self._delimiter = "\t"
680 self._delimiter = None
682 if ColNames is not None: self._colnames = tuple(ColNames)
683 else: self._colnames = None
685 if ColIndex is not None: self._colindex = str(ColIndex)
686 else: self._colindex = None
688 self.__allowvoid = bool(AllowVoidNameList)
690 def __getentete(self, __nblines = 3):
691 "Lit l'entête du fichier pour trouver la définition des variables"
692 __header, __varsline, __skiprows = [], "", 1
693 if self._format in ("application/numpy.npy", "application/numpy.npz"):
696 with open(self._filename,'r') as fid:
697 __line = fid.readline().strip()
698 while "#" in __line or len(__line) < 1:
699 __header.append(__line)
701 __line = fid.readline().strip()
702 __varsline = __line # Première ligne non commentée non vide
703 for i in range(max(0,__nblines)):
704 __header.append(fid.readline())
705 return (__header, __varsline, __skiprows)
707 def __getindices(self, __colnames, __colindex, __delimiter=None ):
708 "Indices de colonnes correspondants à l'index et aux variables"
709 if __delimiter is None:
710 __varserie = self._varsline.strip('#').strip().split()
712 __varserie = self._varsline.strip('#').strip().split(str(__delimiter))
714 if __colnames is not None:
716 __colnames = tuple(__colnames)
718 for i, n in enumerate(__varserie):
719 if v == n: __usecols.append(i)
720 __usecols = tuple(__usecols)
721 if len(__usecols) == 0:
725 raise ValueError("Can not found any column corresponding to the required names %s"%(__colnames,))
729 if __colindex is not None:
731 __colindex = str(__colindex)
732 for i, n in enumerate(__varserie):
733 if __colindex == n: __useindex = i
737 return (__usecols, __useindex)
739 def getvalue(self, ColNames=None, ColIndex=None ):
740 "Renvoie la ou les variables demandees par la liste de leurs noms"
741 # Uniquement si mise à jour
742 if ColNames is not None: self._colnames = tuple(ColNames)
743 if ColIndex is not None: self._colindex = str(ColIndex)
746 if self._format == "application/numpy.npy":
747 __columns = numpy.load(self._filename)
748 elif self._format == "application/numpy.npz":
750 with numpy.load(self._filename) as __allcolumns:
751 if self._colnames is None:
752 self._colnames = __allcolumns.files
753 for nom in self._colnames:
754 if nom in __allcolumns.files:
755 if __columns is not None:
756 # Attention : toutes les variables doivent avoir la même taille
757 __columns = numpy.vstack((__columns, numpy.reshape(__allcolumns[nom], (1,-1))))
760 __columns = numpy.reshape(__allcolumns[nom], (1,-1))
761 if self._colindex is not None and self._colindex in __allcolumns.files:
762 __index = numpy.array(numpy.reshape(__allcolumns[self._colindex], (1,-1)), dtype=bytes)
763 elif self._format == "text/plain":
764 __usecols, __useindex = self.__getindices(self._colnames, self._colindex)
765 __columns = numpy.loadtxt(self._filename, usecols = __usecols, skiprows=self._skiprows)
766 if __useindex is not None:
767 __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), skiprows=self._skiprows)
769 elif self._format == "text/csv":
770 __usecols, __useindex = self.__getindices(self._colnames, self._colindex, self._delimiter)
771 __columns = numpy.loadtxt(self._filename, usecols = __usecols, delimiter = self._delimiter, skiprows=self._skiprows)
772 if __useindex is not None:
773 __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), delimiter = self._delimiter, skiprows=self._skiprows)
775 elif self._format == "text/tab-separated-values":
776 __usecols, __useindex = self.__getindices(self._colnames, self._colindex, self._delimiter)
777 __columns = numpy.loadtxt(self._filename, usecols = __usecols, delimiter = self._delimiter, skiprows=self._skiprows)
778 if __useindex is not None:
779 __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), delimiter = self._delimiter, skiprows=self._skiprows)
781 raise ValueError("Unkown file format \"%s\""%self._format)
782 if __columns is None: __columns = ()
786 return value.decode()
789 if __index is not None:
790 __index = tuple([toString(v) for v in __index])
792 return (self._colnames, __columns, self._colindex, __index)
795 "Renvoie le fichier complet"
796 with open(self._filename,'r') as fid:
802 # ==============================================================================
803 class ImportScalarLinesFromFile(ImportFromFile):
805 Importation de fichier contenant des variables scalaires nommées. Le
806 fichier comporte soit 2, soit 4 colonnes, obligatoirement nommées "Name",
807 "Value", "Minimum", "Maximum" si les noms sont précisés. Sur chaque ligne
808 est indiqué le nom, la valeur, et éventuelement deux bornes min et max (ou
809 None si nécessaire pour une borne).
811 Seule la méthode "getvalue" est changée.
813 def __enter__(self): return self
814 def __exit__(self, exc_type, exc_val, exc_tb): return False
816 def __init__(self, Filename=None, ColNames=None, ColIndex=None, Format="Guess"):
817 ImportFromFile.__init__(self, Filename, ColNames, ColIndex, Format)
818 if self._format not in ["text/plain", "text/csv", "text/tab-separated-values"]:
819 raise ValueError("Unkown file format \"%s\""%self._format)
821 def getvalue(self, VarNames = None, HeaderNames=()):
822 "Renvoie la ou les variables demandees par la liste de leurs noms"
823 if VarNames is not None: __varnames = tuple(VarNames)
824 else: __varnames = None
826 if "Name" in self._varsline and "Value" in self._varsline and "Minimum" in self._varsline and "Maximum" in self._varsline:
827 __ftype = "NamValMinMax"
828 __dtypes = {'names' : ('Name', 'Value', 'Minimum', 'Maximum'),
829 'formats': ('S128', 'g', 'g', 'g')}
830 __usecols = (0, 1, 2, 3)
831 def __replaceNoneN( s ):
832 if s.strip() == b'None': return numpy.NINF
834 def __replaceNoneP( s ):
835 if s.strip() == b'None': return numpy.PINF
837 __converters = {2: __replaceNoneN, 3: __replaceNoneP}
838 elif "Name" in self._varsline and "Value" in self._varsline and ("Minimum" not in self._varsline or "Maximum" not in self._varsline):
840 __dtypes = {'names' : ('Name', 'Value'),
841 'formats': ('S128', 'g')}
844 elif len(HeaderNames)>0 and numpy.all([kw in self._varsline for kw in HeaderNames]):
845 __ftype = "NamLotOfVals"
846 __dtypes = {'names' : HeaderNames,
847 'formats': tuple(['S128',]+['g']*(len(HeaderNames)-1))}
848 __usecols = tuple(range(len(HeaderNames)))
849 def __replaceNone( s ):
850 if s.strip() == b'None': return numpy.NAN
852 __converters = dict()
853 for i in range(1,len(HeaderNames)):
854 __converters[i] = __replaceNone
856 raise ValueError("Can not find names of columns for initial values. Wrong first line is:\n \"%s\""%__firstline)
858 if self._format == "text/plain":
859 __content = numpy.loadtxt(self._filename, dtype = __dtypes, usecols = __usecols, skiprows = self._skiprows, converters = __converters)
860 elif self._format in ["text/csv", "text/tab-separated-values"]:
861 __content = numpy.loadtxt(self._filename, dtype = __dtypes, usecols = __usecols, skiprows = self._skiprows, converters = __converters, delimiter = self._delimiter)
863 raise ValueError("Unkown file format \"%s\""%self._format)
865 __names, __background, __bounds = [], [], []
866 for sub in __content:
867 if len(__usecols) == 4:
869 if numpy.isneginf(mi): mi = None # Réattribue les variables None
870 elif numpy.isnan(mi): mi = None # Réattribue les variables None
871 if numpy.isposinf(ma): ma = None # Réattribue les variables None
872 elif numpy.isnan(ma): ma = None # Réattribue les variables None
873 elif len(__usecols) == 2 and __ftype == "NamVal":
879 for i, v in enumerate(nsub[1:]):
880 if numpy.isnan(v): nsub[i+1] = None
884 if (__varnames is None or na in __varnames) and (na not in __names):
885 # Ne stocke que la premiere occurence d'une variable
887 __background.append(va)
888 __bounds.append((mi,ma))
890 __names = tuple(__names)
891 __background = numpy.array(__background)
892 __bounds = tuple(__bounds)
894 return (__names, __background, __bounds)
896 # ==============================================================================
897 if __name__ == "__main__":
898 print('\n AUTODIAGNOSTIC \n')