1 # -*- coding: utf-8 -*-
3 # Copyright (C) 2008-2018 EDF R&D
5 # This library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Lesser General Public
7 # License as published by the Free Software Foundation; either
8 # version 2.1 of the License.
10 # This library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # Lesser General Public License for more details.
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with this library; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 # See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
21 # Author: Jean-Philippe Argaud, jean-philippe.argaud@edf.fr, EDF R&D
24 Définit les outils d'interfaces normalisées de cas.
26 __author__ = "Jean-Philippe ARGAUD"
35 from daCore import Persistence
36 from daCore import PlatformInfo
37 from daCore import Templates
39 # ==============================================================================
40 class GenericCaseViewer(object):
42 Gestion des commandes de creation d'une vue de cas
44 def __init__(self, __name="", __objname="case", __content=None, __object=None):
45 "Initialisation et enregistrement de l'entete"
46 self._name = str(__name)
47 self._objname = str(__objname)
49 self._switchoff = False
50 self._numobservers = 2
51 self._content = __content
52 self._object = __object
53 self._missing = """raise ValueError("This case requires beforehand to import or define the variable named <%s>. When corrected, remove this command, correct and uncomment the following one.")\n# """
54 def _append(self, *args):
55 "Transformation de commande individuelle en enregistrement"
56 raise NotImplementedError()
57 def _extract(self, *args):
58 "Transformation d'enregistrement en commande individuelle"
59 raise NotImplementedError()
60 def _finalize(self, __upa=None):
61 "Enregistrement du final"
62 if __upa is not None and len(__upa)>0:
63 self._lineSerie.append("%s.execute()"%(self._objname,))
64 self._lineSerie.append(__upa)
65 def _addLine(self, line=""):
66 "Ajoute un enregistrement individuel"
67 self._lineSerie.append(line)
68 def _get_objname(self):
70 def dump(self, __filename=None, __upa=None):
71 "Restitution normalisée des commandes"
73 __text = "\n".join(self._lineSerie)
75 if __filename is not None:
76 __file = os.path.abspath(__filename)
77 __fid = open(__file,"w")
81 def load(self, __filename=None, __content=None, __object=None):
82 "Chargement normalisé des commandes"
83 if __filename is not None and os.path.exists(__filename):
84 self._content = open(__filename, 'r').read()
85 elif __content is not None and type(__content) is str:
86 self._content = __content
87 elif __object is not None and type(__object) is dict:
88 self._object = copy.deepcopy(__object)
90 pass # use "self._content" from initialization
91 __commands = self._extract(self._content, self._object)
94 class _TUIViewer(GenericCaseViewer):
96 Etablissement des commandes d'un cas ADAO TUI (Cas<->TUI)
98 def __init__(self, __name="", __objname="case", __content=None, __object=None):
99 "Initialisation et enregistrement de l'entete"
100 GenericCaseViewer.__init__(self, __name, __objname, __content, __object)
101 self._addLine("# -*- coding: utf-8 -*-")
102 self._addLine("#\n# Python script using ADAO TUI\n#")
103 self._addLine("from numpy import array, matrix")
104 self._addLine("import adaoBuilder")
105 self._addLine("%s = adaoBuilder.New('%s')"%(self._objname, self._name))
106 if self._content is not None:
107 for command in self._content:
108 self._append(*command)
109 def _append(self, __command=None, __keys=None, __local=None, __pre=None, __switchoff=False):
110 "Transformation d'une commande individuelle en un enregistrement"
111 if __command is not None and __keys is not None and __local is not None:
113 if __pre is not None:
114 __text += "%s = "%__pre
115 __text += "%s.%s( "%(self._objname,str(__command))
116 if "self" in __keys: __keys.remove("self")
117 if __command not in ("set","get") and "Concept" in __keys: __keys.remove("Concept")
120 if __v is None: continue
121 if k == "Checked" and not __v: continue
122 if k == "Stored" and not __v: continue
123 if k == "AvoidRC" and __v: continue
124 if k == "noDetails": continue
125 if isinstance(__v,Persistence.Persistence): __v = __v.values()
126 if callable(__v): __text = self._missing%__v.__name__+__text
127 if isinstance(__v,dict):
128 for val in __v.values():
129 if callable(val): __text = self._missing%val.__name__+__text
130 numpy.set_printoptions(precision=15,threshold=1000000,linewidth=1000*15)
131 __text += "%s=%s, "%(k,repr(__v))
132 numpy.set_printoptions(precision=8,threshold=1000,linewidth=75)
135 self._addLine(__text)
136 def _extract(self, __multilines="", __object=None):
137 "Transformation un enregistrement en une commande individuelle"
140 __multilines = __multilines.replace("\r\n","\n")
141 for line in __multilines.split("\n"):
142 if "adaoBuilder.New" in line and "=" in line:
143 self._objname = line.split("=")[0].strip()
145 logging.debug("TUI Extracting commands of '%s' object..."%(self._objname,))
149 if self._objname+".set" in line:
150 __commands.append( line.replace(self._objname+".","",1) )
151 logging.debug("TUI Extracted command: %s"%(__commands[-1],))
154 class _COMViewer(GenericCaseViewer):
156 Etablissement des commandes d'un cas COMM (Eficas Native Format/Cas<-COM)
158 def __init__(self, __name="", __objname="case", __content=None, __object=None):
159 "Initialisation et enregistrement de l'entete"
160 GenericCaseViewer.__init__(self, __name, __objname, __content, __object)
161 self._observerIndex = 0
162 self._addLine("# -*- coding: utf-8 -*-")
163 self._addLine("#\n# Python script using ADAO COMM\n#")
164 self._addLine("from numpy import array, matrix")
166 self._addLine("%s = {}"%__objname)
167 if self._content is not None:
168 for command in self._content:
169 self._append(*command)
170 def _extract(self, __multilines=None, __object=None):
171 "Transformation un enregistrement en une commande individuelle"
172 if __multilines is not None:
173 __multilines = __multilines.replace("ASSIMILATION_STUDY","dict")
174 __multilines = __multilines.replace("CHECKING_STUDY", "dict")
175 __multilines = __multilines.replace("_F(", "dict(")
176 __multilines = __multilines.replace(",),);", ",),)")
178 for line in __multilines.split("\n"):
179 if len(line) < 1: continue
180 __fulllines += line + "\n"
181 __multilines = __fulllines
182 self._objname = "case"
184 exec("self._objdata = "+__multilines)
186 if self._objdata is None or not(type(self._objdata) is dict) or not('AlgorithmParameters' in self._objdata):
187 raise ValueError("Impossible to load given content as an ADAO COMM one (no dictionnary or no 'AlgorithmParameters' key found).")
188 # ----------------------------------------------------------------------
189 logging.debug("COMM Extracting commands of '%s' object..."%(self._objname,))
191 __UserPostAnalysis = ""
192 for k,r in self._objdata.items():
194 logging.debug("COMM Extracted command: %s:%s"%(k, r))
195 if __command == "StudyName" and len(str(r))>0:
196 __commands.append( "set( Concept='Name', String='%s')"%(str(r),) )
197 elif __command == "StudyRepertory":
198 __commands.append( "set( Concept='Directory', String='%s')"%(str(r),) )
200 elif __command == "UserPostAnalysis" and type(r) is dict:
202 __UserPostAnalysis = r['STRING']
203 elif 'SCRIPT_FILE' in r and os.path.exists(r['SCRIPT_FILE']):
204 __UserPostAnalysis = open(r['SCRIPT_FILE'],'r').read()
205 elif 'Template' in r and 'ValueTemplate' in r:
207 __UserPostAnalysis = r['ValueTemplate']
209 __UserPostAnalysis = ""
210 __UserPostAnalysis = __UserPostAnalysis.replace("ADD",self._objname)
212 elif __command == "AlgorithmParameters" and type(r) is dict and 'Algorithm' in r:
213 if 'data' in r and r['Parameters'] == 'Dict':
215 if 'STRING' in __from:
216 __parameters = ", Parameters=%s"%(repr(eval(__from['STRING'])),)
217 elif 'SCRIPT_FILE' in __from and os.path.exists(__from['SCRIPT_FILE']):
218 __parameters = ", Script='%s'"%(__from['SCRIPT_FILE'],)
219 else: # if 'Parameters' in r and r['Parameters'] == 'Defaults':
220 __Dict = copy.deepcopy(r)
221 __Dict.pop('Algorithm','')
222 __Dict.pop('Parameters','')
223 if 'SetSeed' in __Dict:__Dict['SetSeed'] = int(__Dict['SetSeed'])
224 if 'BoxBounds' in __Dict and type(__Dict['BoxBounds']) is str:
225 __Dict['BoxBounds'] = eval(__Dict['BoxBounds'])
227 __parameters = ', Parameters=%s'%(repr(__Dict),)
230 __commands.append( "set( Concept='AlgorithmParameters', Algorithm='%s'%s )"%(r['Algorithm'],__parameters) )
232 elif __command == "Observers" and type(r) is dict and 'SELECTION' in r:
233 if type(r['SELECTION']) is str:
234 __selection = (r['SELECTION'],)
236 __selection = tuple(r['SELECTION'])
237 for sk in __selection:
238 __idata = r['%s_data'%sk]
239 if __idata['NodeType'] == 'Template' and 'Template' in __idata:
240 __template = __idata['Template']
241 if 'Info' in __idata:
242 __info = ", Info='%s'"%(__idata['Info'],)
245 __commands.append( "set( Concept='Observer', Variable='%s', Template='%s'%s )"%(sk,__template,__info) )
246 if __idata['NodeType'] == 'String' and 'Value' in __idata:
247 __value =__idata['Value']
248 __commands.append( "set( Concept='Observer', Variable='%s', String='%s' )"%(sk,__value) )
250 # Background, ObservationError, ObservationOperator...
251 elif type(r) is dict:
253 if 'Stored' in r and bool(r['Stored']):
254 __argumentsList.append(['Stored',True])
255 if 'INPUT_TYPE' in r and 'data' in r:
256 # Vector, Matrix, ScalarSparseMatrix, DiagonalSparseMatrix, Function
257 __itype = r['INPUT_TYPE']
259 if 'FROM' in __idata:
260 # String, Script, Template, ScriptWithOneFunction, ScriptWithFunctions
261 __ifrom = __idata['FROM']
262 __idata.pop('FROM','')
263 if __ifrom == 'String' or __ifrom == 'Template':
264 __argumentsList.append([__itype,__idata['STRING']])
265 if __ifrom == 'Script':
266 __argumentsList.append([__itype,True])
267 __argumentsList.append(['Script',__idata['SCRIPT_FILE']])
268 if __ifrom == 'ScriptWithOneFunction':
269 __argumentsList.append(['OneFunction',True])
270 __argumentsList.append(['Script',__idata.pop('SCRIPTWITHONEFUNCTION_FILE')])
272 __argumentsList.append(['Parameters',__idata])
273 if __ifrom == 'ScriptWithFunctions':
274 __argumentsList.append(['ThreeFunctions',True])
275 __argumentsList.append(['Script',__idata.pop('SCRIPTWITHFUNCTIONS_FILE')])
277 __argumentsList.append(['Parameters',__idata])
278 __arguments = ["%s = %s"%(k,repr(v)) for k,v in __argumentsList]
279 __commands.append( "set( Concept='%s', %s )"%(__command, ", ".join(__arguments)))
281 # ----------------------------------------------------------------------
282 __commands.sort() # Pour commencer par 'AlgorithmParameters'
283 __commands.append(__UserPostAnalysis)
286 class _SCDViewer(GenericCaseViewer):
288 Etablissement des commandes d'un cas SCD (Study Config Dictionary/Cas->SCD)
290 def __init__(self, __name="", __objname="case", __content=None, __object=None):
291 "Initialisation et enregistrement de l'entete"
292 GenericCaseViewer.__init__(self, __name, __objname, __content, __object)
293 self._addLine("# -*- coding: utf-8 -*-")
294 self._addLine("#\n# Input for ADAO converter to YACS\n#")
295 self._addLine("from numpy import array, matrix")
297 self._addLine("study_config = {}")
298 self._addLine("study_config['StudyType'] = 'ASSIMILATION_STUDY'")
299 self._addLine("study_config['Name'] = '%s'"%self._name)
300 self._addLine("observers = {}")
301 self._addLine("study_config['Observers'] = observers")
303 self._addLine("inputvariables_config = {}")
304 self._addLine("inputvariables_config['Order'] =['adao_default']")
305 self._addLine("inputvariables_config['adao_default'] = -1")
306 self._addLine("study_config['InputVariables'] = inputvariables_config")
308 self._addLine("outputvariables_config = {}")
309 self._addLine("outputvariables_config['Order'] = ['adao_default']")
310 self._addLine("outputvariables_config['adao_default'] = -1")
311 self._addLine("study_config['OutputVariables'] = outputvariables_config")
312 if __content is not None:
313 for command in __content:
314 self._append(*command)
315 def _append(self, __command=None, __keys=None, __local=None, __pre=None, __switchoff=False):
316 "Transformation d'une commande individuelle en un enregistrement"
317 if __command == "set": __command = __local["Concept"]
318 else: __command = __command.replace("set", "", 1)
321 if __command in (None, 'execute', 'executePythonScheme', 'executeYACSScheme', 'get', 'Name'):
323 elif __command in ['Debug', 'setDebug']:
324 __text = "#\nstudy_config['Debug'] = '1'"
325 elif __command in ['NoDebug', 'setNoDebug']:
326 __text = "#\nstudy_config['Debug'] = '0'"
327 elif __command in ['Observer', 'setObserver']:
328 __obs = __local['Variable']
329 self._numobservers += 1
331 __text += "observers['%s'] = {}\n"%__obs
332 if __local['String'] is not None:
333 __text += "observers['%s']['nodetype'] = '%s'\n"%(__obs, 'String')
334 __text += "observers['%s']['String'] = \"\"\"%s\"\"\"\n"%(__obs, __local['String'])
335 if __local['Script'] is not None:
336 __text += "observers['%s']['nodetype'] = '%s'\n"%(__obs, 'Script')
337 __text += "observers['%s']['Script'] = \"%s\"\n"%(__obs, __local['Script'])
338 if __local['Template'] is not None and __local['Template'] in Templates.ObserverTemplates:
339 __text += "observers['%s']['nodetype'] = '%s'\n"%(__obs, 'String')
340 __text += "observers['%s']['String'] = \"\"\"%s\"\"\"\n"%(__obs, Templates.ObserverTemplates[__local['Template']])
341 if __local['Info'] is not None:
342 __text += "observers['%s']['info'] = \"\"\"%s\"\"\"\n"%(__obs, __local['Info'])
344 __text += "observers['%s']['info'] = \"\"\"%s\"\"\"\n"%(__obs, __obs)
345 __text += "observers['%s']['number'] = %s"%(__obs, self._numobservers)
346 elif __local is not None: # __keys is not None and
347 numpy.set_printoptions(precision=15,threshold=1000000,linewidth=1000*15)
349 __text += "%s_config = {}\n"%__command
350 __local.pop('self','')
352 for __k,__v in __local.items():
353 if __v is None: __to_be_removed.append(__k)
354 for __k in __to_be_removed:
356 for __k,__v in __local.items():
357 if __k == "Concept": continue
358 if __k in ['ScalarSparseMatrix','DiagonalSparseMatrix','Matrix','OneFunction','ThreeFunctions'] and 'Script' in __local: continue
359 if __k == 'Algorithm':
360 __text += "study_config['Algorithm'] = %s\n"%(repr(__v))
361 elif __k == 'Script':
364 __v = "'"+repr(__v)+"'"
365 for __lk in ['ScalarSparseMatrix','DiagonalSparseMatrix','Matrix']:
366 if __lk in __local and __local[__lk]: __k = __lk
367 if __command == "AlgorithmParameters": __k = "Dict"
368 if 'OneFunction' in __local and __local['OneFunction']:
369 __text += "%s_ScriptWithOneFunction = {}\n"%(__command,)
370 __text += "%s_ScriptWithOneFunction['Function'] = ['Direct', 'Tangent', 'Adjoint']\n"%(__command,)
371 __text += "%s_ScriptWithOneFunction['Script'] = {}\n"%(__command,)
372 __text += "%s_ScriptWithOneFunction['Script']['Direct'] = %s\n"%(__command,__v)
373 __text += "%s_ScriptWithOneFunction['Script']['Tangent'] = %s\n"%(__command,__v)
374 __text += "%s_ScriptWithOneFunction['Script']['Adjoint'] = %s\n"%(__command,__v)
375 __text += "%s_ScriptWithOneFunction['DifferentialIncrement'] = 1e-06\n"%(__command,)
376 __text += "%s_ScriptWithOneFunction['CenteredFiniteDifference'] = 0\n"%(__command,)
378 __f = 'ScriptWithOneFunction'
379 __v = '%s_ScriptWithOneFunction'%(__command,)
380 if 'ThreeFunctions' in __local and __local['ThreeFunctions']:
381 __text += "%s_ScriptWithFunctions = {}\n"%(__command,)
382 __text += "%s_ScriptWithFunctions['Function'] = ['Direct', 'Tangent', 'Adjoint']\n"%(__command,)
383 __text += "%s_ScriptWithFunctions['Script'] = {}\n"%(__command,)
384 __text += "%s_ScriptWithFunctions['Script']['Direct'] = %s\n"%(__command,__v)
385 __text += "%s_ScriptWithFunctions['Script']['Tangent'] = %s\n"%(__command,__v)
386 __text += "%s_ScriptWithFunctions['Script']['Adjoint'] = %s\n"%(__command,__v)
388 __f = 'ScriptWithFunctions'
389 __v = '%s_ScriptWithFunctions'%(__command,)
390 __text += "%s_config['Type'] = '%s'\n"%(__command,__k)
391 __text += "%s_config['From'] = '%s'\n"%(__command,__f)
392 __text += "%s_config['Data'] = %s\n"%(__command,__v)
393 __text = __text.replace("''","'")
394 elif __k in ('Stored', 'Checked'):
396 __text += "%s_config['%s'] = '%s'\n"%(__command,__k,int(bool(__v)))
397 elif __k in ('AvoidRC', 'noDetails'):
399 __text += "%s_config['%s'] = '%s'\n"%(__command,__k,int(bool(__v)))
401 if __k == 'Parameters': __k = "Dict"
402 if isinstance(__v,Persistence.Persistence): __v = __v.values()
403 if callable(__v): __text = self._missing%__v.__name__+__text
404 if isinstance(__v,dict):
405 for val in __v.values():
406 if callable(val): __text = self._missing%val.__name__+__text
407 __text += "%s_config['Type'] = '%s'\n"%(__command,__k)
408 __text += "%s_config['From'] = '%s'\n"%(__command,'String')
409 __text += "%s_config['Data'] = \"\"\"%s\"\"\"\n"%(__command,repr(__v))
410 __text += "study_config['%s'] = %s_config"%(__command,__command)
411 numpy.set_printoptions(precision=8,threshold=1000,linewidth=75)
413 self._switchoff = True
414 if __text is not None: self._addLine(__text)
416 self._switchoff = False
417 def _finalize(self, *__args):
418 self.__loadVariablesByScript()
420 self._addLine("Analysis_config = {}")
421 self._addLine("Analysis_config['From'] = 'String'")
422 self._addLine("Analysis_config['Data'] = \"\"\"import numpy")
423 self._addLine("xa=numpy.ravel(ADD.get('Analysis')[-1])")
424 self._addLine("print 'Analysis:',xa\"\"\"")
425 self._addLine("study_config['UserPostAnalysis'] = Analysis_config")
426 def __loadVariablesByScript(self):
427 __ExecVariables = {} # Necessaire pour recuperer la variable
428 exec("\n".join(self._lineSerie), __ExecVariables)
429 study_config = __ExecVariables['study_config']
430 # Pour Python 3 : self.__hasAlgorithm = bool(study_config['Algorithm'])
431 if 'Algorithm' in study_config:
432 self.__hasAlgorithm = True
434 self.__hasAlgorithm = False
435 if not self.__hasAlgorithm and \
436 "AlgorithmParameters" in study_config and \
437 isinstance(study_config['AlgorithmParameters'], dict) and \
438 "From" in study_config['AlgorithmParameters'] and \
439 "Data" in study_config['AlgorithmParameters'] and \
440 study_config['AlgorithmParameters']['From'] == 'Script':
441 __asScript = study_config['AlgorithmParameters']['Data']
442 __var = ImportFromScript(__asScript).getvalue( "Algorithm" )
443 __text = "#\nstudy_config['Algorithm'] = '%s'"%(__var,)
444 self._addLine(__text)
445 if self.__hasAlgorithm and \
446 "AlgorithmParameters" in study_config and \
447 isinstance(study_config['AlgorithmParameters'], dict) and \
448 "From" not in study_config['AlgorithmParameters'] and \
449 "Data" not in study_config['AlgorithmParameters']:
451 __text += "AlgorithmParameters_config['Type'] = 'Dict'\n"
452 __text += "AlgorithmParameters_config['From'] = 'String'\n"
453 __text += "AlgorithmParameters_config['Data'] = '{}'\n"
454 self._addLine(__text)
457 class _YACSViewer(GenericCaseViewer):
459 Etablissement des commandes d'un cas YACS (Cas->SCD->YACS)
461 def __init__(self, __name="", __objname="case", __content=None, __object=None):
462 "Initialisation et enregistrement de l'entete"
463 GenericCaseViewer.__init__(self, __name, __objname, __content, __object)
464 self.__internalSCD = _SCDViewer(__name, __objname, __content, __object)
465 self._append = self.__internalSCD._append
466 def dump(self, __filename=None, __upa=None):
467 "Restitution normalisée des commandes"
469 if __filename is None:
470 raise ValueError("A file name has to be given for YACS XML output.")
472 __file = os.path.abspath(__filename)
473 if os.path.isfile(__file) or os.path.islink(__file):
476 if not PlatformInfo.has_salome or \
477 not PlatformInfo.has_adao:
479 "Unable to get SALOME or ADAO environnement variables for YACS conversion.\n"+\
480 "Please load the right environnement before trying to use it.")
482 from daYacsSchemaCreator.run import create_schema_from_content
484 self.__internalSCD._finalize(__upa)
485 __SCDdump = self.__internalSCD.dump()
486 create_schema_from_content(__SCDdump, __file)
488 if not os.path.exists(__file):
489 __msg = "An error occured during the ADAO YACS Schema build for\n"
490 __msg += "the target output file:\n"
491 __msg += " %s\n"%__file
492 __msg += "See errors details in your launching terminal log.\n"
493 raise ValueError(__msg)
495 __fid = open(__file,"r")
496 __text = __fid.read()
500 # ==============================================================================
501 class ImportFromScript(object):
503 Obtention d'une variable nommee depuis un fichier script importé
505 __slots__ = ("__basename", "__filenspace", "__filestring")
506 def __init__(self, __filename=None):
507 "Verifie l'existence et importe le script"
508 if __filename is None:
509 raise ValueError("The name of the file, containing the variable to be read, has to be specified.")
510 if not os.path.isfile(__filename):
511 raise ValueError("The file containing the variable to be imported doesn't seem to exist. Please check the file. The given file name is:\n \"%s\""%str(__filename))
512 if os.path.dirname(__filename) != '':
513 sys.path.insert(0, os.path.dirname(__filename))
514 __basename = os.path.basename(__filename).rstrip(".py")
516 __basename = __filename.rstrip(".py")
517 self.__basename = __basename
518 self.__filenspace = __import__(__basename, globals(), locals(), [])
519 self.__filestring = open(__filename,'r').read()
520 def getvalue(self, __varname=None, __synonym=None ):
521 "Renvoie la variable demandee par son nom ou son synonyme"
522 if __varname is None:
523 raise ValueError("The name of the variable to be read has to be specified. Please check the content of the file and the syntax.")
524 if not hasattr(self.__filenspace, __varname):
525 if __synonym is None:
526 raise ValueError("The imported script file \"%s\" doesn't contain the mandatory variable \"%s\" to be read. Please check the content of the file and the syntax."%(str(self.__basename)+".py",__varname))
527 elif not hasattr(self.__filenspace, __synonym):
528 raise ValueError("The imported script file \"%s\" doesn't contain the mandatory variable \"%s\" to be read. Please check the content of the file and the syntax."%(str(self.__basename)+".py",__synonym))
530 return getattr(self.__filenspace, __synonym)
532 return getattr(self.__filenspace, __varname)
534 "Renvoie le script complet"
535 return self.__filestring
537 # ==============================================================================
538 class ImportDetector(object):
540 Détection des caractéristiques de fichiers ou objets en entrée
543 "__url", "__usr", "__root", "__end")
544 def __enter__(self): return self
545 def __exit__(self, exc_type, exc_val, exc_tb): return False
547 def __init__(self, __url, UserMime=""):
549 raise ValueError("The name or url of the file object has to be specified.")
551 self.__url = __url.decode()
553 self.__url = str(__url)
554 if UserMime is bytes:
555 self.__usr = UserMime.decode().lower()
557 self.__usr = str(UserMime).lower()
558 (self.__root, self.__end) = os.path.splitext(self.__url)
560 mimetypes.add_type('application/numpy.npy', '.npy')
561 mimetypes.add_type('application/numpy.npz', '.npz')
562 if sys.platform.startswith("win"):
563 mimetypes.add_type('text/plain', '.txt')
564 mimetypes.add_type('text/csv', '.csv')
565 mimetypes.add_type('text/tab-separated-values', '.tsv')
569 def is_local_file(self):
570 if os.path.isfile(os.path.realpath(self.__url)):
574 def is_not_local_file(self):
575 if not os.path.isfile(os.path.realpath(self.__url)):
579 def raise_error_if_not_local_file(self):
580 if not os.path.isfile(os.path.realpath(self.__url)):
581 raise ValueError("The name or the url of the file object doesn't seem to exist. The given name is:\n \"%s\""%str(self.__url))
584 # Directory related tests
585 # -----------------------
586 def is_local_dir(self):
587 if os.path.isdir(self.__url):
591 def is_not_local_dir(self):
592 if not os.path.isdir(self.__url):
596 def raise_error_if_not_local_dir(self):
597 if not os.path.isdir(self.__url):
598 raise ValueError("The name or the url of the directory object doesn't seem to exist. The given name is:\n \"%s\""%str(self.__url))
601 # Mime related functions
602 # ------------------------
603 def get_standard_mime(self):
604 (__mtype, __encoding) = mimetypes.guess_type(self.__url, strict=False)
606 def get_user_mime(self):
607 __fake = "fake."+self.__usr.lower()
608 (__mtype, __encoding) = mimetypes.guess_type(__fake, strict=False)
610 def get_comprehensive_mime(self):
611 if self.get_standard_mime() is not None:
612 return self.get_standard_mime()
613 elif self.get_user_mime() is not None:
614 return self.get_user_mime()
617 # Name related functions
618 # ----------------------
619 def get_user_name(self):
621 def get_absolute_name(self):
622 return os.path.abspath(os.path.realpath(self.__url))
623 def get_extension(self):
626 # ==============================================================================
627 class ImportFromFile(object):
629 Obtention de variables disrétisées en 1D, définies par une ou des variables
630 nommées, et sous la forme d'une série de points éventuellement indexés. La
631 lecture d'un fichier au format spécifié (ou intuité) permet de charger ces
633 - des fichiers textes en colonnes de type TXT, CSV, TSV...
634 - des fichiers de données binaires NPY, NPZ...
635 La lecture du fichier complet ne se fait que si nécessaire, pour assurer la
636 performance tout en disposant de l'interprétation du contenu. Les fichiers
637 textes doivent présenter en première ligne (hors commentaire ou ligne vide)
638 les noms des variables de colonnes. Les commentaires commencent par un "#".
641 "_filename", "_colnames", "_colindex", "_varsline", "_format",
642 "_delimiter", "_skiprows", "__url", "__filestring", "__header",
644 def __enter__(self): return self
645 def __exit__(self, exc_type, exc_val, exc_tb): return False
647 def __init__(self, Filename=None, ColNames=None, ColIndex=None, Format="Guess", AllowVoidNameList=True):
649 Verifie l'existence et les informations de définition du fichier. Les
650 noms de colonnes ou de variables sont ignorées si le format ne permet
653 - Filename : nom du fichier
654 - ColNames : noms de la ou des colonnes/variables à lire
655 - ColIndex : nom unique de la colonne/variable servant d'index
656 - Format : format du fichier et/ou des données inclues
657 - AllowVoidNameList : permet, si la liste de noms est vide, de
658 prendre par défaut toutes les colonnes
660 self.__url = ImportDetector( Filename, Format)
661 self.__url.raise_error_if_not_local_file()
662 self._filename = self.__url.get_absolute_name()
664 self._format = self.__url.get_comprehensive_mime()
666 self.__header, self._varsline, self._skiprows = self.__getentete()
668 if self._format == "text/csv" or Format.upper() == "CSV":
669 self.__filestring = "".join(self.__header)
670 if self.__filestring.count(",") > 1:
671 self._delimiter = ","
672 elif self.__filestring.count(";") > 1:
673 self._delimiter = ";"
674 elif self._format == "text/tab-separated-values" or Format.upper() == "TSV":
675 self._delimiter = "\t"
677 self._delimiter = None
679 if ColNames is not None: self._colnames = tuple(ColNames)
680 else: self._colnames = None
682 if ColIndex is not None: self._colindex = str(ColIndex)
683 else: self._colindex = None
685 self.__allowvoid = bool(AllowVoidNameList)
687 def __getentete(self, __nblines = 3):
688 "Lit l'entête du fichier pour trouver la définition des variables"
689 __header, __varsline, __skiprows = [], "", 1
690 if self._format in ("application/numpy.npy", "application/numpy.npz"):
693 with open(self._filename,'r') as fid:
694 __line = fid.readline().strip()
695 while "#" in __line or len(__line) < 1:
696 __header.append(__line)
698 __line = fid.readline().strip()
699 __varsline = __line # Première ligne non commentée non vide
700 for i in range(max(0,__nblines)):
701 __header.append(fid.readline())
702 return (__header, __varsline, __skiprows)
704 def __getindices(self, __colnames, __colindex, __delimiter=None ):
705 "Indices de colonnes correspondants à l'index et aux variables"
706 if __delimiter is None:
707 __varserie = self._varsline.strip('#').strip().split()
709 __varserie = self._varsline.strip('#').strip().split(str(__delimiter))
711 if __colnames is not None:
713 __colnames = tuple(__colnames)
715 for i, n in enumerate(__varserie):
716 if v == n: __usecols.append(i)
717 __usecols = tuple(__usecols)
718 if len(__usecols) == 0:
722 raise ValueError("Can not found any column corresponding to the required names %s"%(__colnames,))
726 if __colindex is not None:
728 __colindex = str(__colindex)
729 for i, n in enumerate(__varserie):
730 if __colindex == n: __useindex = i
734 return (__usecols, __useindex)
736 def getvalue(self, ColNames=None, ColIndex=None ):
737 "Renvoie la ou les variables demandees par la liste de leurs noms"
738 # Uniquement si mise à jour
739 if ColNames is not None: self._colnames = tuple(ColNames)
740 if ColIndex is not None: self._colindex = str(ColIndex)
743 if self._format == "application/numpy.npy":
744 __columns = numpy.load(self._filename)
745 elif self._format == "application/numpy.npz":
747 with numpy.load(self._filename) as __allcolumns:
748 if self._colnames is None:
749 self._colnames = __allcolumns.files
750 for nom in self._colnames:
751 if nom in __allcolumns.files:
752 if __columns is not None:
753 # Attention : toutes les variables doivent avoir la même taille
754 __columns = numpy.vstack((__columns, numpy.reshape(__allcolumns[nom], (1,-1))))
757 __columns = numpy.reshape(__allcolumns[nom], (1,-1))
758 if self._colindex is not None and self._colindex in __allcolumns.files:
759 __index = numpy.array(numpy.reshape(__allcolumns[self._colindex], (1,-1)), dtype=bytes)
760 elif self._format == "text/plain":
761 __usecols, __useindex = self.__getindices(self._colnames, self._colindex)
762 __columns = numpy.loadtxt(self._filename, usecols = __usecols, skiprows=self._skiprows)
763 if __useindex is not None:
764 __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), skiprows=self._skiprows)
766 elif self._format == "text/csv":
767 __usecols, __useindex = self.__getindices(self._colnames, self._colindex, self._delimiter)
768 __columns = numpy.loadtxt(self._filename, usecols = __usecols, delimiter = self._delimiter, skiprows=self._skiprows)
769 if __useindex is not None:
770 __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), delimiter = self._delimiter, skiprows=self._skiprows)
772 elif self._format == "text/tab-separated-values":
773 __usecols, __useindex = self.__getindices(self._colnames, self._colindex, self._delimiter)
774 __columns = numpy.loadtxt(self._filename, usecols = __usecols, delimiter = self._delimiter, skiprows=self._skiprows)
775 if __useindex is not None:
776 __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), delimiter = self._delimiter, skiprows=self._skiprows)
778 raise ValueError("Unkown file format \"%s\""%self._format)
782 return value.decode()
785 if __index is not None:
786 __index = tuple([toString(v) for v in __index])
788 return (self._colnames, __columns, self._colindex, __index)
791 "Renvoie le fichier complet"
792 with open(self._filename,'r') as fid:
795 # ==============================================================================
796 class ImportScalarLinesFromFile(ImportFromFile):
798 Importation de fichier contenant des variables scalaires nommées. Le
799 fichier comporte soit 2, soit 4 colonnes, obligatoirement nommées "Name",
800 "Value", "Minimum", "Maximum" si les noms sont précisés. Sur chaque ligne
801 est indiqué le nom, la valeur, et éventuelement deux bornes min et max (ou
802 None si nécessaire pour une borne).
804 Seule la méthode "getvalue" est changée.
806 def __enter__(self): return self
807 def __exit__(self, exc_type, exc_val, exc_tb): return False
809 def __init__(self, Filename=None, ColNames=None, ColIndex=None, Format="Guess"):
810 ImportFromFile.__init__(self, Filename, ColNames, ColIndex, Format)
811 if self._format not in ["text/plain", "text/csv", "text/tab-separated-values"]:
812 raise ValueError("Unkown file format \"%s\""%self._format)
814 def getvalue(self, VarNames = None, HeaderNames=()):
815 "Renvoie la ou les variables demandees par la liste de leurs noms"
816 if VarNames is not None: __varnames = tuple(VarNames)
817 else: __varnames = None
819 if "Name" in self._varsline and "Value" in self._varsline and "Minimum" in self._varsline and "Maximum" in self._varsline:
820 __ftype = "NamValMinMax"
821 __dtypes = {'names' : ('Name', 'Value', 'Minimum', 'Maximum'),
822 'formats': ('S128', 'g', 'g', 'g')}
823 __usecols = (0, 1, 2, 3)
824 def __replaceNoneN( s ):
825 if s.strip() == b'None': return numpy.NINF
827 def __replaceNoneP( s ):
828 if s.strip() == b'None': return numpy.PINF
830 __converters = {2: __replaceNoneN, 3: __replaceNoneP}
831 elif "Name" in self._varsline and "Value" in self._varsline and ("Minimum" not in self._varsline or "Maximum" not in self._varsline):
833 __dtypes = {'names' : ('Name', 'Value'),
834 'formats': ('S128', 'g')}
837 elif len(HeaderNames)>0 and numpy.all([kw in self._varsline for kw in HeaderNames]):
838 __ftype = "NamLotOfVals"
839 __dtypes = {'names' : HeaderNames,
840 'formats': tuple(['S128',]+['g']*(len(HeaderNames)-1))}
841 __usecols = tuple(range(len(HeaderNames)))
842 def __replaceNone( s ):
843 if s.strip() == b'None': return numpy.NAN
845 __converters = dict()
846 for i in range(1,len(HeaderNames)):
847 __converters[i] = __replaceNone
849 raise ValueError("Can not find names of columns for initial values. Wrong first line is:\n \"%s\""%__firstline)
851 if self._format == "text/plain":
852 __content = numpy.loadtxt(self._filename, dtype = __dtypes, usecols = __usecols, skiprows = self._skiprows, converters = __converters)
853 elif self._format in ["text/csv", "text/tab-separated-values"]:
854 __content = numpy.loadtxt(self._filename, dtype = __dtypes, usecols = __usecols, skiprows = self._skiprows, converters = __converters, delimiter = self._delimiter)
856 raise ValueError("Unkown file format \"%s\""%self._format)
858 __names, __background, __bounds = [], [], []
859 for sub in __content:
860 if len(__usecols) == 4:
862 if numpy.isneginf(mi): mi = None # Réattribue les variables None
863 elif numpy.isnan(mi): mi = None # Réattribue les variables None
864 if numpy.isposinf(ma): ma = None # Réattribue les variables None
865 elif numpy.isnan(ma): ma = None # Réattribue les variables None
866 elif len(__usecols) == 2 and __ftype == "NamVal":
872 for i, v in enumerate(nsub[1:]):
873 if numpy.isnan(v): nsub[i+1] = None
877 if (__varnames is None or na in __varnames) and (na not in __names):
878 # Ne stocke que la premiere occurence d'une variable
880 __background.append(va)
881 __bounds.append((mi,ma))
883 __names = tuple(__names)
884 __background = numpy.array(__background)
885 __bounds = tuple(__bounds)
887 return (__names, __background, __bounds)
889 # ==============================================================================
890 if __name__ == "__main__":
891 print('\n AUTODIAGNOSTIC \n')