# -*- coding: utf-8 -*-
#
-# Copyright (C) 2008-2018 EDF R&D
+# Copyright (C) 2008-2019 EDF R&D
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
import os
import sys
+import numpy
+import mimetypes
import logging
import copy
-import numpy
from daCore import Persistence
from daCore import PlatformInfo
from daCore import Templates
__text +="\n"
if __filename is not None:
__file = os.path.abspath(__filename)
- __fid = open(__file,"w",0)
+ __fid = open(__file,"w")
__fid.write(__text)
__fid.close()
return __text
self._addLine("# -*- coding: utf-8 -*-")
self._addLine("#\n# Python script using ADAO TUI\n#")
self._addLine("from numpy import array, matrix")
- self._addLine("import adaoBuilder")
+ self._addLine("from adao import adaoBuilder")
self._addLine("%s = adaoBuilder.New('%s')"%(self._objname, self._name))
if self._content is not None:
for command in self._content:
for k in __keys:
__v = __local[k]
if __v is None: continue
- if k == "Checked" and not __v: continue
- if k == "Stored" and not __v: continue
- if k == "AvoidRC" and __v: continue
- if k == "noDetails": continue
+ if k == "Checked" and not __v: continue
+ if k == "Stored" and not __v: continue
+ if k == "ColMajor" and not __v: continue
+ if k == "InputFunctionAsMulti" and not __v: continue
+ if k == "AvoidRC" and __v: continue
+ if k == "noDetails": continue
if isinstance(__v,Persistence.Persistence): __v = __v.values()
if callable(__v): __text = self._missing%__v.__name__+__text
if isinstance(__v,dict):
__text += "%s_config['From'] = '%s'\n"%(__command,__f)
__text += "%s_config['Data'] = %s\n"%(__command,__v)
__text = __text.replace("''","'")
- elif __k in ('Stored', 'Checked'):
+ elif __k in ('Stored', 'Checked', 'ColMajor', 'InputFunctionAsMulti'):
if bool(__v):
__text += "%s_config['%s'] = '%s'\n"%(__command,__k,int(bool(__v)))
elif __k in ('AvoidRC', 'noDetails'):
self._addLine("Analysis_config['From'] = 'String'")
self._addLine("Analysis_config['Data'] = \"\"\"import numpy")
self._addLine("xa=numpy.ravel(ADD.get('Analysis')[-1])")
- self._addLine("print 'Analysis:',xa\"\"\"")
+ self._addLine("print('Analysis:',xa)\"\"\"")
self._addLine("study_config['UserPostAnalysis'] = Analysis_config")
def __loadVariablesByScript(self):
__ExecVariables = {} # Necessaire pour recuperer la variable
__fid.close()
return __text
+# ==============================================================================
class ImportFromScript(object):
"""
- Obtention d'une variable nommee depuis un fichier script importe
+ Obtention d'une variable nommee depuis un fichier script importé
"""
+ __slots__ = ("__basename", "__filenspace", "__filestring")
def __init__(self, __filename=None):
"Verifie l'existence et importe le script"
if __filename is None:
raise ValueError("The name of the file, containing the variable to be read, has to be specified.")
if not os.path.isfile(__filename):
- raise ValueError("The file containing the variable to be imported doesn't seem to exist. Please check the file. The given file name is:\n \"%s\""%__filename)
+ raise ValueError("The file containing the variable to be imported doesn't seem to exist. Please check the file. The given file name is:\n \"%s\""%str(__filename))
if os.path.dirname(__filename) != '':
sys.path.insert(0, os.path.dirname(__filename))
__basename = os.path.basename(__filename).rstrip(".py")
else:
__basename = __filename.rstrip(".py")
+ PlatformInfo.checkFileNameImportability( __basename+".py" )
self.__basename = __basename
- self.__scriptfile = __import__(__basename, globals(), locals(), [])
- self.__scriptstring = open(__filename,'r').read()
+ self.__filenspace = __import__(__basename, globals(), locals(), [])
+ self.__filestring = open(__filename,'r').read()
def getvalue(self, __varname=None, __synonym=None ):
- "Renvoie la variable demandee"
+ "Renvoie la variable demandee par son nom ou son synonyme"
if __varname is None:
raise ValueError("The name of the variable to be read has to be specified. Please check the content of the file and the syntax.")
- if not hasattr(self.__scriptfile, __varname):
+ if not hasattr(self.__filenspace, __varname):
if __synonym is None:
raise ValueError("The imported script file \"%s\" doesn't contain the mandatory variable \"%s\" to be read. Please check the content of the file and the syntax."%(str(self.__basename)+".py",__varname))
- elif not hasattr(self.__scriptfile, __synonym):
+ elif not hasattr(self.__filenspace, __synonym):
raise ValueError("The imported script file \"%s\" doesn't contain the mandatory variable \"%s\" to be read. Please check the content of the file and the syntax."%(str(self.__basename)+".py",__synonym))
else:
- return getattr(self.__scriptfile, __synonym)
+ return getattr(self.__filenspace, __synonym)
else:
- return getattr(self.__scriptfile, __varname)
+ return getattr(self.__filenspace, __varname)
def getstring(self):
"Renvoie le script complet"
- return self.__scriptstring
+ return self.__filestring
+
+# ==============================================================================
+class ImportDetector(object):
+ """
+ Détection des caractéristiques de fichiers ou objets en entrée
+ """
+ __slots__ = (
+ "__url", "__usr", "__root", "__end")
+ def __enter__(self): return self
+ def __exit__(self, exc_type, exc_val, exc_tb): return False
+ #
+ def __init__(self, __url, UserMime=""):
+ if __url is None:
+ raise ValueError("The name or url of the file object has to be specified.")
+ if __url is bytes:
+ self.__url = __url.decode()
+ else:
+ self.__url = str(__url)
+ if UserMime is bytes:
+ self.__usr = UserMime.decode().lower()
+ else:
+ self.__usr = str(UserMime).lower()
+ (self.__root, self.__end) = os.path.splitext(self.__url)
+ #
+ mimetypes.add_type('application/numpy.npy', '.npy')
+ mimetypes.add_type('application/numpy.npz', '.npz')
+ mimetypes.add_type('application/dymola.sdf', '.sdf')
+ if sys.platform.startswith("win"):
+ mimetypes.add_type('text/plain', '.txt')
+ mimetypes.add_type('text/csv', '.csv')
+ mimetypes.add_type('text/tab-separated-values', '.tsv')
+ #
+ # File related tests
+ # ------------------
+ def is_local_file(self):
+ if os.path.isfile(os.path.realpath(self.__url)):
+ return True
+ else:
+ return False
+ def is_not_local_file(self):
+ if not os.path.isfile(os.path.realpath(self.__url)):
+ return True
+ else:
+ return False
+ def raise_error_if_not_local_file(self):
+ if not os.path.isfile(os.path.realpath(self.__url)):
+ raise ValueError("The name or the url of the file object doesn't seem to exist. The given name is:\n \"%s\""%str(self.__url))
+ else:
+ return False
+ #
+ # Directory related tests
+ # -----------------------
+ def is_local_dir(self):
+ if os.path.isdir(self.__url):
+ return True
+ else:
+ return False
+ def is_not_local_dir(self):
+ if not os.path.isdir(self.__url):
+ return True
+ else:
+ return False
+ def raise_error_if_not_local_dir(self):
+ if not os.path.isdir(self.__url):
+ raise ValueError("The name or the url of the directory object doesn't seem to exist. The given name is:\n \"%s\""%str(self.__url))
+ else:
+ return False
+ #
+ # Mime related functions
+ # ------------------------
+ def get_standard_mime(self):
+ (__mtype, __encoding) = mimetypes.guess_type(self.__url, strict=False)
+ return __mtype
+ def get_user_mime(self):
+ __fake = "fake."+self.__usr.lower()
+ (__mtype, __encoding) = mimetypes.guess_type(__fake, strict=False)
+ return __mtype
+ def get_comprehensive_mime(self):
+ if self.get_standard_mime() is not None:
+ return self.get_standard_mime()
+ elif self.get_user_mime() is not None:
+ return self.get_user_mime()
+ else:
+ return None
+ #
+ # Name related functions
+ # ----------------------
+ def get_user_name(self):
+ return self.__url
+ def get_absolute_name(self):
+ return os.path.abspath(os.path.realpath(self.__url))
+ def get_extension(self):
+ return self.__end
+
+# ==============================================================================
+class ImportFromFile(object):
+ """
+ Obtention de variables disrétisées en 1D, définies par une ou des variables
+ nommées, et sous la forme d'une série de points éventuellement indexés. La
+ lecture d'un fichier au format spécifié (ou intuité) permet de charger ces
+ fonctions depuis :
+ - des fichiers textes en colonnes de type TXT, CSV, TSV...
+ - des fichiers de données binaires NPY, NPZ, SDF...
+ La lecture du fichier complet ne se fait que si nécessaire, pour assurer la
+ performance tout en disposant de l'interprétation du contenu. Les fichiers
+ textes doivent présenter en première ligne (hors commentaire ou ligne vide)
+ les noms des variables de colonnes. Les commentaires commencent par un "#".
+ """
+ __slots__ = (
+ "_filename", "_colnames", "_colindex", "_varsline", "_format",
+ "_delimiter", "_skiprows", "__url", "__filestring", "__header",
+ "__allowvoid", "__binaryformats", "__supportedformats")
+ def __enter__(self): return self
+ def __exit__(self, exc_type, exc_val, exc_tb): return False
+ #
+ def __init__(self, Filename=None, ColNames=None, ColIndex=None, Format="Guess", AllowVoidNameList=True):
+ """
+ Verifie l'existence et les informations de définition du fichier. Les
+ noms de colonnes ou de variables sont ignorées si le format ne permet
+ pas de les indiquer.
+ Arguments :
+ - Filename : nom du fichier
+ - ColNames : noms de la ou des colonnes/variables à lire
+ - ColIndex : nom unique de la colonne/variable servant d'index
+ - Format : format du fichier et/ou des données inclues
+ - AllowVoidNameList : permet, si la liste de noms est vide, de
+ prendre par défaut toutes les colonnes
+ """
+ self.__binaryformats =(
+ "application/numpy.npy",
+ "application/numpy.npz",
+ "application/dymola.sdf",
+ )
+ self.__url = ImportDetector( Filename, Format)
+ self.__url.raise_error_if_not_local_file()
+ self._filename = self.__url.get_absolute_name()
+ PlatformInfo.checkFileNameConformity( self._filename )
+ #
+ self._format = self.__url.get_comprehensive_mime()
+ #
+ self.__header, self._varsline, self._skiprows = self.__getentete()
+ #
+ if self._format == "text/csv" or Format.upper() == "CSV":
+ self._format = "text/csv"
+ self.__filestring = "".join(self.__header)
+ if self.__filestring.count(",") > 1:
+ self._delimiter = ","
+ elif self.__filestring.count(";") > 1:
+ self._delimiter = ";"
+ elif self._format == "text/tab-separated-values" or Format.upper() == "TSV":
+ self._format = "text/tab-separated-values"
+ self._delimiter = "\t"
+ else:
+ self._delimiter = None
+ #
+ if ColNames is not None: self._colnames = tuple(ColNames)
+ else: self._colnames = None
+ #
+ if ColIndex is not None: self._colindex = str(ColIndex)
+ else: self._colindex = None
+ #
+ self.__allowvoid = bool(AllowVoidNameList)
+
+ def __getentete(self, __nblines = 3):
+ "Lit l'entête du fichier pour trouver la définition des variables"
+ __header, __varsline, __skiprows = [], "", 1
+ if self._format in self.__binaryformats:
+ pass
+ else:
+ with open(self._filename,'r') as fid:
+ __line = fid.readline().strip()
+ while "#" in __line or len(__line) < 1:
+ __header.append(__line)
+ __skiprows += 1
+ __line = fid.readline().strip()
+ __varsline = __line # Première ligne non commentée non vide
+ for i in range(max(0,__nblines)):
+ __header.append(fid.readline())
+ return (__header, __varsline, __skiprows)
+
+ def __getindices(self, __colnames, __colindex, __delimiter=None ):
+ "Indices de colonnes correspondants à l'index et aux variables"
+ if __delimiter is None:
+ __varserie = self._varsline.strip('#').strip().split()
+ else:
+ __varserie = self._varsline.strip('#').strip().split(str(__delimiter))
+ #
+ if __colnames is not None:
+ __usecols = []
+ __colnames = tuple(__colnames)
+ for v in __colnames:
+ for i, n in enumerate(__varserie):
+ if v == n: __usecols.append(i)
+ __usecols = tuple(__usecols)
+ if len(__usecols) == 0:
+ if self.__allowvoid:
+ __usecols = None
+ else:
+ raise ValueError("Can not found any column corresponding to the required names %s"%(__colnames,))
+ else:
+ __usecols = None
+ #
+ if __colindex is not None:
+ __useindex = None
+ __colindex = str(__colindex)
+ for i, n in enumerate(__varserie):
+ if __colindex == n: __useindex = i
+ else:
+ __useindex = None
+ #
+ return (__usecols, __useindex)
+
+ def getsupported(self):
+ self.__supportedformats = {}
+ self.__supportedformats["text/plain"] = True
+ self.__supportedformats["text/csv"] = True
+ self.__supportedformats["text/tab-separated-values"] = True
+ self.__supportedformats["application/numpy.npy"] = True
+ self.__supportedformats["application/numpy.npz"] = True
+ self.__supportedformats["application/dymola.sdf"] = PlatformInfo.has_sdf
+ return self.__supportedformats
+
+ def getvalue(self, ColNames=None, ColIndex=None ):
+ "Renvoie la ou les variables demandees par la liste de leurs noms"
+ # Uniquement si mise à jour
+ if ColNames is not None: self._colnames = tuple(ColNames)
+ if ColIndex is not None: self._colindex = str(ColIndex)
+ #
+ __index = None
+ if self._format == "application/numpy.npy":
+ __columns = numpy.load(self._filename)
+ #
+ elif self._format == "application/numpy.npz":
+ __columns = None
+ with numpy.load(self._filename) as __allcolumns:
+ if self._colnames is None:
+ self._colnames = __allcolumns.files
+ for nom in self._colnames:
+ if nom in __allcolumns.files:
+ if __columns is not None:
+ # Attention : toutes les variables doivent avoir la même taille
+ __columns = numpy.vstack((__columns, numpy.reshape(__allcolumns[nom], (1,-1))))
+ else:
+ # Première colonne
+ __columns = numpy.reshape(__allcolumns[nom], (1,-1))
+ if self._colindex is not None and self._colindex in __allcolumns.files:
+ __index = numpy.array(numpy.reshape(__allcolumns[self._colindex], (1,-1)), dtype=bytes)
+ elif self._format == "text/plain":
+ __usecols, __useindex = self.__getindices(self._colnames, self._colindex)
+ __columns = numpy.loadtxt(self._filename, usecols = __usecols, skiprows=self._skiprows)
+ if __useindex is not None:
+ __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), skiprows=self._skiprows)
+ #
+ elif self._format == "application/dymola.sdf" and PlatformInfo.has_sdf:
+ import sdf
+ __content = sdf.load(self._filename)
+ __columns = None
+ if self._colnames is None:
+ self._colnames = [__content.datasets[i].name for i in range(len(__content.datasets))]
+ for nom in self._colnames:
+ if nom in __content:
+ if __columns is not None:
+ # Attention : toutes les variables doivent avoir la même taille
+ __columns = numpy.vstack((__columns, numpy.reshape(__content[nom].data, (1,-1))))
+ else:
+ # Première colonne
+ __columns = numpy.reshape(__content[nom].data, (1,-1))
+ if self._colindex is not None and self._colindex in __content:
+ __index = __content[self._colindex].data
+ #
+ elif self._format == "text/csv":
+ __usecols, __useindex = self.__getindices(self._colnames, self._colindex, self._delimiter)
+ __columns = numpy.loadtxt(self._filename, usecols = __usecols, delimiter = self._delimiter, skiprows=self._skiprows)
+ if __useindex is not None:
+ __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), delimiter = self._delimiter, skiprows=self._skiprows)
+ #
+ elif self._format == "text/tab-separated-values":
+ __usecols, __useindex = self.__getindices(self._colnames, self._colindex, self._delimiter)
+ __columns = numpy.loadtxt(self._filename, usecols = __usecols, delimiter = self._delimiter, skiprows=self._skiprows)
+ if __useindex is not None:
+ __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), delimiter = self._delimiter, skiprows=self._skiprows)
+ else:
+ raise ValueError("Unkown file format \"%s\" or no reader available"%self._format)
+ if __columns is None: __columns = ()
+ #
+ def toString(value):
+ try:
+ return value.decode()
+ except ValueError:
+ return value
+ if __index is not None:
+ __index = tuple([toString(v) for v in __index])
+ #
+ return (self._colnames, __columns, self._colindex, __index)
+
+ def getstring(self):
+ "Renvoie le fichier texte complet"
+ if self._format in self.__binaryformats:
+ return ""
+ else:
+ with open(self._filename,'r') as fid:
+ return fid.read()
+
+ def getformat(self):
+ return self._format
+
+# ==============================================================================
+class ImportScalarLinesFromFile(ImportFromFile):
+ """
+ Importation de fichier contenant des variables scalaires nommées. Le
+ fichier comporte soit 2, soit 4 colonnes, obligatoirement nommées "Name",
+ "Value", "Minimum", "Maximum" si les noms sont précisés. Sur chaque ligne
+ est indiqué le nom, la valeur, et éventuelement deux bornes min et max (ou
+ None si nécessaire pour une borne).
+
+ Seule la méthode "getvalue" est changée.
+ """
+ def __enter__(self): return self
+ def __exit__(self, exc_type, exc_val, exc_tb): return False
+ #
+ def __init__(self, Filename=None, ColNames=None, ColIndex=None, Format="Guess"):
+ ImportFromFile.__init__(self, Filename, ColNames, ColIndex, Format)
+ if self._format not in ["text/plain", "text/csv", "text/tab-separated-values"]:
+ raise ValueError("Unkown file format \"%s\""%self._format)
+ #
+ def getvalue(self, VarNames = None, HeaderNames=()):
+ "Renvoie la ou les variables demandees par la liste de leurs noms"
+ if VarNames is not None: __varnames = tuple(VarNames)
+ else: __varnames = None
+ #
+ if "Name" in self._varsline and "Value" in self._varsline and "Minimum" in self._varsline and "Maximum" in self._varsline:
+ __ftype = "NamValMinMax"
+ __dtypes = {'names' : ('Name', 'Value', 'Minimum', 'Maximum'),
+ 'formats': ('S128', 'g', 'g', 'g')}
+ __usecols = (0, 1, 2, 3)
+ def __replaceNoneN( s ):
+ if s.strip() == b'None': return numpy.NINF
+ else: return s
+ def __replaceNoneP( s ):
+ if s.strip() == b'None': return numpy.PINF
+ else: return s
+ __converters = {2: __replaceNoneN, 3: __replaceNoneP}
+ elif "Name" in self._varsline and "Value" in self._varsline and ("Minimum" not in self._varsline or "Maximum" not in self._varsline):
+ __ftype = "NamVal"
+ __dtypes = {'names' : ('Name', 'Value'),
+ 'formats': ('S128', 'g')}
+ __converters = None
+ __usecols = (0, 1)
+ elif len(HeaderNames)>0 and numpy.all([kw in self._varsline for kw in HeaderNames]):
+ __ftype = "NamLotOfVals"
+ __dtypes = {'names' : HeaderNames,
+ 'formats': tuple(['S128',]+['g']*(len(HeaderNames)-1))}
+ __usecols = tuple(range(len(HeaderNames)))
+ def __replaceNone( s ):
+ if s.strip() == b'None': return numpy.NAN
+ else: return s
+ __converters = dict()
+ for i in range(1,len(HeaderNames)):
+ __converters[i] = __replaceNone
+ else:
+ raise ValueError("Can not find names of columns for initial values. Wrong first line is:\n \"%s\""%__firstline)
+ #
+ if self._format == "text/plain":
+ __content = numpy.loadtxt(self._filename, dtype = __dtypes, usecols = __usecols, skiprows = self._skiprows, converters = __converters)
+ elif self._format in ["text/csv", "text/tab-separated-values"]:
+ __content = numpy.loadtxt(self._filename, dtype = __dtypes, usecols = __usecols, skiprows = self._skiprows, converters = __converters, delimiter = self._delimiter)
+ else:
+ raise ValueError("Unkown file format \"%s\""%self._format)
+ #
+ __names, __thevalue, __bounds = [], [], []
+ for sub in __content:
+ if len(__usecols) == 4:
+ na, va, mi, ma = sub
+ if numpy.isneginf(mi): mi = None # Réattribue les variables None
+ elif numpy.isnan(mi): mi = None # Réattribue les variables None
+ if numpy.isposinf(ma): ma = None # Réattribue les variables None
+ elif numpy.isnan(ma): ma = None # Réattribue les variables None
+ elif len(__usecols) == 2 and __ftype == "NamVal":
+ na, va = sub
+ mi, ma = None, None
+ else:
+ nsub = list(sub)
+ na = sub[0]
+ for i, v in enumerate(nsub[1:]):
+ if numpy.isnan(v): nsub[i+1] = None
+ va = nsub[1:]
+ mi, ma = None, None
+ na = na.decode()
+ if (__varnames is None or na in __varnames) and (na not in __names):
+ # Ne stocke que la premiere occurence d'une variable
+ __names.append(na)
+ __thevalue.append(va)
+ __bounds.append((mi,ma))
+ #
+ __names = tuple(__names)
+ __thevalue = numpy.array(__thevalue)
+ __bounds = tuple(__bounds)
+ #
+ return (__names, __thevalue, __bounds)
# ==============================================================================
if __name__ == "__main__":
- print('\n AUTODIAGNOSTIC \n')
+ print('\n AUTODIAGNOSTIC\n')
