# -*- coding: utf-8 -*-
#
-# Copyright (C) 2008-2018 EDF R&D
+# Copyright (C) 2008-2019 EDF R&D
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
import os
import sys
+import numpy
+import mimetypes
import logging
import copy
-import numpy
from daCore import Persistence
from daCore import PlatformInfo
from daCore import Templates
self._addLine("# -*- coding: utf-8 -*-")
self._addLine("#\n# Python script using ADAO TUI\n#")
self._addLine("from numpy import array, matrix")
- self._addLine("import adaoBuilder")
+ self._addLine("from adao import adaoBuilder")
self._addLine("%s = adaoBuilder.New('%s')"%(self._objname, self._name))
if self._content is not None:
for command in self._content:
for k in __keys:
__v = __local[k]
if __v is None: continue
- if k == "Checked" and not __v: continue
- if k == "Stored" and not __v: continue
- if k == "AvoidRC" and __v: continue
- if k == "noDetails": continue
+ if k == "Checked" and not __v: continue
+ if k == "Stored" and not __v: continue
+ if k == "ColMajor" and not __v: continue
+ if k == "InputFunctionAsMulti" and not __v: continue
+ if k == "AvoidRC" and __v: continue
+ if k == "noDetails": continue
if isinstance(__v,Persistence.Persistence): __v = __v.values()
if callable(__v): __text = self._missing%__v.__name__+__text
if isinstance(__v,dict):
__text += "%s_config['From'] = '%s'\n"%(__command,__f)
__text += "%s_config['Data'] = %s\n"%(__command,__v)
__text = __text.replace("''","'")
- elif __k in ('Stored', 'Checked'):
+ elif __k in ('Stored', 'Checked', 'ColMajor', 'InputFunctionAsMulti'):
if bool(__v):
__text += "%s_config['%s'] = '%s'\n"%(__command,__k,int(bool(__v)))
elif __k in ('AvoidRC', 'noDetails'):
self._addLine("Analysis_config['From'] = 'String'")
self._addLine("Analysis_config['Data'] = \"\"\"import numpy")
self._addLine("xa=numpy.ravel(ADD.get('Analysis')[-1])")
- self._addLine("print 'Analysis:',xa\"\"\"")
+ self._addLine("print('Analysis:',xa)\"\"\"")
self._addLine("study_config['UserPostAnalysis'] = Analysis_config")
def __loadVariablesByScript(self):
__ExecVariables = {} # Necessaire pour recuperer la variable
__basename = os.path.basename(__filename).rstrip(".py")
else:
__basename = __filename.rstrip(".py")
+ PlatformInfo.checkFileNameImportability( __basename+".py" )
self.__basename = __basename
self.__filenspace = __import__(__basename, globals(), locals(), [])
self.__filestring = open(__filename,'r').read()
"Renvoie le script complet"
return self.__filestring
+# ==============================================================================
+class ImportDetector(object):
+ """
+ Détection des caractéristiques de fichiers ou objets en entrée
+ """
+ __slots__ = (
+ "__url", "__usr", "__root", "__end")
+ def __enter__(self): return self
+ def __exit__(self, exc_type, exc_val, exc_tb): return False
+ #
+ def __init__(self, __url, UserMime=""):
+ if __url is None:
+ raise ValueError("The name or url of the file object has to be specified.")
+ if __url is bytes:
+ self.__url = __url.decode()
+ else:
+ self.__url = str(__url)
+ if UserMime is bytes:
+ self.__usr = UserMime.decode().lower()
+ else:
+ self.__usr = str(UserMime).lower()
+ (self.__root, self.__end) = os.path.splitext(self.__url)
+ #
+ mimetypes.add_type('application/numpy.npy', '.npy')
+ mimetypes.add_type('application/numpy.npz', '.npz')
+ mimetypes.add_type('application/dymola.sdf', '.sdf')
+ if sys.platform.startswith("win"):
+ mimetypes.add_type('text/plain', '.txt')
+ mimetypes.add_type('text/csv', '.csv')
+ mimetypes.add_type('text/tab-separated-values', '.tsv')
+ #
+ # File related tests
+ # ------------------
+ def is_local_file(self):
+ if os.path.isfile(os.path.realpath(self.__url)):
+ return True
+ else:
+ return False
+ def is_not_local_file(self):
+ if not os.path.isfile(os.path.realpath(self.__url)):
+ return True
+ else:
+ return False
+ def raise_error_if_not_local_file(self):
+ if not os.path.isfile(os.path.realpath(self.__url)):
+ raise ValueError("The name or the url of the file object doesn't seem to exist. The given name is:\n \"%s\""%str(self.__url))
+ else:
+ return False
+ #
+ # Directory related tests
+ # -----------------------
+ def is_local_dir(self):
+ if os.path.isdir(self.__url):
+ return True
+ else:
+ return False
+ def is_not_local_dir(self):
+ if not os.path.isdir(self.__url):
+ return True
+ else:
+ return False
+ def raise_error_if_not_local_dir(self):
+ if not os.path.isdir(self.__url):
+ raise ValueError("The name or the url of the directory object doesn't seem to exist. The given name is:\n \"%s\""%str(self.__url))
+ else:
+ return False
+ #
+ # Mime related functions
+ # ------------------------
+ def get_standard_mime(self):
+ (__mtype, __encoding) = mimetypes.guess_type(self.__url, strict=False)
+ return __mtype
+ def get_user_mime(self):
+ __fake = "fake."+self.__usr.lower()
+ (__mtype, __encoding) = mimetypes.guess_type(__fake, strict=False)
+ return __mtype
+ def get_comprehensive_mime(self):
+ if self.get_standard_mime() is not None:
+ return self.get_standard_mime()
+ elif self.get_user_mime() is not None:
+ return self.get_user_mime()
+ else:
+ return None
+ #
+ # Name related functions
+ # ----------------------
+ def get_user_name(self):
+ return self.__url
+ def get_absolute_name(self):
+ return os.path.abspath(os.path.realpath(self.__url))
+ def get_extension(self):
+ return self.__end
+
# ==============================================================================
class ImportFromFile(object):
"""
lecture d'un fichier au format spécifié (ou intuité) permet de charger ces
fonctions depuis :
- des fichiers textes en colonnes de type TXT, CSV, TSV...
- - des fichiers de données binaires NPY, NPZ...
+ - des fichiers de données binaires NPY, NPZ, SDF...
La lecture du fichier complet ne se fait que si nécessaire, pour assurer la
performance tout en disposant de l'interprétation du contenu. Les fichiers
textes doivent présenter en première ligne (hors commentaire ou ligne vide)
les noms des variables de colonnes. Les commentaires commencent par un "#".
"""
__slots__ = (
- "_filename", "_varsline", "_format", "_delimiter", "_skiprows",
- "__colnames", "__colindex", "__filestring", "__header")
+ "_filename", "_colnames", "_colindex", "_varsline", "_format",
+ "_delimiter", "_skiprows", "__url", "__filestring", "__header",
+ "__allowvoid", "__binaryformats", "__supportedformats")
def __enter__(self): return self
def __exit__(self, exc_type, exc_val, exc_tb): return False
#
- def __init__(self, Filename=None, ColNames=None, ColIndex=None, Format="Guess"):
+ def __init__(self, Filename=None, ColNames=None, ColIndex=None, Format="Guess", AllowVoidNameList=True):
"""
Verifie l'existence et les informations de définition du fichier. Les
noms de colonnes ou de variables sont ignorées si le format ne permet
- ColNames : noms de la ou des colonnes/variables à lire
- ColIndex : nom unique de la colonne/variable servant d'index
- Format : format du fichier et/ou des données inclues
+ - AllowVoidNameList : permet, si la liste de noms est vide, de
+ prendre par défaut toutes les colonnes
"""
- if Filename is None:
- raise ValueError("The name of the file, containing the variables to be read, has to be specified.")
- if not os.path.isfile(Filename):
- raise ValueError("The file, containing the variables to be read, doesn't seem to exist. Please check the file. The given file name is:\n \"%s\""%str(Filename))
- self._filename = os.path.abspath(Filename)
+ self.__binaryformats =(
+ "application/numpy.npy",
+ "application/numpy.npz",
+ "application/dymola.sdf",
+ )
+ self.__url = ImportDetector( Filename, Format)
+ self.__url.raise_error_if_not_local_file()
+ self._filename = self.__url.get_absolute_name()
+ PlatformInfo.checkFileNameConformity( self._filename )
#
- self.__header, self._varsline, self._skiprows = self.__getentete(self._filename)
+ self._format = self.__url.get_comprehensive_mime()
#
- self._delimiter = None
- self.__filestring = "".join(self.__header)
- if Format.upper() == "GUESS":
- if self._filename.split(".")[-1].lower() == "npy":
- self._format = "NPY"
- elif self._filename.split(".")[-1].lower() == "npz":
- self._format = "NPZ"
- elif self.__filestring.count(",") > 1 and self._filename.split(".")[-1].lower() == "csv":
- self._format = "CSV"
- self._delimiter = ","
- elif self.__filestring.count(";") > 1 and self._filename.split(".")[-1].lower() == "csv":
- self._format = "CSV"
- self._delimiter = ";"
- elif self.__filestring.count("\t") > 1 and self._filename.split(".")[-1].lower() == "tsv":
- self._format = "TSV"
- self._delimiter = "\t"
- elif self.__filestring.count(" ") > 1 and self._filename.split(".")[-1].lower() == "txt":
- self._format = "TXT"
- else:
- raise ValueError("Can not guess the file format, please specify the good one")
- elif Format.upper() == "CSV" and self._delimiter is None:
- if self.__filestring.count(",") > 1 and self._filename.split(".")[-1].lower() == "csv":
- self._format = "CSV"
+ self.__header, self._varsline, self._skiprows = self.__getentete()
+ #
+ if self._format == "text/csv" or Format.upper() == "CSV":
+ self._format = "text/csv"
+ self.__filestring = "".join(self.__header)
+ if self.__filestring.count(",") > 1:
self._delimiter = ","
- elif self.__filestring.count(";") > 1 and self._filename.split(".")[-1].lower() == "csv":
- self._format = "CSV"
+ elif self.__filestring.count(";") > 1:
self._delimiter = ";"
- elif Format.upper() == "TSV" and self._delimiter is None:
- self._format = "TSV"
+ elif self._format == "text/tab-separated-values" or Format.upper() == "TSV":
+ self._format = "text/tab-separated-values"
self._delimiter = "\t"
else:
- self._format = str(Format).upper()
+ self._delimiter = None
+ #
+ if ColNames is not None: self._colnames = tuple(ColNames)
+ else: self._colnames = None
#
- if ColNames is not None: self.__colnames = tuple(ColNames)
- else: self.__colnames = None
+ if ColIndex is not None: self._colindex = str(ColIndex)
+ else: self._colindex = None
#
- if ColIndex is not None: self.__colindex = str(ColIndex)
- else: self.__colindex = None
+ self.__allowvoid = bool(AllowVoidNameList)
- def __getentete(self, __filename, __nblines = 3):
+ def __getentete(self, __nblines = 3):
"Lit l'entête du fichier pour trouver la définition des variables"
__header, __varsline, __skiprows = [], "", 1
- if __filename.split(".")[-1].lower() in ("npy", "npz"):
+ if self._format in self.__binaryformats:
pass
else:
- with open(__filename,'r') as fid:
+ with open(self._filename,'r') as fid:
__line = fid.readline().strip()
while "#" in __line or len(__line) < 1:
__header.append(__line)
for i, n in enumerate(__varserie):
if v == n: __usecols.append(i)
__usecols = tuple(__usecols)
- if len(__usecols) == 0: __usecols = None
+ if len(__usecols) == 0:
+ if self.__allowvoid:
+ __usecols = None
+ else:
+ raise ValueError("Can not found any column corresponding to the required names %s"%(__colnames,))
else:
__usecols = None
#
#
return (__usecols, __useindex)
+ def getsupported(self):
+ self.__supportedformats = {}
+ self.__supportedformats["text/plain"] = True
+ self.__supportedformats["text/csv"] = True
+ self.__supportedformats["text/tab-separated-values"] = True
+ self.__supportedformats["application/numpy.npy"] = True
+ self.__supportedformats["application/numpy.npz"] = True
+ self.__supportedformats["application/dymola.sdf"] = PlatformInfo.has_sdf
+ return self.__supportedformats
+
def getvalue(self, ColNames=None, ColIndex=None ):
"Renvoie la ou les variables demandees par la liste de leurs noms"
# Uniquement si mise à jour
- if ColNames is not None: self.__colnames = tuple(ColNames)
- if ColIndex is not None: self.__colindex = str(ColIndex)
+ if ColNames is not None: self._colnames = tuple(ColNames)
+ if ColIndex is not None: self._colindex = str(ColIndex)
#
__index = None
- if self._format == "NPY":
+ if self._format == "application/numpy.npy":
__columns = numpy.load(self._filename)
- elif self._format == "NPZ":
+ #
+ elif self._format == "application/numpy.npz":
__columns = None
with numpy.load(self._filename) as __allcolumns:
- if self.__colnames is None:
- self.__colnames = __allcolumns.files
- for nom in self.__colnames:
+ if self._colnames is None:
+ self._colnames = __allcolumns.files
+ for nom in self._colnames:
if nom in __allcolumns.files:
if __columns is not None:
# Attention : toutes les variables doivent avoir la même taille
else:
# Première colonne
__columns = numpy.reshape(__allcolumns[nom], (1,-1))
- if self.__colindex is not None and self.__colindex in __allcolumns.files:
- __index = numpy.reshape(__allcolumns[self.__colindex], (1,-1))
- elif self._format == "TXT":
- __usecols, __useindex = self.__getindices(self.__colnames, self.__colindex)
+ if self._colindex is not None and self._colindex in __allcolumns.files:
+ __index = numpy.array(numpy.reshape(__allcolumns[self._colindex], (1,-1)), dtype=bytes)
+ elif self._format == "text/plain":
+ __usecols, __useindex = self.__getindices(self._colnames, self._colindex)
__columns = numpy.loadtxt(self._filename, usecols = __usecols, skiprows=self._skiprows)
if __useindex is not None:
- __index = numpy.loadtxt(self._filename, usecols = __useindex, skiprows=self._skiprows)
+ __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), skiprows=self._skiprows)
#
- elif self._format == "CSV":
- __usecols, __useindex = self.__getindices(self.__colnames, self.__colindex, self._delimiter)
+ elif self._format == "application/dymola.sdf" and PlatformInfo.has_sdf:
+ import sdf
+ __content = sdf.load(self._filename)
+ __columns = None
+ if self._colnames is None:
+ self._colnames = [__content.datasets[i].name for i in range(len(__content.datasets))]
+ for nom in self._colnames:
+ if nom in __content:
+ if __columns is not None:
+ # Attention : toutes les variables doivent avoir la même taille
+ __columns = numpy.vstack((__columns, numpy.reshape(__content[nom].data, (1,-1))))
+ else:
+ # Première colonne
+ __columns = numpy.reshape(__content[nom].data, (1,-1))
+ if self._colindex is not None and self._colindex in __content:
+ __index = __content[self._colindex].data
+ #
+ elif self._format == "text/csv":
+ __usecols, __useindex = self.__getindices(self._colnames, self._colindex, self._delimiter)
__columns = numpy.loadtxt(self._filename, usecols = __usecols, delimiter = self._delimiter, skiprows=self._skiprows)
if __useindex is not None:
- __index = numpy.loadtxt(self._filename, usecols = __useindex, delimiter = self._delimiter, skiprows=self._skiprows)
+ __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), delimiter = self._delimiter, skiprows=self._skiprows)
#
- elif self._format == "TSV":
- __usecols, __useindex = self.__getindices(self.__colnames, self.__colindex, self._delimiter)
+ elif self._format == "text/tab-separated-values":
+ __usecols, __useindex = self.__getindices(self._colnames, self._colindex, self._delimiter)
__columns = numpy.loadtxt(self._filename, usecols = __usecols, delimiter = self._delimiter, skiprows=self._skiprows)
if __useindex is not None:
- __index = numpy.loadtxt(self._filename, usecols = __useindex, delimiter = self._delimiter, skiprows=self._skiprows)
+ __index = numpy.loadtxt(self._filename, dtype = bytes, usecols = (__useindex,), delimiter = self._delimiter, skiprows=self._skiprows)
else:
- raise ValueError("Unkown file format %s"%self._format)
+ raise ValueError("Unkown file format \"%s\" or no reader available"%self._format)
+ if __columns is None: __columns = ()
+ #
+ def toString(value):
+ try:
+ return value.decode()
+ except ValueError:
+ return value
+ if __index is not None:
+ __index = tuple([toString(v) for v in __index])
#
- return (self.__colnames, __columns, self.__colindex, __index)
+ return (self._colnames, __columns, self._colindex, __index)
def getstring(self):
- "Renvoie le fichier complet"
- with open(self._filename,'r') as fid:
- return fid.read()
+ "Renvoie le fichier texte complet"
+ if self._format in self.__binaryformats:
+ return ""
+ else:
+ with open(self._filename,'r') as fid:
+ return fid.read()
+
+ def getformat(self):
+ return self._format
# ==============================================================================
class ImportScalarLinesFromFile(ImportFromFile):
#
def __init__(self, Filename=None, ColNames=None, ColIndex=None, Format="Guess"):
ImportFromFile.__init__(self, Filename, ColNames, ColIndex, Format)
- if self._format not in ["TXT", "CSV", "TSV"]:
+ if self._format not in ["text/plain", "text/csv", "text/tab-separated-values"]:
raise ValueError("Unkown file format \"%s\""%self._format)
#
def getvalue(self, VarNames = None, HeaderNames=()):
else:
raise ValueError("Can not find names of columns for initial values. Wrong first line is:\n \"%s\""%__firstline)
#
- if self._format == "TXT":
+ if self._format == "text/plain":
__content = numpy.loadtxt(self._filename, dtype = __dtypes, usecols = __usecols, skiprows = self._skiprows, converters = __converters)
- elif self._format in ["CSV", "TSV"]:
+ elif self._format in ["text/csv", "text/tab-separated-values"]:
__content = numpy.loadtxt(self._filename, dtype = __dtypes, usecols = __usecols, skiprows = self._skiprows, converters = __converters, delimiter = self._delimiter)
else:
raise ValueError("Unkown file format \"%s\""%self._format)
#
- __names, __background, __bounds = [], [], []
+ __names, __thevalue, __bounds = [], [], []
for sub in __content:
if len(__usecols) == 4:
na, va, mi, ma = sub
if (__varnames is None or na in __varnames) and (na not in __names):
# Ne stocke que la premiere occurence d'une variable
__names.append(na)
- __background.append(va)
+ __thevalue.append(va)
__bounds.append((mi,ma))
#
__names = tuple(__names)
- __background = numpy.array(__background)
+ __thevalue = numpy.array(__thevalue)
__bounds = tuple(__bounds)
#
- return (__names, __background, __bounds)
+ return (__names, __thevalue, __bounds)
# ==============================================================================
if __name__ == "__main__":
- print('\n AUTODIAGNOSTIC \n')
+ print('\n AUTODIAGNOSTIC\n')
