5 from io import StringIO
9 logConfigParser = logging.getLogger(__name__)
11 ADD_TO_PREFIX = 'ADD_TO_'
12 UNSET_KEYWORD = 'UNSET'
15 # :TRICKY: So ugly solution...
16 class MultiOptSafeConfigParser(ConfigParser.SafeConfigParser):
18 ConfigParser.SafeConfigParser.__init__(self)
20 # copied from python 2.6.8 Lib.ConfigParser.py
21 # modified (see code comments) to handle duplicate keys
22 def _read(self, fp, fpname):
23 """Parse a sectioned setup file.
25 The sections in setup file contains a title line at the top,
26 indicated by a name in square brackets (`[]'), plus key/value
27 options lines, indicated by `name: value' format lines.
28 Continuations are represented by an embedded newline then
29 leading whitespace. Blank lines, lines beginning with a '#',
30 and just about everything else are ignored.
32 cursect = None # None, or a dictionary
35 e = None # None, or an exception
41 # comment or blank line?
42 if line.strip() == '' or line[0] in '#;':
44 if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
45 # no leading whitespace
48 if line[0].isspace() and cursect is not None and optname:
51 cursect[optname].append(value)
52 # a section header or option header?
54 # is it a section header?
55 mo = self.SECTCRE.match(line)
57 sectname = mo.group('header')
58 if sectname in self._sections:
59 cursect = self._sections[sectname]
60 elif sectname == ConfigParser.DEFAULTSECT:
61 cursect = self._defaults
63 cursect = self._dict()
64 cursect['__name__'] = sectname
65 self._sections[sectname] = cursect
66 # So sections can't start with a continuation line
68 # no section header in the file?
70 raise MissingSectionHeaderError(fpname, lineno, line)
73 mo = self.OPTCRE.match(line)
75 optname, vi, optval = mo.group('option', 'vi', 'value')
76 optname = self.optionxform(optname.rstrip())
77 # This check is fine because the OPTCRE cannot
78 # match if it would set optval to None
79 if optval is not None:
80 if vi in ('=', ':') and ';' in optval:
81 # ';' is a comment delimiter only if it follows
83 pos = optval.find(';')
84 if pos != -1 and optval[pos-1].isspace():
86 optval = optval.strip()
90 # REPLACE following line (original):
91 #cursect[optname] = [optval]
93 # Check if this optname already exists
94 if (optname in cursect) and (cursect[optname] is not None):
95 cursect[optname][0] += ','+optval
97 cursect[optname] = [optval]
100 # valueless option handling
101 cursect[optname] = optval
103 # a non-fatal parsing error occurred. set up the
104 # exception but keep going. the exception will be
105 # raised at the end of the file and will contain a
106 # list of all bogus lines
108 e = ConfigParser.ParsingError(fpname)
109 e.append(lineno, repr(line))
110 # if any parsing errors occurred, raise an exception
114 # join the multi-line values collected while reading
115 all_sections = [self._defaults]
116 all_sections.extend(self._sections.values())
117 for options in all_sections:
118 for name, val in options.items():
119 if isinstance(val, list):
120 options[name] = '\n'.join(val)
124 # Parse configuration file
125 # Input: filename, and a list of reserved keywords (environment variables)
126 # Output: a list of pairs (variable, value), and a dictionary associating a list of user-defined values to each reserved keywords
127 # Note: Does not support duplicate keys in a same section
128 def parseConfigFile(filename, reserved = []):
129 config = MultiOptSafeConfigParser()
130 config.optionxform = str # case sensitive
132 # :TODO: test file existence
136 config.read(filename)
137 except ConfigParser.MissingSectionHeaderError:
138 logConfigParser.error("No section found in file: %s"%(filename))
141 return __processConfigFile(config, reserved, filename)
144 def __processConfigFile(config, reserved = [], filename="UNKNOWN FILENAME"):
145 # :TODO: may detect duplicated variables in the same section (raise a warning)
146 # or even duplicate sections
150 # Get raw items for each section, and make some processing for environment variables management
151 reservedKeys = [ADD_TO_PREFIX+str(x) for x in reserved] # produce [ 'ADD_TO_reserved_1', 'ADD_TO_reserved_2', ..., ADD_TO_reserved_n ]
152 reservedValues = dict([str(i),[]] for i in reserved) # create a dictionary in which keys are the 'ADD_TO_reserved_i' and associated values are empty lists: { 'reserved_1':[], 'reserved_2':[], ..., reserved_n:[] }
153 sections = config.sections()
154 for section in sections:
155 entries = config.items(section, raw=False) # use interpolation
156 if len(entries) == 0: # empty section
157 logConfigParser.warning("Empty section: %s in file: %s"%(section, filename))
159 for key,val in entries:
161 logConfigParser.error("Invalid use of reserved variable: %s in file: %s"%(key, filename))
162 elif key == UNSET_KEYWORD:
163 unsetVariables += val.replace(',', ' ').split()
165 expandedVal = os.path.expandvars(val) # expand environment variables
166 # Search for not expanded variables (i.e. non-existing environment variables)
167 pattern = re.compile('\${ ( [^}]* ) }', re.VERBOSE) # string enclosed in ${ and }
168 expandedVal = pattern.sub(r'', expandedVal) # remove matching patterns
170 if not "DLIM8VAR" in key: # special case: DISTENE licence key can contain double clons (::)
171 expandedVal = _trimColons(expandedVal)
173 if key in reservedKeys:
174 shortKey = key[len(ADD_TO_PREFIX):]
175 vals = expandedVal.split(',')
176 reservedValues[shortKey] += vals
177 # remove left&right spaces on each element
178 vals = [v.strip(' \t\n\r') for v in vals]
180 outputVariables.append((key, expandedVal))
183 pass # end for key,val
184 pass # end for section
186 return unsetVariables, outputVariables, reservedValues
189 def _trimColons(var):
191 # Remove leading and trailing colons (:)
192 pattern = re.compile('^:+ | :+$', re.VERBOSE)
193 v = pattern.sub(r'', v) # remove matching patterns
194 # Remove multiple colons
195 pattern = re.compile('::+', re.VERBOSE)
196 v = pattern.sub(r':', v) # remove matching patterns
200 # This class is used to parse .sh environment file
201 # It deals with specific treatments:
202 # - virtually add a section to configuration file
203 # - process shell keywords (if, then...)
204 class EnvFileConverter(object):
205 def __init__(self, fp, section_name, reserved = [], outputFile=None):
207 self.sechead = '[' + section_name + ']\n'
208 self.reserved = reserved
209 self.outputFile = outputFile
210 self.allParsedVariableNames=[]
211 # exclude line that begin with:
212 self.exclude = [ 'if', 'then', 'else', 'fi', '#', 'echo', 'exit' ]
213 self.exclude.append('$gconfTool') # QUICK FIX :TODO: provide method to extend this variable
214 # discard the following keywords if at the beginning of line:
215 self.discard = [ 'export' ]
216 # the following keywords imply a special processing if at the beginning of line:
217 self.special = [ 'unset' ]
222 if self.outputFile is not None:
223 self.outputFile.write(self.sechead)
228 line = self.fp.readline()
230 line = line.strip(' \t\n\r')
231 # line of interest? (not beginning by a keyword of self.exclude)
232 for k in self.exclude:
233 if line.startswith(k):
235 # look for substrinsg beginning with sharp charcter ('#')
236 line = re.sub(r'#.*$', r'', line)
237 # line to be pre-processed? (beginning by a keyword of self.special)
238 for k in self.special:
239 if k == "unset" and line.startswith(k):
241 line = line.strip(' \t\n\r')
242 line = UNSET_KEYWORD + ": " + line
243 # line to be pre-processed? (beginning by a keyword of self.discard)
244 for k in self.discard:
245 if line.startswith(k):
247 line = line.strip(' \t\n\r')
248 # process reserved keywords
249 for k in self.reserved:
250 if line.startswith(k) and "=" in line:
251 variable, value = line.split('=')
252 value = self._purgeValue(value, k)
253 line = ADD_TO_PREFIX + k + ": " + value
254 # Update list of variable names
255 # :TODO: define excludeBlock variable (similar to exclude) and provide method to extend it
256 if line.startswith("LOGNAME="):
258 if "cleandup()" in line:
259 print "WARNING: parseConfigFile.py: skip cleandup and look for '# PRODUCT environment'"
261 line = self.fp.readline()
262 if "# PRODUCT environment" in line:
263 print "WARNING: parseConfigFile.py: '# PRODUCT environment' found"
265 while "clean " in line[0:6]: #skip clean calls with ending ";" crash
266 line = self.fp.readline()
267 # Extract variable=value
270 variable, value = line.split('=')
271 except: #avoid error for complicated sh line xx=`...=...`, but warning
272 print "WARNING: parseConfigFile.py: line with multiples '=' character are hazardous: '"+line+"'"
273 variable, value = line.split('=',1)
276 # Self-extending variables that are not in reserved keywords
277 # Example: FOO=something:${FOO}
278 # In this case, remove the ${FOO} in value
279 if variable in value:
280 value = self._purgeValue(value, variable)
281 line = "%s=%s"%(variable,value)
283 self.allParsedVariableNames.append(variable)
290 # replace "${FOO}" and "$FOO" and ${FOO} and $FOO by %(FOO)s if FOO is
291 # defined in current file (i.e. it is not an external environment variable)
292 for k in self.allParsedVariableNames:
293 key = r'\$\{?'+k+'\}?'
294 pattern = re.compile(key, re.VERBOSE)
295 line = pattern.sub(r'%('+k+')s', line)
296 # Remove quotes (if line does not contain whitespaces)
298 variable, value = line.split('=', 1)
300 variable, value = line.split(':', 1)
301 if not ' ' in value.strip():
302 pattern = re.compile(r'\"', re.VERBOSE)
303 line = pattern.sub(r'', line)
306 # Replace `shell_command` by its result
308 obj = re.sub('`', r'', obj.group(0)) # remove quotes
310 res = subprocess.Popen(obj, stdout=subprocess.PIPE).communicate()[0]
311 res = res.strip(' \t\n\r') # trim whitespaces
314 line = re.sub('`[^`]+`', myrep, line)
316 if self.outputFile is not None:
317 self.outputFile.write(line+'\n')
320 def _purgeValue(self, value, name):
321 # Replace foo:${PATTERN}:bar or foo:$PATTERN:bar by foo:bar
322 key = r'\$\{?'+name+'\}?'
323 pattern = re.compile(key, re.VERBOSE)
324 value = pattern.sub(r'', value)
327 value = _trimColons(value)
332 # Convert .sh environment file to configuration file format
333 def convertEnvFileToConfigFile(envFilename, configFilename, reserved=[]):
334 logConfigParser.debug('convert env file %s to %s'%(envFilename, configFilename))
335 fileContents = open(envFilename, 'r').read()
337 pattern = re.compile('\n[\n]+', re.VERBOSE) # multiple '\n'
338 fileContents = pattern.sub(r'\n', fileContents) # replace by a single '\n'
340 finput = StringIO(unicode(fileContents))
341 foutput = open(configFilename, 'w')
343 config = MultiOptSafeConfigParser()
344 config.optionxform = str # case sensitive
345 config.readfp(EnvFileConverter(finput, 'SALOME Configuration', reserved, outputFile=foutput))
349 logConfigParser.info('Configuration file generated: %s'%configFilename)