2 #!/tools/net/app/Python-1.5.2/bin/python1.5
4 """Translate - a first attempt at parsing my little language
6 Usage: Translate [switches] <infile> [<outfile>]
8 -stdout -- write to standard output instead of a file
9 -force -- write to the <outfile> even if it already
10 exists (overwrite any existing file)
12 -import -- import tag table from Translate_tags.py,
13 instead of using the internal table
15 -compare -- compare the imported and internal tag tables
18 -test -- use our internal test data and write to stdout
19 -pytag -- use the interpreted tagging engine
20 -debug -- if -pytag, enable its debugger
21 -diag -- enable general debugging
22 Beware that this currently also writes line
23 numbers to the start of each line in the output,
24 so it doesn't emit legal Python...
26 -help -- show this text
27 -history -- show the module history
28 -version -- show the module version
30 If <outfile> is not specified, <infile> will be used with its extension
34 __author__ = """Tibs (Tony J Ibbs)
35 tony@lsl.co.uk or tibs@tibsnjoan.demon.co.uk or tibs@bigfoot.com
36 http://www.tibsnjoan.demon.co.uk/
38 __version__ = "0.3 (tiepin) of 1999-11-15"
40 Originally created 1999-08-13
42 First released version is 0.2 (bootstrap)/1999-09-09, which gave
43 an idea of how the thing would work, and nearly did.
45 Second released version is 0.3 (tiepin)/1999-11-15, which is sufficient
46 to allow the parser used within this utility to be written in the little
47 language, translated and then used as such.
54 # ............................................................
55 # How we want to work things - this is fudge for me in initial development
56 if os.name == "posix":
62 TEXTTOOLS_PATH = "C:\\Program Files\\Python"
63 PYTAG_PATH = "C:\\Program Files\\Python\\TextTools\\Examples"
67 if TEXTTOOLS_PATH not in sys.path:
68 print "Adding",TEXTTOOLS_PATH
69 sys.path.append(TEXTTOOLS_PATH)
71 if PYTAG_PATH not in sys.path:
72 print "Adding",PYTAG_PATH
73 sys.path.append(PYTAG_PATH)
74 # ............................................................
76 # Import the TextTools themselves
77 # - I'm not personally too keen on import *, but it seems to be
78 # the recommended thing, so I'll leave it for now...
80 from TextTools import *
82 from mx.TextTools import *
83 #from TextTools.Constants.TagTables import *
84 #from TextTools.Constants.Sets import *
87 # ------------------------------------------------------------
88 # Useful little constants for unpicking the parsed tuples
94 # We want to align inline comments when possible - so this is
95 # the column at which we will try to place their "#" marks...
98 # Are we (generally) debugging?
101 # Do we want a comma after the last tuple (or item) in a table?
105 # ------------------------------------------------------------
106 def define_tagtable():
107 """Returns our tag table, if we're not importing it."""
109 # We are not, initially, going to try for anything very sophisticated
110 # - just something that will get us bootstrapped, so that I can use the
111 # "little language" to write more sophisticated stuff (without having
112 # to worry about dropped commas between tuples, and so on!)
115 # Whitespace is always useful
116 t_whitespace = (None,AllIn,' \t')
117 t_opt_whitespace = t_whitespace + (+1,)
119 # Comments are fairly simple
120 t_comment = ('comment',Table,
122 (None,AllNotIn,'\n\r',MatchOk))
125 # We care about the "content" of the indentation at the start of a line,
126 # but note that it is optional
127 t_indent = ('indent',AllIn,' \t')
128 t_indentation = t_indent + (+1,) # zero indentation doesn't show
130 # A string is text within single or double quotes
131 # (of course, this is an oversimplification, because we should also
132 # deal with things like "This is a \"substring\"", and it would be
133 # nice to be able to cope with triple-quoted strings too, but it
134 # will do for a start)
136 # Major bug - doesn't recognised zero length strings...
137 # (since "AllNotIn" must match at least one character)
138 t_string = ('str',Table,
139 ((None,Is,"'",+3,+1),
140 ('text',AllNotIn,"'"),
141 (None,Is,"'",MatchFail,MatchOk),
143 ('text',AllNotIn,'"'),
147 # An integer is a series of digits...
148 t_integer = ('int',AllIn,number)
150 t_signed_integer = ('signed_int',Table,
151 (('sign',Is,"+",+1,+2),
152 ('sign',Is,"-",+1,+1),
156 # Remember to be careful to specify the LONGEST possible match first,
157 # so that we try for "IsIn" before we try for "Is" (because "IsIn"
158 # would *match* "Is", leaving us with a spurious "In" hanging around...)
159 t_operation = ('op',Table,
160 (('op',Word,"AllInSet", +1,MatchOk),
161 ('op',Word,"AllIn", +1,MatchOk),
162 ('op',Word,"AllNotIn", +1,MatchOk),
163 ('op',Word,"CallArg", +1,MatchOk),
164 ('op',Word,"Call", +1,MatchOk),
165 ('op',Word,"EOF", +1,MatchOk),
166 ('op',Word,"Fail", +1,MatchOk),
167 ('op',Word,"IsInSet", +1,MatchOk),
168 ('op',Word,"IsIn", +1,MatchOk),
169 ('op',Word,"IsNotIn", +1,MatchOk),
170 ('op',Word,"IsNot", +1,MatchOk),
171 ('op',Word,"Is", +1,MatchOk),
172 ('op',Word,"Jump", +1,MatchOk),
173 ('op',Word,"LoopControl",+1,MatchOk),
174 ('op',Word,"Loop", +1,MatchOk),
175 ('op',Word,"Move", +1,MatchOk),
176 ('op',Word,"NoWord", +1,MatchOk), # alias for WordStart
177 ('op',Word,"Skip", +1,MatchOk),
178 ('op',Word,"SubTableInList",+1,MatchOk),
179 ('op',Word,"SubTable", +1,MatchOk),
180 ('op',Word,"sFindWord", +1,MatchOk),
181 ('op',Word,"sWordStart", +1,MatchOk),
182 ('op',Word,"sWordEnd", +1,MatchOk),
183 ('op',Word,"TableInList",+1,MatchOk),
184 ('op',Word,"Table", +1,MatchOk),
185 ('op',Word,"WordStart", +1,MatchOk),
186 ('op',Word,"WordEnd", +1,MatchOk),
187 ('op',Word,"Word", MatchFail,MatchOk),
191 t_keyword = ('keyword',Table,
192 ((None,Word,"and", +1,+28),
193 (None,Word,"assert", +1,+27),
194 (None,Word,"break", +1,+26),
195 (None,Word,"class", +1,+25),
196 (None,Word,"continue",+1,+24),
197 (None,Word,"def", +1,+23),
198 (None,Word,"del", +1,+22),
199 (None,Word,"elif", +1,+21),
200 (None,Word,"else", +1,+20),
201 (None,Word,"except", +1,+19),
202 (None,Word,"exec", +1,+18),
203 (None,Word,"finally", +1,+17),
204 (None,Word,"for", +1,+16),
205 (None,Word,"from", +1,+15),
206 (None,Word,"global", +1,+14),
207 (None,Word,"if", +1,+13),
208 (None,Word,"import", +1,+12),
209 (None,Word,"in", +1,+11),
210 (None,Word,"is", +1,+10),
211 (None,Word,"lambda", +1,+9),
212 (None,Word,"not", +1,+8),
213 (None,Word,"or", +1,+7),
214 (None,Word,"pass", +1,+6),
215 (None,Word,"print", +1,+5),
216 (None,Word,"raise", +1,+4),
217 (None,Word,"return", +1,+3),
218 (None,Word,"try", +1,+2),
219 (None,Word,"while", MatchFail,+1),
220 # In order to not recognise things like "in_THIS_CASE"
221 # we must check that the next character is not legitimate
222 # within an identifier
223 (None,IsIn,alpha+'_'+number,+1,MatchFail),
224 # If it wasn't another identifier character, we need to
225 # unread it so that it can be recognised as something else
226 # (so that, for instance, "else:" is seen as "else" followed
231 # Do the same for mxText commands
232 t_mxkeyword = ('mxKeyword',Table,
234 (None,IsIn,alpha+'_'+number,+1,MatchFail),
238 # Traditional identifiers
239 t_identifier = ('identifier',Table,
240 (t_keyword + (+1,MatchFail), # don't allow Python keywords
241 t_mxkeyword + (+1,MatchFail), # don't allow mxText commands
242 (None,IsIn,alpha+'_'), # can't start with a digit
243 (None,AllIn,alpha+'_'+number,MatchOk))
246 # We don't yet deal with the following with anything in parentheses,
247 # which means we can't handle functions or command lists, or other
248 # things which "look like" a tuple
249 t_argument = ('arg',Table,
250 (('arg',Word,"Here", +1,MatchOk), # EOF Here, Fail Here
251 ('arg',Word,"ToEOF", +1,MatchOk), # Move ToEOF
252 ('arg',Word,"To", +1,MatchOk), # Jump To
253 ('arg',Word,"ThisTable",+1,MatchOk), # [Sub]Table ThisTable
254 ('arg',Word,"back", +1,MatchOk), # Skip back
255 ('arg',Word,"Break", +1,MatchOk), # LoopControl Break
256 ('arg',Word,"Reset", +1,MatchOk), # LoopControl Reset
257 t_string + (+1,MatchOk), # e.g., Word "Fred"
258 t_signed_integer + (+1,MatchOk), # e.g., Skip -4, Move 3
259 t_identifier # e.g., Table Fred
262 t_plus = ('plus',Table,
268 # Arguments can contain "+"
269 t_plus_arg = ('plusarg',Table,
270 (t_argument, # start with a single argument
271 t_plus + (MatchOk,), # if we have a "+"
272 t_argument, # then we expect another argument
273 (None,Jump,To,-2), # then look for another "+"
276 # Match, for example:
278 t_label = ('label',Table,
284 # Targets for Jump and F:/T:
285 t_target = ('target',Table,
286 (('tgt',Word,"next", +1,MatchOk),
287 ('tgt',Word,"previous", +1,MatchOk),
288 ('tgt',Word,"repeat", +1,MatchOk),
289 ('tgt',Word,"MatchOk", +1,MatchOk),
290 ('tgt',Word,"MatchOK", +1,MatchOk), # For kindness sake
291 ('tgt',Word,"MatchFail",+1,MatchOk),
295 # A value is either an identifier, or a string, or an integer
296 t_value = ('val',Table,
297 (t_identifier +(+1,MatchOk),
298 t_string +(+1,MatchOk),
302 # An assignment is (optionally) used in Tuple and Table definitions...
303 t_assignment = ('assignment',Table,
309 # A common error when writing tuples is to miss off the "=" sign
310 # - the following is used in diagnosing that (see t_bad_tuple below)
311 # (it's useful to have something with identical structure to the
313 t_bad_tagobj = ('tagobj',Table,
317 t_bad_assignment = ('assignment',Table,
321 # This is the line that starts the definition of a single tuple.
322 # For the moment, restrict what it gets assigned to to a simple identifier.
323 # Match, for example:
325 t_tupleblock = ('tupleblock',Table,
331 # This is the line that starts a new table or sub-table.
332 # For the moment, we only cope with full Tables.
333 # NOTE that this is used for the "outer" declaration of a tag table,
334 # and also for the "inner" declaration of an inner table or sub-table.
335 # The discrimination between these is done after initial parsing.
336 # Match, for example:
337 # 'keyword' = Table is: (inner)
338 # tagtable = Table is: (outer)
339 t_tableblock = ('tableblock',Table,
340 (t_assignment + (+2,+1), # left hand side is optional
342 ('type',Word,"Table",+1,+2), # Either "Table"
343 ('type',Word,"SubTable"), # or "SubTable" is required
344 t_whitespace, # whitespace is required
345 (None,Word,"is:") # "is:" is required
348 # This is the line that starts an "if" block
349 # Match, for example:
352 t_ifblock = ('ifblock',Table,
353 (t_assignment + (+2,+1), # left hand side is optional
355 t_operation + (+4,+1),
358 (None,Is,":",MatchFail,MatchOk),
363 # Note that we don't allow spaces WITHIN our false and true thingies
365 t_onfalse = ('onfalse',Table,
371 t_ontrue = ('ontrue',Table,
377 # Valid examples are things like:
378 # 'fred' = Is "xxx" F:<wow> T:MatchOk
379 # AllIn jim T:<foundJim>
381 # For the moment, we're not trying to recognise things in any detail
382 t_tuple = ('tuple',Table,
383 (t_assignment + (+2,+1), # left hand side is optional
385 t_operation, # operation is required
386 t_whitespace, # for the moment, we always require space here
387 t_plus_arg, # argument is required
388 t_onfalse + (+1,+1), # F:target is optional
389 t_ontrue + (MatchOk,MatchOk) # T:target is also optional
392 # If the user has defined a "partial" tuple, they might use something
394 # match_fred F:MatchFalse T:MatchOk
395 t_tupleplus = ('tupleplus',Table,
397 t_onfalse + (+1,+1), # F:target is optional
398 t_ontrue + (MatchOk,MatchOk) # T:target is also optional
401 # Treat Jump To specially - for example:
403 # so that they don't have to do the less obvious "Jump To F:<label>"
404 # (although that will still be recognised, of course, for people who
405 # are used to the tag tuple format itself)
406 t_jumpto = ('jumpto',Table,
414 # Is it worth coping with these?
415 t_bad_jumpto = ('jumpto',Table,
416 ((None,Word,"Jump",+2), # cope with Jump to
417 (None,Word,"to",MatchFail,+2),
418 (None,Word,"JumpTo"), # and with JumpTo
422 # The "content" of a line is the bit after any indentation, and before
424 # For the moment, we won't try to maintain ANY context, so it is up to the
425 # user of the tuples produced to see if they make sense...
426 t_content = ('content',Table,
427 (t_label + (+1,MatchOk),
428 t_tableblock + (+1,MatchOk), # [<value> =] [Sub]Table is:
429 t_tupleblock + (+1,MatchOk), # <identifier> is:
430 t_ifblock + (+1,MatchOk), # <cmd> <arg>: OR <identifier>:
431 t_jumpto + (+1,MatchOk), # Jump To <target>
432 t_tuple + (+1,MatchOk),
433 t_tupleplus + (+1,MatchOk), # name [F:<label> [T:<label>]]
436 t_contentline = ('contentline',Table,
437 (t_content, # something that we care about
439 t_comment +(+1,+1), # always allow a comment
440 (None,IsIn,newline) # the end of the line
443 # Sometimes, the user (e.g., me) writes:
447 # Unfortunately, without the "is", it would get too confusing whether
448 # we actually wanted an if block...
449 t_bad_tableblock = ('tableblock',Table,
450 (t_assignment + (+2,+1), # left hand side is optional
452 (None,Word,"Table"), # "Table" is required
453 (None,Is,":") # "is" is needed before the ":"
456 # Sometimes, the use (e.g., me again) write:
460 # Whilst I'm not entirely convinced that "=" is the best character
461 # to use here, I think we do need something!
462 t_bad_tuple = ('tuple',Table,
463 (t_bad_assignment, # obviously we have to have this!
464 t_whitespace, # in which case the whitespace IS needed
465 t_operation, # operation is required
466 t_whitespace, # for the moment, we must have space here
467 t_plus_arg, # argument is required
468 t_onfalse + (+1,+1), # F:target is optional
469 t_ontrue + (MatchOk,MatchOk) # T:target is also optional
472 # Make some attempt to recognise common errors...
473 t_badcontent = ('badcontent',Table,
474 (t_bad_tableblock +(+1,MatchOk),
478 t_badline = ('badline',Table,
479 (t_badcontent, # something that we sort of care about
481 t_comment +(+1,+1), # always allow a comment
482 (None,IsIn,newline) # the end of the line
485 t_emptyline = ('emptyline',Table,
487 (None,IsIn,newline) # the end of the line
490 t_commentline = ('commentline',Table,
492 (None,IsIn,newline) # the end of the line
495 t_passthruline = ('passthruline',Table,
496 (('passthru',AllNotIn,newline,+1), # owt else on the line
497 (None,IsIn,newline) # the end of the line
500 # Basically, a file is a series of lines
501 t_line = ('line',Table,
502 (t_emptyline +(+1,MatchOk), # empty lines are simple enough
503 t_indent +(+1,+1), # optional indentation
504 t_commentline +(+1,MatchOk), # always allow a comment
505 t_contentline +(+1,MatchOk), # a line we care about
506 t_badline +(+1,MatchOk), # a line we think is wrong
507 t_passthruline # a line we don't care about
517 # ------------------------------------------------------------
518 # We'll define some moderately interesting test data
521 # This example isn't *meant* to make any sense!
522 # It's just an accumulation of things that got checked for various reasons
523 from TextTools import *
532 # And the rest is our business...
534 'int' = AllIn '0123456789'
538 # A comment here is OK
539 <label> # Strangely enough, so is a label
540 'indent' = AllIn ' \t'
542 'int' AllIn number # BUGGY LINE (missing "=")
543 (None,"AllIn",number) # BUGGY LINE (an actual tuple)
544 fred = jim # BUGGY LINE (not our business)
545 tagobj F:<op> T:next # label <op> is undefined
546 # The next line is totally empty
548 # The next line contains just indentation
550 # This line is just a comment
551 # And this comment should be JUST after the preceding block...
552 t_indentation is: # This should be "= Table is:"
556 t_indent F:previous T:previous
560 'this' = Table ThisTable
566 'a' = AllIn 'a' F:previous
567 'a' = AllIn 'a' T:previous
568 'a' = AllIn 'a' F:previous T:previous
570 AllIn 'xyz' F:<later> T:<top>
577 AllIn number+"_"+alpha
582 # ------------------------------------------------------------
585 class OutsideError(Exception):
586 """The entity is not permitted outside a block."""
589 class IndentError(Exception):
590 """An indentation error has been detected."""
593 class NoIdentifier(Exception):
594 """We're missing an identifier (to assign to)."""
598 # ------------------------------------------------------------
599 def LineFactory(lineno,tagtuple,text):
600 """Take some tagged data and return an appropriate line class.
602 lineno -- the line number in the "file". Note that the first line
603 in the file is line 1
604 tagtuple -- a tag tuple for a single line of data
605 text -- the text for the "file". All the "left" and "right" offsets
606 are relative to this text (i.e., it is the entire content
609 The tag tuples we get back from the parser will be of the form:
612 ('indent',left,right,None), -- this is optional
613 ('content',left,right,[<data>])
616 Looking at <type> should enable us to decide what to do with
620 # Extract the list of tuples from this 'line'
621 tuples = tagtuple[SUBLIST]
623 # First off, do we have any indentation?
625 if tup[OBJECT] == "indent":
626 # This is inefficient, because it actually copies strings
627 # around - better would be to duplicate the calculation
628 # that string.expandtabs does internally...
629 indent_str = string.expandtabs(text[tup[LEFT]:tup[RIGHT]])
635 # Now, work out which class we want an instance of
636 # (this is the 'fun' bit)
638 type = tuples[0][OBJECT]
639 if type == 'emptyline':
640 return EmptyLine(lineno,indent_str,tuples[0],text)
641 elif type == 'commentline':
642 return CommentLine(lineno,indent_str,tuples[0],text)
643 elif type == 'passthruline':
644 return PassThruLine(lineno,indent_str,tuples[0],text)
645 elif type == 'contentline':
646 # OK - we need to go down another level
647 sublist = tuples[0][SUBLIST]
649 # Do we also have an in-line comment?
655 # And the actual DATA for our line is down yet another level...
656 sublist = sublist[0][SUBLIST]
657 type = sublist[0][OBJECT]
659 return LabelLine(lineno,indent_str,sublist[0],comment,text)
660 elif type == 'tableblock':
661 return TableBlockLine(lineno,indent_str,sublist[0],comment,text)
662 elif type == 'tupleblock':
663 return TupleBlockLine(lineno,indent_str,sublist[0],comment,text)
664 elif type == 'ifblock':
665 return IfBlockLine(lineno,indent_str,sublist[0],comment,text)
666 elif type == 'tuple':
667 return TupleLine(lineno,indent_str,sublist[0],comment,text)
668 elif type == 'tupleplus':
669 return TuplePlusLine(lineno,indent_str,sublist[0],comment,text)
670 elif type == 'jumpto':
671 return JumpToLine(lineno,indent_str,sublist[0],comment,text)
674 "Line %d is of unexpected type 'contentline/%s'"%(lineno,
676 elif type == 'badline':
677 # OK - we need to go down another level
678 sublist = tuples[0][SUBLIST]
680 # Do we also have an in-line comment?
686 # And the actual DATA for our line is down yet another level...
687 sublist = sublist[0][SUBLIST]
688 type = sublist[0][OBJECT]
689 if type == 'tableblock':
690 return BadTableBlockLine(lineno,indent_str,sublist[0],comment,text)
692 return BadTupleLine(lineno,indent_str,sublist[0],comment,text)
695 "Line %d is of unexpected type 'badline/%s'"%(lineno,type)
697 raise ValueError,"Line %d is of unexpected type '%s'"%(lineno,type)
701 # ------------------------------------------------------------
703 """The base class on which the various line types depend
707 tagtuple -- the tagtuple we (our subclass instance) represent(s)
708 lineno -- the line number in the file (first line is line 1)
709 indent -- our indentation (integer)
710 indent_str -- our indentation (a string of spaces)
711 text -- the text of the "file" we're within
712 class_name -- the name of the actual class this instance belongs to
713 (i.e., the name of the subclass, suitable for printing)
715 Some things only get useful values after we've been instantiated
717 next_indent -- the indentation of the next line
718 index -- for a line in a block, its index therein
721 def __init__(self,lineno,indent_str,tagtuple,text):
722 """Instantiate a BaseLine.
724 lineno -- the line number in the "file". Note that the first line
725 in the file is line 1
726 indent_str -- the indentation of the line (a string of spaces)
727 tagtuple -- the tag tuple for this line of data
728 text -- the text for the "file". All the "left" and "right"
729 offsets are relative to this text (i.e., it is the
730 entire content of the file)
732 The content of the tagtuple depends on which of our subclasses
733 is being used. Refer to the relevant doc string.
736 self.tagtuple = tagtuple
740 self.class_name = self._class_name()
741 self.indent_str = indent_str
742 self.indent = len(indent_str)
744 # OK - we don't really know! (but this will do for "EOF")
747 # We don't always HAVE a sensible value for this
751 # print "Line %3d: %s%s"%(lineno,indent_str,self.class_name)
753 def change_indent(self,count=None,spaces=""):
754 """Change our indentation.
756 Specify either "count" or "spaces" (if both are given,
757 "count" will be used, if neither is given, then the
758 indentation will be set to zero)
760 count -- the number of spaces we're indented by
761 spaces -- a string of spaces
765 self.indent_str = count * " "
767 self.indent_str = spaces
768 self.indent = len(spaces)
770 def _class_name(self):
771 """Return a representation of the class name."""
773 full_name = "%s"%self.__class__
774 bits = string.split(full_name,".")
777 def starts_block(self):
778 """Return true if we start a new block."""
781 def only_in_block(self):
782 """Return true if we can only occur inside a block."""
785 def our_business(self):
786 """Return true if we are a line we understand."""
790 return "%3d %s%-10s"%(self.lineno,self.indent_str,self.class_name)
793 """Returns a useful 'introductory' string."""
794 return "%3d %-10s %s"%(self.lineno,self.class_name,self.indent_str)
797 """Returns a "truncated" representation of our text."""
799 text = "%s %s"%(self._intro(),
800 `self.text[self.tagtuple[LEFT]:self.tagtuple[RIGHT]]`)
803 return text[:60]+"..."
807 def resolve_labels(self,block):
808 """Called to resolve any labels use in this line.
810 block -- the block that contains us
812 # The default is to do nothing as we don't HAVE any labels...
815 def expand(self,stream,block=None):
816 """Write out the expanded equivalent of ourselves.
818 stream -- an object with a "write" method, e.g., a file
819 newline -- true if we should output a terminating newline
820 block -- used to pass the containing Block down to lines
821 within a block, or None if we're not in a block
825 stream.write("Line %3d: "%self.lineno)
827 stream.write(self.indent_str)
828 stream.write(self.text[self.tagtuple[LEFT]:self.tagtuple[RIGHT]])
831 def warning(self,text):
832 """Report a warning message.
834 text -- the text to report
837 lines = string.split(text,"\n")
838 print "###WARNING: line %d (%s)"%(self.lineno,self.class_name)
842 def error(self,text):
845 text -- the error text to report
848 lines = string.split(text,"\n")
849 print "###ERROR: line %d (%s)"%(self.lineno,self.class_name)
854 # ------------------------------------------------------------
855 class EmptyLine(BaseLine):
858 Note that the indentation of an empty line is taken to be the
859 same as that of the next (non-empty) line. This is because it
860 seems to me that (a) an empty line should not per-se close a
861 block (which it would do if it had indentation 0) and (b) we
862 don't remember any whitespace in an empty line, so the user
863 can't assign an indentation themselves (which is a Good Thing!)
866 def __init__(self,lineno,indent_str,tagtuple,text):
867 """Instantiate an EmptyLine.
869 The content of the tagtuple is:
873 BaseLine.__init__(self,lineno,indent_str,tagtuple,text)
875 def expand(self,stream,block=None):
876 """Write out the expanded equivalent of ourselves.
878 stream -- an object with a "write" method, e.g., a file
879 block -- used to pass the containing Block down to lines
880 within a block, or None if we're not in a block
884 stream.write("Line %3d: "%self.lineno)
886 # um - there's nothing to do, folks
889 def our_business(self):
890 """Return true if we are a line we understand."""
894 """Returns a "truncated" representation of our text."""
899 # ------------------------------------------------------------
900 class CommentLine(BaseLine):
901 """A comment line."""
903 def __init__(self,lineno,indent_str,tagtuple,text):
904 """Instantiate a CommentLine.
906 The content of the tagtuple is:
907 ('comment',left,right,None)
908 and the demarcated text includes the initial '#' character
911 BaseLine.__init__(self,lineno,indent_str,tagtuple,text)
913 # We actually want the next tuple down (so to speak) so that
914 # we lose the trailing newline...
915 tup = self.tagtuple[SUBLIST][0]
916 self.data = self.text[tup[LEFT]:tup[RIGHT]]
918 def our_business(self):
919 """Return true if we are a line we understand."""
922 def expand(self,stream,block=None):
923 """Write out the expanded equivalent of ourselves.
925 stream -- an object with a "write" method, e.g., a file
926 block -- used to pass the containing Block down to lines
927 within a block, or None if we're not in a block
931 stream.write("Line %3d: "%self.lineno)
933 stream.write(self.indent_str)
934 stream.write("%s\n"%self.data)
937 # ------------------------------------------------------------
938 class PassThruLine(BaseLine):
939 """A line we just pass throught without interpretation."""
941 def __init__(self,lineno,indent_str,tagtuple,text):
942 """Instantiate a PassThruLine.
944 The content of the tagtuple is:
945 ('passthru',left,right,None)
948 BaseLine.__init__(self,lineno,indent_str,tagtuple,text)
950 # We actually want the next tuple down (so to speak) so that
951 # we lose the trailing newline...
952 tup = self.tagtuple[SUBLIST][0]
953 self.data = self.text[tup[LEFT]:tup[RIGHT]]
955 def our_business(self):
956 """Return true if we are a line we understand."""
959 def expand(self,stream,block=None):
960 """Write out the expanded equivalent of ourselves.
962 stream -- an object with a "write" method, e.g., a file
963 block -- used to pass the containing Block down to lines
964 within a block, or None if we're not in a block
968 stream.write("Line %3d: "%self.lineno)
971 err_str = "Unparsed line inside a block"\
972 " - it has been commented out"
973 # Hmm - the following advice is less often useful than I
974 # had hoped - leave it out for now...
975 #if string.find(self.data,",") != -1:
976 # err_str = err_str + "\nCheck for a trailing comma?"
980 # Always output the indentation, 'cos otherwise it looks silly
981 stream.write(self.indent_str)
984 stream.write("#[ignored]#")
986 stream.write("%s\n"%self.data)
989 # ------------------------------------------------------------
990 class ContentLine(BaseLine):
991 """A line we have to interpret - another base class.
993 Adds the following variables:
995 comment -- any in-line comment on this line
998 def __init__(self,lineno,indent_str,tagtuple,comment,text):
999 """Instantiate a ContentLine.
1001 comment -- either a comment tuple or None
1003 The content of the tagtuple is:
1004 ('contentline',left,right,
1005 [('content',left,right,[<data>]),
1006 ('comment',left,right,None) -- optional
1008 where <data> is used in the internals of one of our subclasses
1009 (i.e., it is what is passed down in the "tagtuple" argument)
1012 BaseLine.__init__(self,lineno,indent_str,tagtuple,text)
1013 self.comment = comment
1015 # Assume we're not the last "our business" line in a block...
1018 def _write_comment(self,stream,sofar):
1019 """Write out the in-line comment string.
1021 Since we're the only people to call this, we can safely
1022 rely on it only being called when there IS a comment tuple
1025 stream -- an object with a "write" method, e.g., a file
1026 sofar -- the number of characters written to the line
1029 if sofar < COMMENT_COLUMN:
1030 stream.write(" "*(COMMENT_COLUMN - sofar))
1032 # always write at least one space...
1034 stream.write(self.text[self.comment[LEFT]:self.comment[RIGHT]])
1036 def _write_text(self,stream,block):
1037 """Write out the main tuple text.
1039 stream -- an object with a "write" method, e.g., a file
1040 block -- used to pass the containing Block down to lines
1041 within a block, or None if we're not in a block
1043 This should generally be the method that subclasses override.
1044 It returns the number of characters written, or -1 if we had
1047 stream.write(self.text[self.tagtuple[LEFT]:self.tagtuple[RIGHT]])
1048 return self.tagtuple[RIGHT] - self.tagtuple[LEFT]
1050 def expand(self,stream,block=None):
1051 """Write out the expanded equivalent of ourselves.
1053 stream -- an object with a "write" method, e.g., a file
1054 block -- used to pass the containing Block down to lines
1055 within a block, or None if we're not in a block
1059 stream.write("Line %3d: "%self.lineno)
1061 stream.write(self.indent_str)
1062 nchars = self._write_text(stream,block)
1063 # Don't write any in-line comment out if we had an error,
1064 # as the layout won't work!
1065 if nchars > -1 and self.comment:
1066 self._write_comment(stream,sofar=nchars+self.indent)
1070 # ------------------------------------------------------------
1071 class LabelLine(ContentLine):
1072 """A line containing a label.
1075 label -- our label string
1078 def __init__(self,lineno,indent_str,tagtuple,comment,text):
1079 """Instantiate a LabelLine.
1085 The content of the tagtuple is:
1087 ('label',left,right,[
1088 ('identifier',left,right,None)
1092 ContentLine.__init__(self,lineno,indent_str,tagtuple,comment,text)
1094 self.label = self.text[self.tagtuple[LEFT]:self.tagtuple[RIGHT]]
1096 def _write_text(self,stream,block):
1097 """Write out the main tuple text.
1099 stream -- an object with a "write" method, e.g., a file
1100 block -- used to pass the containing Block down to lines
1101 within a block, or None if we're not in a block
1103 # Enough difficult length calculation - let's do this one
1106 text = "# Label %s at index %d"%(self.label,self.index)
1108 text = "# %s"%(self.label) # surely enough for most people...
1112 def translate(self,index,block):
1113 """Return the translation of a use of this label as a target.
1115 index -- the index of the line which uses the label as a target
1116 block -- the Block we are within
1119 # Hmm - I don't think this CAN go wrong at this point...
1120 return block.translate_label(self.label,self)
1122 def only_in_block(self):
1123 """Return true if we can only occur inside a block."""
1127 # ------------------------------------------------------------
1128 class TableBlockLine(ContentLine):
1129 """A line starting a table block."""
1131 def __init__(self,lineno,indent_str,tagtuple,comment,text):
1132 """Instantiate a TableBlockLine.
1139 This is used for two purposes:
1140 1. To define the actual tag table itself (i.e., at the outer
1141 level). Only "Table" is allowed in this instance, but since
1142 that is all we recognised for now, we shan't worry about it...
1143 2. To define an inner table (i.e., at an inner level)
1145 The content of the tagtuple is:
1147 ('tableblock',left,right,[
1148 ('assignment',left,right,[ -- optional if inner
1151 ('identifier',left,right,[])
1154 ('text',left,right,None)
1157 ('int',left,right,[])
1161 ('type',left,right,[]) -- either "Table" or "SubTable"
1164 NOTE: as an "emergency" measure (so we can `pretend' that a
1165 TupleBlock was actually a TableBlock as part of attempted
1166 error correction), if tagtuple == ("error",tagobj) then we
1167 short-circuit some of the initialisation...
1170 ContentLine.__init__(self,lineno,indent_str,tagtuple,comment,text)
1172 if tagtuple[0] == "error":
1173 # We're "bluffing" at the creation of a TableBlock
1174 self.tagobj = tagtuple[1]
1175 self.is_subtable = 0
1176 elif len(self.tagtuple[SUBLIST]) == 1:
1177 self.tagobj = "None"
1178 tup = self.tagtuple[SUBLIST][0]
1179 self.is_subtable = (self.text[tup[LEFT]:tup[RIGHT]] == "SubTable")
1181 # The first tuple down gives us the "<value> = " string
1182 tup = self.tagtuple[SUBLIST][0]
1183 # The next tuple down gives us "<value>" which is what we want
1184 tup = tup[SUBLIST][0]
1185 self.tagobj = self.text[tup[LEFT]:tup[RIGHT]]
1186 # Then we have the type of table
1187 tup = self.tagtuple[SUBLIST][1]
1188 self.is_subtable = (self.text[tup[LEFT]:tup[RIGHT]] == "SubTable")
1190 def got_tagobj(self):
1191 return (self.tagobj != "None")
1193 def starts_block(self):
1194 """Return true if we start a new block."""
1197 def _write_text(self,stream,block):
1198 """Write out the main tuple text.
1200 stream -- an object with a "write" method, e.g., a file
1201 block -- used to pass the containing Block down to lines
1202 within a block, or None if we're not in a block
1204 It returns the number of characters written, or -1 if we had
1209 if self.is_subtable:
1210 stream.write("(%s,SubTable,("%self.tagobj)
1211 return len(self.tagobj) + 11
1213 stream.write("(%s,Table,("%self.tagobj)
1214 return len(self.tagobj) + 8
1216 stream.write("%s = ("%self.tagobj)
1217 return len(self.tagobj) + 4
1220 # ------------------------------------------------------------
1221 class TupleBlockLine(ContentLine):
1222 """A line starting a tuple block (i.e., defining a single tuple)
1226 name -- the "name" of this tuple (i.e., what comes
1230 def __init__(self,lineno,indent_str,tagtuple,comment,text):
1231 """Instantiate a TupleBlockLine.
1237 The content of the tagtuple is:
1239 ('tupleblock',left,right,[
1240 ('identifier',left,right,None)
1244 ContentLine.__init__(self,lineno,indent_str,tagtuple,comment,text)
1246 tup = self.tagtuple[SUBLIST][0]
1247 self.name = self.text[tup[LEFT]:tup[RIGHT]]
1249 def starts_block(self):
1250 """Return true if we start a new block."""
1253 def only_in_block(self):
1254 """Return true if we can only occur inside a block."""
1257 def _write_text(self,stream,block):
1258 """Write out the main tuple text.
1260 stream -- an object with a "write" method, e.g., a file
1261 block -- used to pass the containing Block down to lines
1262 within a block, or None if we're not in a block
1264 It returns the number of characters written, or -1 if we had
1267 # The "\" at the end is somewhat clumsy looking, but the
1268 # only obvious way of preserving layout...
1269 stream.write("%s = \\"%self.name)
1270 return len(self.name) + 5
1273 # ------------------------------------------------------------
1274 class IfBlockLine(ContentLine):
1275 """A line starting an if block.
1278 cmd -- the command within this if block
1279 arg -- the argument for said command
1281 name -- the name within this if block
1284 def __init__(self,lineno,indent_str,tagtuple,comment,text):
1285 """Instantiate an IfBlockLine.
1293 The content of the tagtuple is:
1295 ('ifblock',left,right,[
1296 ('assignment',left,right,[
1299 ('identifier',left,right,[])
1302 ('text',left,right,None)
1305 ('int',left,right,[])
1309 ('op',left,right,None),
1310 ('arg',left,right,None),
1313 ('ifblock',left,right,[
1314 ('op',left,right,None),
1315 ('arg',left,right,None),
1318 ('ifblock',left,right,[
1319 ('identifier',left,right,None)
1323 ContentLine.__init__(self,lineno,indent_str,tagtuple,comment,text)
1325 tuples = self.tagtuple[SUBLIST]
1326 if tuples[0][OBJECT] == 'op':
1329 self.tagobj = "None"
1330 self.cmd = self.text[tup1[LEFT]:tup1[RIGHT]]
1331 self.arg = self.text[tup2[LEFT]:tup2[RIGHT]]
1333 elif tuples[0][OBJECT] == 'assignment':
1334 # The "<value>" in the "<value> = " string is down
1335 # one level more than the others
1336 tup0 = tuples[0][SUBLIST][0]
1337 self.tagobj = self.text[tup0[LEFT]:tup0[RIGHT]]
1340 self.cmd = self.text[tup1[LEFT]:tup1[RIGHT]]
1341 self.arg = self.text[tup2[LEFT]:tup2[RIGHT]]
1343 elif tuples[0][OBJECT] == 'identifier':
1345 self.name = self.text[tup[LEFT]:tup[RIGHT]]
1350 # Hmm - try to continue with anything unexpected
1352 self.error("Unexpected IfBlock subtype %s"%tup[OBJECT])
1353 self.name = self.text[tup[LEFT]:tup[RIGHT]]
1358 # Currently, we have one 'special' argument
1359 if self.arg == "back": self.arg = "-1"
1361 # We don't yet know the offset of the "virtual label" at the
1362 # end of this if block...
1363 self.end_label = None
1365 def starts_block(self):
1366 """Return true if we start a new block."""
1369 def only_in_block(self):
1370 """Return true if we can only occur inside a block."""
1373 def resolve_labels(self,block):
1374 """Called to resolve any labels used in this line.
1376 block -- the block that contains us
1378 Note that this only does something the first time it
1379 is called - this will be when the IF block's startline
1380 is asked to resolve its labels. If it is called again,
1381 as a 'normal' line, it will do nothing...
1383 if not self.end_label:
1384 self.end_label = "%+d"%(len(block.business)+1)
1386 def _write_text(self,stream,block):
1387 """Write out the main tuple text.
1389 stream -- an object with a "write" method, e.g., a file
1390 block -- used to pass the containing Block down to lines
1391 within a block, or None if we're not in a block
1393 It returns the number of characters written, or -1 if we had
1396 if not self.end_label:
1397 # This should never happen, but just in case, warn the user!
1398 self.error("Unable to determine 'onFalse' destination in IF")
1401 stream.write("%s + (%s,+1),"%(self.name,
1402 self.end_label or "<undefined>"))
1403 return len(self.name) + 20
1405 stream.write("(%s,%s,%s,%s,+1),"%(self.tagobj,self.cmd,self.arg,
1406 self.end_label or "<undefined>"))
1407 return len(self.tagobj) + len(self.cmd) + len(self.arg) + \
1408 len(self.end_label) + 20
1411 # ------------------------------------------------------------
1412 class TupleLine(ContentLine):
1413 """A line containing a basic tuple.
1420 ontrue -- what to do if true
1421 onfalse -- ditto false
1424 def __init__(self,lineno,indent_str,tagtuple,comment,text):
1425 """Instantiate a TupleLine.
1427 The content of the tagtuple is:
1429 ('tuple',left,right,[
1430 ('tagobj',left,right,[ -- optional
1432 ('text',left,right,None)
1435 ('op',left,right,None),
1436 ('arg',left,right,None),
1437 ('onfalse',left,right,[ -- optional
1438 ('target',left,right,[
1439 ('tgt',left,right,None)
1441 ('ontrue',left,right,[ -- optional
1442 ('target',left,right,[
1443 ('tgt',left,right,None)
1449 ContentLine.__init__(self,lineno,indent_str,tagtuple,comment,text)
1455 """Unpack our contents from our tagtuple."""
1457 # This is doubtless not the most efficient way of doing this,
1458 # but it IS relatively simple...
1460 #for key in ("assignment","op","arg","onfalse","ontrue"):
1461 for key in ("assignment","op","plusarg","onfalse","ontrue"):
1464 tuples = self.tagtuple[SUBLIST]
1467 if name == "onfalse" or name == "ontrue" or name == "assignment":
1468 # For these, we need to go "down one level" for our data
1469 tup = item[SUBLIST][0]
1470 dict[name] = (tup[LEFT],tup[RIGHT])
1472 dict[name] = (item[LEFT],item[RIGHT])
1474 # The tag object is optional
1475 if dict["assignment"]:
1476 left,right = dict["assignment"]
1477 self.tagobj = self.text[left:right]
1479 self.tagobj = "None"
1481 # The operation (command) and argument are required
1482 left,right = dict["op"]
1483 self.cmd = self.text[left:right]
1485 #left,right = dict["arg"]
1486 left,right = dict["plusarg"]
1487 self.arg = self.text[left:right]
1489 # Currently, we have one 'special' argument
1490 if self.arg == "back": self.arg = "-1"
1492 # Actually, we don't want the F and T jumps explicit if not
1493 # given, since we mustn't output them for a single tuple if
1494 # they're not given (so they can be "added in" later on)
1496 left,right = dict["onfalse"]
1497 self.onfalse = self.text[left:right]
1499 self.onfalse = None # "MatchFail"
1501 left,right = dict["ontrue"]
1502 self.ontrue = self.text[left:right]
1504 self.ontrue = None # "next"
1506 def only_in_block(self):
1507 """Return true if we can only occur inside a block."""
1510 def resolve_labels(self,block):
1511 """Called to resolve any labels use in this line.
1513 block -- the block that contains us
1516 self.onfalse = block.translate_label(self.onfalse,self)
1518 self.ontrue = block.translate_label(self.ontrue,self)
1520 def _write_text(self,stream,block):
1521 """Write out the main tuple text.
1523 stream -- an object with a "write" method, e.g., a file
1524 block -- used to pass the containing Block down to lines
1525 within a block, or None if we're not in a block
1527 It returns the number of characters written, or -1 if we had
1531 # Start with the stuff we must have...
1532 stream.write("(%s,%s,%s"%(self.tagobj,self.cmd,self.arg))
1533 length = len(self.tagobj) + len(self.cmd) + len(self.arg) + 3
1536 if not self.onfalse:
1537 # OK, we didn't get an explicit F, but because it comes
1538 # before the T jump in the tuple, we need to fake it
1540 stream.write(",%s,%s)"%("MatchFail",self.ontrue))
1541 length = length + len("MatchFail") + len(self.ontrue) + 3
1543 # We had both F and T
1544 stream.write(",%s,%s)"%(self.onfalse,self.ontrue))
1545 length = length + len(self.onfalse) + len(self.ontrue) + 3
1547 # We only had F. We shan't "fake" the T jump, *just* in case
1548 # the user is defining a single tuple that they'll add the
1549 # T jump to later on (although that *is* a bit dodgy, I think)
1550 # [[The option would be to "fake" it if we're IN a block - I may
1551 # go for that approach later on]]
1552 stream.write(",%s)"%self.onfalse)
1553 length = length + len(self.onfalse) + 2
1555 # Neither F nor T - so don't write the defaults for either,
1556 # in case this is a top level tuple they're going to add to
1558 # [[Comments as for the case above, I think]]
1562 if block and not self.is_last:
1568 # ------------------------------------------------------------
1569 class TuplePlusLine(ContentLine):
1570 """A line containing a tuple "plus" (e.g., "fred + (+1,+1)").
1574 name -- the name/identifier
1575 ontrue -- what to do if true
1576 onfalse -- ditto false
1579 def __init__(self,lineno,indent_str,tagtuple,comment,text):
1580 """Instantiate a TuplePlusLine.
1582 <identifier> + (onF,onT)
1584 The content of the tagtuple is:
1586 ('tupleplus',left,right,[
1587 ('identifier',left,right,None)
1588 ('onfalse',left,right,[ -- optional
1589 ('target',left,right,[
1590 ('tgt',left,right,None)
1592 ('ontrue',left,right,[ -- optional
1593 ('target',left,right,[
1594 ('tgt',left,right,None)
1600 ContentLine.__init__(self,lineno,indent_str,tagtuple,comment,text)
1606 """Unpack our contents from our tagtuple."""
1608 # This is doubtless not the most efficient way of doing this,
1609 # but it IS relatively simple...
1611 for key in ("identifier","onfalse","ontrue"):
1614 tuples = self.tagtuple[SUBLIST]
1617 if name == "onfalse" or name == "ontrue":
1618 # For these, we need to go "down one level" for our data
1619 tup = item[SUBLIST][0]
1620 dict[name] = (tup[LEFT],tup[RIGHT])
1622 dict[name] = (item[LEFT],item[RIGHT])
1624 # Start with the identifier
1625 left,right = dict["identifier"]
1626 self.name = self.text[left:right]
1628 # Actually, we don't want the F and T jumps explicit if not
1629 # given, since we mustn't output them for a single tuple if
1630 # they're not given (so they can be "added in" later on)
1632 left,right = dict["onfalse"]
1633 self.onfalse = self.text[left:right]
1635 self.onfalse = None # "MatchFail"
1637 left,right = dict["ontrue"]
1638 self.ontrue = self.text[left:right]
1640 self.ontrue = None # "next"
1642 def only_in_block(self):
1643 """Return true if we can only occur inside a block."""
1646 def resolve_labels(self,block):
1647 """Called to resolve any labels use in this line.
1649 block -- the block that contains us
1652 self.onfalse = block.translate_label(self.onfalse,self)
1654 self.ontrue = block.translate_label(self.ontrue,self)
1656 def _write_text(self,stream,block):
1657 """Write out the main tuple text.
1659 stream -- an object with a "write" method, e.g., a file
1660 block -- used to pass the containing Block down to lines
1661 within a block, or None if we're not in a block
1663 It returns the number of characters written, or -1 if we had
1667 if not self.onfalse and not self.ontrue:
1668 stream.write("%s"%self.name)
1669 length = len(self.name)
1671 # Make a feeble attempt to cause successive such lines to
1672 # look neater, by aligning the "+" signs (if we output them)
1673 stream.write("%-15s + ("%(self.name))
1674 length = max(len(self.name),15) + 4
1675 if self.ontrue and self.onfalse:
1676 stream.write("%s,%s)"%(self.onfalse,self.ontrue))
1677 length = length + len(self.onfalse) + len(self.ontrue) + 2
1679 stream.write("MatchFail,%s)"%(self.ontrue))
1680 length = length + len(self.ontrue) + 11
1682 # Don't forget that comma to make this a tuple!
1683 stream.write("%s,)"%(self.onfalse))
1684 length = length + len(self.onfalse) + 1
1686 if not self.is_last:
1693 # ------------------------------------------------------------
1694 class JumpToLine(ContentLine):
1695 """A line containing "Jump To <label>"
1699 name -- the name/identifier
1700 onfalse -- the target (which is technically an "on false" jump)
1703 def __init__(self,lineno,indent_str,tagtuple,comment,text):
1704 """Instantiate a JumpLine.
1708 The content of the tagtuple is:
1710 ('jumpto',left,right,[
1711 ('target',left,right,[
1712 ('tgt',left,right,None)
1717 ContentLine.__init__(self,lineno,indent_str,tagtuple,comment,text)
1719 tup = self.tagtuple[SUBLIST][0]
1720 self.onfalse = self.text[tup[LEFT]:tup[RIGHT]]
1722 def only_in_block(self):
1723 """Return true if we can only occur inside a block."""
1726 def resolve_labels(self,block):
1727 """Called to resolve any labels use in this line.
1729 block -- the block that contains us
1731 self.onfalse = block.translate_label(self.onfalse,self)
1733 def _write_text(self,stream,block):
1734 """Write out the main tuple text.
1736 stream -- an object with a "write" method, e.g., a file
1737 block -- used to pass the containing Block down to lines
1738 within a block, or None if we're not in a block
1740 It returns the number of characters written, or -1 if we had
1744 stream.write("(None,Jump,To,%s)"%(self.onfalse))
1745 length = len(self.onfalse) + 15
1747 if not self.is_last:
1754 # ------------------------------------------------------------
1755 class BadTableBlockLine(TableBlockLine):
1756 """We think they MEANT this to be a table block line."""
1758 def __init__(self,lineno,indent_str,tagtuple,comment,text):
1759 """Instantiate a BadTableBlockLine.
1766 TableBlockLine.__init__(self,lineno,indent_str,tagtuple,comment,text)
1767 self.error("Suspected missing 'is' before the colon\n"
1768 "pretending it's there")
1771 # ------------------------------------------------------------
1772 class BadTupleLine(TupleLine):
1773 """We think they MEANT this to be a tuple line."""
1775 def __init__(self,lineno,indent_str,tagtuple,comment,text):
1776 """Instantiate a BadTupleLine.
1782 TupleLine.__init__(self,lineno,indent_str,tagtuple,comment,text)
1783 self.error("Suspected missing '=' between tag object and command\n"
1784 "pretending it's there")
1787 # ------------------------------------------------------------
1788 class Block(ContentLine):
1789 """This class represents a "block".
1791 A "block" is a section of code which starts with a line ending in
1792 a colon (":"), with the next line and subsequent lines ("in" the
1793 block) having an extra indent. The block ends when a dedent is
1796 Each instance "eats" lines from the input until (if) it finds the first
1797 "sub" block. That then "eats" lines until it finds its own end, and
1798 then hands control back to the first instance, which does the same thing
1801 Note that we "pretend" to be a content line - it is convenient to
1802 look like a line class, so that line processing can cope with us,
1803 and indeed what we do is "pretend" to be a clone of our start line
1804 with some extra information...
1807 startline -- the line that "introduces" this block
1808 items -- a list of the lines and blocks within this block
1809 label_dict -- a dictionary of {label name : line index}
1810 inner_indent -- the indentation of our "inner" lines
1811 outer -- true if we are an "outer" block
1812 (i.e., not contained within another block)
1815 def __init__(self,startline=None,outer=0,file=None):
1816 """Instantiate a new block.
1818 startline -- the line that introduces this block
1819 outer -- true if we are an outer block
1820 file -- the "file" we're reading lines from
1823 # Pretend to be our own startline (as a generic)
1824 ContentLine.__init__(self,
1825 startline.lineno,startline.indent_str,
1826 startline.tagtuple,startline.comment,
1829 # But also remember the specifics of the startline
1830 self.startline = startline
1832 # We "fudge" our class name
1833 self.class_name = self._block_class_name(startline)
1838 # If we're an outer table block, do we have a tagobj?
1839 if self.startline.class_name == "TableBlockLine" and outer:
1840 if not self.startline.got_tagobj():
1841 raise NoIdentifier,\
1842 "Tag table at line %d is not assigned to a variable"%\
1844 elif self.startline.is_subtable:
1845 raise OutsideError,\
1846 "SubTable is not allowed outside a block at line %d"%\
1849 self.items = [] # all lines within this block
1850 self.business = [] # just those that are "our business"
1851 self.label_dict = {} # remember our labels and their locations
1852 self.next_index = 0 # 'business' line indices
1853 self.inner_indent = None
1855 # Eat lines until we reach the end of our block...
1856 if DEBUGGING: print "%sStart %s"%(self.indent_str,self.class_name)
1860 def _block_class_name(self,startline):
1861 """Return a representation of the class name."""
1863 full_name = "%s"%self.__class__
1864 bits = string.split(full_name,".")
1865 return "%s/%s"%(bits[-1],startline.class_name)
1867 def _eat_lines(self):
1868 """Eat lines until we run out of block..."""
1872 nextline = self.file.next()
1876 # Check the indentation makes sense...
1877 if self.inner_indent:
1878 # We already know how much our block is indented
1879 # - is this line part of the block?
1880 if nextline.indent < self.inner_indent:
1881 # Apparently a dedent - is it what we expect?
1882 if nextline.indent <= self.indent:
1883 # Unread that line - it isn't one of ours!
1888 "Line %d (%s) is indented less than the previous "\
1889 "line, but its indentation doesn't match the "\
1890 "start of the block at line %d"%\
1891 (nextline.lineno,nextline.class_name,self.lineno)
1892 elif nextline.indent > self.inner_indent:
1894 # (note that doing this stops us from coping with,
1895 # for instance, things in (..), but then we also don't
1896 # cope with any form of continued line, or lots of other
1897 # things, so let's not worry too much for now!)
1899 "Line %d (%s) is indented more than the previous line"%\
1900 (nextline.lineno,nextline.class_name)
1902 # This is the first line of the (inside of) the block
1903 # - check its indentation makes sense...
1904 self.inner_indent = nextline.indent
1905 if self.inner_indent <= self.indent:
1907 "Line %d (%s) should be indented more than line %d (%s)"%\
1908 (nextline.lineno,nextline.class_name,
1909 self.lineno,self.startline.class_name)
1911 # Is it a line or the start of another block?
1912 if nextline.starts_block():
1913 # Heh - it's the start of an inner block - add it
1914 # (remember that instantiating it causes it to
1915 # "eat" the lines that belong to it)
1916 self.items.append(Block(startline=nextline,
1917 outer=0,file=self.file))
1919 self.items.append(nextline)
1921 def _end_block(self):
1924 if DEBUGGING: print "%sEnd %s"%(self.indent_str,self.class_name)
1926 # If we're a tuple block, we should only have one line...
1927 # (that is, one "business" line)
1928 if self.startline.class_name == "TupleBlockLine" and \
1929 len(self.items) > 1:
1930 # Are all but one of them not "our business"?
1932 for item in self.items:
1933 if item.our_business():
1937 self.error("Tuple declaration can only contain one 'business'"
1939 "Assuming it's a table instead (i.e.,"
1940 "'Table is:' instead of 'is:')"%len(self.items))
1941 # Can we correct this by "pretending" its a table?
1942 temp = TableBlockLine(self.startline.lineno,
1943 self.startline.indent_str,
1944 ("error",self.startline.name),
1945 self.startline.comment,
1947 self.startline = temp
1949 # We've now got all of our lines, and so we can go back over
1950 # them, expanding out any IF blocks (whose content is actually
1951 # within this block's scope, so who need to have their labels
1952 # (come from or go to) in that scope), working out the label
1953 # indices, and so on...
1954 # This uses "next_index" to calculate the indices of business
1955 # lines (needed for label calculation), and also populates the
1956 # "business" list with just the items that are "our_business()"
1958 print "Expanding IF blocks, sorting out labels, etc."
1963 if item.class_name == "Block/IfBlockLine":
1964 self._add(item.startline)
1965 for thing in item.items:
1970 # Go back through our contents and resolve any labels
1972 print "%s...processing labels (next_index=%d)"%(self.indent_str,
1974 self.startline.resolve_labels(self)
1975 # If we're an IF block, we mustn't try to resolve our component
1976 # lines' labels, as they're actually in our parent block's scope...
1977 if self.startline.class_name != "IfBlockLine":
1978 for item in self.items:
1979 item.resolve_labels(self)
1981 # If we're in a block that wants to suppress the comma at the
1982 # end of the last item in that block, tell the last item so...
1983 # (this is debatable for [Bad]TableBlockLine - it might be
1984 # better to leave the last comma there - so we have an option
1985 # to determine it...
1986 if self.startline.class_name == "TupleBlockLine" or \
1987 (not WANT_LAST_COMMA and \
1988 (self.startline.class_name == "TableBlockLine" or \
1989 self.startline.class_name == "BadTableBlockLine")):
1990 if len(self.business) > 0:
1991 self.business[-1].is_last = 1
1993 def _add(self,item):
1994 """Add a line or block to our list of items.
1996 item -- the Line or Block instance to add
1998 NB: Also adds it to our "business" list if it is our business
2002 if item.class_name == "LabelLine":
2003 self.label_dict[item.label] = self.next_index
2005 print "%sadd [%2d] %s"%(item.indent_str,self.next_index,item)
2006 # Might as well give it the index it is labelling
2007 item.index = self.next_index
2008 self.items.append(item)
2009 elif item.our_business():
2010 item.index = self.next_index
2011 self.items.append(item)
2012 self.business.append(item)
2014 print "%sadd %2d %s"%(item.indent_str,
2015 self.next_index,item)
2016 self.next_index = self.next_index + 1
2018 # It's not something we can assign a sensible index to, so don't
2020 print "%sadd xx %s"%(item.indent_str,item)
2021 self.items.append(item)
2023 def translate_label(self,label,line):
2024 """Given a label, return its translation.
2026 label -- either a string of the form "<...>" to look up in
2027 this block's label dictionary, or one of the special
2028 targets (e.g., next, MatchOk, etc.)
2029 line -- the line using this label
2031 Reports an error and just returns the original "label" if it
2034 if self.label_dict.has_key(label):
2035 # How far do we have to jump?
2036 offset = self.label_dict[label] - line.index
2038 elif label == "MatchOk":
2040 elif label == "MatchOK":
2041 line.warning("Label 'MatchOK' should be spelt 'MatchOk'"
2042 " (using 'MatchOk')")
2044 elif label == "MatchFail":
2046 elif label == "next":
2048 elif label == "previous":
2050 elif label == "repeat":
2053 line.error("Undefined label '%s'"%label)
2056 def expand(self,stream,block=None):
2057 """Write out the expanded equivalent of ourselves.
2059 stream -- an object with a "write" method, e.g., a file
2060 block -- if we're in a block, this is it, otherwise None
2063 self.startline.expand(stream,block=block)
2064 for item in self.items[:-1]:
2065 item.expand(stream,block=self)
2067 self.items[-1].expand(stream,block=self)
2069 # Deal with closing any block parentheses
2070 if self.startline.class_name == "TableBlockLine" or \
2071 self.startline.class_name == "BadTableBlockLine":
2073 stream.write("Line ...: ")
2075 stream.write(self.indent_str)
2077 # Outer block - just close it
2080 # Inner block is a Table block, and we need to close both
2081 # the tuple-of-tuples, and also the tuple containing the
2084 if not self.is_last:
2089 # ------------------------------------------------------------
2091 """This is the class that holds our processed data
2094 lines -- a list of the line instances for each "line" in our text
2095 items -- a list of lines and BLOCKs
2098 def __init__(self,tagtuples,text):
2099 """Instantiate a File
2101 tagtuples -- the list of mxTextTools tag tuples generated by
2102 parsing the data in "text"
2103 text -- the text we parsed
2107 self.tagtuples = tagtuples
2109 # Assemble our list of lines
2110 print "Pass 1: assembling lines"
2111 if DEBUGGING: print "~~~~~~~~~~~~~~~~~~~~~~~~"
2115 for tagtuple in tagtuples:
2117 thisline = LineFactory(lineno,tagtuple,text)
2120 prevline.next_indent = thisline.indent
2122 self.lines.append(thisline)
2125 #if DEBUGGING: print
2127 # The indentation of an empty line is taken to be the same
2128 # as the indentation of the first following non-empty line
2129 # The easiest way to do that is to work backwards through
2130 # the list (is it better to take a copy and reverse THAT,
2131 # or to reverse our original list twice?)
2132 print "Pass 2: sorting out indentation of empty lines"
2133 if DEBUGGING: print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
2134 revlist = self.lines[:]
2137 for line in revlist:
2138 if line.class_name == "EmptyLine":
2139 line.change_indent(indent)
2141 indent = line.indent
2145 print "Pass 2.5 - the contents of those lines..."
2146 print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
2147 for line in self.lines:
2148 print "Line %d %s"%(line.lineno,line.class_name)
2149 #print_tuples([line.tagtuple],self.text," ")
2152 # Now we need to assemble blocks
2153 print "Pass 3: assembling blocks"
2154 if DEBUGGING: print "~~~~~~~~~~~~~~~~~~~~~~~~~"
2165 print "%sTOP %s"%(item.indent_str,item)
2166 if item.starts_block():
2167 block = Block(startline=item,outer=1,file=self)
2168 self.items.append(block)
2169 block.is_last = 1 # Everything at outer level is "last"
2171 if item.only_in_block():
2172 item.error("This line is not allowed outside a block "
2173 "- continuing anyway")
2174 self.items.append(item)
2175 if item.our_business():
2176 item.is_last = 1 # Everything at outer level is "last"
2182 """Ensure that the next call of "nextline" returns the first line."""
2186 """Unread the current line."""
2187 self.index = self.index - 1
2192 """Retrieve the next line from the list of lines in this "file".
2194 Raises EOFError if there is no next line (i.e., "end of file")
2196 self.index = self.index + 1
2198 return self.lines[self.index]
2200 # leave the index off the end, so we get EOF again if
2201 # we're called again - but there's no point courting overflow...
2202 self.index = self.index -1
2205 def expand(self,stream):
2206 """Expand out the result."""
2207 for item in self.items:
2211 # ------------------------------------------------------------
2212 def print_tuples(tuples,text,indent=""):
2213 """Print out a list of tuples in a neat form
2215 tuples -- our tuple list
2216 text -- the text it tags
2217 indent -- our current indentation
2220 # Tuples are of the form:
2221 # (object,left_index,right_index,sublist)
2223 for obj,left,right,sub in tuples:
2225 print "%s%s"%(indent,obj)
2226 print_tuples(sub,text,indent+" ")
2228 # Terminal node - show the actual text we've tagged!
2229 print "%s%s = %s"%(indent,obj,`text[left:right]`)
2232 # ------------------------------------------------------------
2233 def print_text(text):
2234 """Print out text with line numbers."""
2235 lines = string.split(text,"\n")
2238 print "Original text"
2239 print "============="
2242 print "%3d: %s"%(lineno,`line`)
2245 # ------------------------------------------------------------
2246 def print_usage(argv0):
2247 #script_name = string.split(argv0, os.sep)[-1]
2248 #print __doc__%(script_name)
2253 # ------------------------------------------------------------
2254 def show_tup(indent,nn,tup):
2257 if type(item) == type((1,)) or type(item) == type([]):
2263 print "%s%d: (%s)"%(indent,nn,string.join(ll,","))
2265 print "%s(%s)"%(indent,string.join(ll,","))
2267 def comp_sub(indent,one,two):
2269 if len(two) != len(one):
2270 print "%sTuple lengths differ - 1:%d, 2:%d"%(indent,len1,len(two))
2271 show_tup(indent,1,one)
2272 show_tup(indent,2,two)
2273 # If this is all, let's try to continue...
2274 len1 = min(len1,len(two))
2276 for count in range(len1):
2279 if type(a) != type(b):
2280 print "%sValue types differ, item %d: 1:%s, 2:%s"%(indent,count,
2282 show_tupe(indent,1,one)
2283 show_tupe(indent2,two)
2285 if type(a) == type((1,)) or type(a) == type([]):
2286 if not comp_sub(indent+" ",a,b):
2287 # They're the same at this level, so show only one...
2288 show_tup(indent,0,one)
2292 print "%sValues differ, item %d: 1:%s, 2:%s"%(indent,count,
2294 show_tup(indent,1,one)
2295 show_tup(indent,2,two)
2299 def compare_tagtables(one,two):
2300 # Each table is made up of tuples of the form
2301 # (tagobj,action,arg,onfalse,ontrue)
2302 # but if action is Table or SubTable then arg may be a tuple
2304 if comp_sub("",one,two):
2305 print "They appear to be the same"
2308 # ------------------------------------------------------------
2310 """Used to test the module."""
2312 debug_pytag = DEFAULT_DEBUG
2313 use_pytag = DEFAULT_PYTAG
2319 if os.name == "posix":
2328 # Do we have command line arguments?
2329 arg_list = sys.argv[1:]
2333 if len(arg_list) == 0:
2338 if word == "-pytag":
2340 elif word == "-debug":
2342 elif word == "-stdout":
2344 elif word == "-force":
2346 elif word == "-import":
2348 elif word == "-compare":
2350 elif word == "-diag":
2353 elif word == "-test":
2356 elif word == "-help":
2357 print_usage(sys.argv[0])
2359 elif word == "-version":
2360 print "Version:",__version__
2362 elif word == "-history":
2369 arg_list = arg_list[1:]
2373 from Translate_tags import t_file
2374 i_file = define_tagtable()
2375 print "Comparing internal table (1) against external (2)"
2376 compare_tagtables(i_file,t_file)
2379 if not use_testdata and (not args or len(args) > 2):
2380 print_usage(sys.argv[0])
2383 if not use_testdata:
2387 print "Importing tag table definition"
2388 from Translate_tags import t_file
2390 print "Using internal tag table definition"
2391 t_file = define_tagtable()
2394 outfile = "standard output"
2398 base,ext = os.path.splitext(infile)
2400 outfile = base + ".py"
2402 print "Input file has extension .py so won't guess"\
2406 if outfile != "standard output":
2407 if outfile == infile:
2408 print "The output file is the same as the input file"
2409 print "Refusing to overwrite %s"%outfile
2411 elif os.path.exists(outfile):
2413 print "Output file %s already exists"\
2414 " - overwriting it"%outfile
2416 print "Output file %s already exists"%outfile
2419 # Read the input file
2422 print "Using test data"
2423 if DEBUGGING: print "==============="
2427 print "Reading text from %s"%infile
2428 if DEBUGGING: print "=================="+"="*len(infile)
2429 file = open(infile,"r")
2433 # Show what we are trying to parse
2434 if DEBUGGING or use_testdata:
2440 print "Tagging text"
2441 if DEBUGGING: print "============"
2444 pytag.set_verbosity(0)
2446 pytag.set_verbosity(1)
2447 pytag.use_debugger()
2448 result,taglist,next = pytag.pytag(text,t_file)
2450 timer = TextTools._timer()
2452 result, taglist, next = tag(text,t_file)
2453 #result, taglist, next = tag(text,t_file,0,len(text),taglist)
2454 print "Tagging took",timer.stop()[0],"seconds"
2456 # Now print out the result of the tagging
2458 print "Manipulating tagged data"
2459 if DEBUGGING: print "========================"
2460 tagfile = File(taglist,text)
2463 print "Writing translation to %s"%outfile
2464 if DEBUGGING: print "======================="+"="*len(outfile)
2466 # Open the output file, if necessary
2470 file = open(outfile,"w")
2472 tagfile.expand(file)
2475 # ------------------------------------------------------------
2476 if __name__ == '__main__':