doc/salome/tui/pythfilter.py

   1 #!/usr/bin/env python
   2
   3 import getopt
   4 import glob
   5 import os.path
   6 import shutil
   7 import string
   8 import sys
   9 import token
  10 import tokenize
  11
  12 from stat import *
  13
  14 OUTSIDE          = 0
  15 BUILD_COMMENT    = 1
  16 BUILD_CLASS_DECL = 2
  17 BUILD_CLASS_BODY = 3
  18 BUILD_DEF_DECL   = 4
  19 BUILD_DEF_BODY   = 5
  20 IMPORT           = 6
  21 IMPORT_OP        = 7
  22 IMPORT_APPEND    = 8
  23
  24 # Output file stream
  25 outfile = sys.stdout
  26
  27 # Output buffer
  28 outbuffer = []
  29
  30 out_row = 0
  31 out_col = 0
  32
  33 # Variables used by rec_name_n_param()
  34 name         = ""
  35 param        = ""
  36 doc_string   = ""
  37 record_state = 0
  38 bracket_counter = 0
  39
  40 # Tuple: (row,column)
  41 class_spos  = (0,0)
  42 def_spos    = (0,0)
  43 import_spos = (0,0)
  44
  45 # Which import was used? ("import" or "from")
  46 import_token = ""
  47
  48 # Comment block buffer
  49 comment_block = []
  50 comment_finished = 0
  51
  52 # Imported modules
  53 modules = []
  54
  55 # Program state
  56 stateStack = [OUTSIDE]
  57
  58 # Keep track of whether module has a docstring
  59 module_has_docstring = False
  60
  61 # Keep track of member protection
  62 protection_level = "public"
  63 private_member = False
  64
  65 # Keep track of the module namespace
  66 namespace = ""
  67
  68 ######################################################################
  69 # Output string s. '\n' may only be at the end of the string (not
  70 # somewhere in the middle).
  71 #
  72 # In: s    - String
  73 #     spos - Startpos
  74 ######################################################################
  75 def output(s,spos, immediate=0):
  76     global outbuffer, out_row, out_col, outfile
  77
  78     os = string.rjust(s,spos[1]-out_col+len(s))
  79     if immediate:
  80         outfile.write(os)
  81     else:
  82         outbuffer.append(os)
  83     if (s[-1:]=="\n"):
  84         out_row = out_row+1
  85         out_col = 0
  86     else:
  87         out_col = spos[1]+len(s)
  88
  89
  90 ######################################################################
  91 # Records a name and parameters. The name is either a class name or
  92 # a function name. Then the parameter is either the base class or
  93 # the function parameters.
  94 # The name is stored in the global variable "name", the parameters
  95 # in "param".
  96 # The variable "record_state" holds the current state of this internal
  97 # state machine.
  98 # The recording is started by calling start_recording().
  99 #
 100 # In: type, tok
 101 ######################################################################
 102 def rec_name_n_param(type, tok):
 103     global record_state,name,param,doc_string,bracket_counter
 104     s = record_state
 105     # State 0: Do nothing.
 106     if   (s==0):
 107          return
 108     # State 1: Remember name.
 109     elif (s==1):
 110         name = tok
 111         record_state = 2
 112     # State 2: Wait for opening bracket or colon
 113     elif (s==2):
 114         if (tok=='('):
 115             bracket_counter = 1
 116             record_state=3
 117         if (tok==':'): record_state=4
 118     # State 3: Store parameter (or base class) and wait for an ending bracket
 119     elif (s==3):
 120         if (tok=='*' or tok=='**'):
 121             tok=''
 122         if (tok=='('):
 123             bracket_counter = bracket_counter+1
 124         if (tok==')'):
 125             bracket_counter = bracket_counter-1
 126         if bracket_counter==0:
 127             record_state=4
 128         else:
 129             param=param+tok
 130     # State 4: Look for doc string
 131     elif (s==4):
 132         if (type==token.NEWLINE or type==token.INDENT or type==token.SLASHEQUAL):
 133             return
 134         elif (tok==":"):
 135             return
 136         elif (type==token.STRING):
 137             while tok[:1]=='r' or tok[:1]=='u':
 138                 tok=tok[1:]
 139             while tok[:1]=='"':
 140                 tok=tok[1:]
 141             while tok[-1:]=='"':
 142                 tok=tok[:-1]
 143             doc_string=tok
 144         record_state=0
 145
 146 ######################################################################
 147 # Starts the recording of a name & param part.
 148 # The function rec_name_n_param() has to be fed with tokens. After
 149 # the necessary tokens are fed the name and parameters can be found
 150 # in the global variables "name" und "param".
 151 ######################################################################
 152 def start_recording():
 153     global record_state,param,name, doc_string
 154     record_state=1
 155     name=""
 156     param=""
 157     doc_string=""
 158
 159 ######################################################################
 160 # Test if recording is finished
 161 ######################################################################
 162 def is_recording_finished():
 163     global record_state
 164     return record_state==0
 165
 166 ######################################################################
 167 ## Gather comment block
 168 ######################################################################
 169 def gather_comment(type,tok,spos):
 170     global comment_block,comment_finished
 171     if (type!=tokenize.COMMENT):
 172         comment_finished = 1
 173     else:
 174         # Output old comment block if a new one is started.
 175         if (comment_finished):
 176             print_comment(spos)
 177             comment_finished=0
 178         if (tok[0:2]=="##" and tok[0:3]!="###"):
 179             comment_block.append(tok[2:])
 180
 181 ######################################################################
 182 ## Output comment block and empty buffer.
 183 ######################################################################
 184 def print_comment(spos):
 185     global comment_block,comment_finished
 186     if (comment_block!=[]):
 187         output("/**\n",spos)
 188         for c in comment_block:
 189             output(c,spos)
 190         output("*/\n",spos)
 191     comment_block    = []
 192     comment_finished = 0
 193
 194 ######################################################################
 195 def set_state(s):
 196     global stateStack
 197     stateStack[len(stateStack)-1]=s
 198
 199 ######################################################################
 200 def get_state():
 201     global stateStack
 202     return stateStack[len(stateStack)-1]
 203
 204 ######################################################################
 205 def push_state(s):
 206     global stateStack
 207     stateStack.append(s)
 208
 209 ######################################################################
 210 def pop_state():
 211     global stateStack
 212     stateStack.pop()
 213
 214
 215 ######################################################################
 216 def tok_eater(type, tok, spos, epos, line):
 217     global stateStack,name,param,class_spos,def_spos,import_spos
 218     global doc_string, modules, import_token, module_has_docstring
 219     global protection_level, private_member
 220
 221     rec_name_n_param(type,tok)
 222     if (string.replace(string.strip(tok)," ","")=="##private:"):
 223          protection_level = "private"
 224          output("private:\n",spos)
 225     elif (string.replace(string.strip(tok)," ","")=="##protected:"):
 226          protection_level = "protected"
 227          output("protected:\n",spos)
 228     elif (string.replace(string.strip(tok)," ","")=="##public:"):
 229          protection_level = "public"
 230          output("public:\n",spos)
 231     else:
 232          gather_comment(type,tok,spos)
 233
 234     state = get_state()
 235
 236 #    sys.stderr.write("%d: %s\n"%(state, tok))
 237
 238     # OUTSIDE
 239     if   (state==OUTSIDE):
 240         if  (tok=="class"):
 241             start_recording()
 242             class_spos = spos
 243             push_state(BUILD_CLASS_DECL)
 244         elif (tok=="def"):
 245             start_recording()
 246             def_spos = spos
 247             push_state(BUILD_DEF_DECL)
 248         elif (tok=="import") or (tok=="from"):
 249             import_token = tok
 250             import_spos = spos
 251             modules     = []
 252             push_state(IMPORT)
 253         elif (spos[1] == 0 and tok[:3] == '"""'):
 254             # Capture module docstring as namespace documentation
 255             module_has_docstring = True
 256             #comment_block.append("\\namespace %s\n" % namespace)
 257             comment_block.append(tok[3:-3])
 258             print_comment(spos)
 259
 260     # IMPORT
 261     elif (state==IMPORT):
 262         if (type==token.NAME):
 263             modules.append(tok)
 264             set_state(IMPORT_OP)
 265     # IMPORT_OP
 266     elif (state==IMPORT_OP):
 267         if (tok=="."):
 268             set_state(IMPORT_APPEND)
 269         elif (tok==","):
 270             set_state(IMPORT)
 271         else:
 272             for m in modules:
 273                 output('#include "'+m.replace('.',os.sep)+'.py"\n', import_spos, immediate=1)
 274                 if import_token=="from":
 275                     output('using namespace '+m.replace('.', '::')+';\n', import_spos)
 276             pop_state()
 277     # IMPORT_APPEND
 278     elif (state==IMPORT_APPEND):
 279         if (type==token.NAME):
 280             modules[len(modules)-1]+="."+tok
 281             set_state(IMPORT_OP)
 282     # BUILD_CLASS_DECL
 283     elif (state==BUILD_CLASS_DECL):
 284         if (is_recording_finished()):
 285             s = "class "+name
 286             if (param!=""): s = s+" : public "+param.replace('.','::')
 287             if (doc_string!=""): comment_block.append(doc_string)
 288             print_comment(class_spos)
 289             output(s+"\n",class_spos)
 290             output("{\n",(class_spos[0]+1,class_spos[1]))
 291             protection_level = "public"
 292             output("  public:\n",(class_spos[0]+2,class_spos[1]))
 293             set_state(BUILD_CLASS_BODY)
 294     # BUILD_CLASS_BODY
 295     elif (state==BUILD_CLASS_BODY):
 296         if (type!=token.INDENT and type!=token.NEWLINE and type!=40 and
 297             type!=tokenize.NL and type!=tokenize.COMMENT and
 298             (spos[1]<=class_spos[1])):
 299             output("}; // end of class\n",(out_row+1,class_spos[1]))
 300             pop_state()
 301         elif (tok=="def"):
 302             start_recording()
 303             def_spos = spos
 304             push_state(BUILD_DEF_DECL)
 305     # BUILD_DEF_DECL
 306     elif (state==BUILD_DEF_DECL):
 307         if (is_recording_finished()):
 308             s = ''
 309             # Do we document a class method? then remove the 'self' parameter
 310             if BUILD_CLASS_BODY in stateStack:
 311                 params = param.split(",")
 312                 if params[0] == 'self':
 313                     param = string.join(params[1:], ",")
 314                 else:
 315                     s = 'static '
 316                     if params[0] == 'cls':
 317                         param = string.join(params[1:], ",")
 318                 s = s+name+"("+param+");\n"
 319                 if len(name) > 1 \
 320                    and name[0:2] == '__' \
 321                    and name[len(name)-2:len(name)] != '__' \
 322                    and protection_level != 'private':
 323                        private_member = True
 324                        output("  private:\n",(def_spos[0]+2,def_spos[1]))
 325             else:
 326                 s = name+"("+param+");\n"
 327             if (doc_string!=""): comment_block.append(doc_string)
 328             print_comment(def_spos)
 329             output(s,def_spos)
 330 #       output("{\n",(def_spos[0]+1,def_spos[1]))
 331             set_state(BUILD_DEF_BODY)
 332     # BUILD_DEF_BODY
 333     elif (state==BUILD_DEF_BODY):
 334         if (type!=token.INDENT and type!=token.NEWLINE \
 335             and type!=40 and type!=tokenize.NL \
 336             and (spos[1]<=def_spos[1])):
 337 #            output("} // end of method/function\n",(out_row+1,def_spos[1]))
 338             if private_member and protection_level != 'private':
 339                 private_member = False
 340                 output("  " + protection_level + ":\n",(def_spos[0]+2,def_spos[1]))
 341             pop_state()
 342 #       else:
 343 #            output(tok,spos)
 344
 345
 346 def dump(filename):
 347     f = open(filename)
 348     r = f.readlines()
 349     for s in r:
 350         sys.stdout.write(s)
 351
 352 def filter(filename):
 353     global name, module_has_docstring
 354     global namespace,outbuffer
 355     outbuffer=[]
 356
 357     path,name = os.path.split(filename)
 358     root,ext  = os.path.splitext(name)
 359
 360
 361     if namespace:
 362        if root == "__init__":
 363           root=namespace
 364        else:
 365           root=namespace+"::"+root
 366     else:
 367        root=root
 368
 369     output("namespace "+root+" {\n",(0,0))
 370
 371     # set module name for tok_eater to use if there's a module doc string
 372     package=namespace
 373     name = root
 374     namespace = root
 375
 376     sys.stderr.write("namespace: "+namespace+'\n')
 377     sys.stderr.write("root: "+root+'\n')
 378     sys.stderr.write('Filtering "'+filename+'"...')
 379
 380     f = open(filename)
 381     tokenize.tokenize(f.readline, tok_eater)
 382     f.close()
 383     print_comment((0,0))
 384
 385     output("\n",(0,0))
 386     output("}  // end of namespace\n",(0,0))
 387
 388     if not module_has_docstring:
 389         # Put in default namespace documentation
 390         output('/** \\namespace '+root+' \n',(0,0))
 391         output('    \\brief Module "%s" */\n'%(root),(0,0))
 392
 393     for s in outbuffer:
 394         outfile.write(s)
 395     namespace=package
 396     module_has_docstring = False
 397
 398
 399 def filterFile(filename, out=sys.stdout):
 400     global outfile
 401
 402     outfile = out
 403
 404     try:
 405         root,ext  = os.path.splitext(filename)
 406
 407         if ext==".py":
 408             filter(filename)
 409         else:
 410             dump(filename)
 411
 412         sys.stderr.write("OK\n")
 413     except IOError,e:
 414         sys.stderr.write(e[1]+"\n")
 415
 416
 417 ######################################################################
 418
 419 # preparePath
 420 def preparePath(path):
 421     """Prepare a path.
 422
 423     Checks if the path exists and creates it if it does not exist.
 424     """
 425     if not os.path.exists(path):
 426         parent = os.path.dirname(path)
 427         if parent!="":
 428             preparePath(parent)
 429         os.mkdir(path)
 430
 431 # isNewer
 432 def isNewer(file1,file2):
 433     """Check if file1 is newer than file2.
 434
 435     file1 must be an existing file.
 436     """
 437     if not os.path.exists(file2):
 438         return True
 439     return os.stat(file1)[ST_MTIME]>os.stat(file2)[ST_MTIME]
 440
 441 # convert
 442 def convert(srcpath, destpath):
 443     """Convert a Python source tree into a C+ stub tree.
 444
 445     All *.py files in srcpath (including sub-directories) are filtered
 446     and written to destpath. If destpath exists, only the files
 447     that have been modified are filtered again. Files that were deleted
 448     from srcpath are also deleted in destpath if they are still present.
 449     The function returns the number of processed *.py files.
 450     """
 451     global namespace
 452     count=0
 453     l=os.listdir(srcpath)
 454     if "__init__.py" in l:
 455        if namespace:
 456          namespace=namespace+"::"+os.path.split(srcpath)[1]
 457        else:
 458          namespace=os.path.split(srcpath)[1]
 459        print "It's a package:",namespace
 460     sp = os.path.join(srcpath,"*")
 461     sfiles = glob.glob(sp)
 462     dp = os.path.join(destpath,"*")
 463     dfiles = glob.glob(dp)
 464     leftovers={}
 465     for df in dfiles:
 466         leftovers[os.path.basename(df)]=1
 467
 468     for srcfile in sfiles:
 469         basename = os.path.basename(srcfile)
 470         if basename in leftovers:
 471             del leftovers[basename]
 472
 473         # Is it a subdirectory?
 474         if os.path.isdir(srcfile):
 475             package=namespace
 476             sdir = os.path.join(srcpath,basename)
 477             ddir = os.path.join(destpath,basename)
 478             count+=convert(sdir, ddir)
 479             namespace=package
 480             continue
 481         # Check the extension (only *.py will be converted)
 482         root, ext = os.path.splitext(srcfile)
 483         if ext.lower()!=".py":
 484             continue
 485
 486         destfile = os.path.join(destpath,basename)
 487         if destfile==srcfile:
 488             print "WARNING: Input and output names are identical!"
 489             sys.exit(1)
 490
 491         count+=1
 492 #        sys.stdout.write("%s\015"%(srcfile))
 493
 494         if isNewer(srcfile, destfile):
 495             preparePath(os.path.dirname(destfile))
 496             out=open(destfile,"w")
 497             filterFile(srcfile, out)
 498             out.close()
 499 #            os.system("python %s -f %s>%s"%(sys.argv[0],srcfile,destfile))
 500
 501     # Delete obsolete files in destpath
 502     for df in leftovers:
 503         dname=os.path.join(destpath,df)
 504         if os.path.isdir(dname):
 505             try:
 506                 shutil.rmtree(dname)
 507             except:
 508                 print "Can't remove obsolete directory '%s'"%dname
 509         else:
 510             try:
 511                 os.remove(dname)
 512             except:
 513                 print "Can't remove obsolete file '%s'"%dname
 514
 515     return count
 516
 517
 518 ######################################################################
 519 ######################################################################
 520 ######################################################################
 521
 522 filter_file = False
 523
 524 try:
 525     opts, args = getopt.getopt(sys.argv[1:], "hf", ["help"])
 526 except getopt.GetoptError,e:
 527     print e
 528     sys.exit(1)
 529
 530 for o,a in opts:
 531     if o=="-f":
 532         filter_file = True
 533
 534 if filter_file:
 535     # Filter the specified file and print the result to stdout
 536     filename = string.join(args)
 537     filterFile(filename)
 538 else:
 539
 540     if len(args)!=2:
 541         sys.stderr.write("%s options input output\n"%(os.path.basename(sys.argv[0])))
 542         sys.exit(1)
 543
 544     # Filter an entire Python source tree
 545     print '"%s" -> "%s"\n'%(args[0],args[1])
 546     c=convert(args[0],args[1])
 547     print "%d files"%(c)