doc/salome/tui/pythfilter.py

   1 # Copyright (C) 2005  OPEN CASCADE, CEA, EDF R&D, LEG
   2 #           PRINCIPIA R&D, EADS CCR, Lip6, BV, CEDRAT
   3 # This library is free software; you can redistribute it and/or
   4 # modify it under the terms of the GNU Lesser General Public
   5 # License as published by the Free Software Foundation; either
   6 # version 2.1 of the License.
   7 #
   8 # This library is distributed in the hope that it will be useful
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11 # Lesser General Public License for more details.
  12 #
  13 # You should have received a copy of the GNU Lesser General Public
  14 # License along with this library; if not, write to the Free Software
  15 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  16 #
  17 # See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
  18 #
  19 #!/usr/bin/env python
  20
  21 import getopt
  22 import glob
  23 import os.path
  24 import shutil
  25 import string
  26 import sys
  27 import token
  28 import tokenize
  29
  30 from stat import *
  31
  32 OUTSIDE          = 0
  33 BUILD_COMMENT    = 1
  34 BUILD_CLASS_DECL = 2
  35 BUILD_CLASS_BODY = 3
  36 BUILD_DEF_DECL   = 4
  37 BUILD_DEF_BODY   = 5
  38 IMPORT           = 6
  39 IMPORT_OP        = 7
  40 IMPORT_APPEND    = 8
  41
  42 # Output file stream
  43 outfile = sys.stdout
  44
  45 # Output buffer
  46 outbuffer = []
  47
  48 out_row = 0
  49 out_col = 0
  50
  51 # Variables used by rec_name_n_param()
  52 name         = ""
  53 param        = ""
  54 doc_string   = ""
  55 record_state = 0
  56 bracket_counter = 0
  57
  58 # Tuple: (row,column)
  59 class_spos  = (0,0)
  60 def_spos    = (0,0)
  61 import_spos = (0,0)
  62
  63 # Which import was used? ("import" or "from")
  64 import_token = ""
  65
  66 # Comment block buffer
  67 comment_block = []
  68 comment_finished = 0
  69
  70 # Imported modules
  71 modules = []
  72
  73 # Program state
  74 stateStack = [OUTSIDE]
  75
  76 # Keep track of whether module has a docstring
  77 module_has_docstring = False
  78
  79 # Keep track of member protection
  80 protection_level = "public"
  81 private_member = False
  82
  83 # Keep track of the module namespace
  84 namespace = ""
  85
  86 ######################################################################
  87 # Output string s. '\n' may only be at the end of the string (not
  88 # somewhere in the middle).
  89 #
  90 # In: s    - String
  91 #     spos - Startpos
  92 ######################################################################
  93 def output(s,spos, immediate=0):
  94     global outbuffer, out_row, out_col, outfile
  95
  96     os = string.rjust(s,spos[1]-out_col+len(s))
  97     if immediate:
  98         outfile.write(os)
  99     else:
 100         outbuffer.append(os)
 101     if (s[-1:]=="\n"):
 102         out_row = out_row+1
 103         out_col = 0
 104     else:
 105         out_col = spos[1]+len(s)
 106
 107
 108 ######################################################################
 109 # Records a name and parameters. The name is either a class name or
 110 # a function name. Then the parameter is either the base class or
 111 # the function parameters.
 112 # The name is stored in the global variable "name", the parameters
 113 # in "param".
 114 # The variable "record_state" holds the current state of this internal
 115 # state machine.
 116 # The recording is started by calling start_recording().
 117 #
 118 # In: type, tok
 119 ######################################################################
 120 def rec_name_n_param(type, tok):
 121     global record_state,name,param,doc_string,bracket_counter
 122     s = record_state
 123     # State 0: Do nothing.
 124     if   (s==0):
 125          return
 126     # State 1: Remember name.
 127     elif (s==1):
 128         name = tok
 129         record_state = 2
 130     # State 2: Wait for opening bracket or colon
 131     elif (s==2):
 132         if (tok=='('):
 133             bracket_counter = 1
 134             record_state=3
 135         if (tok==':'): record_state=4
 136     # State 3: Store parameter (or base class) and wait for an ending bracket
 137     elif (s==3):
 138         if (tok=='*' or tok=='**'):
 139             tok=''
 140         if (tok=='('):
 141             bracket_counter = bracket_counter+1
 142         if (tok==')'):
 143             bracket_counter = bracket_counter-1
 144         if bracket_counter==0:
 145             record_state=4
 146         else:
 147             param=param+tok
 148     # State 4: Look for doc string
 149     elif (s==4):
 150         if (type==token.NEWLINE or type==token.INDENT or type==token.SLASHEQUAL):
 151             return
 152         elif (tok==":"):
 153             return
 154         elif (type==token.STRING):
 155             while tok[:1]=='r' or tok[:1]=='u':
 156                 tok=tok[1:]
 157             while tok[:1]=='"':
 158                 tok=tok[1:]
 159             while tok[-1:]=='"':
 160                 tok=tok[:-1]
 161             doc_string=tok
 162         record_state=0
 163
 164 ######################################################################
 165 # Starts the recording of a name & param part.
 166 # The function rec_name_n_param() has to be fed with tokens. After
 167 # the necessary tokens are fed the name and parameters can be found
 168 # in the global variables "name" und "param".
 169 ######################################################################
 170 def start_recording():
 171     global record_state,param,name, doc_string
 172     record_state=1
 173     name=""
 174     param=""
 175     doc_string=""
 176
 177 ######################################################################
 178 # Test if recording is finished
 179 ######################################################################
 180 def is_recording_finished():
 181     global record_state
 182     return record_state==0
 183
 184 ######################################################################
 185 ## Gather comment block
 186 ######################################################################
 187 def gather_comment(type,tok,spos):
 188     global comment_block,comment_finished
 189     if (type!=tokenize.COMMENT):
 190         comment_finished = 1
 191     else:
 192         # Output old comment block if a new one is started.
 193         if (comment_finished):
 194             print_comment(spos)
 195             comment_finished=0
 196         if (tok[0:2]=="##" and tok[0:3]!="###"):
 197             comment_block.append(tok[2:])
 198
 199 ######################################################################
 200 ## Output comment block and empty buffer.
 201 ######################################################################
 202 def print_comment(spos):
 203     global comment_block,comment_finished
 204     if (comment_block!=[]):
 205         output("/**\n",spos)
 206         for c in comment_block:
 207             output(c,spos)
 208         output("*/\n",spos)
 209     comment_block    = []
 210     comment_finished = 0
 211
 212 ######################################################################
 213 def set_state(s):
 214     global stateStack
 215     stateStack[len(stateStack)-1]=s
 216
 217 ######################################################################
 218 def get_state():
 219     global stateStack
 220     return stateStack[len(stateStack)-1]
 221
 222 ######################################################################
 223 def push_state(s):
 224     global stateStack
 225     stateStack.append(s)
 226
 227 ######################################################################
 228 def pop_state():
 229     global stateStack
 230     stateStack.pop()
 231
 232
 233 ######################################################################
 234 def tok_eater(type, tok, spos, epos, line):
 235     global stateStack,name,param,class_spos,def_spos,import_spos
 236     global doc_string, modules, import_token, module_has_docstring
 237     global protection_level, private_member
 238
 239     rec_name_n_param(type,tok)
 240     if (string.replace(string.strip(tok)," ","")=="##private:"):
 241          protection_level = "private"
 242          output("private:\n",spos)
 243     elif (string.replace(string.strip(tok)," ","")=="##protected:"):
 244          protection_level = "protected"
 245          output("protected:\n",spos)
 246     elif (string.replace(string.strip(tok)," ","")=="##public:"):
 247          protection_level = "public"
 248          output("public:\n",spos)
 249     else:
 250          gather_comment(type,tok,spos)
 251
 252     state = get_state()
 253
 254 #    sys.stderr.write("%d: %s\n"%(state, tok))
 255
 256     # OUTSIDE
 257     if   (state==OUTSIDE):
 258         if  (tok=="class"):
 259             start_recording()
 260             class_spos = spos
 261             push_state(BUILD_CLASS_DECL)
 262         elif (tok=="def"):
 263             start_recording()
 264             def_spos = spos
 265             push_state(BUILD_DEF_DECL)
 266         elif (tok=="import") or (tok=="from"):
 267             import_token = tok
 268             import_spos = spos
 269             modules     = []
 270             push_state(IMPORT)
 271         elif (spos[1] == 0 and tok[:3] == '"""'):
 272             # Capture module docstring as namespace documentation
 273             module_has_docstring = True
 274             #comment_block.append("\\namespace %s\n" % namespace)
 275             comment_block.append(tok[3:-3])
 276             print_comment(spos)
 277
 278     # IMPORT
 279     elif (state==IMPORT):
 280         if (type==token.NAME):
 281             modules.append(tok)
 282             set_state(IMPORT_OP)
 283     # IMPORT_OP
 284     elif (state==IMPORT_OP):
 285         if (tok=="."):
 286             set_state(IMPORT_APPEND)
 287         elif (tok==","):
 288             set_state(IMPORT)
 289         else:
 290             for m in modules:
 291                 output('#include "'+m.replace('.',os.sep)+'.py"\n', import_spos, immediate=1)
 292                 if import_token=="from":
 293                     output('using namespace '+m.replace('.', '::')+';\n', import_spos)
 294             pop_state()
 295     # IMPORT_APPEND
 296     elif (state==IMPORT_APPEND):
 297         if (type==token.NAME):
 298             modules[len(modules)-1]+="."+tok
 299             set_state(IMPORT_OP)
 300     # BUILD_CLASS_DECL
 301     elif (state==BUILD_CLASS_DECL):
 302         if (is_recording_finished()):
 303             s = "class "+name
 304             if (param!=""): s = s+" : public "+param.replace('.','::')
 305             if (doc_string!=""): comment_block.append(doc_string)
 306             print_comment(class_spos)
 307             output(s+"\n",class_spos)
 308             output("{\n",(class_spos[0]+1,class_spos[1]))
 309             protection_level = "public"
 310             output("  public:\n",(class_spos[0]+2,class_spos[1]))
 311             set_state(BUILD_CLASS_BODY)
 312     # BUILD_CLASS_BODY
 313     elif (state==BUILD_CLASS_BODY):
 314         if (type!=token.INDENT and type!=token.NEWLINE and type!=40 and
 315             type!=tokenize.NL and type!=tokenize.COMMENT and
 316             (spos[1]<=class_spos[1])):
 317             output("}; // end of class\n",(out_row+1,class_spos[1]))
 318             pop_state()
 319         elif (tok=="def"):
 320             start_recording()
 321             def_spos = spos
 322             push_state(BUILD_DEF_DECL)
 323     # BUILD_DEF_DECL
 324     elif (state==BUILD_DEF_DECL):
 325         if (is_recording_finished()):
 326             s = ''
 327             # Do we document a class method? then remove the 'self' parameter
 328             if BUILD_CLASS_BODY in stateStack:
 329                 params = param.split(",")
 330                 if params[0] == 'self':
 331                     param = string.join(params[1:], ",")
 332                 else:
 333                     s = 'static '
 334                     if params[0] == 'cls':
 335                         param = string.join(params[1:], ",")
 336                 s = s+name+"("+param+");\n"
 337                 if len(name) > 1 \
 338                    and name[0:2] == '__' \
 339                    and name[len(name)-2:len(name)] != '__' \
 340                    and protection_level != 'private':
 341                        private_member = True
 342                        output("  private:\n",(def_spos[0]+2,def_spos[1]))
 343             else:
 344                 s = name+"("+param+");\n"
 345             if (doc_string!=""): comment_block.append(doc_string)
 346             print_comment(def_spos)
 347             output(s,def_spos)
 348 #       output("{\n",(def_spos[0]+1,def_spos[1]))
 349             set_state(BUILD_DEF_BODY)
 350     # BUILD_DEF_BODY
 351     elif (state==BUILD_DEF_BODY):
 352         if (type!=token.INDENT and type!=token.NEWLINE \
 353             and type!=40 and type!=tokenize.NL \
 354             and (spos[1]<=def_spos[1])):
 355 #            output("} // end of method/function\n",(out_row+1,def_spos[1]))
 356             if private_member and protection_level != 'private':
 357                 private_member = False
 358                 output("  " + protection_level + ":\n",(def_spos[0]+2,def_spos[1]))
 359             pop_state()
 360 #       else:
 361 #            output(tok,spos)
 362
 363
 364 def dump(filename):
 365     f = open(filename)
 366     r = f.readlines()
 367     for s in r:
 368         sys.stdout.write(s)
 369
 370 def filter(filename):
 371     global name, module_has_docstring
 372     global namespace,outbuffer
 373     outbuffer=[]
 374
 375     path,name = os.path.split(filename)
 376     root,ext  = os.path.splitext(name)
 377
 378
 379     if namespace:
 380        if root == "__init__":
 381           root=namespace
 382        else:
 383           root=namespace+"::"+root
 384     else:
 385        root=root
 386
 387     output("namespace "+root+" {\n",(0,0))
 388
 389     # set module name for tok_eater to use if there's a module doc string
 390     package=namespace
 391     name = root
 392     namespace = root
 393
 394     sys.stderr.write("namespace: "+namespace+'\n')
 395     sys.stderr.write("root: "+root+'\n')
 396     sys.stderr.write('Filtering "'+filename+'"...')
 397
 398     f = open(filename)
 399     tokenize.tokenize(f.readline, tok_eater)
 400     f.close()
 401     print_comment((0,0))
 402
 403     output("\n",(0,0))
 404     output("}  // end of namespace\n",(0,0))
 405
 406     if not module_has_docstring:
 407         # Put in default namespace documentation
 408         output('/** \\namespace '+root+' \n',(0,0))
 409         output('    \\brief Module "%s" */\n'%(root),(0,0))
 410
 411     for s in outbuffer:
 412         outfile.write(s)
 413     namespace=package
 414     module_has_docstring = False
 415
 416
 417 def filterFile(filename, out=sys.stdout):
 418     global outfile
 419
 420     outfile = out
 421
 422     try:
 423         root,ext  = os.path.splitext(filename)
 424
 425         if ext==".py":
 426             filter(filename)
 427         else:
 428             dump(filename)
 429
 430         sys.stderr.write("OK\n")
 431     except IOError,e:
 432         sys.stderr.write(e[1]+"\n")
 433
 434
 435 ######################################################################
 436
 437 # preparePath
 438 def preparePath(path):
 439     """Prepare a path.
 440
 441     Checks if the path exists and creates it if it does not exist.
 442     """
 443     if not os.path.exists(path):
 444         parent = os.path.dirname(path)
 445         if parent!="":
 446             preparePath(parent)
 447         os.mkdir(path)
 448
 449 # isNewer
 450 def isNewer(file1,file2):
 451     """Check if file1 is newer than file2.
 452
 453     file1 must be an existing file.
 454     """
 455     if not os.path.exists(file2):
 456         return True
 457     return os.stat(file1)[ST_MTIME]>os.stat(file2)[ST_MTIME]
 458
 459 # convert
 460 def convert(srcpath, destpath):
 461     """Convert a Python source tree into a C+ stub tree.
 462
 463     All *.py files in srcpath (including sub-directories) are filtered
 464     and written to destpath. If destpath exists, only the files
 465     that have been modified are filtered again. Files that were deleted
 466     from srcpath are also deleted in destpath if they are still present.
 467     The function returns the number of processed *.py files.
 468     """
 469     global namespace
 470     count=0
 471     l=os.listdir(srcpath)
 472     if "__init__.py" in l:
 473        if namespace:
 474          namespace=namespace+"::"+os.path.split(srcpath)[1]
 475        else:
 476          namespace=os.path.split(srcpath)[1]
 477        print "It's a package:",namespace
 478     sp = os.path.join(srcpath,"*")
 479     sfiles = glob.glob(sp)
 480     dp = os.path.join(destpath,"*")
 481     dfiles = glob.glob(dp)
 482     leftovers={}
 483     for df in dfiles:
 484         leftovers[os.path.basename(df)]=1
 485
 486     for srcfile in sfiles:
 487         basename = os.path.basename(srcfile)
 488         if basename in leftovers:
 489             del leftovers[basename]
 490
 491         # Is it a subdirectory?
 492         if os.path.isdir(srcfile):
 493             package=namespace
 494             sdir = os.path.join(srcpath,basename)
 495             ddir = os.path.join(destpath,basename)
 496             count+=convert(sdir, ddir)
 497             namespace=package
 498             continue
 499         # Check the extension (only *.py will be converted)
 500         root, ext = os.path.splitext(srcfile)
 501         if ext.lower()!=".py":
 502             continue
 503
 504         destfile = os.path.join(destpath,basename)
 505         if destfile==srcfile:
 506             print "WARNING: Input and output names are identical!"
 507             sys.exit(1)
 508
 509         count+=1
 510 #        sys.stdout.write("%s\015"%(srcfile))
 511
 512         if isNewer(srcfile, destfile):
 513             preparePath(os.path.dirname(destfile))
 514             out=open(destfile,"w")
 515             filterFile(srcfile, out)
 516             out.close()
 517 #            os.system("python %s -f %s>%s"%(sys.argv[0],srcfile,destfile))
 518
 519     # Delete obsolete files in destpath
 520     for df in leftovers:
 521         dname=os.path.join(destpath,df)
 522         if os.path.isdir(dname):
 523             try:
 524                 shutil.rmtree(dname)
 525             except:
 526                 print "Can't remove obsolete directory '%s'"%dname
 527         else:
 528             try:
 529                 os.remove(dname)
 530             except:
 531                 print "Can't remove obsolete file '%s'"%dname
 532
 533     return count
 534
 535
 536 ######################################################################
 537 ######################################################################
 538 ######################################################################
 539
 540 filter_file = False
 541
 542 try:
 543     opts, args = getopt.getopt(sys.argv[1:], "hf", ["help"])
 544 except getopt.GetoptError,e:
 545     print e
 546     sys.exit(1)
 547
 548 for o,a in opts:
 549     if o=="-f":
 550         filter_file = True
 551
 552 if filter_file:
 553     # Filter the specified file and print the result to stdout
 554     filename = string.join(args)
 555     filterFile(filename)
 556 else:
 557
 558     if len(args)!=2:
 559         sys.stderr.write("%s options input output\n"%(os.path.basename(sys.argv[0])))
 560         sys.exit(1)
 561
 562     # Filter an entire Python source tree
 563     print '"%s" -> "%s"\n'%(args[0],args[1])
 564     c=convert(args[0],args[1])
 565     print "%d files"%(c)