doc/salome/tui/pythfilter.py

   1 #  Copyright (C) 2007-2008  CEA/DEN, EDF R&D, OPEN CASCADE
   2 #
   3 #  Copyright (C) 2003-2007  OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN,
   4 #  CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS
   5 #
   6 #  This library is free software; you can redistribute it and/or
   7 #  modify it under the terms of the GNU Lesser General Public
   8 #  License as published by the Free Software Foundation; either
   9 #  version 2.1 of the License.
  10 #
  11 #  This library is distributed in the hope that it will be useful,
  12 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 #  Lesser General Public License for more details.
  15 #
  16 #  You should have received a copy of the GNU Lesser General Public
  17 #  License along with this library; if not, write to the Free Software
  18 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  19 #
  20 #  See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
  21 #
  22 #!/usr/bin/env python
  23 #
  24 import getopt
  25 import glob
  26 import os.path
  27 import shutil
  28 import string
  29 import sys
  30 import token
  31 import tokenize
  32
  33 from stat import *
  34
  35 OUTSIDE          = 0
  36 BUILD_COMMENT    = 1
  37 BUILD_CLASS_DECL = 2
  38 BUILD_CLASS_BODY = 3
  39 BUILD_DEF_DECL   = 4
  40 BUILD_DEF_BODY   = 5
  41 IMPORT           = 6
  42 IMPORT_OP        = 7
  43 IMPORT_APPEND    = 8
  44
  45 # Output file stream
  46 outfile = sys.stdout
  47
  48 # Output buffer
  49 outbuffer = []
  50
  51 out_row = 0
  52 out_col = 0
  53
  54 # Variables used by rec_name_n_param()
  55 name         = ""
  56 param        = ""
  57 doc_string   = ""
  58 record_state = 0
  59 bracket_counter = 0
  60
  61 # Tuple: (row,column)
  62 class_spos  = (0,0)
  63 def_spos    = (0,0)
  64 import_spos = (0,0)
  65
  66 # Which import was used? ("import" or "from")
  67 import_token = ""
  68
  69 # Comment block buffer
  70 comment_block = []
  71 comment_finished = 0
  72
  73 # Imported modules
  74 modules = []
  75
  76 # Program state
  77 stateStack = [OUTSIDE]
  78
  79 # Keep track of whether module has a docstring
  80 module_has_docstring = False
  81
  82 # Keep track of member protection
  83 protection_level = "public"
  84 private_member = False
  85
  86 # Keep track of the module namespace
  87 namespace = ""
  88
  89 ######################################################################
  90 # Output string s. '\n' may only be at the end of the string (not
  91 # somewhere in the middle).
  92 #
  93 # In: s    - String
  94 #     spos - Startpos
  95 ######################################################################
  96 def output(s,spos, immediate=0):
  97     global outbuffer, out_row, out_col, outfile
  98
  99     os = string.rjust(s,spos[1]-out_col+len(s))
 100     if immediate:
 101         outfile.write(os)
 102     else:
 103         outbuffer.append(os)
 104     if (s[-1:]=="\n"):
 105         out_row = out_row+1
 106         out_col = 0
 107     else:
 108         out_col = spos[1]+len(s)
 109
 110
 111 ######################################################################
 112 # Records a name and parameters. The name is either a class name or
 113 # a function name. Then the parameter is either the base class or
 114 # the function parameters.
 115 # The name is stored in the global variable "name", the parameters
 116 # in "param".
 117 # The variable "record_state" holds the current state of this internal
 118 # state machine.
 119 # The recording is started by calling start_recording().
 120 #
 121 # In: type, tok
 122 ######################################################################
 123 def rec_name_n_param(type, tok):
 124     global record_state,name,param,doc_string,bracket_counter
 125     s = record_state
 126     # State 0: Do nothing.
 127     if   (s==0):
 128          return
 129     # State 1: Remember name.
 130     elif (s==1):
 131         name = tok
 132         record_state = 2
 133     # State 2: Wait for opening bracket or colon
 134     elif (s==2):
 135         if (tok=='('):
 136             bracket_counter = 1
 137             record_state=3
 138         if (tok==':'): record_state=4
 139     # State 3: Store parameter (or base class) and wait for an ending bracket
 140     elif (s==3):
 141         if (tok=='*' or tok=='**'):
 142             tok=''
 143         if (tok=='('):
 144             bracket_counter = bracket_counter+1
 145         if (tok==')'):
 146             bracket_counter = bracket_counter-1
 147         if bracket_counter==0:
 148             record_state=4
 149         else:
 150             param=param+tok
 151     # State 4: Look for doc string
 152     elif (s==4):
 153         if (type==token.NEWLINE or type==token.INDENT or type==token.SLASHEQUAL):
 154             return
 155         elif (tok==":"):
 156             return
 157         elif (type==token.STRING):
 158             while tok[:1]=='r' or tok[:1]=='u':
 159                 tok=tok[1:]
 160             while tok[:1]=='"':
 161                 tok=tok[1:]
 162             while tok[-1:]=='"':
 163                 tok=tok[:-1]
 164             doc_string=tok
 165         record_state=0
 166
 167 ######################################################################
 168 # Starts the recording of a name & param part.
 169 # The function rec_name_n_param() has to be fed with tokens. After
 170 # the necessary tokens are fed the name and parameters can be found
 171 # in the global variables "name" und "param".
 172 ######################################################################
 173 def start_recording():
 174     global record_state,param,name, doc_string
 175     record_state=1
 176     name=""
 177     param=""
 178     doc_string=""
 179
 180 ######################################################################
 181 # Test if recording is finished
 182 ######################################################################
 183 def is_recording_finished():
 184     global record_state
 185     return record_state==0
 186
 187 ######################################################################
 188 ## Gather comment block
 189 ######################################################################
 190 def gather_comment(type,tok,spos):
 191     global comment_block,comment_finished
 192     if (type!=tokenize.COMMENT):
 193         comment_finished = 1
 194     else:
 195         # Output old comment block if a new one is started.
 196         if (comment_finished):
 197             print_comment(spos)
 198             comment_finished=0
 199         if (tok[0:2]=="##" and tok[0:3]!="###"):
 200             comment_block.append(tok[2:])
 201
 202 ######################################################################
 203 ## Output comment block and empty buffer.
 204 ######################################################################
 205 def print_comment(spos):
 206     global comment_block,comment_finished
 207     if (comment_block!=[]):
 208         output("/**\n",spos)
 209         for c in comment_block:
 210             output(c,spos)
 211         output("*/\n",spos)
 212     comment_block    = []
 213     comment_finished = 0
 214
 215 ######################################################################
 216 def set_state(s):
 217     global stateStack
 218     stateStack[len(stateStack)-1]=s
 219
 220 ######################################################################
 221 def get_state():
 222     global stateStack
 223     return stateStack[len(stateStack)-1]
 224
 225 ######################################################################
 226 def push_state(s):
 227     global stateStack
 228     stateStack.append(s)
 229
 230 ######################################################################
 231 def pop_state():
 232     global stateStack
 233     stateStack.pop()
 234
 235
 236 ######################################################################
 237 def tok_eater(type, tok, spos, epos, line):
 238     global stateStack,name,param,class_spos,def_spos,import_spos
 239     global doc_string, modules, import_token, module_has_docstring
 240     global protection_level, private_member
 241
 242     rec_name_n_param(type,tok)
 243     if (string.replace(string.strip(tok)," ","")=="##private:"):
 244          protection_level = "private"
 245          output("private:\n",spos)
 246     elif (string.replace(string.strip(tok)," ","")=="##protected:"):
 247          protection_level = "protected"
 248          output("protected:\n",spos)
 249     elif (string.replace(string.strip(tok)," ","")=="##public:"):
 250          protection_level = "public"
 251          output("public:\n",spos)
 252     else:
 253          gather_comment(type,tok,spos)
 254
 255     state = get_state()
 256
 257 #    sys.stderr.write("%d: %s\n"%(state, tok))
 258
 259     # OUTSIDE
 260     if   (state==OUTSIDE):
 261         if  (tok=="class"):
 262             start_recording()
 263             class_spos = spos
 264             push_state(BUILD_CLASS_DECL)
 265         elif (tok=="def"):
 266             start_recording()
 267             def_spos = spos
 268             push_state(BUILD_DEF_DECL)
 269         elif (tok=="import") or (tok=="from"):
 270             import_token = tok
 271             import_spos = spos
 272             modules     = []
 273             push_state(IMPORT)
 274         elif (spos[1] == 0 and tok[:3] == '"""'):
 275             # Capture module docstring as namespace documentation
 276             module_has_docstring = True
 277             #comment_block.append("\\namespace %s\n" % namespace)
 278             comment_block.append(tok[3:-3])
 279             print_comment(spos)
 280
 281     # IMPORT
 282     elif (state==IMPORT):
 283         if (type==token.NAME):
 284             modules.append(tok)
 285             set_state(IMPORT_OP)
 286     # IMPORT_OP
 287     elif (state==IMPORT_OP):
 288         if (tok=="."):
 289             set_state(IMPORT_APPEND)
 290         elif (tok==","):
 291             set_state(IMPORT)
 292         else:
 293             for m in modules:
 294                 output('#include "'+m.replace('.',os.sep)+'.py"\n', import_spos, immediate=1)
 295                 if import_token=="from":
 296                     output('using namespace '+m.replace('.', '::')+';\n', import_spos)
 297             pop_state()
 298     # IMPORT_APPEND
 299     elif (state==IMPORT_APPEND):
 300         if (type==token.NAME):
 301             modules[len(modules)-1]+="."+tok
 302             set_state(IMPORT_OP)
 303     # BUILD_CLASS_DECL
 304     elif (state==BUILD_CLASS_DECL):
 305         if (is_recording_finished()):
 306             s = "class "+name
 307             if (param!=""): s = s+" : public "+param.replace('.','::')
 308             if (doc_string!=""): comment_block.append(doc_string)
 309             print_comment(class_spos)
 310             output(s+"\n",class_spos)
 311             output("{\n",(class_spos[0]+1,class_spos[1]))
 312             protection_level = "public"
 313             output("  public:\n",(class_spos[0]+2,class_spos[1]))
 314             set_state(BUILD_CLASS_BODY)
 315     # BUILD_CLASS_BODY
 316     elif (state==BUILD_CLASS_BODY):
 317         if (type!=token.INDENT and type!=token.NEWLINE and type!=40 and
 318             type!=tokenize.NL and type!=tokenize.COMMENT and
 319             (spos[1]<=class_spos[1])):
 320             output("}; // end of class\n",(out_row+1,class_spos[1]))
 321             pop_state()
 322         elif (tok=="def"):
 323             start_recording()
 324             def_spos = spos
 325             push_state(BUILD_DEF_DECL)
 326     # BUILD_DEF_DECL
 327     elif (state==BUILD_DEF_DECL):
 328         if (is_recording_finished()):
 329             s = ''
 330             # Do we document a class method? then remove the 'self' parameter
 331             if BUILD_CLASS_BODY in stateStack:
 332                 params = param.split(",")
 333                 if params[0] == 'self':
 334                     param = string.join(params[1:], ",")
 335                 else:
 336                     s = 'static '
 337                     if params[0] == 'cls':
 338                         param = string.join(params[1:], ",")
 339                 s = s+name+"("+param+");\n"
 340                 if len(name) > 1 \
 341                    and name[0:2] == '__' \
 342                    and name[len(name)-2:len(name)] != '__' \
 343                    and protection_level != 'private':
 344                        private_member = True
 345                        output("  private:\n",(def_spos[0]+2,def_spos[1]))
 346             else:
 347                 s = name+"("+param+");\n"
 348             if (doc_string!=""): comment_block.append(doc_string)
 349             print_comment(def_spos)
 350             output(s,def_spos)
 351 #       output("{\n",(def_spos[0]+1,def_spos[1]))
 352             set_state(BUILD_DEF_BODY)
 353     # BUILD_DEF_BODY
 354     elif (state==BUILD_DEF_BODY):
 355         if (type!=token.INDENT and type!=token.NEWLINE \
 356             and type!=40 and type!=tokenize.NL \
 357             and (spos[1]<=def_spos[1])):
 358 #            output("} // end of method/function\n",(out_row+1,def_spos[1]))
 359             if private_member and protection_level != 'private':
 360                 private_member = False
 361                 output("  " + protection_level + ":\n",(def_spos[0]+2,def_spos[1]))
 362             pop_state()
 363 #       else:
 364 #            output(tok,spos)
 365
 366
 367 def dump(filename):
 368     f = open(filename)
 369     r = f.readlines()
 370     for s in r:
 371         sys.stdout.write(s)
 372
 373 def filter(filename):
 374     global name, module_has_docstring
 375     global namespace,outbuffer
 376     outbuffer=[]
 377
 378     path,name = os.path.split(filename)
 379     root,ext  = os.path.splitext(name)
 380
 381
 382     if namespace:
 383        if root == "__init__":
 384           root=namespace
 385        else:
 386           root=namespace+"::"+root
 387     else:
 388        root=root
 389
 390     output("namespace "+root+" {\n",(0,0))
 391
 392     # set module name for tok_eater to use if there's a module doc string
 393     package=namespace
 394     name = root
 395     namespace = root
 396
 397     sys.stderr.write("namespace: "+namespace+'\n')
 398     sys.stderr.write("root: "+root+'\n')
 399     sys.stderr.write('Filtering "'+filename+'"...')
 400
 401     f = open(filename)
 402     tokenize.tokenize(f.readline, tok_eater)
 403     f.close()
 404     print_comment((0,0))
 405
 406     output("\n",(0,0))
 407     output("}  // end of namespace\n",(0,0))
 408
 409     if not module_has_docstring:
 410         # Put in default namespace documentation
 411         output('/** \\namespace '+root+' \n',(0,0))
 412         output('    \\brief Module "%s" */\n'%(root),(0,0))
 413
 414     for s in outbuffer:
 415         outfile.write(s)
 416     namespace=package
 417     module_has_docstring = False
 418
 419
 420 def filterFile(filename, out=sys.stdout):
 421     global outfile
 422
 423     outfile = out
 424
 425     try:
 426         root,ext  = os.path.splitext(filename)
 427
 428         if ext==".py":
 429             filter(filename)
 430         else:
 431             dump(filename)
 432
 433         sys.stderr.write("OK\n")
 434     except IOError,e:
 435         sys.stderr.write(e[1]+"\n")
 436
 437
 438 ######################################################################
 439
 440 # preparePath
 441 def preparePath(path):
 442     """Prepare a path.
 443
 444     Checks if the path exists and creates it if it does not exist.
 445     """
 446     if not os.path.exists(path):
 447         parent = os.path.dirname(path)
 448         if parent!="":
 449             preparePath(parent)
 450         os.mkdir(path)
 451
 452 # isNewer
 453 def isNewer(file1,file2):
 454     """Check if file1 is newer than file2.
 455
 456     file1 must be an existing file.
 457     """
 458     if not os.path.exists(file2):
 459         return True
 460     return os.stat(file1)[ST_MTIME]>os.stat(file2)[ST_MTIME]
 461
 462 # convert
 463 def convert(srcpath, destpath):
 464     """Convert a Python source tree into a C+ stub tree.
 465
 466     All *.py files in srcpath (including sub-directories) are filtered
 467     and written to destpath. If destpath exists, only the files
 468     that have been modified are filtered again. Files that were deleted
 469     from srcpath are also deleted in destpath if they are still present.
 470     The function returns the number of processed *.py files.
 471     """
 472     global namespace
 473     count=0
 474     l=os.listdir(srcpath)
 475     if "__init__.py" in l:
 476        if namespace:
 477          namespace=namespace+"::"+os.path.split(srcpath)[1]
 478        else:
 479          namespace=os.path.split(srcpath)[1]
 480        print "It's a package:",namespace
 481     sp = os.path.join(srcpath,"*")
 482     sfiles = glob.glob(sp)
 483     dp = os.path.join(destpath,"*")
 484     dfiles = glob.glob(dp)
 485     leftovers={}
 486     for df in dfiles:
 487         leftovers[os.path.basename(df)]=1
 488
 489     for srcfile in sfiles:
 490         basename = os.path.basename(srcfile)
 491         if basename in leftovers:
 492             del leftovers[basename]
 493
 494         # Is it a subdirectory?
 495         if os.path.isdir(srcfile):
 496             package=namespace
 497             sdir = os.path.join(srcpath,basename)
 498             ddir = os.path.join(destpath,basename)
 499             count+=convert(sdir, ddir)
 500             namespace=package
 501             continue
 502         # Check the extension (only *.py will be converted)
 503         root, ext = os.path.splitext(srcfile)
 504         if ext.lower()!=".py":
 505             continue
 506
 507         destfile = os.path.join(destpath,basename)
 508         if destfile==srcfile:
 509             print "WARNING: Input and output names are identical!"
 510             sys.exit(1)
 511
 512         count+=1
 513 #        sys.stdout.write("%s\015"%(srcfile))
 514
 515         if isNewer(srcfile, destfile):
 516             preparePath(os.path.dirname(destfile))
 517             out=open(destfile,"w")
 518             filterFile(srcfile, out)
 519             out.close()
 520 #            os.system("python %s -f %s>%s"%(sys.argv[0],srcfile,destfile))
 521
 522     # Delete obsolete files in destpath
 523     for df in leftovers:
 524         dname=os.path.join(destpath,df)
 525         if os.path.isdir(dname):
 526             try:
 527                 shutil.rmtree(dname)
 528             except:
 529                 print "Can't remove obsolete directory '%s'"%dname
 530         else:
 531             try:
 532                 os.remove(dname)
 533             except:
 534                 print "Can't remove obsolete file '%s'"%dname
 535
 536     return count
 537
 538
 539 ######################################################################
 540 ######################################################################
 541 ######################################################################
 542
 543 filter_file = False
 544
 545 try:
 546     opts, args = getopt.getopt(sys.argv[1:], "hf", ["help"])
 547 except getopt.GetoptError,e:
 548     print e
 549     sys.exit(1)
 550
 551 for o,a in opts:
 552     if o=="-f":
 553         filter_file = True
 554
 555 if filter_file:
 556     # Filter the specified file and print the result to stdout
 557     filename = string.join(args)
 558     filterFile(filename)
 559 else:
 560
 561     if len(args)!=2:
 562         sys.stderr.write("%s options input output\n"%(os.path.basename(sys.argv[0])))
 563         sys.exit(1)
 564
 565     # Filter an entire Python source tree
 566     print '"%s" -> "%s"\n'%(args[0],args[1])
 567     c=convert(args[0],args[1])
 568     print "%d files"%(c)