doc/salome/tui/pythfilter.py

   1 #!/usr/bin/env python3
   2 #  -*- coding: iso-8859-1 -*-
   3 # Copyright (C) 2007-2016  CEA/DEN, EDF R&D, OPEN CASCADE
   4 #
   5 # Copyright (C) 2003-2007  OPEN CASCADE, EADS/CCR, LIP6, CEA/DEN,
   6 # CEDRAT, EDF R&D, LEG, PRINCIPIA R&D, BUREAU VERITAS
   7 #
   8 # This library is free software; you can redistribute it and/or
   9 # modify it under the terms of the GNU Lesser General Public
  10 # License as published by the Free Software Foundation; either
  11 # version 2.1 of the License, or (at your option) any later version.
  12 #
  13 # This library is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16 # Lesser General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Lesser General Public
  19 # License along with this library; if not, write to the Free Software
  20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  21 #
  22 # See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
  23 #
  24
  25 import getopt
  26 import glob
  27 import os.path
  28 import shutil
  29 import string
  30 import sys
  31 import token
  32 import tokenize
  33
  34 from stat import *
  35
  36 OUTSIDE          = 0
  37 BUILD_COMMENT    = 1
  38 BUILD_CLASS_DECL = 2
  39 BUILD_CLASS_BODY = 3
  40 BUILD_DEF_DECL   = 4
  41 BUILD_DEF_BODY   = 5
  42 IMPORT           = 6
  43 IMPORT_OP        = 7
  44 IMPORT_APPEND    = 8
  45
  46 # Output file stream
  47 outfile = sys.stdout
  48
  49 # Output buffer
  50 outbuffer = []
  51
  52 out_row = 0
  53 out_col = 0
  54
  55 # Variables used by rec_name_n_param()
  56 name         = ""
  57 param        = ""
  58 doc_string   = ""
  59 record_state = 0
  60 bracket_counter = 0
  61
  62 # Tuple: (row,column)
  63 class_spos  = (0,0)
  64 def_spos    = (0,0)
  65 import_spos = (0,0)
  66
  67 # Which import was used? ("import" or "from")
  68 import_token = ""
  69
  70 # Comment block buffer
  71 comment_block = []
  72 comment_finished = 0
  73
  74 # Imported modules
  75 modules = []
  76
  77 # Program state
  78 stateStack = [OUTSIDE]
  79
  80 # Keep track of whether module has a docstring
  81 module_has_docstring = False
  82
  83 # Keep track of member protection
  84 protection_level = "public"
  85 private_member = False
  86
  87 # Keep track of the module namespace
  88 namespace = ""
  89
  90 ######################################################################
  91 # Output string s. '\n' may only be at the end of the string (not
  92 # somewhere in the middle).
  93 #
  94 # In: s    - String
  95 #     spos - Startpos
  96 ######################################################################
  97 def output(s,spos, immediate=0):
  98     global outbuffer, out_row, out_col, outfile
  99
 100     os = string.rjust(s,spos[1]-out_col+len(s))
 101     if immediate:
 102         outfile.write(os)
 103     else:
 104         outbuffer.append(os)
 105     if (s[-1:]=="\n"):
 106         out_row = out_row+1
 107         out_col = 0
 108     else:
 109         out_col = spos[1]+len(s)
 110
 111
 112 ######################################################################
 113 # Records a name and parameters. The name is either a class name or
 114 # a function name. Then the parameter is either the base class or
 115 # the function parameters.
 116 # The name is stored in the global variable "name", the parameters
 117 # in "param".
 118 # The variable "record_state" holds the current state of this internal
 119 # state machine.
 120 # The recording is started by calling start_recording().
 121 #
 122 # In: type, tok
 123 ######################################################################
 124 def rec_name_n_param(type, tok):
 125     global record_state,name,param,doc_string,bracket_counter
 126     s = record_state
 127     # State 0: Do nothing.
 128     if   (s==0):
 129          return
 130     # State 1: Remember name.
 131     elif (s==1):
 132         name = tok
 133         record_state = 2
 134     # State 2: Wait for opening bracket or colon
 135     elif (s==2):
 136         if (tok=='('):
 137             bracket_counter = 1
 138             record_state=3
 139         if (tok==':'): record_state=4
 140     # State 3: Store parameter (or base class) and wait for an ending bracket
 141     elif (s==3):
 142         if (tok=='*' or tok=='**'):
 143             tok=''
 144         if (tok=='('):
 145             bracket_counter = bracket_counter+1
 146         if (tok==')'):
 147             bracket_counter = bracket_counter-1
 148         if bracket_counter==0:
 149             record_state=4
 150         else:
 151             param=param+tok
 152     # State 4: Look for doc string
 153     elif (s==4):
 154         if (type==token.NEWLINE or type==token.INDENT or type==token.SLASHEQUAL):
 155             return
 156         elif (tok==":"):
 157             return
 158         elif (type==token.STRING):
 159             while tok[:1]=='r' or tok[:1]=='u':
 160                 tok=tok[1:]
 161             while tok[:1]=='"':
 162                 tok=tok[1:]
 163             while tok[-1:]=='"':
 164                 tok=tok[:-1]
 165             doc_string=tok
 166         record_state=0
 167
 168 ######################################################################
 169 # Starts the recording of a name & param part.
 170 # The function rec_name_n_param() has to be fed with tokens. After
 171 # the necessary tokens are fed the name and parameters can be found
 172 # in the global variables "name" und "param".
 173 ######################################################################
 174 def start_recording():
 175     global record_state,param,name, doc_string
 176     record_state=1
 177     name=""
 178     param=""
 179     doc_string=""
 180
 181 ######################################################################
 182 # Test if recording is finished
 183 ######################################################################
 184 def is_recording_finished():
 185     global record_state
 186     return record_state==0
 187
 188 ######################################################################
 189 ## Gather comment block
 190 ######################################################################
 191 def gather_comment(type,tok,spos):
 192     global comment_block,comment_finished
 193     if (type!=tokenize.COMMENT):
 194         comment_finished = 1
 195     else:
 196         # Output old comment block if a new one is started.
 197         if (comment_finished):
 198             print_comment(spos)
 199             comment_finished=0
 200         if (tok[0:2]=="##" and tok[0:3]!="###"):
 201             comment_block.append(tok[2:])
 202
 203 ######################################################################
 204 ## Output comment block and empty buffer.
 205 ######################################################################
 206 def print_comment(spos):
 207     global comment_block,comment_finished
 208     if (comment_block!=[]):
 209         output("/**\n",spos)
 210         for c in comment_block:
 211             output(c,spos)
 212         output("*/\n",spos)
 213     comment_block    = []
 214     comment_finished = 0
 215
 216 ######################################################################
 217 def set_state(s):
 218     global stateStack
 219     stateStack[len(stateStack)-1]=s
 220
 221 ######################################################################
 222 def get_state():
 223     global stateStack
 224     return stateStack[len(stateStack)-1]
 225
 226 ######################################################################
 227 def push_state(s):
 228     global stateStack
 229     stateStack.append(s)
 230
 231 ######################################################################
 232 def pop_state():
 233     global stateStack
 234     stateStack.pop()
 235
 236
 237 ######################################################################
 238 def tok_eater(type, tok, spos, epos, line):
 239     global stateStack,name,param,class_spos,def_spos,import_spos
 240     global doc_string, modules, import_token, module_has_docstring
 241     global protection_level, private_member
 242
 243     rec_name_n_param(type,tok)
 244     if (string.replace(string.strip(tok)," ","")=="##private:"):
 245          protection_level = "private"
 246          output("private:\n",spos)
 247     elif (string.replace(string.strip(tok)," ","")=="##protected:"):
 248          protection_level = "protected"
 249          output("protected:\n",spos)
 250     elif (string.replace(string.strip(tok)," ","")=="##public:"):
 251          protection_level = "public"
 252          output("public:\n",spos)
 253     else:
 254          gather_comment(type,tok,spos)
 255
 256     state = get_state()
 257
 258 #    sys.stderr.write("%d: %s\n"%(state, tok))
 259
 260     # OUTSIDE
 261     if   (state==OUTSIDE):
 262         if  (tok=="class"):
 263             start_recording()
 264             class_spos = spos
 265             push_state(BUILD_CLASS_DECL)
 266         elif (tok=="def"):
 267             start_recording()
 268             def_spos = spos
 269             push_state(BUILD_DEF_DECL)
 270         elif (tok=="import") or (tok=="from"):
 271             import_token = tok
 272             import_spos = spos
 273             modules     = []
 274             push_state(IMPORT)
 275         elif (spos[1] == 0 and tok[:3] == '"""'):
 276             # Capture module docstring as namespace documentation
 277             module_has_docstring = True
 278             #comment_block.append("\\namespace %s\n" % namespace)
 279             comment_block.append(tok[3:-3])
 280             print_comment(spos)
 281
 282     # IMPORT
 283     elif (state==IMPORT):
 284         if (type==token.NAME):
 285             modules.append(tok)
 286             set_state(IMPORT_OP)
 287     # IMPORT_OP
 288     elif (state==IMPORT_OP):
 289         if (tok=="."):
 290             set_state(IMPORT_APPEND)
 291         elif (tok==","):
 292             set_state(IMPORT)
 293         else:
 294             for m in modules:
 295                 output('#include "'+m.replace('.',os.sep)+'.py"\n', import_spos, immediate=1)
 296                 if import_token=="from":
 297                     output('using namespace '+m.replace('.', '::')+';\n', import_spos)
 298             pop_state()
 299     # IMPORT_APPEND
 300     elif (state==IMPORT_APPEND):
 301         if (type==token.NAME):
 302             modules[len(modules)-1]+="."+tok
 303             set_state(IMPORT_OP)
 304     # BUILD_CLASS_DECL
 305     elif (state==BUILD_CLASS_DECL):
 306         if (is_recording_finished()):
 307             s = "class "+name
 308             if (param!=""): s = s+" : public "+param.replace('.','::')
 309             if (doc_string!=""): comment_block.append(doc_string)
 310             print_comment(class_spos)
 311             output(s+"\n",class_spos)
 312             output("{\n",(class_spos[0]+1,class_spos[1]))
 313             protection_level = "public"
 314             output("  public:\n",(class_spos[0]+2,class_spos[1]))
 315             set_state(BUILD_CLASS_BODY)
 316     # BUILD_CLASS_BODY
 317     elif (state==BUILD_CLASS_BODY):
 318         if (type!=token.INDENT and type!=token.NEWLINE and type!=40 and
 319             type!=tokenize.NL and type!=tokenize.COMMENT and
 320             (spos[1]<=class_spos[1])):
 321             output("}; // end of class\n",(out_row+1,class_spos[1]))
 322             pop_state()
 323         elif (tok=="def"):
 324             start_recording()
 325             def_spos = spos
 326             push_state(BUILD_DEF_DECL)
 327     # BUILD_DEF_DECL
 328     elif (state==BUILD_DEF_DECL):
 329         if (is_recording_finished()):
 330             s = ''
 331             # Do we document a class method? then remove the 'self' parameter
 332             if BUILD_CLASS_BODY in stateStack:
 333                 params = param.split(",")
 334                 if params[0] == 'self':
 335                     param = string.join(params[1:], ",")
 336                 else:
 337                     s = 'static '
 338                     if params[0] == 'cls':
 339                         param = string.join(params[1:], ",")
 340                 s = s+name+"("+param+");\n"
 341                 if len(name) > 1 \
 342                    and name[0:2] == '__' \
 343                    and name[len(name)-2:len(name)] != '__' \
 344                    and protection_level != 'private':
 345                        private_member = True
 346                        output("  private:\n",(def_spos[0]+2,def_spos[1]))
 347             else:
 348                 s = name+"("+param+");\n"
 349             if (doc_string!=""): comment_block.append(doc_string)
 350             print_comment(def_spos)
 351             output(s,def_spos)
 352 #       output("{\n",(def_spos[0]+1,def_spos[1]))
 353             set_state(BUILD_DEF_BODY)
 354     # BUILD_DEF_BODY
 355     elif (state==BUILD_DEF_BODY):
 356         if (type!=token.INDENT and type!=token.NEWLINE \
 357             and type!=40 and type!=tokenize.NL \
 358             and (spos[1]<=def_spos[1])):
 359 #            output("} // end of method/function\n",(out_row+1,def_spos[1]))
 360             if private_member and protection_level != 'private':
 361                 private_member = False
 362                 output("  " + protection_level + ":\n",(def_spos[0]+2,def_spos[1]))
 363             pop_state()
 364 #       else:
 365 #            output(tok,spos)
 366
 367
 368 def dump(filename):
 369     f = open(filename)
 370     r = f.readlines()
 371     for s in r:
 372         sys.stdout.write(s)
 373
 374 def filter(filename):
 375     global name, module_has_docstring
 376     global namespace,outbuffer
 377     outbuffer=[]
 378
 379     path,name = os.path.split(filename)
 380     root,ext  = os.path.splitext(name)
 381
 382
 383     if namespace:
 384        if root == "__init__":
 385           root=namespace
 386        else:
 387           root=namespace+"::"+root
 388     else:
 389        root=root
 390
 391     output("namespace "+root+" {\n",(0,0))
 392
 393     # set module name for tok_eater to use if there's a module doc string
 394     package=namespace
 395     name = root
 396     namespace = root
 397
 398     sys.stderr.write("namespace: "+namespace+'\n')
 399     sys.stderr.write("root: "+root+'\n')
 400     sys.stderr.write('Filtering "'+filename+'"...')
 401
 402     f = open(filename)
 403     tokenize.tokenize(f.readline, tok_eater)
 404     f.close()
 405     print_comment((0,0))
 406
 407     output("\n",(0,0))
 408     output("}  // end of namespace\n",(0,0))
 409
 410     if not module_has_docstring:
 411         # Put in default namespace documentation
 412         output('/** \\namespace '+root+' \n',(0,0))
 413         output('    \\brief Module "%s" */\n'%(root),(0,0))
 414
 415     for s in outbuffer:
 416         outfile.write(s)
 417     namespace=package
 418     module_has_docstring = False
 419
 420
 421 def filterFile(filename, out=sys.stdout):
 422     global outfile
 423
 424     outfile = out
 425
 426     try:
 427         root,ext  = os.path.splitext(filename)
 428
 429         if ext==".py":
 430             list(filter(filename))
 431         else:
 432             dump(filename)
 433
 434         sys.stderr.write("OK\n")
 435     except IOError as e:
 436         sys.stderr.write(e[1]+"\n")
 437
 438
 439 ######################################################################
 440
 441 # preparePath
 442 def preparePath(path):
 443     """Prepare a path.
 444
 445     Checks if the path exists and creates it if it does not exist.
 446     """
 447     if not os.path.exists(path):
 448         parent = os.path.dirname(path)
 449         if parent!="":
 450             preparePath(parent)
 451         os.mkdir(path)
 452
 453 # isNewer
 454 def isNewer(file1,file2):
 455     """Check if file1 is newer than file2.
 456
 457     file1 must be an existing file.
 458     """
 459     if not os.path.exists(file2):
 460         return True
 461     return os.stat(file1)[ST_MTIME]>os.stat(file2)[ST_MTIME]
 462
 463 # convert
 464 def convert(srcpath, destpath):
 465     """Convert a Python source tree into a C+ stub tree.
 466
 467     All *.py files in srcpath (including sub-directories) are filtered
 468     and written to destpath. If destpath exists, only the files
 469     that have been modified are filtered again. Files that were deleted
 470     from srcpath are also deleted in destpath if they are still present.
 471     The function returns the number of processed *.py files.
 472     """
 473     global namespace
 474     count=0
 475     l=os.listdir(srcpath)
 476     if "__init__.py" in l:
 477        if namespace:
 478          namespace=namespace+"::"+os.path.split(srcpath)[1]
 479        else:
 480          namespace=os.path.split(srcpath)[1]
 481        print("It's a package:",namespace)
 482     sp = os.path.join(srcpath,"*")
 483     sfiles = glob.glob(sp)
 484     dp = os.path.join(destpath,"*")
 485     dfiles = glob.glob(dp)
 486     leftovers={}
 487     for df in dfiles:
 488         leftovers[os.path.basename(df)]=1
 489
 490     for srcfile in sfiles:
 491         basename = os.path.basename(srcfile)
 492         if basename in leftovers:
 493             del leftovers[basename]
 494
 495         # Is it a subdirectory?
 496         if os.path.isdir(srcfile):
 497             package=namespace
 498             sdir = os.path.join(srcpath,basename)
 499             ddir = os.path.join(destpath,basename)
 500             count+=convert(sdir, ddir)
 501             namespace=package
 502             continue
 503         # Check the extension (only *.py will be converted)
 504         root, ext = os.path.splitext(srcfile)
 505         if ext.lower()!=".py":
 506             continue
 507
 508         destfile = os.path.join(destpath,basename)
 509         if destfile==srcfile:
 510             print("WARNING: Input and output names are identical!")
 511             sys.exit(1)
 512
 513         count+=1
 514 #        sys.stdout.write("%s\015"%(srcfile))
 515
 516         if isNewer(srcfile, destfile):
 517             preparePath(os.path.dirname(destfile))
 518             out=open(destfile,"w")
 519             filterFile(srcfile, out)
 520             out.close()
 521 #            os.system("python %s -f %s>%s"%(sys.argv[0],srcfile,destfile))
 522
 523     # Delete obsolete files in destpath
 524     for df in leftovers:
 525         dname=os.path.join(destpath,df)
 526         if os.path.isdir(dname):
 527             try:
 528                 shutil.rmtree(dname)
 529             except:
 530                 print("Can't remove obsolete directory '%s'"%dname)
 531         else:
 532             try:
 533                 os.remove(dname)
 534             except:
 535                 print("Can't remove obsolete file '%s'"%dname)
 536
 537     return count
 538
 539
 540 ######################################################################
 541 ######################################################################
 542 ######################################################################
 543
 544 filter_file = False
 545
 546 try:
 547     opts, args = getopt.getopt(sys.argv[1:], "hf", ["help"])
 548 except getopt.GetoptError as e:
 549     print(e)
 550     sys.exit(1)
 551
 552 for o,a in opts:
 553     if o=="-f":
 554         filter_file = True
 555
 556 if filter_file:
 557     # Filter the specified file and print the result to stdout
 558     filename = string.join(args)
 559     filterFile(filename)
 560 else:
 561
 562     if len(args)!=2:
 563         sys.stderr.write("%s options input output\n"%(os.path.basename(sys.argv[0])))
 564         sys.exit(1)
 565
 566     # Filter an entire Python source tree
 567     print('"%s" -> "%s"\n'%(args[0],args[1]))
 568     c=convert(args[0],args[1])
 569     print("%d files"%(c))