src/Container/SALOME_PyNode.py

   1 #  -*- coding: iso-8859-1 -*-
   2 # Copyright (C) 2007-2023  CEA, EDF, OPEN CASCADE
   3 #
   4 # This library is free software; you can redistribute it and/or
   5 # modify it under the terms of the GNU Lesser General Public
   6 # License as published by the Free Software Foundation; either
   7 # version 2.1 of the License, or (at your option) any later version.
   8 #
   9 # This library is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12 # Lesser General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Lesser General Public
  15 # License along with this library; if not, write to the Free Software
  16 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  17 #
  18 # See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
  19 #
  20
  21 #  File   : SALOME_PyNode.py
  22 #  Author : Christian CAREMOLI, EDF
  23 #  Module : SALOME
  24 #  $Header$
  25 #
  26 import sys,traceback
  27 import linecache
  28 import pickle
  29 import Engines__POA
  30 import SALOME__POA
  31 import SALOME
  32
  33 MY_CONTAINER_ENTRY_IN_GLBS = "my_container"
  34
  35 class Generic(SALOME__POA.GenericObj):
  36   """A Python implementation of the GenericObj CORBA IDL"""
  37   def __init__(self,poa):
  38     self.poa=poa
  39     self.cnt=1
  40
  41   def Register(self):
  42     #print("Register called : %d"%self.cnt)
  43     self.cnt+=1
  44
  45   def UnRegister(self):
  46     #print("UnRegister called : %d"%self.cnt)
  47     self.cnt-=1
  48     if self.cnt <= 0:
  49       oid=self.poa.servant_to_id(self)
  50       self.poa.deactivate_object(oid)
  51
  52   def Destroy(self):
  53     print("WARNING SALOME::GenericObj::Destroy() function is obsolete! Use UnRegister() instead.")
  54     self.UnRegister()
  55
  56   def __del__(self):
  57     #print("Destuctor called")
  58     pass
  59
  60 class PyNode_i (Engines__POA.PyNode,Generic):
  61   """The implementation of the PyNode CORBA IDL"""
  62   def __init__(self, nodeName,code,poa,my_container):
  63     """Initialize the node : compilation in the local context"""
  64     Generic.__init__(self,poa)
  65     self.nodeName=nodeName
  66     self.code=code
  67     self.my_container=my_container._container
  68     linecache.cache[nodeName]=0,None,code.split('\n'),nodeName
  69     ccode=compile(code,nodeName,'exec')
  70     self.context={}
  71     self.context[MY_CONTAINER_ENTRY_IN_GLBS] = self.my_container
  72     exec(ccode, self.context)
  73
  74   def getContainer(self):
  75     return self.my_container
  76
  77   def getCode(self):
  78     return self.code
  79
  80   def getName(self):
  81     return self.nodeName
  82
  83   def defineNewCustomVar(self,varName,valueOfVar):
  84     self.context[varName] = pickle.loads(valueOfVar)
  85     pass
  86
  87   def executeAnotherPieceOfCode(self,code):
  88     """Called for initialization of container lodging self."""
  89     try:
  90       ccode=compile(code,self.nodeName,'exec')
  91       exec(ccode, self.context)
  92     except Exception:
  93       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"","PyScriptNode (%s) : code to be executed \"%s\"" %(self.nodeName,code),0))
  94
  95   def execute(self,funcName,argsin):
  96     """Execute the function funcName found in local context with pickled args (argsin)"""
  97     try:
  98       argsin,kws=pickle.loads(argsin)
  99       func=self.context[funcName]
 100       argsout=func(*argsin,**kws)
 101       argsout=pickle.dumps(argsout,-1)
 102       return argsout
 103     except Exception:
 104       exc_typ,exc_val,exc_fr=sys.exc_info()
 105       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 106       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyNode: %s, function: %s" % (self.nodeName,funcName),0))
 107
 108 class SenderByte_i(SALOME__POA.SenderByte,Generic):
 109   def __init__(self,poa,bytesToSend):
 110     Generic.__init__(self,poa)
 111     self.bytesToSend = bytesToSend
 112
 113   def getSize(self):
 114     return len(self.bytesToSend)
 115
 116   def sendPart(self,n1,n2):
 117     return self.bytesToSend[n1:n2]
 118
 119 SALOME_FILE_BIG_OBJ_DIR = "SALOME_FILE_BIG_OBJ_DIR"
 120
 121 SALOME_BIG_OBJ_ON_DISK_THRES_VAR = "SALOME_BIG_OBJ_ON_DISK_THRES"
 122
 123 # default is 50 MB
 124 SALOME_BIG_OBJ_ON_DISK_THRES_DFT = 50000000
 125
 126 from ctypes import c_int
 127 TypeCounter = c_int
 128
 129 def GetSizeOfTCnt():
 130   return len( bytes(TypeCounter(0) ) )
 131
 132 def GetObjectFromFile(fname):
 133   with open(fname,"rb") as f:
 134     cntb = f.read( GetSizeOfTCnt() )
 135     cnt = TypeCounter.from_buffer_copy( cntb ).value
 136     obj = pickle.load(f)
 137   return obj,cnt
 138
 139 def DumpInFile(obj,fname):
 140   with open(fname,"wb") as f:
 141     f.write( bytes( TypeCounter(1) ) )
 142     f.write( obj )
 143
 144 def IncrRefInFile(fname):
 145   with open(fname,"rb") as f:
 146     cntb = f.read( GetSizeOfTCnt() )
 147   cnt = TypeCounter.from_buffer_copy( cntb ).value
 148   with open(fname,"rb+") as f:
 149     f.write( bytes( TypeCounter(cnt+1) ) )
 150
 151 def DecrRefInFile(fname):
 152   import os
 153   with open(fname,"rb") as f:
 154     cntb = f.read( GetSizeOfTCnt() )
 155   cnt = TypeCounter.from_buffer_copy( cntb ).value
 156   #
 157   if cnt == 1:
 158     pass
 159     #print("Remove {}".format(fname))
 160     #print("Remove {}".format(str(GetObjectFromFile(fname)[0])))
 161     #os.unlink( fname )
 162   else:
 163     with open(fname,"rb+") as f:
 164         f.write( bytes( TypeCounter(cnt-1) ) )
 165
 166 def GetBigObjectOnDiskThreshold():
 167   import os
 168   if SALOME_BIG_OBJ_ON_DISK_THRES_VAR in os.environ:
 169     return int( os.environ[SALOME_BIG_OBJ_ON_DISK_THRES_VAR] )
 170   else:
 171     return SALOME_BIG_OBJ_ON_DISK_THRES_DFT
 172
 173 def ActivateProxyMecanismOrNot( sizeInByte ):
 174   thres = GetBigObjectOnDiskThreshold()
 175   if thres == -1:
 176     return False
 177   else:
 178     return sizeInByte > thres
 179
 180 def GetBigObjectDirectory():
 181   import os
 182   if SALOME_FILE_BIG_OBJ_DIR not in os.environ:
 183     raise RuntimeError("An object of size higher than limit detected and no directory specified to dump it in file !")
 184   return os.path.expanduser( os.path.expandvars( os.environ[SALOME_FILE_BIG_OBJ_DIR] ) )
 185
 186 def GetBigObjectFileName():
 187   """
 188   Return a filename in the most secure manner (see tempfile documentation)
 189   """
 190   import tempfile
 191   with tempfile.NamedTemporaryFile(dir=GetBigObjectDirectory(),prefix="mem_",suffix=".pckl") as f:
 192     ret = f.name
 193   return ret
 194
 195 class BigObjectOnDiskBase:
 196   def __init__(self, fileName, objSerialized):
 197     """
 198     :param fileName: the file used to dump into.
 199     :param objSerialized: the object in pickeled form
 200     :type objSerialized: bytes
 201     """
 202     self._filename = fileName
 203     # attribute _destroy is here to tell client side or server side
 204     # only client side can be with _destroy set to True. server side due to risk of concurrency
 205     # so pickled form of self must be done with this attribute set to False.
 206     self._destroy = False
 207     self.__dumpIntoFile(objSerialized)
 208
 209   def getDestroyStatus(self):
 210     return self._destroy
 211
 212   def incrRef(self):
 213     if self._destroy:
 214       IncrRefInFile( self._filename )
 215     else:
 216       # should never happen !
 217       RuntimeError("Invalid call to incrRef !")
 218
 219   def decrRef(self):
 220     if self._destroy:
 221       DecrRefInFile( self._filename )
 222     else:
 223       # should never happen !
 224       RuntimeError("Invalid call to decrRef !")
 225
 226   def unlinkOnDestructor(self):
 227     self._destroy = True
 228
 229   def doNotTouchFile(self):
 230     """
 231     Method called slave side. The life cycle management of file is client side not slave side.
 232     """
 233     self._destroy = False
 234
 235   def __del__(self):
 236     if self._destroy:
 237       DecrRefInFile( self._filename )
 238
 239   def delDebug(self):
 240     import os
 241     if self._destroy:
 242       if os.path.exists( self._filename ):
 243         DecrRefInFile( self._filename )
 244       else:
 245         import KernelServices
 246         KernelServices.GenerateViolentMemoryFaultForTestPurpose()
 247
 248   def getFileName(self):
 249     return self._filename
 250
 251   def __dumpIntoFile(self, objSerialized):
 252     DumpInFile( objSerialized, self._filename )
 253
 254   def get(self):
 255     obj, _ = GetObjectFromFile( self._filename )
 256     return obj
 257
 258   def __float__(self):
 259     return float( self.get() )
 260
 261   def __int__(self):
 262     return int( self.get() )
 263
 264   def __str__(self):
 265     obj = self.get()
 266     if isinstance(obj,str):
 267         return obj
 268     else:
 269         raise RuntimeError("Not a string")
 270
 271 class BigObjectOnDisk(BigObjectOnDiskBase):
 272   def __init__(self, fileName, objSerialized):
 273     BigObjectOnDiskBase.__init__(self, fileName, objSerialized)
 274
 275 class BigObjectOnDiskListElement(BigObjectOnDiskBase):
 276   def __init__(self, pos, length, fileName):
 277     self._filename = fileName
 278     self._destroy = False
 279     self._pos = pos
 280     self._length = length
 281
 282   def get(self):
 283     fullObj = BigObjectOnDiskBase.get(self)
 284     return fullObj[ self._pos ]
 285
 286   def __getitem__(self, i):
 287     return self.get()[i]
 288
 289   def __len__(self):
 290     return len(self.get())
 291
 292 class BigObjectOnDiskSequence(BigObjectOnDiskBase):
 293   def __init__(self, length, fileName, objSerialized):
 294     BigObjectOnDiskBase.__init__(self, fileName, objSerialized)
 295     self._length = length
 296
 297   def __getitem__(self, i):
 298     return BigObjectOnDiskListElement(i, self._length, self.getFileName())
 299
 300   def __len__(self):
 301     return self._length
 302
 303 class BigObjectOnDiskList(BigObjectOnDiskSequence):
 304   def __init__(self, length, fileName, objSerialized):
 305     BigObjectOnDiskSequence.__init__(self, length, fileName, objSerialized)
 306
 307 class BigObjectOnDiskTuple(BigObjectOnDiskSequence):
 308   def __init__(self, length, fileName, objSerialized):
 309     BigObjectOnDiskSequence.__init__(self, length, fileName, objSerialized)
 310
 311 def SpoolPickleObject( obj ):
 312   import pickle
 313   pickleObjInit = pickle.dumps( obj , pickle.HIGHEST_PROTOCOL )
 314   if not ActivateProxyMecanismOrNot( len(pickleObjInit) ):
 315     return pickleObjInit
 316   else:
 317     if isinstance( obj, list):
 318       proxyObj = BigObjectOnDiskList( len(obj), GetBigObjectFileName() , pickleObjInit )
 319     elif isinstance( obj, tuple):
 320       proxyObj = BigObjectOnDiskTuple( len(obj), GetBigObjectFileName() , pickleObjInit )
 321     else:
 322       proxyObj = BigObjectOnDisk( GetBigObjectFileName() , pickleObjInit )
 323     pickleProxy = pickle.dumps( proxyObj , pickle.HIGHEST_PROTOCOL )
 324     return pickleProxy
 325
 326 def UnProxyObjectSimple( obj ):
 327   """
 328   Method to be called in Remote mode. Alterate the obj _status attribute.
 329   Because the slave process does not participate in the reference counting
 330   """
 331   if isinstance(obj,BigObjectOnDiskBase):
 332     obj.doNotTouchFile()
 333     return obj.get()
 334   elif isinstance( obj, list):
 335     retObj = []
 336     for elt in obj:
 337       retObj.append( UnProxyObjectSimple(elt) )
 338     return retObj
 339   else:
 340     return obj
 341
 342 def UnProxyObjectSimpleLocal( obj ):
 343   """
 344   Method to be called in Local mode. Do not alterate the PyObj counter
 345   """
 346   if isinstance(obj,BigObjectOnDiskBase):
 347     return obj.get()
 348   elif isinstance( obj, list):
 349     retObj = []
 350     for elt in obj:
 351       retObj.append( UnProxyObjectSimpleLocal(elt) )
 352     return retObj
 353   else:
 354     return obj
 355
 356 class SeqByteReceiver:
 357   # 2GB limit to trigger split into chunks
 358   CHUNK_SIZE = 2000000000
 359   def __init__(self,sender):
 360     self._obj = sender
 361   def __del__(self):
 362     self._obj.UnRegister()
 363     pass
 364   def data(self):
 365     size = self._obj.getSize()
 366     if size <= SeqByteReceiver.CHUNK_SIZE:
 367       return self.fetchOneShot( size )
 368     else:
 369       return self.fetchByChunks( size )
 370   def fetchOneShot(self,size):
 371     return self._obj.sendPart(0,size)
 372   def fetchByChunks(self,size):
 373       """
 374       To avoid memory peak parts over 2GB are sent using EFF_CHUNK_SIZE size.
 375       """
 376       data_for_split_case = bytes(0)
 377       EFF_CHUNK_SIZE = SeqByteReceiver.CHUNK_SIZE // 8
 378       iStart = 0 ; iEnd = EFF_CHUNK_SIZE
 379       while iStart!=iEnd and iEnd <= size:
 380         part = self._obj.sendPart(iStart,iEnd)
 381         data_for_split_case = bytes(0).join( [data_for_split_case,part] )
 382         iStart = iEnd; iEnd = min(iStart + EFF_CHUNK_SIZE,size)
 383       return data_for_split_case
 384
 385 class PyScriptNode_i (Engines__POA.PyScriptNode,Generic):
 386   """The implementation of the PyScriptNode CORBA IDL that executes a script"""
 387   def __init__(self, nodeName,code,poa,my_container):
 388     """Initialize the node : compilation in the local context"""
 389     Generic.__init__(self,poa)
 390     self.nodeName=nodeName
 391     self.code=code
 392     self.my_container=my_container._container
 393     linecache.cache[nodeName]=0,None,code.split('\n'),nodeName
 394     self.ccode=compile(code,nodeName,'exec')
 395     self.context={}
 396     self.context[MY_CONTAINER_ENTRY_IN_GLBS] = self.my_container
 397
 398   def __del__(self):
 399     # force removal of self.context. Don t know why it s not done by default
 400     self.removeAllVarsInContext()
 401     pass
 402
 403   def getContainer(self):
 404     return self.my_container
 405
 406   def getCode(self):
 407     return self.code
 408
 409   def getName(self):
 410     return self.nodeName
 411
 412   def defineNewCustomVar(self,varName,valueOfVar):
 413     self.context[varName] = pickle.loads(valueOfVar)
 414     pass
 415
 416   def executeAnotherPieceOfCode(self,code):
 417     """Called for initialization of container lodging self."""
 418     try:
 419       ccode=compile(code,self.nodeName,'exec')
 420       exec(ccode, self.context)
 421     except Exception:
 422       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"","PyScriptNode (%s) : code to be executed \"%s\"" %(self.nodeName,code),0))
 423
 424   def assignNewCompiledCode(self,codeStr):
 425     try:
 426       self.code=codeStr
 427       self.ccode=compile(codeStr,self.nodeName,'exec')
 428     except Exception:
 429       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"","PyScriptNode.assignNewCompiledCode (%s) : code to be executed \"%s\"" %(self.nodeName,codeStr),0))
 430
 431   def executeSimple(self, key, val):
 432     """
 433     Same as execute method except that no pickelization mecanism is implied here. No output is expected
 434     """
 435     try:
 436       self.context.update({ "env" : [(k,v) for k,v in zip(key,val)]})
 437       exec(self.ccode,self.context)
 438     except Exception:
 439       exc_typ,exc_val,exc_fr=sys.exc_info()
 440       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 441       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" % (self.nodeName),0))
 442
 443   def execute(self,outargsname,argsin):
 444     """Execute the script stored in attribute ccode with pickled args (argsin)"""
 445     try:
 446       argsname,kws=pickle.loads(argsin)
 447       self.context.update(kws)
 448       exec(self.ccode, self.context)
 449       argsout=[]
 450       for arg in outargsname:
 451         if arg not in self.context:
 452           raise KeyError("There is no variable %s in context" % arg)
 453         argsout.append(self.context[arg])
 454       argsout=pickle.dumps(tuple(argsout),-1)
 455       return argsout
 456     except Exception:
 457       exc_typ,exc_val,exc_fr=sys.exc_info()
 458       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 459       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s, outargsname: %s" % (self.nodeName,outargsname),0))
 460
 461   def executeFirst(self,argsin):
 462     """ Same than first part of self.execute to reduce memory peak."""
 463     import time
 464     try:
 465       data = None
 466       if True: # to force call of SeqByteReceiver's destructor
 467         argsInPy = SeqByteReceiver( argsin )
 468         data = argsInPy.data()
 469       _,kws=pickle.loads(data)
 470       for elt in kws:
 471         # fetch real data if necessary
 472         kws[elt] = UnProxyObjectSimple( kws[elt] )
 473       self.context.update(kws)
 474     except Exception:
 475       exc_typ,exc_val,exc_fr=sys.exc_info()
 476       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 477       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode:First %s" % (self.nodeName),0))
 478
 479   def executeSecond(self,outargsname):
 480     """ Same than second part of self.execute to reduce memory peak."""
 481     try:
 482       exec(self.ccode, self.context)
 483       argsout=[]
 484       for arg in outargsname:
 485         if arg not in self.context:
 486           raise KeyError("There is no variable %s in context" % arg)
 487         argsout.append(self.context[arg])
 488       ret = [ ]
 489       for arg in argsout:
 490         # the proxy mecanism is catched here
 491         argPickle = SpoolPickleObject( arg )
 492         retArg = SenderByte_i( self.poa,argPickle )
 493         id_o = self.poa.activate_object(retArg)
 494         retObj = self.poa.id_to_reference(id_o)
 495         ret.append( retObj._narrow( SALOME.SenderByte ) )
 496       return ret
 497     except Exception:
 498       exc_typ,exc_val,exc_fr=sys.exc_info()
 499       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 500       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode:Second %s, outargsname: %s" % (self.nodeName,outargsname),0))
 501
 502   def listAllVarsInContext(self):
 503       import re
 504       pat = re.compile("^__([a-z]+)__$")
 505       return [elt for elt in self.context if not pat.match(elt) and elt != MY_CONTAINER_ENTRY_IN_GLBS]
 506
 507   def removeAllVarsInContext(self):
 508       for elt in self.listAllVarsInContext():
 509         del self.context[elt]
 510
 511   def getValueOfVarInContext(self,varName):
 512     try:
 513       return pickle.dumps(self.context[varName],-1)
 514     except Exception:
 515       exc_typ,exc_val,exc_fr=sys.exc_info()
 516       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 517       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" %self.nodeName,0))
 518     pass
 519
 520   def assignVarInContext(self, varName, value):
 521     try:
 522       self.context[varName][0] = pickle.loads(value)
 523     except Exception:
 524       exc_typ,exc_val,exc_fr=sys.exc_info()
 525       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 526       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" %self.nodeName,0))
 527     pass
 528
 529   def callMethodOnVarInContext(self, varName, methodName, args):
 530     try:
 531       return pickle.dumps( getattr(self.context[varName][0],methodName)(*pickle.loads(args)),-1 )
 532     except Exception:
 533       exc_typ,exc_val,exc_fr=sys.exc_info()
 534       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 535       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" %self.nodeName,0))
 536     pass