src/Container/SALOME_PyNode.py

   1 #  -*- coding: iso-8859-1 -*-
   2 # Copyright (C) 2007-2023  CEA, EDF, OPEN CASCADE
   3 #
   4 # This library is free software; you can redistribute it and/or
   5 # modify it under the terms of the GNU Lesser General Public
   6 # License as published by the Free Software Foundation; either
   7 # version 2.1 of the License, or (at your option) any later version.
   8 #
   9 # This library is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12 # Lesser General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Lesser General Public
  15 # License along with this library; if not, write to the Free Software
  16 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  17 #
  18 # See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
  19 #
  20
  21 #  File   : SALOME_PyNode.py
  22 #  Author : Christian CAREMOLI, EDF
  23 #  Module : SALOME
  24 #  $Header$
  25 #
  26 import sys,traceback
  27 import linecache
  28 import pickle
  29 import Engines__POA
  30 import SALOME__POA
  31 import SALOME
  32
  33 class Generic(SALOME__POA.GenericObj):
  34   """A Python implementation of the GenericObj CORBA IDL"""
  35   def __init__(self,poa):
  36     self.poa=poa
  37     self.cnt=1
  38
  39   def Register(self):
  40     #print("Register called : %d"%self.cnt)
  41     self.cnt+=1
  42
  43   def UnRegister(self):
  44     #print("UnRegister called : %d"%self.cnt)
  45     self.cnt-=1
  46     if self.cnt <= 0:
  47       oid=self.poa.servant_to_id(self)
  48       self.poa.deactivate_object(oid)
  49
  50   def Destroy(self):
  51     print("WARNING SALOME::GenericObj::Destroy() function is obsolete! Use UnRegister() instead.")
  52     self.UnRegister()
  53
  54   def __del__(self):
  55     #print("Destuctor called")
  56     pass
  57
  58 class PyNode_i (Engines__POA.PyNode,Generic):
  59   """The implementation of the PyNode CORBA IDL"""
  60   def __init__(self, nodeName,code,poa,my_container):
  61     """Initialize the node : compilation in the local context"""
  62     Generic.__init__(self,poa)
  63     self.nodeName=nodeName
  64     self.code=code
  65     self.my_container=my_container._container
  66     linecache.cache[nodeName]=0,None,code.split('\n'),nodeName
  67     ccode=compile(code,nodeName,'exec')
  68     self.context={}
  69     self.context["my_container"] = self.my_container
  70     exec(ccode, self.context)
  71
  72   def getContainer(self):
  73     return self.my_container
  74
  75   def getCode(self):
  76     return self.code
  77
  78   def getName(self):
  79     return self.nodeName
  80
  81   def defineNewCustomVar(self,varName,valueOfVar):
  82     self.context[varName] = pickle.loads(valueOfVar)
  83     pass
  84
  85   def executeAnotherPieceOfCode(self,code):
  86     """Called for initialization of container lodging self."""
  87     try:
  88       ccode=compile(code,self.nodeName,'exec')
  89       exec(ccode, self.context)
  90     except Exception:
  91       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"","PyScriptNode (%s) : code to be executed \"%s\"" %(self.nodeName,code),0))
  92
  93   def execute(self,funcName,argsin):
  94     """Execute the function funcName found in local context with pickled args (argsin)"""
  95     try:
  96       argsin,kws=pickle.loads(argsin)
  97       func=self.context[funcName]
  98       argsout=func(*argsin,**kws)
  99       argsout=pickle.dumps(argsout,-1)
 100       return argsout
 101     except Exception:
 102       exc_typ,exc_val,exc_fr=sys.exc_info()
 103       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 104       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyNode: %s, function: %s" % (self.nodeName,funcName),0))
 105
 106 class SenderByte_i(SALOME__POA.SenderByte,Generic):
 107   def __init__(self,poa,bytesToSend):
 108     Generic.__init__(self,poa)
 109     self.bytesToSend = bytesToSend
 110
 111   def getSize(self):
 112     return len(self.bytesToSend)
 113
 114   def sendPart(self,n1,n2):
 115     return self.bytesToSend[n1:n2]
 116
 117 SALOME_FILE_BIG_OBJ_DIR = "SALOME_FILE_BIG_OBJ_DIR"
 118
 119 SALOME_BIG_OBJ_ON_DISK_THRES_VAR = "SALOME_BIG_OBJ_ON_DISK_THRES"
 120
 121 # default is 50 MB
 122 SALOME_BIG_OBJ_ON_DISK_THRES_DFT = 50000000
 123
 124 from ctypes import c_int
 125 TypeCounter = c_int
 126
 127 def GetSizeOfTCnt():
 128   return len( bytes(TypeCounter(0) ) )
 129
 130 def GetObjectFromFile(fname):
 131   with open(fname,"rb") as f:
 132     cntb = f.read( GetSizeOfTCnt() )
 133     cnt = TypeCounter.from_buffer_copy( cntb ).value
 134     obj = pickle.load(f)
 135   return obj,cnt
 136
 137 def DumpInFile(obj,fname):
 138   with open(fname,"wb") as f:
 139     f.write( bytes( TypeCounter(1) ) )
 140     f.write( obj )
 141
 142 def IncrRefInFile(fname):
 143   with open(fname,"rb") as f:
 144     cntb = f.read( GetSizeOfTCnt() )
 145   cnt = TypeCounter.from_buffer_copy( cntb ).value
 146   with open(fname,"rb+") as f:
 147     f.write( bytes( TypeCounter(cnt+1) ) )
 148
 149 def DecrRefInFile(fname):
 150   import os
 151   with open(fname,"rb") as f:
 152     cntb = f.read( GetSizeOfTCnt() )
 153   cnt = TypeCounter.from_buffer_copy( cntb ).value
 154   #
 155   if cnt == 1:
 156     os.unlink( fname )
 157   else:
 158     with open(fname,"rb+") as f:
 159         f.write( bytes( TypeCounter(cnt-1) ) )
 160
 161 def GetBigObjectOnDiskThreshold():
 162   import os
 163   if SALOME_BIG_OBJ_ON_DISK_THRES_VAR in os.environ:
 164     return int( os.environ[SALOME_BIG_OBJ_ON_DISK_THRES_VAR] )
 165   else:
 166     return SALOME_BIG_OBJ_ON_DISK_THRES_DFT
 167
 168 def GetBigObjectDirectory():
 169   import os
 170   if SALOME_FILE_BIG_OBJ_DIR not in os.environ:
 171     raise RuntimeError("An object of size higher than limit detected and no directory specified to dump it in file !")
 172   return os.path.expanduser( os.path.expandvars( os.environ[SALOME_FILE_BIG_OBJ_DIR] ) )
 173
 174 def GetBigObjectFileName():
 175   """
 176   Return a filename in the most secure manner (see tempfile documentation)
 177   """
 178   import tempfile
 179   with tempfile.NamedTemporaryFile(dir=GetBigObjectDirectory(),prefix="mem_",suffix=".pckl") as f:
 180     ret = f.name
 181   return ret
 182
 183 class BigObjectOnDiskBase:
 184   def __init__(self, fileName, objSerialized):
 185     """
 186     :param fileName: the file used to dump into.
 187     :param objSerialized: the object in pickeled form
 188     :type objSerialized: bytes
 189     """
 190     self._filename = fileName
 191     # attribute _destroy is here to tell client side or server side
 192     # only client side can be with _destroy set to True. server side due to risk of concurrency
 193     # so pickled form of self must be done with this attribute set to False.
 194     self._destroy = False
 195     self.__dumpIntoFile(objSerialized)
 196
 197   def getDestroyStatus(self):
 198     return self._destroy
 199
 200   def incrRef(self):
 201     if self._destroy:
 202       IncrRefInFile( self._filename )
 203     else:
 204       # should never happen !
 205       RuntimeError("Invalid call to incrRef !")
 206
 207   def decrRef(self):
 208     if self._destroy:
 209       DecrRefInFile( self._filename )
 210     else:
 211       # should never happen !
 212       RuntimeError("Invalid call to decrRef !")
 213
 214   def unlinkOnDestructor(self):
 215     self._destroy = True
 216
 217   def doNotTouchFile(self):
 218     """
 219     Method called slave side. The life cycle management of file is client side not slave side.
 220     """
 221     self._destroy = False
 222
 223   def __del__(self):
 224     if self._destroy:
 225       DecrRefInFile( self._filename )
 226
 227   def getFileName(self):
 228     return self._filename
 229
 230   def __dumpIntoFile(self, objSerialized):
 231     DumpInFile( objSerialized, self._filename )
 232
 233   def get(self):
 234     obj, _ = GetObjectFromFile( self._filename )
 235     return obj
 236
 237   def __float__(self):
 238     return float( self.get() )
 239
 240   def __int__(self):
 241     return int( self.get() )
 242
 243   def __str__(self):
 244     obj = self.get()
 245     if isinstance(obj,str):
 246         return obj
 247     else:
 248         raise RuntimeError("Not a string")
 249
 250 class BigObjectOnDisk(BigObjectOnDiskBase):
 251   def __init__(self, fileName, objSerialized):
 252     BigObjectOnDiskBase.__init__(self, fileName, objSerialized)
 253
 254 class BigObjectOnDiskListElement(BigObjectOnDiskBase):
 255   def __init__(self, pos, length, fileName):
 256     self._filename = fileName
 257     self._destroy = False
 258     self._pos = pos
 259     self._length = length
 260
 261   def get(self):
 262     fullObj = BigObjectOnDiskBase.get(self)
 263     return fullObj[ self._pos ]
 264
 265   def __getitem__(self, i):
 266     return self.get()[i]
 267
 268   def __len__(self):
 269     return len(self.get())
 270
 271 class BigObjectOnDiskSequence(BigObjectOnDiskBase):
 272   def __init__(self, length, fileName, objSerialized):
 273     BigObjectOnDiskBase.__init__(self, fileName, objSerialized)
 274     self._length = length
 275
 276   def __getitem__(self, i):
 277     return BigObjectOnDiskListElement(i, self._length, self.getFileName())
 278
 279   def __len__(self):
 280     return self._length
 281
 282 class BigObjectOnDiskList(BigObjectOnDiskSequence):
 283   def __init__(self, length, fileName, objSerialized):
 284     BigObjectOnDiskSequence.__init__(self, length, fileName, objSerialized)
 285
 286 class BigObjectOnDiskTuple(BigObjectOnDiskSequence):
 287   def __init__(self, length, fileName, objSerialized):
 288     BigObjectOnDiskSequence.__init__(self, length, fileName, objSerialized)
 289
 290 def SpoolPickleObject( obj ):
 291   import pickle
 292   pickleObjInit = pickle.dumps( obj , pickle.HIGHEST_PROTOCOL )
 293   if len(pickleObjInit) < GetBigObjectOnDiskThreshold():
 294     return pickleObjInit
 295   else:
 296     if isinstance( obj, list):
 297       proxyObj = BigObjectOnDiskList( len(obj), GetBigObjectFileName() , pickleObjInit )
 298     elif isinstance( obj, tuple):
 299       proxyObj = BigObjectOnDiskTuple( len(obj), GetBigObjectFileName() , pickleObjInit )
 300     else:
 301       proxyObj = BigObjectOnDisk( GetBigObjectFileName() , pickleObjInit )
 302     pickleProxy = pickle.dumps( proxyObj , pickle.HIGHEST_PROTOCOL )
 303     return pickleProxy
 304
 305 def UnProxyObjectSimple( obj ):
 306   """
 307   Method to be called in Remote mode. Alterate the obj _status attribute.
 308   Because the slave process does not participate in the reference counting
 309   """
 310   if isinstance(obj,BigObjectOnDiskBase):
 311     obj.doNotTouchFile()
 312     return obj.get()
 313   elif isinstance( obj, list):
 314     retObj = []
 315     for elt in obj:
 316       retObj.append( UnProxyObjectSimple(elt) )
 317     return retObj
 318   else:
 319     return obj
 320
 321 def UnProxyObjectSimpleLocal( obj ):
 322   """
 323   Method to be called in Local mode. Do not alterate the PyObj counter
 324   """
 325   if isinstance(obj,BigObjectOnDiskBase):
 326     return obj.get()
 327   elif isinstance( obj, list):
 328     retObj = []
 329     for elt in obj:
 330       retObj.append( UnProxyObjectSimpleLocal(elt) )
 331     return retObj
 332   else:
 333     return obj
 334
 335 class SeqByteReceiver:
 336   # 2GB limit to trigger split into chunks
 337   CHUNK_SIZE = 2000000000
 338   def __init__(self,sender):
 339     self._obj = sender
 340   def __del__(self):
 341     self._obj.UnRegister()
 342     pass
 343   def data(self):
 344     size = self._obj.getSize()
 345     if size <= SeqByteReceiver.CHUNK_SIZE:
 346       return self.fetchOneShot( size )
 347     else:
 348       return self.fetchByChunks( size )
 349   def fetchOneShot(self,size):
 350     return self._obj.sendPart(0,size)
 351   def fetchByChunks(self,size):
 352       """
 353       To avoid memory peak parts over 2GB are sent using EFF_CHUNK_SIZE size.
 354       """
 355       data_for_split_case = bytes(0)
 356       EFF_CHUNK_SIZE = SeqByteReceiver.CHUNK_SIZE // 8
 357       iStart = 0 ; iEnd = EFF_CHUNK_SIZE
 358       while iStart!=iEnd and iEnd <= size:
 359         part = self._obj.sendPart(iStart,iEnd)
 360         data_for_split_case = bytes(0).join( [data_for_split_case,part] )
 361         iStart = iEnd; iEnd = min(iStart + EFF_CHUNK_SIZE,size)
 362       return data_for_split_case
 363
 364 class PyScriptNode_i (Engines__POA.PyScriptNode,Generic):
 365   """The implementation of the PyScriptNode CORBA IDL that executes a script"""
 366   def __init__(self, nodeName,code,poa,my_container):
 367     """Initialize the node : compilation in the local context"""
 368     Generic.__init__(self,poa)
 369     self.nodeName=nodeName
 370     self.code=code
 371     self.my_container=my_container._container
 372     linecache.cache[nodeName]=0,None,code.split('\n'),nodeName
 373     self.ccode=compile(code,nodeName,'exec')
 374     self.context={}
 375     self.context["my_container"] = self.my_container
 376
 377   def getContainer(self):
 378     return self.my_container
 379
 380   def getCode(self):
 381     return self.code
 382
 383   def getName(self):
 384     return self.nodeName
 385
 386   def defineNewCustomVar(self,varName,valueOfVar):
 387     self.context[varName] = pickle.loads(valueOfVar)
 388     pass
 389
 390   def executeAnotherPieceOfCode(self,code):
 391     """Called for initialization of container lodging self."""
 392     try:
 393       ccode=compile(code,self.nodeName,'exec')
 394       exec(ccode, self.context)
 395     except Exception:
 396       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"","PyScriptNode (%s) : code to be executed \"%s\"" %(self.nodeName,code),0))
 397
 398   def assignNewCompiledCode(self,codeStr):
 399     try:
 400       self.code=codeStr
 401       self.ccode=compile(codeStr,self.nodeName,'exec')
 402     except Exception:
 403       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"","PyScriptNode.assignNewCompiledCode (%s) : code to be executed \"%s\"" %(self.nodeName,codeStr),0))
 404
 405   def execute(self,outargsname,argsin):
 406     """Execute the script stored in attribute ccode with pickled args (argsin)"""
 407     try:
 408       argsname,kws=pickle.loads(argsin)
 409       self.context.update(kws)
 410       exec(self.ccode, self.context)
 411       argsout=[]
 412       for arg in outargsname:
 413         if arg not in self.context:
 414           raise KeyError("There is no variable %s in context" % arg)
 415         argsout.append(self.context[arg])
 416       argsout=pickle.dumps(tuple(argsout),-1)
 417       return argsout
 418     except Exception:
 419       exc_typ,exc_val,exc_fr=sys.exc_info()
 420       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 421       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s, outargsname: %s" % (self.nodeName,outargsname),0))
 422
 423   def executeFirst(self,argsin):
 424     """ Same than first part of self.execute to reduce memory peak."""
 425     import time
 426     try:
 427       data = None
 428       if True: # to force call of SeqByteReceiver's destructor
 429         argsInPy = SeqByteReceiver( argsin )
 430         data = argsInPy.data()
 431       _,kws=pickle.loads(data)
 432       for elt in kws:
 433         # fetch real data if necessary
 434         kws[elt] = UnProxyObjectSimple( kws[elt] )
 435       self.context.update(kws)
 436     except Exception:
 437       exc_typ,exc_val,exc_fr=sys.exc_info()
 438       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 439       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode:First %s" % (self.nodeName),0))
 440
 441   def executeSecond(self,outargsname):
 442     """ Same than second part of self.execute to reduce memory peak."""
 443     try:
 444       exec(self.ccode, self.context)
 445       argsout=[]
 446       for arg in outargsname:
 447         if arg not in self.context:
 448           raise KeyError("There is no variable %s in context" % arg)
 449         argsout.append(self.context[arg])
 450       ret = [ ]
 451       for arg in argsout:
 452         # the proxy mecanism is catched here
 453         argPickle = SpoolPickleObject( arg )
 454         retArg = SenderByte_i( self.poa,argPickle )
 455         id_o = self.poa.activate_object(retArg)
 456         retObj = self.poa.id_to_reference(id_o)
 457         ret.append( retObj._narrow( SALOME.SenderByte ) )
 458       return ret
 459     except Exception:
 460       exc_typ,exc_val,exc_fr=sys.exc_info()
 461       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 462       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode:Second %s, outargsname: %s" % (self.nodeName,outargsname),0))
 463
 464   def listAllVarsInContext(self):
 465       import re
 466       pat = re.compile("^__([a-z]+)__$")
 467       return [elt for elt in self.context if not pat.match(elt)]
 468
 469   def removeAllVarsInContext(self):
 470       for elt in self.listAllVarsInContext():
 471         del self.context[elt]
 472
 473   def getValueOfVarInContext(self,varName):
 474     try:
 475       return pickle.dumps(self.context[varName],-1)
 476     except Exception:
 477       exc_typ,exc_val,exc_fr=sys.exc_info()
 478       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 479       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" %self.nodeName,0))
 480     pass
 481
 482   def assignVarInContext(self, varName, value):
 483     try:
 484       self.context[varName][0] = pickle.loads(value)
 485     except Exception:
 486       exc_typ,exc_val,exc_fr=sys.exc_info()
 487       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 488       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" %self.nodeName,0))
 489     pass
 490
 491   def callMethodOnVarInContext(self, varName, methodName, args):
 492     try:
 493       return pickle.dumps( getattr(self.context[varName][0],methodName)(*pickle.loads(args)),-1 )
 494     except Exception:
 495       exc_typ,exc_val,exc_fr=sys.exc_info()
 496       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
 497       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" %self.nodeName,0))
 498     pass