Salome HOME
[EDF29093] : Quick fix resorbed
[modules/kernel.git] / src / Container / SALOME_PyNode.py
1 #  -*- coding: iso-8859-1 -*-
2 # Copyright (C) 2007-2023  CEA, EDF, OPEN CASCADE
3 #
4 # This library is free software; you can redistribute it and/or
5 # modify it under the terms of the GNU Lesser General Public
6 # License as published by the Free Software Foundation; either
7 # version 2.1 of the License, or (at your option) any later version.
8 #
9 # This library is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12 # Lesser General Public License for more details.
13 #
14 # You should have received a copy of the GNU Lesser General Public
15 # License along with this library; if not, write to the Free Software
16 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
17 #
18 # See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
19 #
20
21 #  File   : SALOME_PyNode.py
22 #  Author : Christian CAREMOLI, EDF
23 #  Module : SALOME
24 #  $Header$
25 #
26 import sys,traceback
27 import linecache
28 import pickle
29 import Engines__POA
30 import SALOME__POA
31 import SALOME
32
33 MY_CONTAINER_ENTRY_IN_GLBS = "my_container"
34
35 class Generic(SALOME__POA.GenericObj):
36   """A Python implementation of the GenericObj CORBA IDL"""
37   def __init__(self,poa):
38     self.poa=poa
39     self.cnt=1
40
41   def Register(self):
42     #print("Register called : %d"%self.cnt)
43     self.cnt+=1
44
45   def UnRegister(self):
46     #print("UnRegister called : %d"%self.cnt)
47     self.cnt-=1
48     if self.cnt <= 0:
49       oid=self.poa.servant_to_id(self)
50       self.poa.deactivate_object(oid)
51
52   def Destroy(self):
53     print("WARNING SALOME::GenericObj::Destroy() function is obsolete! Use UnRegister() instead.")
54     self.UnRegister()
55
56   def __del__(self):
57     #print("Destuctor called")
58     pass
59
60 class PyNode_i (Engines__POA.PyNode,Generic):
61   """The implementation of the PyNode CORBA IDL"""
62   def __init__(self, nodeName,code,poa,my_container):
63     """Initialize the node : compilation in the local context"""
64     Generic.__init__(self,poa)
65     self.nodeName=nodeName
66     self.code=code
67     self.my_container=my_container._container
68     linecache.cache[nodeName]=0,None,code.split('\n'),nodeName
69     ccode=compile(code,nodeName,'exec')
70     self.context={}
71     self.context[MY_CONTAINER_ENTRY_IN_GLBS] = self.my_container
72     exec(ccode, self.context)
73
74   def getContainer(self):
75     return self.my_container
76
77   def getCode(self):
78     return self.code
79
80   def getName(self):
81     return self.nodeName
82
83   def defineNewCustomVar(self,varName,valueOfVar):
84     self.context[varName] = pickle.loads(valueOfVar)
85     pass
86
87   def executeAnotherPieceOfCode(self,code):
88     """Called for initialization of container lodging self."""
89     try:
90       ccode=compile(code,self.nodeName,'exec')
91       exec(ccode, self.context)
92     except Exception:
93       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"","PyScriptNode (%s) : code to be executed \"%s\"" %(self.nodeName,code),0))
94
95   def execute(self,funcName,argsin):
96     """Execute the function funcName found in local context with pickled args (argsin)"""
97     try:
98       argsin,kws=pickle.loads(argsin)
99       func=self.context[funcName]
100       argsout=func(*argsin,**kws)
101       argsout=pickle.dumps(argsout,-1)
102       return argsout
103     except Exception:
104       exc_typ,exc_val,exc_fr=sys.exc_info()
105       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
106       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyNode: %s, function: %s" % (self.nodeName,funcName),0))
107
108 class SenderByte_i(SALOME__POA.SenderByte,Generic):
109   def __init__(self,poa,bytesToSend):
110     Generic.__init__(self,poa)
111     self.bytesToSend = bytesToSend
112
113   def getSize(self):
114     return len(self.bytesToSend)
115
116   def sendPart(self,n1,n2):
117     return self.bytesToSend[n1:n2]
118
119 SALOME_FILE_BIG_OBJ_DIR = "SALOME_FILE_BIG_OBJ_DIR"
120     
121 SALOME_BIG_OBJ_ON_DISK_THRES_VAR = "SALOME_BIG_OBJ_ON_DISK_THRES"
122
123 # default is 50 MB
124 SALOME_BIG_OBJ_ON_DISK_THRES_DFT = 50000000
125
126 from ctypes import c_int
127 TypeCounter = c_int
128
129 def GetSizeOfTCnt():
130   return len( bytes(TypeCounter(0) ) )
131
132 def GetObjectFromFile(fname):
133   with open(fname,"rb") as f:
134     cntb = f.read( GetSizeOfTCnt() )
135     cnt = TypeCounter.from_buffer_copy( cntb ).value
136     obj = pickle.load(f)
137   return obj,cnt
138
139 def DumpInFile(obj,fname):
140   with open(fname,"wb") as f:
141     f.write( bytes( TypeCounter(1) ) )
142     f.write( obj )
143
144 def IncrRefInFile(fname):
145   with open(fname,"rb") as f:
146     cntb = f.read( GetSizeOfTCnt() )
147   cnt = TypeCounter.from_buffer_copy( cntb ).value
148   with open(fname,"rb+") as f:
149     #import KernelServices ; KernelServices.EntryForDebuggerBreakPoint()
150     f.write( bytes( TypeCounter(cnt+1) ) )
151
152 def DecrRefInFile(fname):
153   import os
154   with open(fname,"rb") as f:
155     cntb = f.read( GetSizeOfTCnt() )
156   cnt = TypeCounter.from_buffer_copy( cntb ).value
157   #
158   #import KernelServices ; KernelServices.EntryForDebuggerBreakPoint()
159   if cnt == 1:
160     os.unlink( fname )
161   else:
162     with open(fname,"rb+") as f:
163         f.write( bytes( TypeCounter(cnt-1) ) )
164
165 def GetBigObjectOnDiskThreshold():
166   import os
167   if SALOME_BIG_OBJ_ON_DISK_THRES_VAR in os.environ:
168     return int( os.environ[SALOME_BIG_OBJ_ON_DISK_THRES_VAR] )
169   else:
170     return SALOME_BIG_OBJ_ON_DISK_THRES_DFT
171
172 def ActivateProxyMecanismOrNot( sizeInByte ):
173   thres = GetBigObjectOnDiskThreshold()
174   if thres == -1:
175     return False
176   else:
177     return sizeInByte > thres
178
179 def GetBigObjectDirectory():
180   import os
181   if SALOME_FILE_BIG_OBJ_DIR not in os.environ:
182     raise RuntimeError("An object of size higher than limit detected and no directory specified to dump it in file !")
183   return os.path.expanduser( os.path.expandvars( os.environ[SALOME_FILE_BIG_OBJ_DIR] ) )
184
185 def GetBigObjectFileName():
186   """
187   Return a filename in the most secure manner (see tempfile documentation)
188   """
189   import tempfile
190   with tempfile.NamedTemporaryFile(dir=GetBigObjectDirectory(),prefix="mem_",suffix=".pckl") as f:
191     ret = f.name
192   return ret
193
194 class BigObjectOnDiskBase:
195   def __init__(self, fileName, objSerialized):
196     """
197     :param fileName: the file used to dump into.
198     :param objSerialized: the object in pickeled form
199     :type objSerialized: bytes
200     """
201     self._filename = fileName
202     # attribute _destroy is here to tell client side or server side
203     # only client side can be with _destroy set to True. server side due to risk of concurrency
204     # so pickled form of self must be done with this attribute set to False.
205     self._destroy = False
206     self.__dumpIntoFile(objSerialized)
207
208   def getDestroyStatus(self):
209     return self._destroy
210
211   def incrRef(self):
212     if self._destroy:
213       IncrRefInFile( self._filename )
214     else:
215       # should never happen !
216       RuntimeError("Invalid call to incrRef !")
217
218   def decrRef(self):
219     if self._destroy:
220       DecrRefInFile( self._filename )
221     else:
222       # should never happen !
223       RuntimeError("Invalid call to decrRef !")
224
225   def unlinkOnDestructor(self):
226     self._destroy = True
227
228   def doNotTouchFile(self):
229     """
230     Method called slave side. The life cycle management of file is client side not slave side.
231     """
232     self._destroy = False
233
234   def __del__(self):
235     if self._destroy:
236       DecrRefInFile( self._filename )
237
238   def getFileName(self):
239     return self._filename
240   
241   def __dumpIntoFile(self, objSerialized):
242     DumpInFile( objSerialized, self._filename )
243
244   def get(self):
245     obj, _ = GetObjectFromFile( self._filename )
246     return obj
247
248   def __float__(self):
249     return float( self.get() )
250     
251   def __int__(self):
252     return int( self.get() )
253     
254   def __str__(self):
255     obj = self.get()
256     if isinstance(obj,str):
257         return obj
258     else:
259         raise RuntimeError("Not a string")
260       
261 class BigObjectOnDisk(BigObjectOnDiskBase):
262   def __init__(self, fileName, objSerialized):
263     BigObjectOnDiskBase.__init__(self, fileName, objSerialized)
264     
265 class BigObjectOnDiskListElement(BigObjectOnDiskBase):
266   def __init__(self, pos, length, fileName):
267     self._filename = fileName
268     self._destroy = False
269     self._pos = pos
270     self._length = length
271
272   def get(self):
273     fullObj = BigObjectOnDiskBase.get(self)
274     return fullObj[ self._pos ]
275     
276   def __getitem__(self, i):
277     return self.get()[i]
278
279   def __len__(self):
280     return len(self.get())
281     
282 class BigObjectOnDiskSequence(BigObjectOnDiskBase):
283   def __init__(self, length, fileName, objSerialized):
284     BigObjectOnDiskBase.__init__(self, fileName, objSerialized)
285     self._length = length
286
287   def __getitem__(self, i):
288     return BigObjectOnDiskListElement(i, self._length, self.getFileName())
289
290   def __len__(self):
291     return self._length
292
293 class BigObjectOnDiskList(BigObjectOnDiskSequence):
294   def __init__(self, length, fileName, objSerialized):
295     BigObjectOnDiskSequence.__init__(self, length, fileName, objSerialized)
296     
297 class BigObjectOnDiskTuple(BigObjectOnDiskSequence):
298   def __init__(self, length, fileName, objSerialized):
299     BigObjectOnDiskSequence.__init__(self, length, fileName, objSerialized)
300
301 def SpoolPickleObject( obj ):
302   import pickle
303   pickleObjInit = pickle.dumps( obj , pickle.HIGHEST_PROTOCOL )
304   if not ActivateProxyMecanismOrNot( len(pickleObjInit) ):
305     return pickleObjInit
306   else:
307     if isinstance( obj, list):
308       proxyObj = BigObjectOnDiskList( len(obj), GetBigObjectFileName() , pickleObjInit )
309     elif isinstance( obj, tuple):
310       proxyObj = BigObjectOnDiskTuple( len(obj), GetBigObjectFileName() , pickleObjInit )
311     else:
312       proxyObj = BigObjectOnDisk( GetBigObjectFileName() , pickleObjInit )
313     pickleProxy = pickle.dumps( proxyObj , pickle.HIGHEST_PROTOCOL )
314     return pickleProxy
315
316 def UnProxyObjectSimple( obj ):
317   """
318   Method to be called in Remote mode. Alterate the obj _status attribute. 
319   Because the slave process does not participate in the reference counting
320   """
321   if isinstance(obj,BigObjectOnDiskBase):
322     obj.doNotTouchFile()
323     return obj.get()
324   elif isinstance( obj, list):
325     retObj = []
326     for elt in obj:
327       retObj.append( UnProxyObjectSimple(elt) )
328     return retObj
329   else:
330     return obj
331
332 def UnProxyObjectSimpleLocal( obj ):
333   """
334   Method to be called in Local mode. Do not alterate the PyObj counter
335   """
336   if isinstance(obj,BigObjectOnDiskBase):
337     return obj.get()
338   elif isinstance( obj, list):
339     retObj = []
340     for elt in obj:
341       retObj.append( UnProxyObjectSimpleLocal(elt) )
342     return retObj
343   else:
344     return obj
345     
346 class SeqByteReceiver:
347   # 2GB limit to trigger split into chunks
348   CHUNK_SIZE = 2000000000
349   def __init__(self,sender):
350     self._obj = sender
351   def __del__(self):
352     self._obj.UnRegister()
353     pass
354   def data(self):
355     size = self._obj.getSize()
356     if size <= SeqByteReceiver.CHUNK_SIZE:
357       return self.fetchOneShot( size )
358     else:
359       return self.fetchByChunks( size )
360   def fetchOneShot(self,size):
361     return self._obj.sendPart(0,size)
362   def fetchByChunks(self,size):
363       """
364       To avoid memory peak parts over 2GB are sent using EFF_CHUNK_SIZE size.
365       """
366       data_for_split_case = bytes(0)
367       EFF_CHUNK_SIZE = SeqByteReceiver.CHUNK_SIZE // 8
368       iStart = 0 ; iEnd = EFF_CHUNK_SIZE
369       while iStart!=iEnd and iEnd <= size:
370         part = self._obj.sendPart(iStart,iEnd)
371         data_for_split_case = bytes(0).join( [data_for_split_case,part] )
372         iStart = iEnd; iEnd = min(iStart + EFF_CHUNK_SIZE,size)
373       return data_for_split_case
374
375 class PyScriptNode_i (Engines__POA.PyScriptNode,Generic):
376   """The implementation of the PyScriptNode CORBA IDL that executes a script"""
377   def __init__(self, nodeName,code,poa,my_container):
378     """Initialize the node : compilation in the local context"""
379     Generic.__init__(self,poa)
380     self.nodeName=nodeName
381     self.code=code
382     self.my_container=my_container._container
383     linecache.cache[nodeName]=0,None,code.split('\n'),nodeName
384     self.ccode=compile(code,nodeName,'exec')
385     self.context={}
386     self.context[MY_CONTAINER_ENTRY_IN_GLBS] = self.my_container
387     
388   def __del__(self):
389     # force removal of self.context. Don t know why it s not done by default
390     self.removeAllVarsInContext()
391     pass
392
393   def getContainer(self):
394     return self.my_container
395
396   def getCode(self):
397     return self.code
398
399   def getName(self):
400     return self.nodeName
401
402   def defineNewCustomVar(self,varName,valueOfVar):
403     self.context[varName] = pickle.loads(valueOfVar)
404     pass
405
406   def executeAnotherPieceOfCode(self,code):
407     """Called for initialization of container lodging self."""
408     try:
409       ccode=compile(code,self.nodeName,'exec')
410       exec(ccode, self.context)
411     except Exception:
412       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"","PyScriptNode (%s) : code to be executed \"%s\"" %(self.nodeName,code),0))
413
414   def assignNewCompiledCode(self,codeStr):
415     try:
416       self.code=codeStr
417       self.ccode=compile(codeStr,self.nodeName,'exec')
418     except Exception:
419       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"","PyScriptNode.assignNewCompiledCode (%s) : code to be executed \"%s\"" %(self.nodeName,codeStr),0))
420
421   def executeSimple(self, key, val):
422     """
423     Same as execute method except that no pickelization mecanism is implied here. No output is expected
424     """
425     try:
426       self.context.update({ "env" : [(k,v) for k,v in zip(key,val)]})
427       exec(self.ccode,self.context)
428     except Exception:
429       exc_typ,exc_val,exc_fr=sys.exc_info()
430       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
431       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" % (self.nodeName),0))
432     
433   def execute(self,outargsname,argsin):
434     """Execute the script stored in attribute ccode with pickled args (argsin)"""
435     try:
436       argsname,kws=pickle.loads(argsin)
437       self.context.update(kws)
438       exec(self.ccode, self.context)
439       argsout=[]
440       for arg in outargsname:
441         if arg not in self.context:
442           raise KeyError("There is no variable %s in context" % arg)
443         argsout.append(self.context[arg])
444       argsout=pickle.dumps(tuple(argsout),-1)
445       return argsout
446     except Exception:
447       exc_typ,exc_val,exc_fr=sys.exc_info()
448       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
449       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s, outargsname: %s" % (self.nodeName,outargsname),0))
450
451   def executeFirst(self,argsin):
452     """ Same than first part of self.execute to reduce memory peak."""
453     import time
454     try:
455       data = None
456       if True: # to force call of SeqByteReceiver's destructor
457         argsInPy = SeqByteReceiver( argsin )
458         data = argsInPy.data()
459       _,kws=pickle.loads(data)
460       for elt in kws:
461         # fetch real data if necessary
462         kws[elt] = UnProxyObjectSimple( kws[elt] )
463       self.context.update(kws)
464     except Exception:
465       exc_typ,exc_val,exc_fr=sys.exc_info()
466       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
467       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode:First %s" % (self.nodeName),0))
468
469   def executeSecond(self,outargsname):
470     """ Same than second part of self.execute to reduce memory peak."""
471     try:
472       exec(self.ccode, self.context)
473       argsout=[]
474       for arg in outargsname:
475         if arg not in self.context:
476           raise KeyError("There is no variable %s in context" % arg)
477         argsout.append(self.context[arg])
478       ret = [ ]
479       for arg in argsout:
480         # the proxy mecanism is catched here
481         argPickle = SpoolPickleObject( arg )
482         retArg = SenderByte_i( self.poa,argPickle )
483         id_o = self.poa.activate_object(retArg)
484         retObj = self.poa.id_to_reference(id_o)
485         ret.append( retObj._narrow( SALOME.SenderByte ) )
486       return ret
487     except Exception:
488       exc_typ,exc_val,exc_fr=sys.exc_info()
489       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
490       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode:Second %s, outargsname: %s" % (self.nodeName,outargsname),0))
491
492   def listAllVarsInContext(self):
493       import re
494       pat = re.compile("^__([a-z]+)__$")
495       return [elt for elt in self.context if not pat.match(elt) and elt != MY_CONTAINER_ENTRY_IN_GLBS]
496       
497   def removeAllVarsInContext(self):
498       for elt in self.listAllVarsInContext():
499         del self.context[elt]
500
501   def getValueOfVarInContext(self,varName):
502     try:
503       return pickle.dumps(self.context[varName],-1)
504     except Exception:
505       exc_typ,exc_val,exc_fr=sys.exc_info()
506       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
507       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" %self.nodeName,0))
508     pass
509   
510   def assignVarInContext(self, varName, value):
511     try:
512       self.context[varName][0] = pickle.loads(value)
513     except Exception:
514       exc_typ,exc_val,exc_fr=sys.exc_info()
515       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
516       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" %self.nodeName,0))
517     pass
518
519   def callMethodOnVarInContext(self, varName, methodName, args):
520     try:
521       return pickle.dumps( getattr(self.context[varName][0],methodName)(*pickle.loads(args)),-1 )
522     except Exception:
523       exc_typ,exc_val,exc_fr=sys.exc_info()
524       l=traceback.format_exception(exc_typ,exc_val,exc_fr)
525       raise SALOME.SALOME_Exception(SALOME.ExceptionStruct(SALOME.BAD_PARAM,"".join(l),"PyScriptNode: %s" %self.nodeName,0))
526     pass