1 // Copyright (C) 2006-2023 CEA, EDF
3 // This library is free software; you can redistribute it and/or
4 // modify it under the terms of the GNU Lesser General Public
5 // License as published by the Free Software Foundation; either
6 // version 2.1 of the License, or (at your option) any later version.
8 // This library is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 // Lesser General Public License for more details.
13 // You should have received a copy of the GNU Lesser General Public
14 // License along with this library; if not, write to the Free Software
15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
20 #include "RuntimeSALOME.hxx"
21 #include "PythonNode.hxx"
22 #include "PythonPorts.hxx"
23 #include "TypeCode.hxx"
24 #include "PythonCppUtils.hxx"
25 #include "Container.hxx"
26 #include "SalomeContainer.hxx"
27 #include "SalomeHPContainer.hxx"
28 #include "SalomeContainerTmpForHP.hxx"
29 #include "ConversionException.hxx"
30 #include "ReceiverFactory.hxx"
31 #include "SenderByteImpl.hxx"
33 #include "PyStdout.hxx"
41 #define getpid _getpid
44 #if PY_VERSION_HEX < 0x02050000
45 typedef int Py_ssize_t;
49 #include "YacsTrace.hxx"
51 using namespace YACS::ENGINE;
54 const char PythonEntry::SCRIPT_FOR_SIMPLE_SERIALIZATION[]="import pickle\n"
55 "def pickleForVarSimplePyth2009(val):\n"
56 " return pickle.dumps(val,-1)\n"
59 PyObject *PythonEntry::_pyClsBigObject = nullptr;
61 const char PythonNode::IMPL_NAME[]="Python";
62 const char PythonNode::KIND[]="Python";
64 const char PythonNode::SCRIPT_FOR_SERIALIZATION[]="import pickle\n"
65 "def pickleForDistPyth2009(kws):\n"
66 " return pickle.dumps(((),kws),-1)\n"
68 "def unPickleForDistPyth2009(st):\n"
69 " args=pickle.loads(st)\n"
72 const char PythonNode::REMOTE_NAME[]="remote";
74 const char PythonNode::DPL_INFO_NAME[]="my_dpl_localization";
76 const char PyFuncNode::SCRIPT_FOR_SERIALIZATION[]="import pickle\n"
77 "def pickleForDistPyth2009(*args,**kws):\n"
78 " return pickle.dumps((args,kws),-1)\n"
80 "def unPickleForDistPyth2009(st):\n"
81 " args=pickle.loads(st)\n"
84 static char SCRIPT_FOR_BIGOBJECT[]="import SALOME_PyNode\n"
85 "BigObjectOnDiskBase = SALOME_PyNode.BigObjectOnDiskBase\n";
87 // pickle.load concurrency issue : see https://bugs.python.org/issue12680
88 #if PY_VERSION_HEX < 0x03070000
90 static std::mutex data_mutex;
93 PythonEntry::PythonEntry():_context(0),_pyfuncSer(0),_pyfuncUnser(0),_pyfuncSimpleSer(0)
97 PythonEntry::~PythonEntry()
100 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
101 // not Py_XDECREF of _pyfuncUnser because it is returned by PyDict_GetItem -> borrowed
102 // not Py_XDECREF of _pyfuncSer because it is returned by PyDict_GetItem -> borrowed
103 Py_XDECREF(_context);
106 void PythonEntry::loadRemoteContainer(InlineNode *reqNode)
108 DEBTRACE( "---------------PythonEntry::CommonRemoteLoad function---------------" );
109 Container *container(reqNode->getContainer());
110 bool isContAlreadyStarted(false);
115 if(hasImposedResource())
116 container->start(reqNode, _imposedResource, _imposedContainer);
119 isContAlreadyStarted=container->isAlreadyStarted(reqNode);
120 if(!isContAlreadyStarted)
121 container->start(reqNode);
126 reqNode->setErrorDetails(e.what());
132 std::string what("PythonEntry::CommonRemoteLoad : a load operation requested on \"");
133 what+=reqNode->getName(); what+="\" with no container specified.";
134 reqNode->setErrorDetails(what);
135 throw Exception(what);
139 Engines::Container_var GetContainerObj(InlineNode *reqNode, bool& isStandardCont)
141 isStandardCont = false;
142 Container *container(reqNode->getContainer());
143 Engines::Container_var objContainer(Engines::Container::_nil());
145 throw YACS::Exception("No container specified !");
146 SalomeContainer *containerCast0(dynamic_cast<SalomeContainer *>(container));
147 SalomeHPContainer *containerCast1(dynamic_cast<SalomeHPContainer *>(container));
150 isStandardCont = true;
151 objContainer=containerCast0->getContainerPtr(reqNode);
153 else if(containerCast1)
155 YACS::BASES::AutoCppPtr<SalomeContainerTmpForHP> tmpCont(SalomeContainerTmpForHP::BuildFrom(containerCast1,reqNode));
156 objContainer=tmpCont->getContainerPtr(reqNode);
159 throw YACS::Exception("Unrecognized type of container ! Salome one is expected for PythonNode/PyFuncNode !");
160 if(CORBA::is_nil(objContainer))
161 throw YACS::Exception("Container corba pointer is NULL for PythonNode !");
165 Engines::Container_var PythonEntry::loadPythonAdapter(InlineNode *reqNode, bool& isInitializeRequested)
167 bool isStandardCont(true);
168 Engines::Container_var objContainer(GetContainerObj(reqNode,isStandardCont));
169 isInitializeRequested=false;
172 Engines::PyNodeBase_var dftPyScript(retrieveDftRemotePyInterpretorIfAny(objContainer));
173 if(CORBA::is_nil(dftPyScript))
175 isInitializeRequested=!isStandardCont;
176 createRemoteAdaptedPyInterpretor(objContainer);
179 assignRemotePyInterpretor(dftPyScript);
181 catch( const SALOME::SALOME_Exception& ex )
183 std::string msg="Exception on remote python node creation ";
185 msg += ex.details.text.in();
186 reqNode->setErrorDetails(msg);
187 throw Exception(msg);
189 Engines::PyNodeBase_var pynode(getRemoteInterpreterHandle());
190 if(CORBA::is_nil(pynode))
191 throw Exception("In PythonNode the ref in NULL ! ");
195 void PythonEntry::loadRemoteContext(InlineNode *reqNode, Engines::Container_ptr objContainer, bool isInitializeRequested)
197 Container *container(reqNode->getContainer());
198 Engines::PyNodeBase_var pynode(getRemoteInterpreterHandle());
201 #if PY_VERSION_HEX < 0x03070000
202 std::unique_lock<std::mutex> lock(data_mutex);
205 const char *picklizeScript(getSerializationScript());
206 PyObject *res=PyRun_String(picklizeScript,Py_file_input,_context,_context);
207 PyObject *res2(PyRun_String(SCRIPT_FOR_SIMPLE_SERIALIZATION,Py_file_input,_context,_context));
208 if(res == NULL || res2==NULL)
210 std::string errorDetails;
211 PyObject* new_stderr = newPyStdOut(errorDetails);
212 reqNode->setErrorDetails(errorDetails);
213 PySys_SetObject((char*)"stderr", new_stderr);
215 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
216 Py_DECREF(new_stderr);
217 throw Exception("Error during load");
219 Py_DECREF(res); Py_DECREF(res2);
220 AutoPyRef res3(PyRun_String(SCRIPT_FOR_BIGOBJECT,Py_file_input,_context,_context));
221 _pyfuncSer=PyDict_GetItemString(_context,"pickleForDistPyth2009");
222 _pyfuncUnser=PyDict_GetItemString(_context,"unPickleForDistPyth2009");
223 _pyfuncSimpleSer=PyDict_GetItemString(_context,"pickleForVarSimplePyth2009");
224 if(! _pyClsBigObject )
226 _pyClsBigObject=PyDict_GetItemString(_context,"BigObjectOnDiskBase");
227 Py_INCREF(_pyClsBigObject);
229 if(_pyfuncSer == NULL)
231 std::string errorDetails;
232 PyObject *new_stderr(newPyStdOut(errorDetails));
233 reqNode->setErrorDetails(errorDetails);
234 PySys_SetObject((char*)"stderr", new_stderr);
236 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
237 Py_DECREF(new_stderr);
238 throw Exception("Error during load");
240 if(_pyfuncUnser == NULL)
242 std::string errorDetails;
243 PyObject *new_stderr(newPyStdOut(errorDetails));
244 reqNode->setErrorDetails(errorDetails);
245 PySys_SetObject((char*)"stderr", new_stderr);
247 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
248 Py_DECREF(new_stderr);
249 throw Exception("Error during load");
251 if(_pyfuncSimpleSer == NULL)
253 std::string errorDetails;
254 PyObject *new_stderr(newPyStdOut(errorDetails));
255 reqNode->setErrorDetails(errorDetails);
256 PySys_SetObject((char*)"stderr", new_stderr);
258 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
259 Py_DECREF(new_stderr);
260 throw Exception("Error during load");
263 if(isInitializeRequested)
264 {//This one is called only once at initialization in the container if an init-script is specified.
267 std::string zeInitScriptKey(container->getProperty(HomogeneousPoolContainer::INITIALIZE_SCRIPT_KEY));
268 if(!zeInitScriptKey.empty())
269 pynode->executeAnotherPieceOfCode(zeInitScriptKey.c_str());
271 catch( const SALOME::SALOME_Exception& ex )
273 std::string msg="Exception on PythonNode::loadRemote python invocation of initializisation py script !";
275 msg += ex.details.text.in();
276 reqNode->setErrorDetails(msg);
277 throw Exception(msg);
279 DEBTRACE( "---------------End PyNode::loadRemote function---------------" );
283 std::string PythonEntry::GetContainerLog(const std::string& mode, Container *container, const Task *askingTask)
291 SalomeContainer *containerCast(dynamic_cast<SalomeContainer *>(container));
292 SalomeHPContainer *objContainer2(dynamic_cast<SalomeHPContainer *>(container));
295 Engines::Container_var objContainer(containerCast->getContainerPtr(askingTask));
296 CORBA::String_var logname = objContainer->logfilename();
299 std::string::size_type pos = msg.find(":");
300 msg=msg.substr(pos+1);
302 else if(objContainer2)
304 msg="Remote PythonNode is on HP Container : no log because no info of the location by definition of HP Container !";
308 msg="Not implemented yet for container log for that type of container !";
313 msg = "Container no longer reachable";
318 void PythonEntry::commonRemoteLoad(InlineNode *reqNode)
320 loadRemoteContainer(reqNode);
321 bool isInitializeRequested;
322 Engines::Container_var objContainer(loadPythonAdapter(reqNode,isInitializeRequested));
323 loadRemoteContext(reqNode,objContainer,isInitializeRequested);
326 bool PythonEntry::hasImposedResource()const
328 return !_imposedResource.empty() && !_imposedContainer.empty();
331 bool PythonEntry::GetDestroyStatus( PyObject *ob )
335 if( PyObject_IsInstance( ob, _pyClsBigObject) == 1 )
337 AutoPyRef unlinkOnDestructor = PyObject_GetAttrString(ob,"getDestroyStatus");
338 AutoPyRef tmp = PyObject_CallFunctionObjArgs(unlinkOnDestructor,nullptr);
339 if( PyBool_Check(tmp.get()) )
341 return tmp.get() == Py_True;
348 void PythonEntry::IfProxyDoSomething( PyObject *ob, const char *meth )
352 if( PyObject_IsInstance( ob, _pyClsBigObject) == 1 )
354 AutoPyRef unlinkOnDestructor = PyObject_GetAttrString(ob,meth);
355 AutoPyRef tmp = PyObject_CallFunctionObjArgs(unlinkOnDestructor,nullptr);
359 void PythonEntry::DoNotTouchFileIfProxy( PyObject *ob )
361 IfProxyDoSomething(ob,"doNotTouchFile");
364 void PythonEntry::UnlinkOnDestructorIfProxy( PyObject *ob )
366 IfProxyDoSomething(ob,"unlinkOnDestructor");
369 PythonNode::PythonNode(const PythonNode& other, ComposedNode *father):InlineNode(other,father),_autoSqueeze(other._autoSqueeze)
371 _pynode = Engines::PyScriptNode::_nil();
372 _implementation=IMPL_NAME;
375 _context=PyDict_New();
376 if( PyDict_SetItemString( _context, "__builtins__", getSALOMERuntime()->getBuiltins() ))
379 msg << "Impossible to set builtins" << __FILE__ << ":" << __LINE__;
380 _errorDetails=msg.str();
381 throw Exception(msg.str());
386 PythonNode::PythonNode(const std::string& name):InlineNode(name)
388 _pynode = Engines::PyScriptNode::_nil();
389 _implementation=IMPL_NAME;
392 _context=PyDict_New();
393 if( PyDict_SetItemString( _context, "__builtins__", getSALOMERuntime()->getBuiltins() ))
396 msg << "Impossible to set builtins" << __FILE__ << ":" << __LINE__;
397 _errorDetails=msg.str();
398 throw Exception(msg.str());
403 PythonNode::~PythonNode()
408 void PythonNode::checkBasicConsistency() const
410 DEBTRACE("checkBasicConsistency");
411 InlineNode::checkBasicConsistency();
415 res=Py_CompileString(_script.c_str(),getName().c_str(),Py_file_input);
418 std::string error="";
419 PyObject* new_stderr = newPyStdOut(error);
420 PySys_SetObject((char*)"stderr", new_stderr);
422 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
423 Py_DECREF(new_stderr);
424 throw Exception(error);
431 void PythonNode::load()
433 DEBTRACE( "---------------PyNode::load function---------------" );
434 if(_mode==PythonNode::REMOTE_NAME)
440 void PythonNode::loadLocal()
442 DEBTRACE( "---------------PyNode::loadLocal function---------------" );
446 void PythonNode::loadRemote()
448 commonRemoteLoad(this);
451 void PythonNode::execute()
453 if(_mode==PythonNode::REMOTE_NAME)
459 void PythonNode::executeRemote()
461 DEBTRACE( "++++++++++++++ PyNode::executeRemote: " << getName() << " ++++++++++++++++++++" );
463 throw Exception("PythonNode badly loaded");
465 if(dynamic_cast<HomogeneousPoolContainer *>(getContainer()))
468 loadPythonAdapter(this,dummy);
469 _pynode->assignNewCompiledCode(getScript().c_str());
471 // not managed by unique_ptr here because destructed by the order of client.
472 SenderByteImpl *serializationInputCorba = nullptr;
473 AutoPyRef serializationInput;
475 #if PY_VERSION_HEX < 0x03070000
476 std::unique_lock<std::mutex> lock(data_mutex);
479 PyObject *args(0),*ob(0);
480 //===========================================================================
481 // Get inputs in input ports, build a Python dict and pickle it
482 //===========================================================================
484 std::list<InputPort *>::iterator iter2;
486 for(iter2 = _setOfInputPort.begin(); iter2 != _setOfInputPort.end(); ++iter2)
488 InputPyPort *p=(InputPyPort *)*iter2;
490 PyDict_SetItemString(args,p->getName().c_str(),ob);
494 PyObject_Print(args,stderr,Py_PRINT_RAW);
497 serializationInput.set(PyObject_CallFunctionObjArgs(_pyfuncSer,args,nullptr));
499 //The pickled string may contain NULL characters so use PyString_AsStringAndSize
500 char *serializationInputC(nullptr);
502 if (PyBytes_AsStringAndSize(serializationInput, &serializationInputC, &len))
503 throw Exception("DistributedPythonNode problem in python pickle");
504 // no copy here. The C byte array of Python is taken as this into CORBA sequence to avoid copy
505 serializationInputCorba = new SenderByteImpl(serializationInputC,len);
508 //get the list of output argument names
509 std::list<OutputPort *>::iterator iter;
510 Engines::listofstring myseq;
511 myseq.length(getNumberOfOutputPorts());
513 for(iter = _setOfOutputPort.begin(); iter != _setOfOutputPort.end(); ++iter)
515 OutputPyPort *p=(OutputPyPort *)*iter;
516 myseq[pos]=p->getName().c_str();
517 DEBTRACE( "port name: " << p->getName() );
518 DEBTRACE( "port kind: " << p->edGetType()->kind() );
519 DEBTRACE( "port pos : " << pos );
522 //===========================================================================
523 // Execute in remote Python node
524 //===========================================================================
525 DEBTRACE( "-----------------starting remote python invocation-----------------" );
526 std::unique_ptr<SALOME::SenderByteSeq> resultCorba;
529 //pass outargsname and dict serialized
530 SALOME::SenderByte_var serializationInputRef = serializationInputCorba->_this();
531 _pynode->executeFirst(serializationInputRef);
532 //serializationInput and serializationInputCorba are no more needed for server. Release it.
533 serializationInput.set(nullptr);
534 resultCorba.reset( _pynode->executeSecond(myseq) );
535 if( ! this->isUsingPythonCache() )
536 _pynode->removeAllVarsInContext();
538 catch( const SALOME::SALOME_Exception& ex )
540 std::ostringstream msg; msg << "Exception on remote python invocation" << std::endl << ex.details.text.in() << std::endl;
541 msg << "PyScriptNode CORBA ref : ";
543 CORBA::ORB_ptr orb(getSALOMERuntime()->getOrb());
544 if(!CORBA::is_nil(orb))
546 CORBA::String_var IOR(orb->object_to_string(_pynode));
551 _errorDetails=msg.str();
552 throw Exception(msg.str());
554 catch(CORBA::COMM_FAILURE& ex)
556 std::ostringstream msg;
557 msg << "Exception on remote python invocation." << std::endl ;
558 msg << "Caught system exception COMM_FAILURE -- unable to contact the "
559 << "object." << std::endl;
560 _errorDetails=msg.str();
561 throw Exception(msg.str());
563 catch(CORBA::SystemException& ex)
565 std::ostringstream msg;
566 msg << "Exception on remote python invocation." << std::endl ;
567 msg << "Caught a CORBA::SystemException." ;
570 CORBA::TypeCode_var tc = tmp.type();
571 const char *p = tc->name();
577 _errorDetails=msg.str();
578 throw Exception(msg.str());
580 catch(CORBA::Exception& ex)
582 std::ostringstream msg;
583 msg << "Exception on remote python invocation." << std::endl ;
584 msg << "Caught CORBA::Exception. " ;
587 CORBA::TypeCode_var tc = tmp.type();
588 const char *p = tc->name();
594 _errorDetails=msg.str();
595 throw Exception(msg.str());
597 catch(omniORB::fatalException& fe)
599 std::ostringstream msg;
600 msg << "Exception on remote python invocation." << std::endl ;
601 msg << "Caught omniORB::fatalException:" << std::endl;
602 msg << " file: " << fe.file() << std::endl;
603 msg << " line: " << fe.line() << std::endl;
604 msg << " mesg: " << fe.errmsg() << std::endl;
605 _errorDetails=msg.str();
606 throw Exception(msg.str());
608 DEBTRACE( "-----------------end of remote python invocation-----------------" );
609 //===========================================================================
610 // Get results, unpickle and put them in output ports
611 //===========================================================================
613 #if PY_VERSION_HEX < 0x03070000
614 std::unique_lock<std::mutex> lock(data_mutex);
617 DEBTRACE( "-----------------PythonNode::outputs-----------------" );
618 int nres( resultCorba->length() );
620 if(getNumberOfOutputPorts() != nres)
622 std::string msg="Number of output arguments : Mismatch between definition and execution";
624 throw Exception(msg);
629 for(iter = _setOfOutputPort.begin(); iter != _setOfOutputPort.end(); ++iter)
631 OutputPyPort *p=(OutputPyPort *)*iter;
632 DEBTRACE( "port name: " << p->getName() );
633 DEBTRACE( "port kind: " << p->edGetType()->kind() );
634 DEBTRACE( "port pos : " << pos );
635 SALOME::SenderByte_var elt = (*resultCorba)[pos];
636 SeqByteReceiver recv(elt);
637 unsigned long length = 0;
638 char *resultCorbaC = recv.data(length);
640 AutoPyRef resultPython=PyMemoryView_FromMemory(resultCorbaC,length,PyBUF_READ);
641 AutoPyRef args = PyTuple_New(1);
642 PyTuple_SetItem(args,0,resultPython.retn());
643 AutoPyRef ob = PyObject_CallObject(_pyfuncUnser,args);
646 std::stringstream msg;
647 msg << "Conversion with pickle of output ports failed !";
648 msg << " : " << __FILE__ << ":" << __LINE__;
649 _errorDetails=msg.str();
650 throw YACS::ENGINE::ConversionException(msg.str());
652 UnlinkOnDestructorIfProxy(ob);
658 catch(ConversionException& ex)
660 _errorDetails=ex.what();
664 squeezeMemoryRemote();
667 if(!isUsingPythonCache())
671 Engines::Container_var cont(GetContainerObj(this,dummy));
672 cont->removePyScriptNode(getName().c_str());
674 DEBTRACE( "++++++++++++++ ENDOF PyNode::executeRemote: " << getName() << " ++++++++++++++++++++" );
676 void PythonNode::executeLocalInternal(const std::string& codeStr)
679 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
680 std::ostringstream stream;
681 stream << "/tmp/PythonNode_";
683 AutoPyRef code=Py_CompileString(codeStr.c_str(), stream.str().c_str(), Py_file_input);
687 AutoPyRef new_stderr = newPyStdOut(_errorDetails);
688 PySys_SetObject((char*)"stderr", new_stderr);
690 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
691 throw Exception("Error during execution");
694 AutoPyRef res = PyEval_EvalCode( code, _context, _context);
696 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
699 if(PyErr_Occurred ())
702 AutoPyRef new_stderr = newPyStdOut(_errorDetails);
703 PySys_SetObject((char*)"stderr", new_stderr);
704 ofstream errorfile(stream.str().c_str());
705 if (errorfile.is_open())
707 errorfile << codeStr;
711 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
712 throw Exception("Error during execution");
716 void PythonNode::executeLocal()
718 DEBTRACE( "++++++++++++++ PyNode::executeLocal: " << getName() << " ++++++++++++++++++++" );
721 std::ostringstream unpxy; unpxy << "from SALOME_PyNode import UnProxyObjectSimple" << std::endl;
722 DEBTRACE( "---------------PyNode::inputs---------------" );
723 list<InputPort *>::iterator iter2;
724 for(iter2 = _setOfInputPort.begin(); iter2 != _setOfInputPort.end(); iter2++)
726 InputPyPort *p=(InputPyPort *)*iter2;
727 DEBTRACE( "port name: " << p->getName() );
728 DEBTRACE( "port kind: " << p->edGetType()->kind() );
729 PyObject* ob=p->getPyObj();
730 DEBTRACE( "ob refcnt: " << ob->ob_refcnt );
731 unpxy << p->getName() << " = UnProxyObjectSimple( " << p->getName() << " )" << std::endl;
733 PyObject_Print(ob,stderr,Py_PRINT_RAW);
736 int ier=PyDict_SetItemString(_context,p->getName().c_str(),ob);
737 DEBTRACE( "after PyDict_SetItemString:ob refcnt: " << ob->ob_refcnt );
740 DEBTRACE( "---------------End PyNode::inputs---------------" );
743 DEBTRACE( "----------------PyNode::calculation---------------" );
745 executeLocalInternal( unpxy.str() );
747 executeLocalInternal( _script );
749 DEBTRACE( "-----------------PyNode::outputs-----------------" );
750 list<OutputPort *>::iterator iter;
753 for(iter = _setOfOutputPort.begin(); iter != _setOfOutputPort.end(); iter++)
755 OutputPyPort *p=(OutputPyPort *)*iter;
756 DEBTRACE( "port name: " << p->getName() );
757 DEBTRACE( "port kind: " << p->edGetType()->kind() );
758 PyObject *ob=PyDict_GetItemString(_context,p->getName().c_str());
761 std::string msg="Error during execution: there is no variable ";
762 msg=msg+p->getName()+" in node context";
764 throw Exception(msg);
766 DEBTRACE( "PyNode::outputs::ob refcnt: " << ob->ob_refcnt );
768 PyObject_Print(ob,stderr,Py_PRINT_RAW);
774 catch(ConversionException& ex)
776 _errorDetails=ex.what();
781 DEBTRACE( "-----------------End PyNode::outputs-----------------" );
783 DEBTRACE( "++++++++++++++ End PyNode::execute: " << getName() << " ++++++++++++++++++++" );
786 void PythonNode::squeezeMemorySafe()
789 if(_mode==PythonNode::REMOTE_NAME)
790 this->squeezeMemoryRemote();
792 this->squeezeMemory();
795 void PythonNode::squeezeMemory()
797 for(auto p : _setOfInputPort)
799 PyDict_DelItemString(_context,p->getName().c_str());
800 InputPyPort *p2(static_cast<InputPyPort *>(p));
801 if(p2->canSafelySqueezeMemory())
804 for(auto p : _setOfOutputPort)
806 PyDict_DelItemString(_context,p->getName().c_str());
807 OutputPyPort *p2(static_cast<OutputPyPort *>(p));
808 p2->putWithoutForward(Py_None);
812 void PythonNode::squeezeMemoryRemote()
814 for(auto p : _setOfInputPort)
816 InputPyPort *p2(static_cast<InputPyPort *>(p));
817 if(p2->canSafelySqueezeMemory())
820 for(auto p : _setOfOutputPort)
822 OutputPyPort *p2(static_cast<OutputPyPort *>(p));
823 p2->putWithoutForward(Py_None);
827 std::string PythonNode::getContainerLog()
829 return PythonEntry::GetContainerLog(_mode,_container,this);
832 void PythonNode::shutdown(int level)
834 DEBTRACE("PythonNode::shutdown " << level);
835 if(_mode=="local")return;
839 _container->shutdown(level);
843 void PythonNode::imposeResource(const std::string& resource_name,
844 const std::string& container_name)
846 if(!resource_name.empty() && !container_name.empty())
848 _imposedResource = resource_name;
849 _imposedContainer = container_name;
853 bool PythonNode::canAcceptImposedResource()
855 return _container != nullptr && _container->canAcceptImposedResource();
858 bool PythonNode::hasImposedResource()const
860 return PythonEntry::hasImposedResource();
863 std::string PythonNode::pythonEntryName()const
865 if(isUsingPythonCache())
866 return "DEFAULT_NAME_FOR_UNIQUE_PYTHON_NODE_ENTRY";
871 bool PythonNode::isUsingPythonCache()const
875 found = _container->isUsingPythonCache();
879 void PythonNode::freeKernelPynode()
881 if(!CORBA::is_nil(_pynode))
885 _pynode->UnRegister();
889 DEBTRACE("Trouble when pynode->UnRegister!")
891 _pynode = Engines::PyScriptNode::_nil();
895 Node *PythonNode::simpleClone(ComposedNode *father, bool editionOnly) const
897 return new PythonNode(*this,father);
900 void PythonNode::createRemoteAdaptedPyInterpretor(Engines::Container_ptr objContainer)
903 _pynode=objContainer->createPyScriptNode(pythonEntryName().c_str(),getScript().c_str());
907 Engines::PyNodeBase_var PythonNode::retrieveDftRemotePyInterpretorIfAny(Engines::Container_ptr objContainer) const
909 Engines::PyScriptNode_var ret(objContainer->getDefaultPyScriptNode(pythonEntryName().c_str()));
910 if(!CORBA::is_nil(ret))
914 return Engines::PyNodeBase::_narrow(ret);
917 void PythonNode::assignRemotePyInterpretor(Engines::PyNodeBase_var remoteInterp)
919 if(CORBA::is_nil(_pynode))
920 _pynode=Engines::PyScriptNode::_narrow(remoteInterp);
923 Engines::PyScriptNode_var tmpp(Engines::PyScriptNode::_narrow(remoteInterp));
924 if(!_pynode->_is_equivalent(tmpp))
927 _pynode=Engines::PyScriptNode::_narrow(remoteInterp);
930 _pynode->assignNewCompiledCode(getScript().c_str());
933 Engines::PyNodeBase_var PythonNode::getRemoteInterpreterHandle()
935 return Engines::PyNodeBase::_narrow(_pynode);
938 //! Create a new node of same type with a given name
939 PythonNode* PythonNode::cloneNode(const std::string& name)
941 PythonNode* n=new PythonNode(name);
942 n->setScript(_script);
943 list<InputPort *>::iterator iter;
944 for(iter = _setOfInputPort.begin(); iter != _setOfInputPort.end(); iter++)
946 InputPyPort *p=(InputPyPort *)*iter;
947 DEBTRACE( "port name: " << p->getName() );
948 DEBTRACE( "port kind: " << p->edGetType()->kind() );
949 n->edAddInputPort(p->getName(),p->edGetType());
951 list<OutputPort *>::iterator iter2;
952 for(iter2 = _setOfOutputPort.begin(); iter2 != _setOfOutputPort.end(); iter2++)
954 OutputPyPort *p=(OutputPyPort *)*iter2;
955 DEBTRACE( "port name: " << p->getName() );
956 DEBTRACE( "port kind: " << p->edGetType()->kind() );
957 n->edAddOutputPort(p->getName(),p->edGetType());
962 void PythonNode::applyDPLScope(ComposedNode *gfn)
964 std::vector< std::pair<std::string,int> > ret(getDPLScopeInfo(gfn));
971 std::size_t sz(ret.size());
973 for(std::size_t i=0;i<sz;i++)
975 const std::pair<std::string,int>& p(ret[i]);
976 PyObject *elt(PyTuple_New(2));
977 PyTuple_SetItem(elt,0,PyUnicode_FromString(p.first.c_str()));
978 PyTuple_SetItem(elt,1,PyLong_FromLong(p.second));
979 PyList_SetItem(ob,i,elt);
982 if(_mode==REMOTE_NAME)
984 Engines::pickledArgs_var serializationInputCorba(new Engines::pickledArgs);
987 PyObject *serializationInput(PyObject_CallFunctionObjArgs(_pyfuncSimpleSer,ob,NULL));
989 char *serializationInputC(0);
991 if (PyBytes_AsStringAndSize(serializationInput, &serializationInputC, &len))
992 throw Exception("DistributedPythonNode problem in python pickle");
993 serializationInputCorba->length(len);
994 for(int i=0; i < len ; i++)
995 serializationInputCorba[i]=serializationInputC[i];
996 Py_XDECREF(serializationInput);
998 _pynode->defineNewCustomVar(DPL_INFO_NAME,serializationInputCorba);
1003 PyDict_SetItemString(_context,DPL_INFO_NAME,ob);
1008 PyFuncNode::PyFuncNode(const PyFuncNode& other, ComposedNode *father):InlineFuncNode(other,father),_pyfunc(0)
1010 _implementation = PythonNode::IMPL_NAME;
1013 _context=PyDict_New();
1014 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
1015 if( PyDict_SetItemString( _context, "__builtins__", getSALOMERuntime()->getBuiltins() ))
1018 msg << "Not possible to set builtins" << __FILE__ << ":" << __LINE__;
1019 _errorDetails=msg.str();
1020 throw Exception(msg.str());
1025 PyFuncNode::PyFuncNode(const std::string& name): InlineFuncNode(name),_pyfunc(0)
1028 _implementation = PythonNode::IMPL_NAME;
1029 DEBTRACE( "PyFuncNode::PyFuncNode " << name );
1032 _context=PyDict_New();
1033 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
1034 if( PyDict_SetItemString( _context, "__builtins__", getSALOMERuntime()->getBuiltins() ))
1037 msg << "Not possible to set builtins" << __FILE__ << ":" << __LINE__;
1038 _errorDetails=msg.str();
1039 throw Exception(msg.str());
1044 PyFuncNode::~PyFuncNode()
1046 if(!CORBA::is_nil(_pynode))
1048 _pynode->UnRegister();
1052 void PyFuncNode::init(bool start)
1054 initCommonPartWithoutStateManagement(start);
1055 if(_state == YACS::DISABLED)
1057 exDisabledState(); // to refresh propagation of DISABLED state
1060 if(start) //complete initialization
1061 setState(YACS::READY);
1062 else if(_state > YACS::LOADED)// WARNING FuncNode has internal vars (CEA usecase) ! Partial initialization (inside a loop). Exclusivity of funcNode.
1063 setState(YACS::TORECONNECT);
1066 void PyFuncNode::checkBasicConsistency() const
1068 DEBTRACE("checkBasicConsistency");
1069 InlineFuncNode::checkBasicConsistency();
1073 res=Py_CompileString(_script.c_str(),getName().c_str(),Py_file_input);
1076 std::string error="";
1077 PyObject* new_stderr = newPyStdOut(error);
1078 PySys_SetObject((char*)"stderr", new_stderr);
1080 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
1081 Py_DECREF(new_stderr);
1082 throw Exception(error);
1089 void PyFuncNode::load()
1091 DEBTRACE( "---------------PyfuncNode::load function---------------" );
1092 if(_mode==PythonNode::REMOTE_NAME)
1098 void PyFuncNode::loadRemote()
1100 commonRemoteLoad(this);
1103 void PyFuncNode::loadLocal()
1105 DEBTRACE( "---------------PyFuncNode::load function " << getName() << " ---------------" );
1106 DEBTRACE( _script );
1109 list<OutputPort *>::iterator iter;
1110 for(iter = _setOfOutputPort.begin(); iter != _setOfOutputPort.end(); iter++)
1112 OutputPyPort *p=(OutputPyPort *)*iter;
1113 DEBTRACE( "port name: " << p->getName() );
1114 DEBTRACE( "port kind: " << p->edGetType()->kind() );
1120 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
1122 std::ostringstream stream;
1123 stream << "/tmp/PythonNode_";
1126 PyObject* code=Py_CompileString(_script.c_str(), stream.str().c_str(), Py_file_input);
1130 PyObject* new_stderr = newPyStdOut(_errorDetails);
1131 PySys_SetObject((char*)"stderr", new_stderr);
1133 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
1134 Py_DECREF(new_stderr);
1135 throw Exception("Error during execution");
1137 PyObject *res = PyEval_EvalCode( code, _context, _context);
1141 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
1142 if(PyErr_Occurred ())
1145 PyObject* new_stderr = newPyStdOut(_errorDetails);
1146 PySys_SetObject((char*)"stderr", new_stderr);
1147 ofstream errorfile(stream.str().c_str());
1148 if (errorfile.is_open())
1150 errorfile << _script;
1154 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
1155 Py_DECREF(new_stderr);
1156 throw Exception("Error during execution");
1159 _pyfunc=PyDict_GetItemString(_context,_fname.c_str());
1160 DEBTRACE( "_pyfunc refcnt: " << _pyfunc->ob_refcnt );
1164 PyObject* new_stderr = newPyStdOut(_errorDetails);
1165 PySys_SetObject((char*)"stderr", new_stderr);
1167 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
1168 Py_DECREF(new_stderr);
1169 throw Exception("Error during execution");
1171 DEBTRACE( "---------------End PyFuncNode::load function---------------" );
1175 void PyFuncNode::execute()
1177 if(_mode==PythonNode::REMOTE_NAME)
1183 void PyFuncNode::executeRemote()
1185 DEBTRACE( "++++++++++++++ PyFuncNode::executeRemote: " << getName() << " ++++++++++++++++++++" );
1187 throw Exception("DistributedPythonNode badly loaded");
1189 if(dynamic_cast<HomogeneousPoolContainer *>(getContainer()))
1192 loadPythonAdapter(this,dummy);
1193 _pynode->executeAnotherPieceOfCode(getScript().c_str());
1196 Engines::pickledArgs_var serializationInputCorba(new Engines::pickledArgs);;
1198 #if PY_VERSION_HEX < 0x03070000
1199 std::unique_lock<std::mutex> lock(data_mutex);
1203 //===========================================================================
1204 // Get inputs in input ports, build a Python tuple and pickle it
1205 //===========================================================================
1206 PyObject *args(PyTuple_New(getNumberOfInputPorts()));
1208 for(std::list<InputPort *>::iterator iter2 = _setOfInputPort.begin(); iter2 != _setOfInputPort.end(); iter2++,pos++)
1210 InputPyPort *p=(InputPyPort *)*iter2;
1213 PyTuple_SetItem(args,pos,ob);
1216 PyObject_Print(args,stderr,Py_PRINT_RAW);
1219 PyObject *serializationInput=PyObject_CallObject(_pyfuncSer,args);
1221 //The pickled string may contain NULL characters so use PyString_AsStringAndSize
1222 char *serializationInputC(0);
1224 if (PyBytes_AsStringAndSize(serializationInput, &serializationInputC, &len))
1225 throw Exception("DistributedPythonNode problem in python pickle");
1227 serializationInputCorba->length(len);
1228 for(int i=0; i < len ; i++)
1229 serializationInputCorba[i]=serializationInputC[i];
1230 Py_DECREF(serializationInput);
1233 //===========================================================================
1234 // Execute in remote Python node
1235 //===========================================================================
1236 DEBTRACE( "-----------------starting remote python invocation-----------------" );
1237 Engines::pickledArgs_var resultCorba;
1240 resultCorba=_pynode->execute(getFname().c_str(),serializationInputCorba);
1242 catch( const SALOME::SALOME_Exception& ex )
1244 std::string msg="Exception on remote python invocation";
1246 msg += ex.details.text.in();
1248 throw Exception(msg);
1250 catch(CORBA::COMM_FAILURE& ex)
1252 std::ostringstream msg;
1253 msg << "Exception on remote python invocation." << std::endl ;
1254 msg << "Caught system exception COMM_FAILURE -- unable to contact the "
1255 << "object." << std::endl;
1256 _errorDetails=msg.str();
1257 throw Exception(msg.str());
1259 catch(CORBA::SystemException& ex)
1261 std::ostringstream msg;
1262 msg << "Exception on remote python invocation." << std::endl ;
1263 msg << "Caught a CORBA::SystemException." ;
1266 CORBA::TypeCode_var tc = tmp.type();
1267 const char *p = tc->name();
1273 _errorDetails=msg.str();
1274 throw Exception(msg.str());
1276 catch(CORBA::Exception& ex)
1278 std::ostringstream msg;
1279 msg << "Exception on remote python invocation." << std::endl ;
1280 msg << "Caught CORBA::Exception. " ;
1283 CORBA::TypeCode_var tc = tmp.type();
1284 const char *p = tc->name();
1290 _errorDetails=msg.str();
1291 throw Exception(msg.str());
1293 catch(omniORB::fatalException& fe)
1295 std::ostringstream msg;
1296 msg << "Exception on remote python invocation." << std::endl ;
1297 msg << "Caught omniORB::fatalException:" << std::endl;
1298 msg << " file: " << fe.file() << std::endl;
1299 msg << " line: " << fe.line() << std::endl;
1300 msg << " mesg: " << fe.errmsg() << std::endl;
1301 _errorDetails=msg.str();
1302 throw Exception(msg.str());
1304 DEBTRACE( "-----------------end of remote python invocation-----------------" );
1305 //===========================================================================
1306 // Get results, unpickle and put them in output ports
1307 //===========================================================================
1308 char *resultCorbaC=new char[resultCorba->length()+1];
1309 resultCorbaC[resultCorba->length()]='\0';
1310 for(int i=0;i<resultCorba->length();i++)
1311 resultCorbaC[i]=resultCorba[i];
1314 #if PY_VERSION_HEX < 0x03070000
1315 std::unique_lock<std::mutex> lock(data_mutex);
1319 PyObject *resultPython(PyBytes_FromStringAndSize(resultCorbaC,resultCorba->length()));
1320 delete [] resultCorbaC;
1321 PyObject *args(PyTuple_New(1)),*ob(0);
1322 PyTuple_SetItem(args,0,resultPython);
1323 PyObject *finalResult=PyObject_CallObject(_pyfuncUnser,args);
1326 DEBTRACE( "-----------------PythonNode::outputs-----------------" );
1328 if(finalResult == Py_None)
1330 else if(PyTuple_Check(finalResult))
1331 nres=PyTuple_Size(finalResult);
1333 if(getNumberOfOutputPorts() != nres)
1335 std::string msg="Number of output arguments : Mismatch between definition and execution";
1336 Py_DECREF(finalResult);
1338 throw Exception(msg);
1344 for(std::list<OutputPort *>::iterator iter = _setOfOutputPort.begin(); iter != _setOfOutputPort.end(); iter++, pos++)
1346 OutputPyPort *p=(OutputPyPort *)*iter;
1347 DEBTRACE( "port name: " << p->getName() );
1348 DEBTRACE( "port kind: " << p->edGetType()->kind() );
1349 DEBTRACE( "port pos : " << pos );
1350 if(PyTuple_Check(finalResult))
1351 ob=PyTuple_GetItem(finalResult,pos) ;
1354 DEBTRACE( "ob refcnt: " << ob->ob_refcnt );
1357 Py_DECREF(finalResult);
1359 catch(ConversionException& ex)
1361 Py_DECREF(finalResult);
1362 _errorDetails=ex.what();
1367 DEBTRACE( "++++++++++++++ ENDOF PyFuncNode::executeRemote: " << getName() << " ++++++++++++++++++++" );
1370 void PyFuncNode::executeLocal()
1372 DEBTRACE( "++++++++++++++ PyFuncNode::execute: " << getName() << " ++++++++++++++++++++" );
1376 if(!_pyfunc)throw Exception("PyFuncNode badly loaded");
1379 DEBTRACE( "---------------PyFuncNode::inputs---------------" );
1380 PyObject* args = PyTuple_New(getNumberOfInputPorts()) ;
1381 list<InputPort *>::iterator iter2;
1382 for(iter2 = _setOfInputPort.begin(); iter2 != _setOfInputPort.end(); iter2++)
1384 InputPyPort *p=(InputPyPort *)*iter2;
1385 DEBTRACE( "port name: " << p->getName() );
1386 DEBTRACE( "port kind: " << p->edGetType()->kind() );
1389 PyObject_Print(ob,stderr,Py_PRINT_RAW);
1392 DEBTRACE( "ob refcnt: " << ob->ob_refcnt );
1394 PyTuple_SetItem(args,pos,ob);
1395 DEBTRACE( "ob refcnt: " << ob->ob_refcnt );
1398 DEBTRACE( "---------------End PyFuncNode::inputs---------------" );
1400 DEBTRACE( "----------------PyFuncNode::calculation---------------" );
1402 PyObject_Print(_pyfunc,stderr,Py_PRINT_RAW);
1404 PyObject_Print(args,stderr,Py_PRINT_RAW);
1407 DEBTRACE( "_pyfunc refcnt: " << _pyfunc->ob_refcnt );
1408 PyObject* result = PyObject_CallObject( _pyfunc , args ) ;
1409 DEBTRACE( "_pyfunc refcnt: " << _pyfunc->ob_refcnt );
1416 PyObject* new_stderr = newPyStdOut(_errorDetails);
1417 PySys_SetObject((char*)"stderr", new_stderr);
1418 std::ostringstream stream;
1419 stream << "/tmp/PythonNode_";
1421 ofstream errorfile(stream.str().c_str());
1422 if (errorfile.is_open())
1424 errorfile << _script;
1428 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
1429 Py_DECREF(new_stderr);
1430 throw Exception("Error during execution");
1432 DEBTRACE( "----------------End PyFuncNode::calculation---------------" );
1434 DEBTRACE( "-----------------PyFuncNode::outputs-----------------" );
1436 if(result == Py_None)
1438 else if(PyTuple_Check(result))
1439 nres=PyTuple_Size(result);
1441 if(getNumberOfOutputPorts() != nres)
1443 std::string msg="Number of output arguments : Mismatch between definition and execution";
1446 throw Exception(msg);
1451 PyObject_Print(result,stderr,Py_PRINT_RAW);
1454 list<OutputPort *>::iterator iter;
1457 for(iter = _setOfOutputPort.begin(); iter != _setOfOutputPort.end(); iter++)
1459 OutputPyPort *p=(OutputPyPort *)*iter;
1460 DEBTRACE( "port name: " << p->getName() );
1461 DEBTRACE( "port kind: " << p->edGetType()->kind() );
1462 DEBTRACE( "port pos : " << pos );
1463 if(PyTuple_Check(result))ob=PyTuple_GetItem(result,pos) ;
1465 DEBTRACE( "ob refcnt: " << ob->ob_refcnt );
1467 PyObject_Print(ob,stderr,Py_PRINT_RAW);
1474 catch(ConversionException& ex)
1477 _errorDetails=ex.what();
1480 DEBTRACE( "-----------------End PyFuncNode::outputs-----------------" );
1483 DEBTRACE( "++++++++++++++ End PyFuncNode::execute: " << getName() << " ++++++++++++++++++++" );
1486 Node *PyFuncNode::simpleClone(ComposedNode *father, bool editionOnly) const
1488 return new PyFuncNode(*this,father);
1491 void PyFuncNode::createRemoteAdaptedPyInterpretor(Engines::Container_ptr objContainer)
1493 if(!CORBA::is_nil(_pynode))
1494 _pynode->UnRegister();
1495 _pynode=objContainer->createPyNode(getName().c_str(),getScript().c_str());
1498 Engines::PyNodeBase_var PyFuncNode::retrieveDftRemotePyInterpretorIfAny(Engines::Container_ptr objContainer) const
1500 Engines::PyNode_var ret(objContainer->getDefaultPyNode(getName().c_str()));
1501 if(!CORBA::is_nil(ret))
1505 return Engines::PyNodeBase::_narrow(ret);
1508 void PyFuncNode::assignRemotePyInterpretor(Engines::PyNodeBase_var remoteInterp)
1510 if(!CORBA::is_nil(_pynode))
1512 Engines::PyNode_var tmpp(Engines::PyNode::_narrow(remoteInterp));
1513 if(_pynode->_is_equivalent(tmpp))
1516 if(!CORBA::is_nil(_pynode))
1517 _pynode->UnRegister();
1518 _pynode=Engines::PyNode::_narrow(remoteInterp);
1521 Engines::PyNodeBase_var PyFuncNode::getRemoteInterpreterHandle()
1523 return Engines::PyNodeBase::_narrow(_pynode);
1526 //! Create a new node of same type with a given name
1527 PyFuncNode* PyFuncNode::cloneNode(const std::string& name)
1529 PyFuncNode* n=new PyFuncNode(name);
1530 n->setScript(_script);
1531 n->setFname(_fname);
1532 list<InputPort *>::iterator iter;
1533 for(iter = _setOfInputPort.begin(); iter != _setOfInputPort.end(); iter++)
1535 InputPyPort *p=(InputPyPort *)*iter;
1536 n->edAddInputPort(p->getName(),p->edGetType());
1538 list<OutputPort *>::iterator iter2;
1539 for(iter2 = _setOfOutputPort.begin(); iter2 != _setOfOutputPort.end(); iter2++)
1541 OutputPyPort *p=(OutputPyPort *)*iter2;
1542 n->edAddOutputPort(p->getName(),p->edGetType());
1547 std::string PyFuncNode::getContainerLog()
1549 return PythonEntry::GetContainerLog(_mode,_container,this);
1552 void PyFuncNode::shutdown(int level)
1554 DEBTRACE("PyFuncNode::shutdown " << level);
1555 if(_mode=="local")return;
1558 if(!CORBA::is_nil(_pynode)) _pynode->UnRegister();
1559 _pynode=Engines::PyNode::_nil();
1560 _container->shutdown(level);
1564 void PyFuncNode::imposeResource(const std::string& resource_name,
1565 const std::string& container_name)
1567 if(!resource_name.empty() && !container_name.empty())
1569 _imposedResource = resource_name;
1570 _imposedContainer = container_name;
1574 bool PyFuncNode::canAcceptImposedResource()
1576 return _container != nullptr && _container->canAcceptImposedResource();
1579 bool PyFuncNode::hasImposedResource()const
1581 return PythonEntry::hasImposedResource();