1 // Copyright (C) 2006-2022 CEA/DEN, EDF R&D
3 // This library is free software; you can redistribute it and/or
4 // modify it under the terms of the GNU Lesser General Public
5 // License as published by the Free Software Foundation; either
6 // version 2.1 of the License, or (at your option) any later version.
8 // This library is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 // Lesser General Public License for more details.
13 // You should have received a copy of the GNU Lesser General Public
14 // License along with this library; if not, write to the Free Software
15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
20 #include "RuntimeSALOME.hxx"
21 #include "PythonNode.hxx"
22 #include "PythonPorts.hxx"
23 #include "TypeCode.hxx"
24 #include "PythonCppUtils.hxx"
25 #include "Container.hxx"
26 #include "SalomeContainer.hxx"
27 #include "SalomeHPContainer.hxx"
28 #include "SalomeContainerTmpForHP.hxx"
29 #include "ConversionException.hxx"
30 #include "ReceiverFactory.hxx"
31 #include "SenderByteImpl.hxx"
33 #include "PyStdout.hxx"
41 #define getpid _getpid
44 #if PY_VERSION_HEX < 0x02050000
45 typedef int Py_ssize_t;
49 #include "YacsTrace.hxx"
51 using namespace YACS::ENGINE;
54 const char PythonEntry::SCRIPT_FOR_SIMPLE_SERIALIZATION[]="import pickle\n"
55 "def pickleForVarSimplePyth2009(val):\n"
56 " return pickle.dumps(val,-1)\n"
59 const char PythonNode::IMPL_NAME[]="Python";
60 const char PythonNode::KIND[]="Python";
62 const char PythonNode::SCRIPT_FOR_SERIALIZATION[]="import pickle\n"
63 "def pickleForDistPyth2009(kws):\n"
64 " return pickle.dumps(((),kws),-1)\n"
66 "def unPickleForDistPyth2009(st):\n"
67 " args=pickle.loads(st)\n"
70 const char PythonNode::REMOTE_NAME[]="remote";
72 const char PythonNode::DPL_INFO_NAME[]="my_dpl_localization";
74 const char PyFuncNode::SCRIPT_FOR_SERIALIZATION[]="import pickle\n"
75 "def pickleForDistPyth2009(*args,**kws):\n"
76 " return pickle.dumps((args,kws),-1)\n"
78 "def unPickleForDistPyth2009(st):\n"
79 " args=pickle.loads(st)\n"
82 static char SCRIPT_FOR_BIGOBJECT[]="import SALOME_PyNode\n"
83 "BigObjectOnDisk = SALOME_PyNode.BigObjectOnDisk\n";
85 // pickle.load concurrency issue : see https://bugs.python.org/issue12680
86 #if PY_VERSION_HEX < 0x03070000
88 static std::mutex data_mutex;
91 PythonEntry::PythonEntry():_context(0),_pyfuncSer(0),_pyfuncUnser(0),_pyfuncSimpleSer(0)
95 PythonEntry::~PythonEntry()
98 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
99 // not Py_XDECREF of _pyfuncUnser because it is returned by PyDict_GetItem -> borrowed
100 // not Py_XDECREF of _pyfuncSer because it is returned by PyDict_GetItem -> borrowed
101 Py_XDECREF(_context);
104 void PythonEntry::loadRemoteContainer(InlineNode *reqNode)
106 DEBTRACE( "---------------PythonEntry::CommonRemoteLoad function---------------" );
107 Container *container(reqNode->getContainer());
108 bool isContAlreadyStarted(false);
113 if(hasImposedResource())
114 container->start(reqNode, _imposedResource, _imposedContainer);
117 isContAlreadyStarted=container->isAlreadyStarted(reqNode);
118 if(!isContAlreadyStarted)
119 container->start(reqNode);
124 reqNode->setErrorDetails(e.what());
130 std::string what("PythonEntry::CommonRemoteLoad : a load operation requested on \"");
131 what+=reqNode->getName(); what+="\" with no container specified.";
132 reqNode->setErrorDetails(what);
133 throw Exception(what);
137 Engines::Container_var GetContainerObj(InlineNode *reqNode, bool& isStandardCont)
139 isStandardCont = false;
140 Container *container(reqNode->getContainer());
141 Engines::Container_var objContainer(Engines::Container::_nil());
143 throw YACS::Exception("No container specified !");
144 SalomeContainer *containerCast0(dynamic_cast<SalomeContainer *>(container));
145 SalomeHPContainer *containerCast1(dynamic_cast<SalomeHPContainer *>(container));
148 isStandardCont = true;
149 objContainer=containerCast0->getContainerPtr(reqNode);
151 else if(containerCast1)
153 YACS::BASES::AutoCppPtr<SalomeContainerTmpForHP> tmpCont(SalomeContainerTmpForHP::BuildFrom(containerCast1,reqNode));
154 objContainer=tmpCont->getContainerPtr(reqNode);
157 throw YACS::Exception("Unrecognized type of container ! Salome one is expected for PythonNode/PyFuncNode !");
158 if(CORBA::is_nil(objContainer))
159 throw YACS::Exception("Container corba pointer is NULL for PythonNode !");
163 Engines::Container_var PythonEntry::loadPythonAdapter(InlineNode *reqNode, bool& isInitializeRequested)
165 bool isStandardCont(true);
166 Engines::Container_var objContainer(GetContainerObj(reqNode,isStandardCont));
167 isInitializeRequested=false;
170 Engines::PyNodeBase_var dftPyScript(retrieveDftRemotePyInterpretorIfAny(objContainer));
171 if(CORBA::is_nil(dftPyScript))
173 isInitializeRequested=!isStandardCont;
174 createRemoteAdaptedPyInterpretor(objContainer);
177 assignRemotePyInterpretor(dftPyScript);
179 catch( const SALOME::SALOME_Exception& ex )
181 std::string msg="Exception on remote python node creation ";
183 msg += ex.details.text.in();
184 reqNode->setErrorDetails(msg);
185 throw Exception(msg);
187 Engines::PyNodeBase_var pynode(getRemoteInterpreterHandle());
188 if(CORBA::is_nil(pynode))
189 throw Exception("In PythonNode the ref in NULL ! ");
193 void PythonEntry::loadRemoteContext(InlineNode *reqNode, Engines::Container_ptr objContainer, bool isInitializeRequested)
195 Container *container(reqNode->getContainer());
196 Engines::PyNodeBase_var pynode(getRemoteInterpreterHandle());
199 #if PY_VERSION_HEX < 0x03070000
200 std::unique_lock<std::mutex> lock(data_mutex);
203 const char *picklizeScript(getSerializationScript());
204 PyObject *res=PyRun_String(picklizeScript,Py_file_input,_context,_context);
205 PyObject *res2(PyRun_String(SCRIPT_FOR_SIMPLE_SERIALIZATION,Py_file_input,_context,_context));
206 if(res == NULL || res2==NULL)
208 std::string errorDetails;
209 PyObject* new_stderr = newPyStdOut(errorDetails);
210 reqNode->setErrorDetails(errorDetails);
211 PySys_SetObject((char*)"stderr", new_stderr);
213 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
214 Py_DECREF(new_stderr);
215 throw Exception("Error during load");
217 Py_DECREF(res); Py_DECREF(res2);
218 AutoPyRef res3(PyRun_String(SCRIPT_FOR_BIGOBJECT,Py_file_input,_context,_context));
219 _pyfuncSer=PyDict_GetItemString(_context,"pickleForDistPyth2009");
220 _pyfuncUnser=PyDict_GetItemString(_context,"unPickleForDistPyth2009");
221 _pyfuncSimpleSer=PyDict_GetItemString(_context,"pickleForVarSimplePyth2009");
222 _pyClsBigObject=PyDict_GetItemString(_context,"BigObjectOnDisk");
223 if(_pyfuncSer == NULL)
225 std::string errorDetails;
226 PyObject *new_stderr(newPyStdOut(errorDetails));
227 reqNode->setErrorDetails(errorDetails);
228 PySys_SetObject((char*)"stderr", new_stderr);
230 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
231 Py_DECREF(new_stderr);
232 throw Exception("Error during load");
234 if(_pyfuncUnser == NULL)
236 std::string errorDetails;
237 PyObject *new_stderr(newPyStdOut(errorDetails));
238 reqNode->setErrorDetails(errorDetails);
239 PySys_SetObject((char*)"stderr", new_stderr);
241 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
242 Py_DECREF(new_stderr);
243 throw Exception("Error during load");
245 if(_pyfuncSimpleSer == NULL)
247 std::string errorDetails;
248 PyObject *new_stderr(newPyStdOut(errorDetails));
249 reqNode->setErrorDetails(errorDetails);
250 PySys_SetObject((char*)"stderr", new_stderr);
252 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
253 Py_DECREF(new_stderr);
254 throw Exception("Error during load");
257 if(isInitializeRequested)
258 {//This one is called only once at initialization in the container if an init-script is specified.
261 std::string zeInitScriptKey(container->getProperty(HomogeneousPoolContainer::INITIALIZE_SCRIPT_KEY));
262 if(!zeInitScriptKey.empty())
263 pynode->executeAnotherPieceOfCode(zeInitScriptKey.c_str());
265 catch( const SALOME::SALOME_Exception& ex )
267 std::string msg="Exception on PythonNode::loadRemote python invocation of initializisation py script !";
269 msg += ex.details.text.in();
270 reqNode->setErrorDetails(msg);
271 throw Exception(msg);
273 DEBTRACE( "---------------End PyNode::loadRemote function---------------" );
277 std::string PythonEntry::GetContainerLog(const std::string& mode, Container *container, const Task *askingTask)
285 SalomeContainer *containerCast(dynamic_cast<SalomeContainer *>(container));
286 SalomeHPContainer *objContainer2(dynamic_cast<SalomeHPContainer *>(container));
289 Engines::Container_var objContainer(containerCast->getContainerPtr(askingTask));
290 CORBA::String_var logname = objContainer->logfilename();
293 std::string::size_type pos = msg.find(":");
294 msg=msg.substr(pos+1);
296 else if(objContainer2)
298 msg="Remote PythonNode is on HP Container : no log because no info of the location by definition of HP Container !";
302 msg="Not implemented yet for container log for that type of container !";
307 msg = "Container no longer reachable";
312 void PythonEntry::commonRemoteLoad(InlineNode *reqNode)
314 loadRemoteContainer(reqNode);
315 bool isInitializeRequested;
316 Engines::Container_var objContainer(loadPythonAdapter(reqNode,isInitializeRequested));
317 loadRemoteContext(reqNode,objContainer,isInitializeRequested);
320 bool PythonEntry::hasImposedResource()const
322 return !_imposedResource.empty() && !_imposedContainer.empty();
325 PythonNode::PythonNode(const PythonNode& other, ComposedNode *father):InlineNode(other,father),_autoSqueeze(other._autoSqueeze)
327 _pynode = Engines::PyScriptNode::_nil();
328 _implementation=IMPL_NAME;
331 _context=PyDict_New();
332 if( PyDict_SetItemString( _context, "__builtins__", getSALOMERuntime()->getBuiltins() ))
335 msg << "Impossible to set builtins" << __FILE__ << ":" << __LINE__;
336 _errorDetails=msg.str();
337 throw Exception(msg.str());
342 PythonNode::PythonNode(const std::string& name):InlineNode(name)
344 _pynode = Engines::PyScriptNode::_nil();
345 _implementation=IMPL_NAME;
348 _context=PyDict_New();
349 if( PyDict_SetItemString( _context, "__builtins__", getSALOMERuntime()->getBuiltins() ))
352 msg << "Impossible to set builtins" << __FILE__ << ":" << __LINE__;
353 _errorDetails=msg.str();
354 throw Exception(msg.str());
359 PythonNode::~PythonNode()
364 void PythonNode::checkBasicConsistency() const
366 DEBTRACE("checkBasicConsistency");
367 InlineNode::checkBasicConsistency();
371 res=Py_CompileString(_script.c_str(),getName().c_str(),Py_file_input);
374 std::string error="";
375 PyObject* new_stderr = newPyStdOut(error);
376 PySys_SetObject((char*)"stderr", new_stderr);
378 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
379 Py_DECREF(new_stderr);
380 throw Exception(error);
387 void PythonNode::load()
389 DEBTRACE( "---------------PyNode::load function---------------" );
390 if(_mode==PythonNode::REMOTE_NAME)
396 void PythonNode::loadLocal()
398 DEBTRACE( "---------------PyNode::loadLocal function---------------" );
402 void PythonNode::loadRemote()
404 commonRemoteLoad(this);
407 void PythonNode::execute()
409 if(_mode==PythonNode::REMOTE_NAME)
415 void PythonNode::executeRemote()
417 DEBTRACE( "++++++++++++++ PyNode::executeRemote: " << getName() << " ++++++++++++++++++++" );
419 throw Exception("PythonNode badly loaded");
421 if(dynamic_cast<HomogeneousPoolContainer *>(getContainer()))
424 loadPythonAdapter(this,dummy);
425 _pynode->assignNewCompiledCode(getScript().c_str());
427 // not managed by unique_ptr here because destructed by the order of client.
428 SenderByteImpl *serializationInputCorba = nullptr;
429 AutoPyRef serializationInput;
431 #if PY_VERSION_HEX < 0x03070000
432 std::unique_lock<std::mutex> lock(data_mutex);
435 PyObject *args(0),*ob(0);
436 //===========================================================================
437 // Get inputs in input ports, build a Python dict and pickle it
438 //===========================================================================
440 std::list<InputPort *>::iterator iter2;
442 for(iter2 = _setOfInputPort.begin(); iter2 != _setOfInputPort.end(); ++iter2)
444 InputPyPort *p=(InputPyPort *)*iter2;
446 PyDict_SetItemString(args,p->getName().c_str(),ob);
450 PyObject_Print(args,stderr,Py_PRINT_RAW);
453 serializationInput.set(PyObject_CallFunctionObjArgs(_pyfuncSer,args,nullptr));
455 //The pickled string may contain NULL characters so use PyString_AsStringAndSize
456 char *serializationInputC(nullptr);
458 if (PyBytes_AsStringAndSize(serializationInput, &serializationInputC, &len))
459 throw Exception("DistributedPythonNode problem in python pickle");
460 // no copy here. The C byte array of Python is taken as this into CORBA sequence to avoid copy
461 serializationInputCorba = new SenderByteImpl(serializationInputC,len);
464 //get the list of output argument names
465 std::list<OutputPort *>::iterator iter;
466 Engines::listofstring myseq;
467 myseq.length(getNumberOfOutputPorts());
469 for(iter = _setOfOutputPort.begin(); iter != _setOfOutputPort.end(); ++iter)
471 OutputPyPort *p=(OutputPyPort *)*iter;
472 myseq[pos]=p->getName().c_str();
473 DEBTRACE( "port name: " << p->getName() );
474 DEBTRACE( "port kind: " << p->edGetType()->kind() );
475 DEBTRACE( "port pos : " << pos );
478 //===========================================================================
479 // Execute in remote Python node
480 //===========================================================================
481 DEBTRACE( "-----------------starting remote python invocation-----------------" );
482 std::unique_ptr<SALOME::SenderByteSeq> resultCorba;
485 //pass outargsname and dict serialized
486 SALOME::SenderByte_var serializationInputRef = serializationInputCorba->_this();
487 _pynode->executeFirst(serializationInputRef);
488 //serializationInput and serializationInputCorba are no more needed for server. Release it.
489 serializationInput.set(nullptr);
490 resultCorba.reset( _pynode->executeSecond(myseq) );
491 if( ! this->isUsingPythonCache() )
492 _pynode->removeAllVarsInContext();
494 catch( const SALOME::SALOME_Exception& ex )
496 std::ostringstream msg; msg << "Exception on remote python invocation" << std::endl << ex.details.text.in() << std::endl;
497 msg << "PyScriptNode CORBA ref : ";
499 CORBA::ORB_ptr orb(getSALOMERuntime()->getOrb());
500 if(!CORBA::is_nil(orb))
502 CORBA::String_var IOR(orb->object_to_string(_pynode));
507 _errorDetails=msg.str();
508 throw Exception(msg.str());
510 catch(CORBA::COMM_FAILURE& ex)
512 std::ostringstream msg;
513 msg << "Exception on remote python invocation." << std::endl ;
514 msg << "Caught system exception COMM_FAILURE -- unable to contact the "
515 << "object." << std::endl;
516 _errorDetails=msg.str();
517 throw Exception(msg.str());
519 catch(CORBA::SystemException& ex)
521 std::ostringstream msg;
522 msg << "Exception on remote python invocation." << std::endl ;
523 msg << "Caught a CORBA::SystemException." ;
526 CORBA::TypeCode_var tc = tmp.type();
527 const char *p = tc->name();
533 _errorDetails=msg.str();
534 throw Exception(msg.str());
536 catch(CORBA::Exception& ex)
538 std::ostringstream msg;
539 msg << "Exception on remote python invocation." << std::endl ;
540 msg << "Caught CORBA::Exception. " ;
543 CORBA::TypeCode_var tc = tmp.type();
544 const char *p = tc->name();
550 _errorDetails=msg.str();
551 throw Exception(msg.str());
553 catch(omniORB::fatalException& fe)
555 std::ostringstream msg;
556 msg << "Exception on remote python invocation." << std::endl ;
557 msg << "Caught omniORB::fatalException:" << std::endl;
558 msg << " file: " << fe.file() << std::endl;
559 msg << " line: " << fe.line() << std::endl;
560 msg << " mesg: " << fe.errmsg() << std::endl;
561 _errorDetails=msg.str();
562 throw Exception(msg.str());
564 DEBTRACE( "-----------------end of remote python invocation-----------------" );
565 //===========================================================================
566 // Get results, unpickle and put them in output ports
567 //===========================================================================
569 #if PY_VERSION_HEX < 0x03070000
570 std::unique_lock<std::mutex> lock(data_mutex);
573 DEBTRACE( "-----------------PythonNode::outputs-----------------" );
574 int nres( resultCorba->length() );
576 if(getNumberOfOutputPorts() != nres)
578 std::string msg="Number of output arguments : Mismatch between definition and execution";
580 throw Exception(msg);
585 for(iter = _setOfOutputPort.begin(); iter != _setOfOutputPort.end(); ++iter)
587 OutputPyPort *p=(OutputPyPort *)*iter;
588 DEBTRACE( "port name: " << p->getName() );
589 DEBTRACE( "port kind: " << p->edGetType()->kind() );
590 DEBTRACE( "port pos : " << pos );
591 SALOME::SenderByte_var elt = (*resultCorba)[pos];
592 SeqByteReceiver recv(elt);
593 unsigned long length = 0;
594 char *resultCorbaC = recv.data(length);
596 AutoPyRef resultPython=PyMemoryView_FromMemory(resultCorbaC,length,PyBUF_READ);
597 AutoPyRef args = PyTuple_New(1);
598 PyTuple_SetItem(args,0,resultPython.retn());
599 AutoPyRef ob = PyObject_CallObject(_pyfuncUnser,args);
602 std::stringstream msg;
603 msg << "Conversion with pickle of output ports failed !";
604 msg << " : " << __FILE__ << ":" << __LINE__;
605 _errorDetails=msg.str();
606 throw YACS::ENGINE::ConversionException(msg.str());
609 if( PyObject_IsInstance( ob, _pyClsBigObject) == 1 )
611 AutoPyRef unlinkOnDestructor = PyObject_GetAttrString(ob,"unlinkOnDestructor");
612 AutoPyRef tmp = PyObject_CallFunctionObjArgs(unlinkOnDestructor,nullptr);
619 catch(ConversionException& ex)
621 _errorDetails=ex.what();
625 squeezeMemoryRemote();
628 if(!isUsingPythonCache())
632 Engines::Container_var cont(GetContainerObj(this,dummy));
633 cont->removePyScriptNode(getName().c_str());
635 DEBTRACE( "++++++++++++++ ENDOF PyNode::executeRemote: " << getName() << " ++++++++++++++++++++" );
638 void PythonNode::executeLocal()
640 DEBTRACE( "++++++++++++++ PyNode::executeLocal: " << getName() << " ++++++++++++++++++++" );
644 DEBTRACE( "---------------PyNode::inputs---------------" );
645 list<InputPort *>::iterator iter2;
646 for(iter2 = _setOfInputPort.begin(); iter2 != _setOfInputPort.end(); iter2++)
648 InputPyPort *p=(InputPyPort *)*iter2;
649 DEBTRACE( "port name: " << p->getName() );
650 DEBTRACE( "port kind: " << p->edGetType()->kind() );
651 PyObject* ob=p->getPyObj();
652 DEBTRACE( "ob refcnt: " << ob->ob_refcnt );
654 PyObject_Print(ob,stderr,Py_PRINT_RAW);
657 int ier=PyDict_SetItemString(_context,p->getName().c_str(),ob);
658 DEBTRACE( "after PyDict_SetItemString:ob refcnt: " << ob->ob_refcnt );
661 DEBTRACE( "---------------End PyNode::inputs---------------" );
664 DEBTRACE( "----------------PyNode::calculation---------------" );
666 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
668 std::ostringstream stream;
669 stream << "/tmp/PythonNode_";
672 PyObject* code=Py_CompileString(_script.c_str(), stream.str().c_str(), Py_file_input);
676 PyObject* new_stderr = newPyStdOut(_errorDetails);
677 PySys_SetObject((char*)"stderr", new_stderr);
679 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
680 Py_DECREF(new_stderr);
681 throw Exception("Error during execution");
683 PyObject *res = PyEval_EvalCode( code, _context, _context);
687 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
690 if(PyErr_Occurred ())
693 PyObject* new_stderr = newPyStdOut(_errorDetails);
694 PySys_SetObject((char*)"stderr", new_stderr);
695 ofstream errorfile(stream.str().c_str());
696 if (errorfile.is_open())
698 errorfile << _script;
702 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
703 Py_DECREF(new_stderr);
704 throw Exception("Error during execution");
707 DEBTRACE( "-----------------PyNode::outputs-----------------" );
708 list<OutputPort *>::iterator iter;
711 for(iter = _setOfOutputPort.begin(); iter != _setOfOutputPort.end(); iter++)
713 OutputPyPort *p=(OutputPyPort *)*iter;
714 DEBTRACE( "port name: " << p->getName() );
715 DEBTRACE( "port kind: " << p->edGetType()->kind() );
716 PyObject *ob=PyDict_GetItemString(_context,p->getName().c_str());
719 std::string msg="Error during execution: there is no variable ";
720 msg=msg+p->getName()+" in node context";
722 throw Exception(msg);
724 DEBTRACE( "PyNode::outputs::ob refcnt: " << ob->ob_refcnt );
726 PyObject_Print(ob,stderr,Py_PRINT_RAW);
732 catch(ConversionException& ex)
734 _errorDetails=ex.what();
739 DEBTRACE( "-----------------End PyNode::outputs-----------------" );
741 DEBTRACE( "++++++++++++++ End PyNode::execute: " << getName() << " ++++++++++++++++++++" );
744 void PythonNode::squeezeMemorySafe()
747 if(_mode==PythonNode::REMOTE_NAME)
748 this->squeezeMemoryRemote();
750 this->squeezeMemory();
753 void PythonNode::squeezeMemory()
755 for(auto p : _setOfInputPort)
757 PyDict_DelItemString(_context,p->getName().c_str());
758 InputPyPort *p2(static_cast<InputPyPort *>(p));
759 if(p2->canSafelySqueezeMemory())
762 for(auto p : _setOfOutputPort)
764 PyDict_DelItemString(_context,p->getName().c_str());
765 OutputPyPort *p2(static_cast<OutputPyPort *>(p));
766 p2->putWithoutForward(Py_None);
770 void PythonNode::squeezeMemoryRemote()
772 for(auto p : _setOfInputPort)
774 InputPyPort *p2(static_cast<InputPyPort *>(p));
775 if(p2->canSafelySqueezeMemory())
778 for(auto p : _setOfOutputPort)
780 OutputPyPort *p2(static_cast<OutputPyPort *>(p));
781 p2->putWithoutForward(Py_None);
785 std::string PythonNode::getContainerLog()
787 return PythonEntry::GetContainerLog(_mode,_container,this);
790 void PythonNode::shutdown(int level)
792 DEBTRACE("PythonNode::shutdown " << level);
793 if(_mode=="local")return;
797 _container->shutdown(level);
801 void PythonNode::imposeResource(const std::string& resource_name,
802 const std::string& container_name)
804 if(!resource_name.empty() && !container_name.empty())
806 _imposedResource = resource_name;
807 _imposedContainer = container_name;
811 bool PythonNode::canAcceptImposedResource()
813 return _container != nullptr && _container->canAcceptImposedResource();
816 bool PythonNode::hasImposedResource()const
818 return PythonEntry::hasImposedResource();
821 std::string PythonNode::pythonEntryName()const
823 if(isUsingPythonCache())
824 return "DEFAULT_NAME_FOR_UNIQUE_PYTHON_NODE_ENTRY";
829 bool PythonNode::isUsingPythonCache()const
833 found = _container->isUsingPythonCache();
837 void PythonNode::freeKernelPynode()
839 if(!CORBA::is_nil(_pynode))
843 _pynode->UnRegister();
847 DEBTRACE("Trouble when pynode->UnRegister!")
849 _pynode = Engines::PyScriptNode::_nil();
853 Node *PythonNode::simpleClone(ComposedNode *father, bool editionOnly) const
855 return new PythonNode(*this,father);
858 void PythonNode::createRemoteAdaptedPyInterpretor(Engines::Container_ptr objContainer)
861 _pynode=objContainer->createPyScriptNode(pythonEntryName().c_str(),getScript().c_str());
865 Engines::PyNodeBase_var PythonNode::retrieveDftRemotePyInterpretorIfAny(Engines::Container_ptr objContainer) const
867 Engines::PyScriptNode_var ret(objContainer->getDefaultPyScriptNode(pythonEntryName().c_str()));
868 if(!CORBA::is_nil(ret))
872 return Engines::PyNodeBase::_narrow(ret);
875 void PythonNode::assignRemotePyInterpretor(Engines::PyNodeBase_var remoteInterp)
877 if(CORBA::is_nil(_pynode))
878 _pynode=Engines::PyScriptNode::_narrow(remoteInterp);
881 Engines::PyScriptNode_var tmpp(Engines::PyScriptNode::_narrow(remoteInterp));
882 if(!_pynode->_is_equivalent(tmpp))
885 _pynode=Engines::PyScriptNode::_narrow(remoteInterp);
888 _pynode->assignNewCompiledCode(getScript().c_str());
891 Engines::PyNodeBase_var PythonNode::getRemoteInterpreterHandle()
893 return Engines::PyNodeBase::_narrow(_pynode);
896 //! Create a new node of same type with a given name
897 PythonNode* PythonNode::cloneNode(const std::string& name)
899 PythonNode* n=new PythonNode(name);
900 n->setScript(_script);
901 list<InputPort *>::iterator iter;
902 for(iter = _setOfInputPort.begin(); iter != _setOfInputPort.end(); iter++)
904 InputPyPort *p=(InputPyPort *)*iter;
905 DEBTRACE( "port name: " << p->getName() );
906 DEBTRACE( "port kind: " << p->edGetType()->kind() );
907 n->edAddInputPort(p->getName(),p->edGetType());
909 list<OutputPort *>::iterator iter2;
910 for(iter2 = _setOfOutputPort.begin(); iter2 != _setOfOutputPort.end(); iter2++)
912 OutputPyPort *p=(OutputPyPort *)*iter2;
913 DEBTRACE( "port name: " << p->getName() );
914 DEBTRACE( "port kind: " << p->edGetType()->kind() );
915 n->edAddOutputPort(p->getName(),p->edGetType());
920 void PythonNode::applyDPLScope(ComposedNode *gfn)
922 std::vector< std::pair<std::string,int> > ret(getDPLScopeInfo(gfn));
929 std::size_t sz(ret.size());
931 for(std::size_t i=0;i<sz;i++)
933 const std::pair<std::string,int>& p(ret[i]);
934 PyObject *elt(PyTuple_New(2));
935 PyTuple_SetItem(elt,0,PyUnicode_FromString(p.first.c_str()));
936 PyTuple_SetItem(elt,1,PyLong_FromLong(p.second));
937 PyList_SetItem(ob,i,elt);
940 if(_mode==REMOTE_NAME)
942 Engines::pickledArgs_var serializationInputCorba(new Engines::pickledArgs);
945 PyObject *serializationInput(PyObject_CallFunctionObjArgs(_pyfuncSimpleSer,ob,NULL));
947 char *serializationInputC(0);
949 if (PyBytes_AsStringAndSize(serializationInput, &serializationInputC, &len))
950 throw Exception("DistributedPythonNode problem in python pickle");
951 serializationInputCorba->length(len);
952 for(int i=0; i < len ; i++)
953 serializationInputCorba[i]=serializationInputC[i];
954 Py_XDECREF(serializationInput);
956 _pynode->defineNewCustomVar(DPL_INFO_NAME,serializationInputCorba);
961 PyDict_SetItemString(_context,DPL_INFO_NAME,ob);
966 PyFuncNode::PyFuncNode(const PyFuncNode& other, ComposedNode *father):InlineFuncNode(other,father),_pyfunc(0)
968 _implementation = PythonNode::IMPL_NAME;
971 _context=PyDict_New();
972 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
973 if( PyDict_SetItemString( _context, "__builtins__", getSALOMERuntime()->getBuiltins() ))
976 msg << "Not possible to set builtins" << __FILE__ << ":" << __LINE__;
977 _errorDetails=msg.str();
978 throw Exception(msg.str());
983 PyFuncNode::PyFuncNode(const std::string& name): InlineFuncNode(name),_pyfunc(0)
986 _implementation = PythonNode::IMPL_NAME;
987 DEBTRACE( "PyFuncNode::PyFuncNode " << name );
990 _context=PyDict_New();
991 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
992 if( PyDict_SetItemString( _context, "__builtins__", getSALOMERuntime()->getBuiltins() ))
995 msg << "Not possible to set builtins" << __FILE__ << ":" << __LINE__;
996 _errorDetails=msg.str();
997 throw Exception(msg.str());
1002 PyFuncNode::~PyFuncNode()
1004 if(!CORBA::is_nil(_pynode))
1006 _pynode->UnRegister();
1010 void PyFuncNode::init(bool start)
1012 initCommonPartWithoutStateManagement(start);
1013 if(_state == YACS::DISABLED)
1015 exDisabledState(); // to refresh propagation of DISABLED state
1018 if(start) //complete initialization
1019 setState(YACS::READY);
1020 else if(_state > YACS::LOADED)// WARNING FuncNode has internal vars (CEA usecase) ! Partial initialization (inside a loop). Exclusivity of funcNode.
1021 setState(YACS::TORECONNECT);
1024 void PyFuncNode::checkBasicConsistency() const
1026 DEBTRACE("checkBasicConsistency");
1027 InlineFuncNode::checkBasicConsistency();
1031 res=Py_CompileString(_script.c_str(),getName().c_str(),Py_file_input);
1034 std::string error="";
1035 PyObject* new_stderr = newPyStdOut(error);
1036 PySys_SetObject((char*)"stderr", new_stderr);
1038 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
1039 Py_DECREF(new_stderr);
1040 throw Exception(error);
1047 void PyFuncNode::load()
1049 DEBTRACE( "---------------PyfuncNode::load function---------------" );
1050 if(_mode==PythonNode::REMOTE_NAME)
1056 void PyFuncNode::loadRemote()
1058 commonRemoteLoad(this);
1061 void PyFuncNode::loadLocal()
1063 DEBTRACE( "---------------PyFuncNode::load function " << getName() << " ---------------" );
1064 DEBTRACE( _script );
1067 list<OutputPort *>::iterator iter;
1068 for(iter = _setOfOutputPort.begin(); iter != _setOfOutputPort.end(); iter++)
1070 OutputPyPort *p=(OutputPyPort *)*iter;
1071 DEBTRACE( "port name: " << p->getName() );
1072 DEBTRACE( "port kind: " << p->edGetType()->kind() );
1078 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
1080 std::ostringstream stream;
1081 stream << "/tmp/PythonNode_";
1084 PyObject* code=Py_CompileString(_script.c_str(), stream.str().c_str(), Py_file_input);
1088 PyObject* new_stderr = newPyStdOut(_errorDetails);
1089 PySys_SetObject((char*)"stderr", new_stderr);
1091 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
1092 Py_DECREF(new_stderr);
1093 throw Exception("Error during execution");
1095 PyObject *res = PyEval_EvalCode( code, _context, _context);
1099 DEBTRACE( "_context refcnt: " << _context->ob_refcnt );
1100 if(PyErr_Occurred ())
1103 PyObject* new_stderr = newPyStdOut(_errorDetails);
1104 PySys_SetObject((char*)"stderr", new_stderr);
1105 ofstream errorfile(stream.str().c_str());
1106 if (errorfile.is_open())
1108 errorfile << _script;
1112 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
1113 Py_DECREF(new_stderr);
1114 throw Exception("Error during execution");
1117 _pyfunc=PyDict_GetItemString(_context,_fname.c_str());
1118 DEBTRACE( "_pyfunc refcnt: " << _pyfunc->ob_refcnt );
1122 PyObject* new_stderr = newPyStdOut(_errorDetails);
1123 PySys_SetObject((char*)"stderr", new_stderr);
1125 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
1126 Py_DECREF(new_stderr);
1127 throw Exception("Error during execution");
1129 DEBTRACE( "---------------End PyFuncNode::load function---------------" );
1133 void PyFuncNode::execute()
1135 if(_mode==PythonNode::REMOTE_NAME)
1141 void PyFuncNode::executeRemote()
1143 DEBTRACE( "++++++++++++++ PyFuncNode::executeRemote: " << getName() << " ++++++++++++++++++++" );
1145 throw Exception("DistributedPythonNode badly loaded");
1147 if(dynamic_cast<HomogeneousPoolContainer *>(getContainer()))
1150 loadPythonAdapter(this,dummy);
1151 _pynode->executeAnotherPieceOfCode(getScript().c_str());
1154 Engines::pickledArgs_var serializationInputCorba(new Engines::pickledArgs);;
1156 #if PY_VERSION_HEX < 0x03070000
1157 std::unique_lock<std::mutex> lock(data_mutex);
1161 //===========================================================================
1162 // Get inputs in input ports, build a Python tuple and pickle it
1163 //===========================================================================
1164 PyObject *args(PyTuple_New(getNumberOfInputPorts()));
1166 for(std::list<InputPort *>::iterator iter2 = _setOfInputPort.begin(); iter2 != _setOfInputPort.end(); iter2++,pos++)
1168 InputPyPort *p=(InputPyPort *)*iter2;
1171 PyTuple_SetItem(args,pos,ob);
1174 PyObject_Print(args,stderr,Py_PRINT_RAW);
1177 PyObject *serializationInput=PyObject_CallObject(_pyfuncSer,args);
1179 //The pickled string may contain NULL characters so use PyString_AsStringAndSize
1180 char *serializationInputC(0);
1182 if (PyBytes_AsStringAndSize(serializationInput, &serializationInputC, &len))
1183 throw Exception("DistributedPythonNode problem in python pickle");
1185 serializationInputCorba->length(len);
1186 for(int i=0; i < len ; i++)
1187 serializationInputCorba[i]=serializationInputC[i];
1188 Py_DECREF(serializationInput);
1191 //===========================================================================
1192 // Execute in remote Python node
1193 //===========================================================================
1194 DEBTRACE( "-----------------starting remote python invocation-----------------" );
1195 Engines::pickledArgs_var resultCorba;
1198 resultCorba=_pynode->execute(getFname().c_str(),serializationInputCorba);
1200 catch( const SALOME::SALOME_Exception& ex )
1202 std::string msg="Exception on remote python invocation";
1204 msg += ex.details.text.in();
1206 throw Exception(msg);
1208 catch(CORBA::COMM_FAILURE& ex)
1210 std::ostringstream msg;
1211 msg << "Exception on remote python invocation." << std::endl ;
1212 msg << "Caught system exception COMM_FAILURE -- unable to contact the "
1213 << "object." << std::endl;
1214 _errorDetails=msg.str();
1215 throw Exception(msg.str());
1217 catch(CORBA::SystemException& ex)
1219 std::ostringstream msg;
1220 msg << "Exception on remote python invocation." << std::endl ;
1221 msg << "Caught a CORBA::SystemException." ;
1224 CORBA::TypeCode_var tc = tmp.type();
1225 const char *p = tc->name();
1231 _errorDetails=msg.str();
1232 throw Exception(msg.str());
1234 catch(CORBA::Exception& ex)
1236 std::ostringstream msg;
1237 msg << "Exception on remote python invocation." << std::endl ;
1238 msg << "Caught CORBA::Exception. " ;
1241 CORBA::TypeCode_var tc = tmp.type();
1242 const char *p = tc->name();
1248 _errorDetails=msg.str();
1249 throw Exception(msg.str());
1251 catch(omniORB::fatalException& fe)
1253 std::ostringstream msg;
1254 msg << "Exception on remote python invocation." << std::endl ;
1255 msg << "Caught omniORB::fatalException:" << std::endl;
1256 msg << " file: " << fe.file() << std::endl;
1257 msg << " line: " << fe.line() << std::endl;
1258 msg << " mesg: " << fe.errmsg() << std::endl;
1259 _errorDetails=msg.str();
1260 throw Exception(msg.str());
1262 DEBTRACE( "-----------------end of remote python invocation-----------------" );
1263 //===========================================================================
1264 // Get results, unpickle and put them in output ports
1265 //===========================================================================
1266 char *resultCorbaC=new char[resultCorba->length()+1];
1267 resultCorbaC[resultCorba->length()]='\0';
1268 for(int i=0;i<resultCorba->length();i++)
1269 resultCorbaC[i]=resultCorba[i];
1272 #if PY_VERSION_HEX < 0x03070000
1273 std::unique_lock<std::mutex> lock(data_mutex);
1277 PyObject *resultPython(PyBytes_FromStringAndSize(resultCorbaC,resultCorba->length()));
1278 delete [] resultCorbaC;
1279 PyObject *args(PyTuple_New(1)),*ob(0);
1280 PyTuple_SetItem(args,0,resultPython);
1281 PyObject *finalResult=PyObject_CallObject(_pyfuncUnser,args);
1284 DEBTRACE( "-----------------PythonNode::outputs-----------------" );
1286 if(finalResult == Py_None)
1288 else if(PyTuple_Check(finalResult))
1289 nres=PyTuple_Size(finalResult);
1291 if(getNumberOfOutputPorts() != nres)
1293 std::string msg="Number of output arguments : Mismatch between definition and execution";
1294 Py_DECREF(finalResult);
1296 throw Exception(msg);
1302 for(std::list<OutputPort *>::iterator iter = _setOfOutputPort.begin(); iter != _setOfOutputPort.end(); iter++, pos++)
1304 OutputPyPort *p=(OutputPyPort *)*iter;
1305 DEBTRACE( "port name: " << p->getName() );
1306 DEBTRACE( "port kind: " << p->edGetType()->kind() );
1307 DEBTRACE( "port pos : " << pos );
1308 if(PyTuple_Check(finalResult))
1309 ob=PyTuple_GetItem(finalResult,pos) ;
1312 DEBTRACE( "ob refcnt: " << ob->ob_refcnt );
1315 Py_DECREF(finalResult);
1317 catch(ConversionException& ex)
1319 Py_DECREF(finalResult);
1320 _errorDetails=ex.what();
1325 DEBTRACE( "++++++++++++++ ENDOF PyFuncNode::executeRemote: " << getName() << " ++++++++++++++++++++" );
1328 void PyFuncNode::executeLocal()
1330 DEBTRACE( "++++++++++++++ PyFuncNode::execute: " << getName() << " ++++++++++++++++++++" );
1334 if(!_pyfunc)throw Exception("PyFuncNode badly loaded");
1337 DEBTRACE( "---------------PyFuncNode::inputs---------------" );
1338 PyObject* args = PyTuple_New(getNumberOfInputPorts()) ;
1339 list<InputPort *>::iterator iter2;
1340 for(iter2 = _setOfInputPort.begin(); iter2 != _setOfInputPort.end(); iter2++)
1342 InputPyPort *p=(InputPyPort *)*iter2;
1343 DEBTRACE( "port name: " << p->getName() );
1344 DEBTRACE( "port kind: " << p->edGetType()->kind() );
1347 PyObject_Print(ob,stderr,Py_PRINT_RAW);
1350 DEBTRACE( "ob refcnt: " << ob->ob_refcnt );
1352 PyTuple_SetItem(args,pos,ob);
1353 DEBTRACE( "ob refcnt: " << ob->ob_refcnt );
1356 DEBTRACE( "---------------End PyFuncNode::inputs---------------" );
1358 DEBTRACE( "----------------PyFuncNode::calculation---------------" );
1360 PyObject_Print(_pyfunc,stderr,Py_PRINT_RAW);
1362 PyObject_Print(args,stderr,Py_PRINT_RAW);
1365 DEBTRACE( "_pyfunc refcnt: " << _pyfunc->ob_refcnt );
1366 PyObject* result = PyObject_CallObject( _pyfunc , args ) ;
1367 DEBTRACE( "_pyfunc refcnt: " << _pyfunc->ob_refcnt );
1374 PyObject* new_stderr = newPyStdOut(_errorDetails);
1375 PySys_SetObject((char*)"stderr", new_stderr);
1376 std::ostringstream stream;
1377 stream << "/tmp/PythonNode_";
1379 ofstream errorfile(stream.str().c_str());
1380 if (errorfile.is_open())
1382 errorfile << _script;
1386 PySys_SetObject((char*)"stderr", PySys_GetObject((char*)"__stderr__"));
1387 Py_DECREF(new_stderr);
1388 throw Exception("Error during execution");
1390 DEBTRACE( "----------------End PyFuncNode::calculation---------------" );
1392 DEBTRACE( "-----------------PyFuncNode::outputs-----------------" );
1394 if(result == Py_None)
1396 else if(PyTuple_Check(result))
1397 nres=PyTuple_Size(result);
1399 if(getNumberOfOutputPorts() != nres)
1401 std::string msg="Number of output arguments : Mismatch between definition and execution";
1404 throw Exception(msg);
1409 PyObject_Print(result,stderr,Py_PRINT_RAW);
1412 list<OutputPort *>::iterator iter;
1415 for(iter = _setOfOutputPort.begin(); iter != _setOfOutputPort.end(); iter++)
1417 OutputPyPort *p=(OutputPyPort *)*iter;
1418 DEBTRACE( "port name: " << p->getName() );
1419 DEBTRACE( "port kind: " << p->edGetType()->kind() );
1420 DEBTRACE( "port pos : " << pos );
1421 if(PyTuple_Check(result))ob=PyTuple_GetItem(result,pos) ;
1423 DEBTRACE( "ob refcnt: " << ob->ob_refcnt );
1425 PyObject_Print(ob,stderr,Py_PRINT_RAW);
1432 catch(ConversionException& ex)
1435 _errorDetails=ex.what();
1438 DEBTRACE( "-----------------End PyFuncNode::outputs-----------------" );
1441 DEBTRACE( "++++++++++++++ End PyFuncNode::execute: " << getName() << " ++++++++++++++++++++" );
1444 Node *PyFuncNode::simpleClone(ComposedNode *father, bool editionOnly) const
1446 return new PyFuncNode(*this,father);
1449 void PyFuncNode::createRemoteAdaptedPyInterpretor(Engines::Container_ptr objContainer)
1451 if(!CORBA::is_nil(_pynode))
1452 _pynode->UnRegister();
1453 _pynode=objContainer->createPyNode(getName().c_str(),getScript().c_str());
1456 Engines::PyNodeBase_var PyFuncNode::retrieveDftRemotePyInterpretorIfAny(Engines::Container_ptr objContainer) const
1458 Engines::PyNode_var ret(objContainer->getDefaultPyNode(getName().c_str()));
1459 if(!CORBA::is_nil(ret))
1463 return Engines::PyNodeBase::_narrow(ret);
1466 void PyFuncNode::assignRemotePyInterpretor(Engines::PyNodeBase_var remoteInterp)
1468 if(!CORBA::is_nil(_pynode))
1470 Engines::PyNode_var tmpp(Engines::PyNode::_narrow(remoteInterp));
1471 if(_pynode->_is_equivalent(tmpp))
1474 if(!CORBA::is_nil(_pynode))
1475 _pynode->UnRegister();
1476 _pynode=Engines::PyNode::_narrow(remoteInterp);
1479 Engines::PyNodeBase_var PyFuncNode::getRemoteInterpreterHandle()
1481 return Engines::PyNodeBase::_narrow(_pynode);
1484 //! Create a new node of same type with a given name
1485 PyFuncNode* PyFuncNode::cloneNode(const std::string& name)
1487 PyFuncNode* n=new PyFuncNode(name);
1488 n->setScript(_script);
1489 n->setFname(_fname);
1490 list<InputPort *>::iterator iter;
1491 for(iter = _setOfInputPort.begin(); iter != _setOfInputPort.end(); iter++)
1493 InputPyPort *p=(InputPyPort *)*iter;
1494 n->edAddInputPort(p->getName(),p->edGetType());
1496 list<OutputPort *>::iterator iter2;
1497 for(iter2 = _setOfOutputPort.begin(); iter2 != _setOfOutputPort.end(); iter2++)
1499 OutputPyPort *p=(OutputPyPort *)*iter2;
1500 n->edAddOutputPort(p->getName(),p->edGetType());
1505 std::string PyFuncNode::getContainerLog()
1507 return PythonEntry::GetContainerLog(_mode,_container,this);
1510 void PyFuncNode::shutdown(int level)
1512 DEBTRACE("PyFuncNode::shutdown " << level);
1513 if(_mode=="local")return;
1516 if(!CORBA::is_nil(_pynode)) _pynode->UnRegister();
1517 _pynode=Engines::PyNode::_nil();
1518 _container->shutdown(level);
1522 void PyFuncNode::imposeResource(const std::string& resource_name,
1523 const std::string& container_name)
1525 if(!resource_name.empty() && !container_name.empty())
1527 _imposedResource = resource_name;
1528 _imposedContainer = container_name;
1532 bool PyFuncNode::canAcceptImposedResource()
1534 return _container != nullptr && _container->canAcceptImposedResource();
1537 bool PyFuncNode::hasImposedResource()const
1539 return PythonEntry::hasImposedResource();