From bc803f251236fa3b020c57975e25f5c6db05434c Mon Sep 17 00:00:00 2001 From: Anthony Geay Date: Tue, 14 Mar 2017 09:01:47 +0100 Subject: [PATCH] [EDF12520] : Optimization of placements of Kernel containers in ForEachLoop/HP Container containers context --- src/CTestTestfileInstall.cmake | 1 + src/bases/AutoRefCnt.hxx | 30 + src/engine/AbstractPoint.hxx | 6 + src/engine/BagPoint.cxx | 54 +- src/engine/BagPoint.hxx | 6 + src/engine/Bloc.cxx | 31 +- src/engine/Bloc.hxx | 9 + src/engine/BlocPoint.cxx | 11 + src/engine/BlocPoint.hxx | 2 + src/engine/Bloc_impl.cxx | 108 +++ src/engine/CMakeLists.txt | 3 + src/engine/DynParaLoop.cxx | 19 +- src/engine/DynParaLoop.hxx | 5 + src/engine/ElementaryNode.hxx | 4 +- src/engine/ElementaryPoint.cxx | 10 + src/engine/ElementaryPoint.hxx | 2 + src/engine/ForkBlocPoint.cxx | 46 ++ src/engine/ForkBlocPoint.hxx | 2 + src/engine/HomogeneousPoolContainer.hxx | 9 + src/engine/InlineNode.cxx | 15 + src/engine/InlineNode.hxx | 1 + src/engine/LinkedBlocPoint.cxx | 14 + src/engine/LinkedBlocPoint.hxx | 2 + src/engine/Loop.cxx | 13 + src/engine/Loop.hxx | 2 + src/engine/Node.cxx | 16 + src/engine/Node.hxx | 7 +- src/engine/PlayGround.cxx | 635 ++++++++++++++++++ src/engine/PlayGround.hxx | 162 +++++ src/engine/RefCounter.cxx | 2 +- src/engine/RefCounter.hxx | 2 +- src/engine/Runtime.hxx | 2 +- src/engine/SetOfPoints.cxx | 15 + src/engine/SetOfPoints.hxx | 9 +- src/engine/Switch.cxx | 14 + src/engine/Switch.hxx | 2 + .../Test/RuntimeForEngineIntegrationTest.cxx | 5 + .../Test/RuntimeForEngineIntegrationTest.hxx | 1 + src/engine/Test/RuntimeForEngineTest.cxx | 5 + src/engine/Test/RuntimeForEngineTest.hxx | 1 + src/engine_swig/CMakeLists.txt | 10 + src/engine_swig/CTestTestfileInstall.cmake | 24 + src/engine_swig/PlayGround.i | 169 +++++ src/engine_swig/pilot.i | 9 +- src/engine_swig/testPlayGround0.py | 76 +++ src/runtime/DistributedPythonNode.cxx | 5 +- src/runtime/PythonNode.cxx | 5 +- src/runtime/RuntimeSALOME.cxx | 52 +- src/runtime/RuntimeSALOME.hxx | 10 +- src/runtime/SalomeContainerHelper.cxx | 7 + src/runtime/SalomeContainerHelper.hxx | 1 + src/runtime/SalomeContainerTools.cxx | 612 ++++++++++------- src/runtime/SalomeContainerTools.hxx | 96 ++- src/runtime/SalomeHPContainer.cxx | 324 ++++++--- src/runtime/SalomeHPContainer.hxx | 116 +++- src/runtime/SalomeHPContainerTools.cxx | 56 +- src/runtime/SalomeHPContainerTools.hxx | 18 +- src/yacsloader_swig/Test/CMakeLists.txt | 4 + .../Test/CTestTestfileInstall.cmake | 6 + src/yacsloader_swig/Test/testHPDecorator.py | 293 ++++++++ 60 files changed, 2742 insertions(+), 434 deletions(-) create mode 100644 src/engine/Bloc_impl.cxx create mode 100644 src/engine/PlayGround.cxx create mode 100644 src/engine/PlayGround.hxx create mode 100644 src/engine_swig/CTestTestfileInstall.cmake create mode 100644 src/engine_swig/PlayGround.i create mode 100644 src/engine_swig/testPlayGround0.py create mode 100644 src/yacsloader_swig/Test/testHPDecorator.py diff --git a/src/CTestTestfileInstall.cmake b/src/CTestTestfileInstall.cmake index 4d5dd69d2..6f6927794 100644 --- a/src/CTestTestfileInstall.cmake +++ b/src/CTestTestfileInstall.cmake @@ -29,6 +29,7 @@ SET(YACS_TEST_LIB "../lib") SUBDIRS( bases engine + engine_swig runtime pmml yacsloader diff --git a/src/bases/AutoRefCnt.hxx b/src/bases/AutoRefCnt.hxx index 003a73e82..346f0406b 100644 --- a/src/bases/AutoRefCnt.hxx +++ b/src/bases/AutoRefCnt.hxx @@ -33,6 +33,9 @@ namespace YACS AutoRefCnt(const AutoRefCnt& other):_ptr(0) { referPtr(other._ptr); } AutoRefCnt(T *ptr=0):_ptr(ptr) { } ~AutoRefCnt() { destroyPtr(); } + bool isNull() const { return _ptr==0; } + bool isNotNull() const { return !isNull(); } + void nullify() { destroyPtr(); _ptr=0; } bool operator==(const AutoRefCnt& other) const { return _ptr==other._ptr; } bool operator==(const T *other) const { return _ptr==other; } AutoRefCnt &operator=(const AutoRefCnt& other) { if(_ptr!=other._ptr) { destroyPtr(); referPtr(other._ptr); } return *this; } @@ -44,6 +47,7 @@ namespace YACS operator T *() { return _ptr; } operator const T *() const { return _ptr; } T *retn() { if(_ptr) _ptr->incrRef(); return _ptr; } + void takeRef(T *ptr) { if(_ptr!=ptr) { destroyPtr(); _ptr=ptr; if(_ptr) _ptr->incrRef(); } } private: void referPtr(T *ptr) { _ptr=ptr; if(_ptr) _ptr->incrRef(); } void destroyPtr() { if(_ptr) _ptr->decrRef(); } @@ -51,6 +55,32 @@ namespace YACS T *_ptr; }; + template + class AutoConstRefCnt + { + public: + AutoConstRefCnt(const AutoConstRefCnt& other):_ptr(0) { referPtr(other._ptr); } + AutoConstRefCnt(const T *ptr=0):_ptr(ptr) { } + ~AutoConstRefCnt() { destroyPtr(); } + bool isNull() const { return _ptr==0; } + bool isNotNull() const { return !isNull(); } + void nullify() { destroyPtr(); _ptr=0; } + bool operator==(const AutoConstRefCnt& other) const { return _ptr==other._ptr; } + bool operator==(const T *other) const { return _ptr==other; } + AutoConstRefCnt &operator=(const AutoConstRefCnt& other) { if(_ptr!=other._ptr) { destroyPtr(); referPtr(other._ptr); } return *this; } + AutoConstRefCnt &operator=(const T *ptr) { if(_ptr!=ptr) { destroyPtr(); _ptr=ptr; } return *this; } + const T *operator->() const { return _ptr; } + const T& operator*() const { return *_ptr; } + operator const T *() const { return _ptr; } + const T *retn() { if(_ptr) _ptr->incrRef(); return _ptr; } + void takeRef(const T *ptr) { if(_ptr!=ptr) { destroyPtr(); _ptr=ptr; if(_ptr) _ptr->incrRef(); } } + private: + void referPtr(const T *ptr) { _ptr=ptr; if(_ptr) _ptr->incrRef(); } + void destroyPtr() { if(_ptr) _ptr->decrRef(); } + private: + const T *_ptr; + }; + template typename YACS::BASES::AutoRefCnt DynamicCast(typename YACS::BASES::AutoRefCnt& autoSubPtr) throw() { diff --git a/src/engine/AbstractPoint.hxx b/src/engine/AbstractPoint.hxx index 9a8d35afa..3b4f5dc64 100644 --- a/src/engine/AbstractPoint.hxx +++ b/src/engine/AbstractPoint.hxx @@ -21,7 +21,10 @@ #define __ABSTRACTPOINT_HXX__ #include "YACSlibEngineExport.hxx" +#include "PlayGround.hxx" +#include "AutoRefCnt.hxx" +#include #include #include #include @@ -34,6 +37,7 @@ namespace YACS class InGate; class OutGate; class BlocPoint; + class ComposedNode; class ForkBlocPoint; class LinkedBlocPoint; @@ -63,6 +67,8 @@ namespace YACS virtual bool contains(Node *node) = 0; virtual int getNumberOfNodes() const = 0; virtual int getMaxLevelOfParallelism() const = 0; + virtual double getWeightRegardingDPL() const = 0; + virtual void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) const = 0; virtual std::string getRepr() const = 0; virtual ~AbstractPoint(); public: diff --git a/src/engine/BagPoint.cxx b/src/engine/BagPoint.cxx index 01d6dc74c..d7727a16d 100644 --- a/src/engine/BagPoint.cxx +++ b/src/engine/BagPoint.cxx @@ -31,28 +31,62 @@ BagPoint::BagPoint(const std::list& nodes, AbstractPoint *fathe { } -Node *BagPoint::getFirstNode() +AbstractPoint *BagPoint::getUnique() { if(_nodes.size()!=1) - throw YACS::Exception("BagPoint::getFirstNode : invalid call !"); + throw YACS::Exception("BagPoint::getUnique : invalid call !"); else - return (*_nodes.begin())->getFirstNode(); + { + AbstractPoint *ret(*_nodes.begin()); + if(!ret) + throw YACS::Exception("BagPoint::getUnique : Ooops !"); + return ret; + } } -Node *BagPoint::getLastNode() +const AbstractPoint *BagPoint::getUnique() const { if(_nodes.size()!=1) - throw YACS::Exception("BagPoint::getLastNode : invalid call !"); + throw YACS::Exception("BagPoint::getUnique const : invalid call !"); else - return (*_nodes.begin())->getLastNode(); + { + AbstractPoint *ret(*_nodes.begin()); + if(!ret) + throw YACS::Exception("BagPoint::getUnique : Ooops !"); + return ret; + } +} + +AbstractPoint *BagPoint::getUniqueAndReleaseIt() +{ + AbstractPoint *ret(getUnique()); + getOutPoint(ret); + return ret; +} + +Node *BagPoint::getFirstNode() +{ + return getUnique()->getFirstNode(); +} + +Node *BagPoint::getLastNode() +{ + return getUnique()->getLastNode(); } int BagPoint::getMaxLevelOfParallelism() const { - if(_nodes.size()!=1) - throw YACS::Exception("BagPoint::getMaxLevelOfParallelism : invalid call !"); - else - return (*_nodes.begin())->getMaxLevelOfParallelism(); + return getUnique()->getMaxLevelOfParallelism(); +} + +double BagPoint::getWeightRegardingDPL() const +{ + return getUnique()->getWeightRegardingDPL(); +} + +void BagPoint::partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) const +{ + getUnique()->partitionRegardingDPL(pd,zeMap); } std::string BagPoint::getRepr() const diff --git a/src/engine/BagPoint.hxx b/src/engine/BagPoint.hxx index f08f52c7e..9568deb07 100644 --- a/src/engine/BagPoint.hxx +++ b/src/engine/BagPoint.hxx @@ -37,7 +37,10 @@ namespace YACS Node *getFirstNode(); Node *getLastNode(); int getMaxLevelOfParallelism() const; + double getWeightRegardingDPL() const; + void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) const; std::string getRepr() const; + AbstractPoint *getUniqueAndReleaseIt(); public: int size() const { return (int)_nodes.size(); } void replaceInMe(BlocPoint *aSet); @@ -45,6 +48,9 @@ namespace YACS void deal2(bool& somethingDone); void deal2Bis(bool& somethingDone); void deal2Ter(bool& somethingDone); + private: + AbstractPoint *getUnique(); + const AbstractPoint *getUnique() const; }; } } diff --git a/src/engine/Bloc.cxx b/src/engine/Bloc.cxx index 7d5e7b045..6fc3240ab 100644 --- a/src/engine/Bloc.cxx +++ b/src/engine/Bloc.cxx @@ -25,7 +25,9 @@ #include "OutputDataStreamPort.hxx" #include "ElementaryNode.hxx" #include "Visitor.hxx" +#include "ForkBlocPoint.hxx" #include "SetOfPoints.hxx" +#include "PlayGround.hxx" #include #include @@ -452,16 +454,30 @@ void Bloc::accept(Visitor* visitor) * for all concurrent branches in \a this. */ int Bloc::getMaxLevelOfParallelism() const +{ + std::list< AbstractPoint * > pts(analyzeParallelism()); + ForkBlocPoint fbp(pts,NULL); + return fbp.getMaxLevelOfParallelism(); +} + +std::list< AbstractPoint * > Bloc::analyzeParallelism() const { std::vector< std::list > r(splitIntoIndependantGraph()); - int ret(0); + std::list< AbstractPoint * > pts; for(std::vector< std::list >::const_iterator it=r.begin();it!=r.end();it++) { SetOfPoints sop(*it); sop.simplify(); - ret+=sop.getMaxLevelOfParallelism(); + pts.push_back(sop.getUniqueAndReleaseIt()); } - return ret; + return pts; +} + +double Bloc::getWeightRegardingDPL() const +{ + std::list< AbstractPoint * > pts(analyzeParallelism()); + ForkBlocPoint fbp(pts,NULL); + return fbp.getWeightRegardingDPL(); } void Bloc::removeRecursivelyRedundantCL() @@ -476,6 +492,15 @@ void Bloc::removeRecursivelyRedundantCL() destructCFComputations(info); } +void Bloc::partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) +{ + if(!pd) + throw Exception("Bloc::partitionRegardingDPL : NULL pointer !"); + std::list< AbstractPoint * > pts(analyzeParallelism()); + ForkBlocPoint fbp(pts,NULL); + fbp.partitionRegardingDPL(pd,zeMap); +} + void Bloc::performCFComputationsOnlyOneLevel(LinkInfo& info) const { delete _fwLinks;//Normally useless diff --git a/src/engine/Bloc.hxx b/src/engine/Bloc.hxx index 3cfec9172..3759fc094 100644 --- a/src/engine/Bloc.hxx +++ b/src/engine/Bloc.hxx @@ -22,11 +22,16 @@ #include "YACSlibEngineExport.hxx" #include "StaticDefinedComposedNode.hxx" +#include "AutoRefCnt.hxx" namespace YACS { namespace ENGINE { + class PartDefinition; + class DynParaLoop; + class AbstractPoint; + class PlayGround; class YACSLIBENGINE_EXPORT Bloc : public StaticDefinedComposedNode { protected: @@ -61,7 +66,10 @@ namespace YACS void findAllNodesStartingFrom(Node *start, std::set& result, std::map >& accelStr, LinkInfo& info) const; virtual std::string typeName() { return "YACS__ENGINE__Bloc"; } int getMaxLevelOfParallelism() const; + double getWeightRegardingDPL() const; void removeRecursivelyRedundantCL(); + void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap); + void fitToPlayGround(const PlayGround *pg); protected: bool areAllSubNodesFinished() const; bool areAllSubNodesDone() const; @@ -89,6 +97,7 @@ namespace YACS void seekOkAndUseless1(std::vector& okAndUseless1, std::set& allNodes) const; void seekUseless2(std::vector& useless2, std::set& allNodes) const; private: + std::list< AbstractPoint * > analyzeParallelism() const; static void findUselessLinksIn(const std::list< std::vector >& res , LinkInfo& info); template static unsigned appendIfAlreadyFound(std::list< std::vector >& res, const std::vector& startRes, Node *node, std::map >& fastFinder); diff --git a/src/engine/BlocPoint.cxx b/src/engine/BlocPoint.cxx index 485bcdf2d..b351d5cdb 100644 --- a/src/engine/BlocPoint.cxx +++ b/src/engine/BlocPoint.cxx @@ -20,6 +20,8 @@ #include "BlocPoint.hxx" #include "Node.hxx" +#include + using namespace YACS::ENGINE; BlocPoint::BlocPoint(const std::list& nodes, AbstractPoint *father):AbstractPoint(father),_nodes(nodes) @@ -28,6 +30,15 @@ BlocPoint::BlocPoint(const std::list& nodes, AbstractPoint *fat (*it)->setFather(this); } +void BlocPoint::getOutPoint(AbstractPoint *node) +{ + std::list::iterator it(std::find(_nodes.begin(),_nodes.end(),node)); + if(it==_nodes.end()) + throw YACS::Exception("BlocPoint::getOutPoint : node not in this !"); + _nodes.erase(it); + node->setFather(NULL); +} + AbstractPoint *BlocPoint::findPointWithNode(Node *node) { for(std::list::iterator it=_nodes.begin();it!=_nodes.end();it++) diff --git a/src/engine/BlocPoint.hxx b/src/engine/BlocPoint.hxx index eea6b9267..c75f8ef48 100644 --- a/src/engine/BlocPoint.hxx +++ b/src/engine/BlocPoint.hxx @@ -42,6 +42,8 @@ namespace YACS int getNumberOfNodes() const; const std::list& getListOfPoints() const { return _nodes; } virtual ~BlocPoint(); + protected: + void getOutPoint(AbstractPoint *node); }; } } diff --git a/src/engine/Bloc_impl.cxx b/src/engine/Bloc_impl.cxx new file mode 100644 index 000000000..d3f3c6f83 --- /dev/null +++ b/src/engine/Bloc_impl.cxx @@ -0,0 +1,108 @@ +// Copyright (C) 2006-2016 CEA/DEN, EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +#include "Bloc.hxx" +#include "Proc.hxx" +#include "Visitor.hxx" +#include "ForEachLoop.hxx" +#include "InlineNode.hxx" +#include "HomogeneousPoolContainer.hxx" + +using namespace YACS::ENGINE; + +void Bloc::fitToPlayGround(const PlayGround *pg) +{ + static const char MSG[]="Bloc::fitToPlayGround : Not implemented yet for this type of node !"; + class MyVisitor : public Visitor + { + public: + MyVisitor(ComposedNode *root):Visitor(root),_lev(0),_max_lev(0) { } + void visitBloc(Bloc *node) { node->ComposedNode::accept(this); } + void visitElementaryNode(ElementaryNode *node) { } + void visitForEachLoop(ForEachLoop *node) + { + _max_lev=std::max(_max_lev,_lev); + { + _lev++; + node->ComposedNode::accept(this); + _lev--; + } + node->edGetNbOfBranchesPort()->edInit(1); + if(_lev==_max_lev) + { + _fes.push_back(node);// locate all leaves ForEach + } + if(_lev==0) + _max_lev=0; + } + void visitOptimizerLoop(OptimizerLoop *node) { throw YACS::Exception(MSG); } + void visitDynParaLoop(DynParaLoop *node) { throw YACS::Exception(MSG); } + void visitForLoop(ForLoop *node) { throw YACS::Exception(MSG); } + void visitInlineNode(InlineNode *node) + { + Container *cont(node->getContainer()); + HomogeneousPoolContainer *cont2(dynamic_cast(cont)); + if(!cont2) + return ; + _cont.push_back(cont2); + HomogeneousPoolContainer *cont3(cont2->getDirectFather()); + if(cont3) + _cont2.insert(cont3); + } + void visitInlineFuncNode(InlineFuncNode *node) { throw YACS::Exception(MSG); } + void visitLoop(Loop *node) { throw YACS::Exception(MSG); } + void visitProc(Proc *node) { node->ComposedNode::accept(this); } + void visitServiceNode(ServiceNode *node) { throw YACS::Exception(MSG); } + void visitServerNode(ServerNode *node) { throw YACS::Exception(MSG); } + void visitServiceInlineNode(ServiceInlineNode *node) { throw YACS::Exception(MSG); } + void visitSwitch(Switch *node) { throw YACS::Exception(MSG); } + void visitWhileLoop(WhileLoop *node) { throw YACS::Exception(MSG); } + void visitPresetNode(DataNode *node) { throw YACS::Exception(MSG); } + void visitOutNode(DataNode *node) { throw YACS::Exception(MSG); } + void visitStudyInNode(DataNode *node) { throw YACS::Exception(MSG); } + void visitStudyOutNode(DataNode *node) { throw YACS::Exception(MSG); } + public: + std::list _fes; + std::list< HomogeneousPoolContainer *> _cont; + std::set< HomogeneousPoolContainer * > _cont2; + int _lev; + int _max_lev; + }; + YACS::BASES::AutoRefCnt pd(new AllPartDefinition(pg,1)); + std::map > zeMap; + this->partitionRegardingDPL(pd,zeMap); + MyVisitor vis(this); + this->accept(&vis); + for(std::list::const_iterator it=vis._fes.begin();it!=vis._fes.end();it++) + { + std::map >::iterator it2(zeMap.find(*it)); + if(it2==zeMap.end()) + throw YACS::Exception("Bloc::fitToPlayGround : internal error !"); + int maxLev((*it)->getExecNode()->getMaxLevelOfParallelism()); + int a((*it2).second->getNumberOfCoresConsumed()); + int res(a/maxLev); + if(a%maxLev!=0) + res++; + (*it)->edGetNbOfBranchesPort()->edInit(res); + } + for(std::set< HomogeneousPoolContainer * >::const_iterator it=vis._cont2.begin();it!=vis._cont2.end();it++) + (*it)->setSizeOfPool(pg->getNumberOfWorkers((*it)->getNumberOfCoresPerWorker())); + for(std::list< HomogeneousPoolContainer *>::const_iterator it=vis._cont.begin();it!=vis._cont.end();it++) + (*it)->prepareMaskForExecution(); +} diff --git a/src/engine/CMakeLists.txt b/src/engine/CMakeLists.txt index 9c6d3fbcf..b1f43ff38 100644 --- a/src/engine/CMakeLists.txt +++ b/src/engine/CMakeLists.txt @@ -106,6 +106,7 @@ SET(YACSlibEngine_HEADERS LinkedBlocPoint.hxx ElementaryPoint.hxx SetOfPoints.hxx + PlayGround.hxx ) # --- sources --- @@ -142,6 +143,7 @@ SET(YACSlibEngine_SOURCES ServiceInlineNode.cxx StaticDefinedComposedNode.cxx Bloc.cxx + Bloc_impl.cxx Proc.cxx Loop.cxx ForLoop.cxx @@ -178,6 +180,7 @@ SET(YACSlibEngine_SOURCES LinkedBlocPoint.cxx ElementaryPoint.cxx SetOfPoints.cxx + PlayGround.cxx ) SET(YACSlibEngine_HEADERS ${YACSlibEngine_HEADERS} PARENT_SCOPE) # Make it visible to src/engine_swig to handle dependencies diff --git a/src/engine/DynParaLoop.cxx b/src/engine/DynParaLoop.cxx index cca32828b..632f72478 100644 --- a/src/engine/DynParaLoop.cxx +++ b/src/engine/DynParaLoop.cxx @@ -44,7 +44,7 @@ const char DynParaLoop::NAME_OF_NUMBER_OF_BRANCHES[]="nbBranches"; DynParaLoop::DynParaLoop(const std::string& name, TypeCode *typeOfDataSplitted) : ComposedNode(name),_node(0),_initNode(0),_finalizeNode(0),_nbOfEltConsumed(0), _nbOfBranches(NAME_OF_NUMBER_OF_BRANCHES,this,Runtime::_tc_int), - _splittedPort(NAME_OF_SPLITTED_SEQ_OUT,this,typeOfDataSplitted),_initializingCounter(0),_unfinishedCounter(0),_failedCounter(0) + _splittedPort(NAME_OF_SPLITTED_SEQ_OUT,this,typeOfDataSplitted),_initializingCounter(0),_unfinishedCounter(0),_failedCounter(0),_weight(1.) { } @@ -58,7 +58,7 @@ DynParaLoop::~DynParaLoop() DynParaLoop::DynParaLoop(const DynParaLoop& other, ComposedNode *father, bool editionOnly) : ComposedNode(other,father), _nbOfBranches(other._nbOfBranches,this), _splittedPort(other._splittedPort,this), _node(0), _initNode(0), _finalizeNode(0), - _nbOfEltConsumed(0),_initializingCounter(0),_unfinishedCounter(0),_failedCounter(0) + _nbOfEltConsumed(0),_initializingCounter(0),_unfinishedCounter(0),_failedCounter(0),_weight(1.) { if(other._node) _node=other._node->clone(this,editionOnly); @@ -423,6 +423,13 @@ DynParaLoop::TypeOfNode DynParaLoop::getIdentityOfNotifyerNode(const Node *node, return FINALIZE_NODE; } +void DynParaLoop::setWeight(double newVal) +{ + if(newVal<=0.) + throw Exception("DynParaLoop::setWeight : invalid input value !"); + _weight=newVal; +} + bool DynParaLoop::isMultiplicitySpecified(unsigned& value) const { if(_nbOfBranches.edIsManuallyInitialized()) @@ -896,6 +903,14 @@ int DynParaLoop::getMaxLevelOfParallelism() const return _nbOfBranches.getIntValue() * _node->getMaxLevelOfParallelism(); } +void DynParaLoop::partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) +{ + YACS::BASES::AutoRefCnt pd2(pd->copy()); + zeMap[this]=pd2; + if(_node) + _node->partitionRegardingDPL(pd,zeMap); +} + void DynParaLoop::shutdown(int level) { if(level==0)return; diff --git a/src/engine/DynParaLoop.hxx b/src/engine/DynParaLoop.hxx index c00a5e8c5..723b27e2e 100644 --- a/src/engine/DynParaLoop.hxx +++ b/src/engine/DynParaLoop.hxx @@ -57,6 +57,7 @@ namespace YACS int _initializingCounter; int _unfinishedCounter; int _failedCounter; + double _weight; protected: static const char NAME_OF_SPLITTED_SEQ_OUT[]; static const char OLD_NAME_OF_SPLITTED_SEQ_OUT[]; @@ -97,6 +98,9 @@ namespace YACS Node *getChildByShortName(const std::string& name) const throw(Exception); Node *getChildByNameExec(const std::string& name, unsigned id) const throw(Exception); std::vector getNodes() const { return _execNodes; } // need to use in GUI part for adding observers for clone nodes + double getWeight() const { return _weight; } + void setWeight(double newVal); + double getWeightRegardingDPL() const { return getWeight(); } bool isMultiplicitySpecified(unsigned& value) const; void forceMultiplicity(unsigned value); virtual void checkBasicConsistency() const throw(Exception); @@ -106,6 +110,7 @@ namespace YACS Node * getExecNode(); Node * getFinalizeNode(); int getMaxLevelOfParallelism() const; + void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap); protected: void buildDelegateOf(InPort * & port, OutPort *initialStart, const std::list& pointsOfView); void buildDelegateOf(std::pair& port, InPort *finalTarget, const std::list& pointsOfView); diff --git a/src/engine/ElementaryNode.hxx b/src/engine/ElementaryNode.hxx index 5544c86fb..d2561e804 100644 --- a/src/engine/ElementaryNode.hxx +++ b/src/engine/ElementaryNode.hxx @@ -107,7 +107,9 @@ namespace YACS virtual void ensureLoading(); int getMaxLevelOfParallelism() const { return 1; } - + double getWeightRegardingDPL() const { return 0.; } + void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) { } + //run part void begin(); bool isReady(); diff --git a/src/engine/ElementaryPoint.cxx b/src/engine/ElementaryPoint.cxx index f476c20e4..aeef403b4 100644 --- a/src/engine/ElementaryPoint.cxx +++ b/src/engine/ElementaryPoint.cxx @@ -59,6 +59,16 @@ int ElementaryPoint::getMaxLevelOfParallelism() const return _node->getMaxLevelOfParallelism(); } +double ElementaryPoint::getWeightRegardingDPL() const +{ + return _node->getWeightRegardingDPL(); +} + +void ElementaryPoint::partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) const +{ + _node->partitionRegardingDPL(pd,zeMap); +} + std::string ElementaryPoint::getRepr() const { return _node->getName(); diff --git a/src/engine/ElementaryPoint.hxx b/src/engine/ElementaryPoint.hxx index 5228dd5e4..08897e80a 100644 --- a/src/engine/ElementaryPoint.hxx +++ b/src/engine/ElementaryPoint.hxx @@ -43,6 +43,8 @@ namespace YACS Node *getLastNode(); int getNumberOfNodes() const; int getMaxLevelOfParallelism() const; + double getWeightRegardingDPL() const; + void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) const; std::string getRepr() const; virtual ~ElementaryPoint(); }; diff --git a/src/engine/ForkBlocPoint.cxx b/src/engine/ForkBlocPoint.cxx index fd7271cd0..79d30fe65 100644 --- a/src/engine/ForkBlocPoint.cxx +++ b/src/engine/ForkBlocPoint.cxx @@ -54,6 +54,52 @@ int ForkBlocPoint::getMaxLevelOfParallelism() const return ret; } +double ForkBlocPoint::getWeightRegardingDPL() const +{ + double ret(0.); + for(std::list::const_iterator it=_nodes.begin();it!=_nodes.end();it++) + ret+=(*it)->getWeightRegardingDPL(); + return ret; +} + +void ForkBlocPoint::partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) const +{ + std::vector< std::pair > parts,parts2; + std::vector v,v2; + std::size_t ii(0); + for(std::list::const_iterator it=_nodes.begin();it!=_nodes.end();it++,ii++) + { + double w((*it)->getWeightRegardingDPL()); + if(w!=0.) + { + parts.push_back(std::pair(pd,w)); + v.push_back(ii); + } + else + { + parts2.push_back(std::pair(pd,1.)); + v2.push_back(ii); + } + } + std::vector nodes2(_nodes.begin(),_nodes.end()); + if(!parts.empty()) + { + const PlayGround *pg(pd->getPlayGround()); + std::vector< YACS::BASES::AutoRefCnt > pds(pg->partition(parts)); + ii=0; + for(std::vector::const_iterator iter=v.begin();iter!=v.end();iter++,ii++) + nodes2[*iter]->partitionRegardingDPL(pds[ii],zeMap); + } + if(!parts2.empty()) + { + const PlayGround *pg(pd->getPlayGround()); + std::vector< YACS::BASES::AutoRefCnt > pds(pg->partition(parts2)); + ii=0; + for(std::vector::const_iterator iter=v2.begin();iter!=v2.end();iter++,ii++) + nodes2[*iter]->partitionRegardingDPL(pds[ii],zeMap); + } +} + std::string ForkBlocPoint::getRepr() const { std::size_t sz(_nodes.size()),ii(0); diff --git a/src/engine/ForkBlocPoint.hxx b/src/engine/ForkBlocPoint.hxx index 97afd34e7..0aa3fbf17 100644 --- a/src/engine/ForkBlocPoint.hxx +++ b/src/engine/ForkBlocPoint.hxx @@ -34,6 +34,8 @@ namespace YACS Node *getFirstNode(); Node *getLastNode(); int getMaxLevelOfParallelism() const; + double getWeightRegardingDPL() const; + void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) const; std::string getRepr() const; virtual ~ForkBlocPoint(); }; diff --git a/src/engine/HomogeneousPoolContainer.hxx b/src/engine/HomogeneousPoolContainer.hxx index fbc0c3fb4..641a0d320 100644 --- a/src/engine/HomogeneousPoolContainer.hxx +++ b/src/engine/HomogeneousPoolContainer.hxx @@ -23,6 +23,7 @@ #include "YACSlibEngineExport.hxx" #include "Exception.hxx" #include "Container.hxx" +#include "PlayGround.hxx" #include @@ -30,6 +31,8 @@ namespace YACS { namespace ENGINE { + class ForTestOmlyHPContCls; + /*! * This is an abstract class, that represents a set of fixed number of worker "kernelcontainers" homegenous in the sense that can be used indifferently each other. * But each of those worker pool can be used once at a time. @@ -47,6 +50,12 @@ namespace YACS virtual std::size_t getNumberOfFreePlace() const = 0; virtual void allocateFor(const std::vector& nodes) = 0; virtual void release(const Task *node) = 0; + virtual YACS::BASES::AutoRefCnt decorate(YACS::BASES::AutoConstRefCnt pd) = 0; + virtual HomogeneousPoolContainer *getDirectFather() = 0; + virtual const HomogeneousPoolContainer *getDirectFather() const = 0; + virtual int getNumberOfCoresPerWorker() const = 0; + virtual void prepareMaskForExecution() const = 0; + virtual void forYourTestsOnly(ForTestOmlyHPContCls *data) const = 0; static const char SIZE_OF_POOL_KEY[]; static const char INITIALIZE_SCRIPT_KEY[]; protected: diff --git a/src/engine/InlineNode.cxx b/src/engine/InlineNode.cxx index 213d74cee..ce696f618 100644 --- a/src/engine/InlineNode.cxx +++ b/src/engine/InlineNode.cxx @@ -20,6 +20,7 @@ #include "InlineNode.hxx" #include "Visitor.hxx" #include "Container.hxx" +#include "HomogeneousPoolContainer.hxx" #include #define _DEVDEBUG_ @@ -156,3 +157,17 @@ int InlineNode::getMaxLevelOfParallelism() const return ret; } +void InlineNode::partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) +{ + if(!isDeployable()) + return ; + if(!_container) + return ; + HomogeneousPoolContainer *contC(dynamic_cast(_container)); + if(!contC) + return ; + YACS::BASES::AutoConstRefCnt zePd; + zePd.takeRef(pd); + YACS::BASES::AutoRefCnt zeCont(contC->decorate(zePd)); + setContainer(zeCont); +} diff --git a/src/engine/InlineNode.hxx b/src/engine/InlineNode.hxx index cd63a222a..c97af1c0d 100644 --- a/src/engine/InlineNode.hxx +++ b/src/engine/InlineNode.hxx @@ -68,6 +68,7 @@ namespace YACS void performShallowDuplicationOfPlacement(const Node& other); bool isDeployable() const; int getMaxLevelOfParallelism() const; + void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap); public: static const char LOCAL_STR[]; static const char REMOTE_STR[]; diff --git a/src/engine/LinkedBlocPoint.cxx b/src/engine/LinkedBlocPoint.cxx index 49f97a1c2..666f757e5 100644 --- a/src/engine/LinkedBlocPoint.cxx +++ b/src/engine/LinkedBlocPoint.cxx @@ -48,6 +48,20 @@ int LinkedBlocPoint::getMaxLevelOfParallelism() const return ret; } +double LinkedBlocPoint::getWeightRegardingDPL() const +{ + double ret(0.); + for(std::list::const_iterator it=_nodes.begin();it!=_nodes.end();it++) + ret=std::max(ret,(*it)->getWeightRegardingDPL()); + return ret; +} + +void LinkedBlocPoint::partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) const +{ + for(std::list::const_iterator it=_nodes.begin();it!=_nodes.end();it++) + (*it)->partitionRegardingDPL(pd,zeMap); +} + std::string LinkedBlocPoint::getRepr() const { std::size_t sz(_nodes.size()),ii(0); diff --git a/src/engine/LinkedBlocPoint.hxx b/src/engine/LinkedBlocPoint.hxx index c56a4b04a..963ca24e0 100644 --- a/src/engine/LinkedBlocPoint.hxx +++ b/src/engine/LinkedBlocPoint.hxx @@ -36,6 +36,8 @@ namespace YACS Node *getFirstNode(); Node *getLastNode(); int getMaxLevelOfParallelism() const; + double getWeightRegardingDPL() const; + void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) const; std::string getRepr() const; virtual ~LinkedBlocPoint(); }; diff --git a/src/engine/Loop.cxx b/src/engine/Loop.cxx index 53e6ff284..cc4f089f9 100644 --- a/src/engine/Loop.cxx +++ b/src/engine/Loop.cxx @@ -419,6 +419,19 @@ int Loop::getMaxLevelOfParallelism() const return _node->getMaxLevelOfParallelism(); } +double Loop::getWeightRegardingDPL() const +{ + if(!_node) + return 0.; + return _node->getWeightRegardingDPL(); +} + +void Loop::partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) +{ + if(_node) + _node->partitionRegardingDPL(pd,zeMap); +} + Node *Loop::getChildByShortName(const std::string& name) const throw(YACS::Exception) { if (_node) diff --git a/src/engine/Loop.hxx b/src/engine/Loop.hxx index dbe320cbe..a1f533af0 100644 --- a/src/engine/Loop.hxx +++ b/src/engine/Loop.hxx @@ -173,6 +173,8 @@ namespace YACS std::list getSetOfInputPort() const; int getNumberOfInputPorts() const; int getMaxLevelOfParallelism() const; + double getWeightRegardingDPL() const; + void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap); Node *getChildByShortName(const std::string& name) const throw(Exception); static TypeCode* MappingDF2DS(TypeCode* type) throw(Exception); static TypeCode* MappingDS2DF(TypeCode* type) throw(Exception); diff --git a/src/engine/Node.cxx b/src/engine/Node.cxx index 0c633c86e..aaa4ed21f 100644 --- a/src/engine/Node.cxx +++ b/src/engine/Node.cxx @@ -408,6 +408,22 @@ const Proc * Node::getProc() const return _father->getProc(); } +DynParaLoop *Node::getClosestDPLAmongAncestors() const +{ + if(!_father) + return NULL; + ComposedNode *iter(_father); + do + { + DynParaLoop *iter2(dynamic_cast(iter)); + if(iter2) + return iter2; + iter=iter->_father; + } + while(iter); + return NULL; +} + ComposedNode *Node::getRootNode() const throw(YACS::Exception) { if(!_father) diff --git a/src/engine/Node.hxx b/src/engine/Node.hxx index f7d462394..f37920482 100644 --- a/src/engine/Node.hxx +++ b/src/engine/Node.hxx @@ -21,6 +21,8 @@ #define __NODE_HXX__ #include "YACSlibEngineExport.hxx" +#include "PlayGround.hxx" +#include "AutoRefCnt.hxx" #include "InGate.hxx" #include "OutGate.hxx" #include "Exception.hxx" @@ -42,8 +44,8 @@ namespace YACS class InputPort; class OutputPort; class InPropertyPort; - class DynParaLoop; class ForEachLoop; + class DynParaLoop; class ComposedNode; class Proc; class ElementaryNode; @@ -161,6 +163,7 @@ namespace YACS bool operator>(const Node& other) const; bool operator<(const Node& other) const; std::string getImplementation() const; + DynParaLoop *getClosestDPLAmongAncestors() const; virtual ComposedNode *getRootNode() const throw(Exception); virtual void setProperty(const std::string& name,const std::string& value); virtual std::string getProperty(const std::string& name); @@ -172,6 +175,8 @@ namespace YACS virtual const Proc *getProc() const; virtual void accept(Visitor *visitor) = 0; virtual int getMaxLevelOfParallelism() const = 0; + virtual double getWeightRegardingDPL() const = 0; + virtual void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) = 0; std::string getQualifiedName() const; int getNumId(); std::vector > getDPLScopeInfo(ComposedNode *gfn); diff --git a/src/engine/PlayGround.cxx b/src/engine/PlayGround.cxx new file mode 100644 index 000000000..bc96d9b3d --- /dev/null +++ b/src/engine/PlayGround.cxx @@ -0,0 +1,635 @@ +// Copyright (C) 2006-2017 CEA/DEN, EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +#include "PlayGround.hxx" +#include "Runtime.hxx" + +#include +#include +#include +#include +#include +#include + +using namespace YACS::ENGINE; + +std::string PlayGround::printSelf() const +{ + std::ostringstream oss; + std::size_t sz(0); + for(std::vector< std::pair >::const_iterator it=_data.begin();it!=_data.end();it++) + sz=std::max(sz,(*it).first.length()); + for(std::vector< std::pair >::const_iterator it=_data.begin();it!=_data.end();it++) + { + oss << " - " << std::setw(10) << (*it).first << " : " << (*it).second << std::endl; + } + return oss.str(); +} + +void PlayGround::loadFromKernelCatalog() +{ + Runtime *r(getRuntime()); + if(!r) + throw Exception("PlayGround::loadFromKernelCatalog : no runtime !"); + std::vector< std::pair > data(r->getCatalogOfComputeNodes()); + setData(data); +} + +void PlayGround::setData(const std::vector< std::pair >& defOfRes) +{ + _data=defOfRes; + checkCoherentInfo(); +} + +int PlayGround::getNumberOfCoresAvailable() const +{ + int ret(0); + for(std::vector< std::pair >::const_iterator it=_data.begin();it!=_data.end();it++) + ret+=(*it).second; + return ret; +} + +int PlayGround::getMaxNumberOfContainersCanBeHostedWithoutOverlap(int nbCoresPerCont) const +{ + if(nbCoresPerCont<1) + throw Exception("PlayGround::getMaxNumberOfContainersCanBeHostedWithoutOverlap : invalid nbCoresPerCont. Must be >=1 !"); + int ret(0); + for(std::vector< std::pair >::const_iterator it=_data.begin();it!=_data.end();it++) + ret+=(*it).second/nbCoresPerCont; + return ret; +} + +std::vector PlayGround::computeOffsets() const +{ + std::size_t sz(_data.size()),i(0); + std::vector ret(sz+1); ret[0]=0; + for(std::vector< std::pair >::const_iterator it=_data.begin();it!=_data.end();it++,i++) + ret[i+1]=ret[i]+(*it).second; + return ret; +} + +void PlayGround::checkCoherentInfo() const +{ + std::set s; + for(std::vector< std::pair >::const_iterator it=_data.begin();it!=_data.end();it++) + { + s.insert((*it).first); + if((*it).second<0) + throw Exception("Presence of negative int value !"); + } + if(s.size()!=_data.size()) + throw Exception("host names entries must be different each other !"); +} + +std::vector PlayGround::FromUItoVB(unsigned int sz, unsigned int v) +{ + std::vector ret(sz); + unsigned int p(1); + for(std::size_t i=0;i& v) +{ + std::size_t sz(v.size()); + unsigned int ret(0); + for(std::size_t i=0;i PlayGround::GetIdsMatching(const std::vector& bigArr, const std::vector& pat) +{ + std::vector ret; + std::size_t szp(pat.size()); + std::size_t sz(bigArr.size()/szp); + for(std::size_t i=0;i t(bigArr.begin()+i*szp,bigArr.begin()+(i+1)*szp); + if(t==pat) + ret.push_back(i); + } + return ret; +} + +std::vector PlayGround::BuildVectOfIdsFromVecBool(const std::vector& v) +{ + std::size_t sz(std::count(v.begin(),v.end(),true)),i(0); + std::vector ret(sz); + std::vector::const_iterator it(v.begin()); + while(i& coreIds, std::vector& v) const +{ + if(v.size()!=getNumberOfCoresAvailable()) + throw Exception("PlayGround::highlightOnIds : oops ! invalid size !"); + for(std::vector::const_iterator it=coreIds.begin();it!=coreIds.end();it++) + v[*it]=true; +} + +/*! + * you must garantee coherence between PlayGround::deduceMachineFrom, PlayGround::getNumberOfWorkers, and PartDefinition::computeWorkerIdsCovered + */ +std::vector PlayGround::getFetchedCores(int nbCoresPerWorker) const +{ + int nbCores(getNumberOfCoresAvailable()); + std::vector ret(nbCores,false); + if(nbCoresPerWorker==1) + std::fill(ret.begin(),ret.end(),true); + else + { + std::size_t posBg(0); + for(std::vector< std::pair >::const_iterator it=_data.begin();it!=_data.end();it++) + { + int nbElemsToPutOn(((*it).second/nbCoresPerWorker)*nbCoresPerWorker); + std::fill(ret.begin()+posBg,ret.begin()+posBg+nbElemsToPutOn,true); + posBg+=(*it).second; + } + } + return ret; +} +/*! + * follow getMaxNumberOfContainersCanBeHostedWithoutOverlap method + */ +std::vector PlayGround::getWorkerIdsFullyFetchedBy(int nbCoresPerComp, const std::vector& coreFlags) const +{ + std::size_t posBg(0),posWorker(0); + std::vector ret; + for(std::vector< std::pair >::const_iterator it=_data.begin();it!=_data.end();it++) + { + int nbWorker((*it).second/nbCoresPerComp); + for(int j=0;j::const_iterator it2(std::find(coreFlags.begin()+posBg+j*nbCoresPerComp,coreFlags.begin()+posBg+(j+1)*nbCoresPerComp,false)); + if(it2==coreFlags.begin()+posBg+(j+1)*nbCoresPerComp) + ret.push_back(posWorker); + } + posBg+=(*it).second; + } + return ret; +} + +std::vector< YACS::BASES::AutoRefCnt > PlayGround::partition(const std::vector< std::pair >& parts) const +{ + std::size_t sz(parts.size()),szs(getNumberOfCoresAvailable()); + if(sz==0) + return std::vector< YACS::BASES::AutoRefCnt >(); + if(sz==1) + { + const PartDefinition *pd(parts[0].first); + if(!pd) + throw Exception("Presence of null pointer as part def 0 !"); + YACS::BASES::AutoRefCnt ret(pd->copy()); + std::vector< YACS::BASES::AutoRefCnt > ret2(1,ret); + return ret2; + } + if(sz>31) + throw Exception("PlayGround::partition : not implemented yet for more than 31 ! You need to pay for it :)"); + std::vector zeArr(szs*sz,false); + std::size_t i(0); + for(std::vector< std::pair >::const_iterator it=parts.begin();it!=parts.end();it++,i++) + { + const PartDefinition *pd((*it).first); + if(!pd) + throw Exception("Presence of null pointer as part def !"); + if(pd->getPlayGround()!=this) + throw Exception("Presence of non homogeneous playground !"); + if((*it).second<=0.) + throw Exception("Invalid weight !"); + std::vector bs(pd->getCoresOn()); + for(std::size_t j=0;j ss; + for(std::size_t i=0;i vb(zeArr.begin()+i*sz,zeArr.begin()+(i+1)*sz); + ss.insert(FromVBtoUI(vb)); + } + std::vector< std::vector > retIds(sz); + for(std::set::const_iterator i=ss.begin();i!=ss.end();i++) + { + std::vector code(FromUItoVB(sz,*i));// for this configuration which parts are considered + std::vector locIds(GetIdsMatching(zeArr,code)); + std::vector partsIds(BuildVectOfIdsFromVecBool(code)); + if(partsIds.empty()) + continue; + std::vector wg; + std::vector nbCores2; + for(std::vector::const_iterator it=partsIds.begin();it!=partsIds.end();it++) + { + wg.push_back(parts[*it].second); + nbCores2.push_back(parts[*it].first->getNbCoresPerCompo()); + } + std::vector< std::vector > ress(splitIntoParts(locIds,nbCores2,wg)); + std::size_t k(0); + for(std::vector::const_iterator it=partsIds.begin();it!=partsIds.end();it++,k++) + { + retIds[*it].insert(retIds[*it].end(),ress[k].begin(),ress[k].end()); + } + } + // + std::vector< YACS::BASES::AutoRefCnt > ret(sz); + for(std::size_t i=0;i s(retIds[i].begin(),retIds[i].end()); + std::vector v(s.begin(),s.end()); + ret[i]=PartDefinition::BuildFrom(this,(parts[i].first)->getNbCoresPerCompo(),v); + } + return ret; +} + +std::vector sortArr(const std::vector& v) +{ + std::multimap m; + int i(v.size()-1); + for(std::vector::const_reverse_iterator it=v.rbegin();it!=v.rend();it++) + m.insert(std::pair(*it,i--)); + std::vector ret(m.size()); + i=0; + for(std::multimap::const_reverse_iterator it=m.rbegin();it!=m.rend();it++)// reverse -> sort from biggest to the finest + ret[i++]=(*it).second; + return ret; +} + +std::vector< std::vector > PlayGround::splitIntoParts(const std::vector& coreIds, const std::vector& nbCoresConso, const std::vector& weights) const +{ + double wgs(std::accumulate(weights.begin(),weights.end(),0.)); + std::size_t sz(nbCoresConso.size()); + if(sz!=weights.size()) + throw Exception("PlayGround::splitIntoParts : internal error !"); + if(sz==0) + return std::vector< std::vector >(); + int totalSpace(coreIds.size()); + std::vector< std::vector > ret(sz); + std::vector fromBigToTiny(sortArr(nbCoresConso));// start to treat the coarse grain to finish with fine grain + std::vector zeArr(getNumberOfCoresAvailable(),false); + highlightOnIds(coreIds,zeArr); + int nbOfCoresToSplit(coreIds.size()); + std::size_t ii(fromBigToTiny.size()); + for(std::vector::const_iterator it=fromBigToTiny.begin();it!=fromBigToTiny.end();it++,ii--) + { + int maxNbOfCores((int)(totalSpace*weights[*it]/wgs));// now try to find in zeArr at most maxNbOfCores cores + ret[*it]=takePlace(maxNbOfCores,nbCoresConso[*it],zeArr,ii==1); + } + return ret; +} + +std::vector PlayGround::takePlace(int maxNbOfCoresToAlloc, int nbCoresPerShot, std::vector& distributionOfCores, bool lastOne) const +{ + if(maxNbOfCoresToAlloc<1) + throw Exception("PlayGround::takePlace : internal error ! no space to alloc !"); + int tmpMaxNbOfCoresToAlloc(maxNbOfCoresToAlloc); + if(lastOne) + tmpMaxNbOfCoresToAlloc=std::max(tmpMaxNbOfCoresToAlloc,(int)std::count(distributionOfCores.begin(),distributionOfCores.end(),true)); + std::vector ret; + std::vector offsets(computeOffsets()); + int nbFullItem(0); + std::size_t sz(offsets.size()-1); + for(std::size_t i=0;i=nbCoresPerShot;i++) + { + int d(offsets[i+1]-offsets[i]); + if(nbCoresPerShot>d) + continue; + std::vector target(nbCoresPerShot,true); + for(int j=0;j<=d-nbCoresPerShot && tmpMaxNbOfCoresToAlloc>=nbCoresPerShot;) + { + std::vector t(distributionOfCores.begin()+offsets[i]+j,distributionOfCores.begin()+offsets[i]+j+nbCoresPerShot); + if(t==target) + { + nbFullItem++; + tmpMaxNbOfCoresToAlloc-=nbCoresPerShot; + std::fill(distributionOfCores.begin()+offsets[i]+j,distributionOfCores.begin()+offsets[i]+j+nbCoresPerShot,false); + for(int k=offsets[i]+j;k0) + return ret; + if(nbCoresPerShot<=1) + throw Exception("PlayGround::takePlace : internal error !"); + // not enough contiguous place. Find the first wider contiguous place + for(int kk=std::min(nbCoresPerShot-1,tmpMaxNbOfCoresToAlloc);kk>=1;kk--) + { + for(std::size_t i=0;i=kk;i++) + { + int d(offsets[i+1]-offsets[i]); + if(kk>d) + continue; + std::vector target(kk,true); + for(int j=0;j<=d-kk && tmpMaxNbOfCoresToAlloc>=kk;) + { + std::vector t(distributionOfCores.begin()+offsets[i]+j,distributionOfCores.begin()+offsets[i]+j+kk); + if(t==target) + { + nbFullItem++; + tmpMaxNbOfCoresToAlloc-=kk; + std::fill(distributionOfCores.begin()+offsets[i]+j,distributionOfCores.begin()+offsets[i]+j+kk,false); + for(int k=offsets[i]+j;k deltas(sz2+1); deltas[0]=0; + for(std::size_t i=0;i=deltas[zePos+1])) + zePos++; + if(zePos==sz2) + zePos=workerId%sz2; + return zePos; +} + +/*! + * you must garantee coherence between PlayGround::deduceMachineFrom, PlayGround::getNumberOfWorkers, and PartDefinition::computeWorkerIdsCovered + */ +std::string PlayGround::deduceMachineFrom(int workerId, int nbProcPerNode) const +{ + int zePos(fromWorkerIdToResId(workerId,nbProcPerNode)); + return _data[zePos].first; +} + +/*! + * you must garantee coherence between PlayGround::deduceMachineFrom, PlayGround::getNumberOfWorkers, PlayGround::getFetchedCores and PartDefinition::computeWorkerIdsCovered + */ +int PlayGround::getNumberOfWorkers(int nbCoresPerWorker) const +{ + return getMaxNumberOfContainersCanBeHostedWithoutOverlap(nbCoresPerWorker); +} + +PlayGround::~PlayGround() +{ +} + +////////////////////// + +PartDefinition::PartDefinition(const PlayGround *pg, int nbOfCoresPerComp):_nbOfCoresPerComp(nbOfCoresPerComp) +{ + _pg.takeRef(pg); +} + +PartDefinition::PartDefinition(const PartDefinition& other):_pg(other._pg),_nbOfCoresPerComp(other._nbOfCoresPerComp) +{ +} + +PartDefinition::~PartDefinition() +{ +} + +std::vector< YACS::BASES::AutoRefCnt > PartDefinition::partition(const std::vector< double >& wgs) const +{ + std::size_t sz(wgs.size()); + std::vector< std::pair > elts(sz); + for(std::size_t i=0;i(this,wgs[i]); + return getPlayGround()->partition(elts); +} + +YACS::BASES::AutoRefCnt PartDefinition::BuildFrom(const PlayGround *pg, int nbOfCoresPerComp, const std::vector& coreIds) +{ + int spaceSz(pg->getNumberOfCoresAvailable()),sz(coreIds.size()); + if(sz>spaceSz) + throw Exception("PartDefinition::BuildFrom : error 1 !"); + if(sz==0) + throw Exception("PartDefinition::BuildFrom : error 2 !"); + int zeStart(coreIds.front()),zeEnd(coreIds.back()); + if(zeStart<0 || zeEnd pd(new NonContigPartDefinition(pg,nbOfCoresPerComp,coreIds)); + return pd; + } + if(sz==spaceSz) + { + YACS::BASES::AutoRefCnt pd(new AllPartDefinition(pg,nbOfCoresPerComp)); + return pd; + } + YACS::BASES::AutoRefCnt pd(new ContigPartDefinition(pg,nbOfCoresPerComp,zeStart,zeEnd+1)); + return pd; +} + +void PartDefinition::stashPart(int nbCoresStashed, double weightOfRemain, YACS::BASES::AutoRefCnt& pdStashed, YACS::BASES::AutoRefCnt& pdRemain) const +{ + if(nbCoresStashed<=0) + throw Exception("stashPart : Invalid nbCoresStashed value !"); + if(weightOfRemain<=0.) + throw Exception("stashPart : Invalid weight !"); + std::vector coresOn(getCoresOn()); + int nbCoresAvailable(std::count(coresOn.begin(),coresOn.end(),true)); + std::vector ids(PlayGround::BuildVectOfIdsFromVecBool(coresOn)); + if(nbCoresAvailable==0) + throw Exception("PartDefinition::stashPart : no available cores !"); + if(nbCoresAvailable<=nbCoresStashed) + { + int n0((int)(1./(1.+weightOfRemain)*nbCoresAvailable)); n0=std::max(n0,1); + int n1(nbCoresAvailable-n0); + if(n1<=0) + { + pdStashed=PartDefinition::BuildFrom(getPlayGround(),1,ids); + pdRemain=PartDefinition::BuildFrom(getPlayGround(),1,ids); + } + else + { + std::vector ids0(ids.begin(),ids.begin()+n0),ids1(ids.begin()+n0,ids.end()); + pdStashed=PartDefinition::BuildFrom(getPlayGround(),1,ids0); + pdRemain=PartDefinition::BuildFrom(getPlayGround(),1,ids1); + } + } + else + { + std::vector ids0(ids.begin(),ids.begin()+nbCoresStashed),ids1(ids.begin()+nbCoresStashed,ids.end()); + pdStashed=PartDefinition::BuildFrom(getPlayGround(),1,ids0); + pdRemain=PartDefinition::BuildFrom(getPlayGround(),1,ids1); + } +} + +/*! + * you must garantee coherence between PlayGround::deduceMachineFrom, PlayGround::getNumberOfWorkers, and PartDefinition::computeWorkerIdsCovered + */ +std::vector PartDefinition::computeWorkerIdsCovered(int nbCoresPerComp) const +{ + std::vector coresOn(getCoresOn()); + return _pg->getWorkerIdsFullyFetchedBy(nbCoresPerComp,coresOn); +} + +////////////////////// + +ContigPartDefinition::ContigPartDefinition(const PlayGround *pg, int nbOfCoresPerComp, int zeStart, int zeStop):PartDefinition(pg,nbOfCoresPerComp),_start(zeStart),_stop(zeStop) +{ + if(_start<0 || _stop<_start || _stop>getSpaceSize()) + throw Exception("ContigPartDefinition constructor : Invalid input values"); +} + +ContigPartDefinition::ContigPartDefinition(const ContigPartDefinition& other):PartDefinition(other),_start(other._start),_stop(other._stop) +{ +} + +std::string ContigPartDefinition::printSelf() const +{ + std::ostringstream oss; + oss << "Contiguous : start=" << _start << " stop=" << _stop; + return oss.str(); +} + +std::vector ContigPartDefinition::getCoresOn() const +{ + std::vector ret(getSpaceSize(),false); + for(int i=_start;i<_stop;i++) + ret[i]=true; + return ret; +} + +ContigPartDefinition *ContigPartDefinition::copy() const +{ + return new ContigPartDefinition(*this); +} + +int ContigPartDefinition::getNumberOfCoresConsumed() const +{ + return _stop-_start; +} + +////////////////////// + +NonContigPartDefinition::NonContigPartDefinition(const PlayGround *pg, int nbOfCoresPerComp, const std::vector& ids):PartDefinition(pg,nbOfCoresPerComp),_ids(ids) +{ + checkOKIds(); +} + +NonContigPartDefinition::NonContigPartDefinition(const ContigPartDefinition& other):PartDefinition(other) +{ +} + +std::string NonContigPartDefinition::printSelf() const +{ + std::ostringstream oss; + oss << "Non contiguous : "; + for(std::vector::const_iterator it=_ids.begin();it!=_ids.end();it++) + oss << *it << ", "; + return oss.str(); +} + +std::vector NonContigPartDefinition::getCoresOn() const +{ + std::vector ret(getSpaceSize(),false); + for(std::vector::const_iterator it=_ids.begin();it!=_ids.end();it++) + ret[*it]=true; + return ret; +} + +NonContigPartDefinition *NonContigPartDefinition::copy() const +{ + return new NonContigPartDefinition(*this); +} + +int NonContigPartDefinition::getNumberOfCoresConsumed() const +{ + return _ids.size(); +} + +void NonContigPartDefinition::checkOKIds() const +{ + int maxVal(getSpaceSize()); + if(_ids.empty()) + return; + int val(_ids.front()); + if(val<0 || val>=maxVal) + throw Exception("checkOKIds : error 2 !"); + std::size_t sz(_ids.size()); + for(std::size_t i=0;i=maxVal) + throw Exception("checkOKIds : error 3 !"); + } +} + +////////////////////// + +AllPartDefinition::AllPartDefinition(const AllPartDefinition& other):PartDefinition(other) +{ +} + +std::string AllPartDefinition::printSelf() const +{ + std::ostringstream oss; + oss << "All"; + return oss.str(); +} + +std::vector AllPartDefinition::getCoresOn() const +{ + std::vector ret(getSpaceSize(),true); + return ret; +} + +AllPartDefinition *AllPartDefinition::copy() const +{ + return new AllPartDefinition(*this); +} + +int AllPartDefinition::getNumberOfCoresConsumed() const +{ + return getSpaceSize(); +} + +////////////////////// + + std::vector ForTestOmlyHPContCls::getIDS() const + { + std::size_t sz(_ids.size()); + std::vector ret(sz); + for(std::size_t i=0;i +#include + +namespace YACS +{ + namespace ENGINE + { + class PartDefinition; + + class YACSLIBENGINE_EXPORT PlayGround : public RefCounter + { + public: + PlayGround(const std::vector< std::pair >& defOfRes):_data(defOfRes) { checkCoherentInfo(); } + PlayGround() { } + std::string printSelf() const; + void loadFromKernelCatalog(); + std::vector< std::pair > getData() const { return _data; } + void setData(const std::vector< std::pair >& defOfRes); + int getNumberOfCoresAvailable() const; + int getMaxNumberOfContainersCanBeHostedWithoutOverlap(int nbCoresPerCont) const; + std::vector computeOffsets() const; + std::vector< YACS::BASES::AutoRefCnt > partition(const std::vector< std::pair >& parts) const; + int fromWorkerIdToResId(int workerId, int nbProcPerNode) const; + std::string deduceMachineFrom(int workerId, int nbProcPerNode) const; + int getNumberOfWorkers(int nbCoresPerWorker) const; + void highlightOnIds(const std::vector& coreIds, std::vector& v) const; + std::vector getFetchedCores(int nbCoresPerWorker) const; + std::vector getWorkerIdsFullyFetchedBy(int nbCoresPerComp, const std::vector& coreFlags) const; + static std::vector BuildVectOfIdsFromVecBool(const std::vector& v); + static std::vector GetIdsMatching(const std::vector& bigArr, const std::vector& pat); + static std::vector FromUItoVB(unsigned int sz, unsigned int v); + static unsigned int FromVBtoUI(const std::vector& v); + private: + std::vector< std::vector > splitIntoParts(const std::vector& coreIds, const std::vector& nbCoresConso, const std::vector& weights) const; + std::vector takePlace(int maxNbOfCoresToAlloc, int nbCoresPerShot, std::vector& distributionOfCores, bool lastOne=false) const; + private: + void checkCoherentInfo() const; + private: + ~PlayGround(); + private: + std::vector< std::pair > _data; + }; + + class YACSLIBENGINE_EXPORT PartDefinition : public RefCounter + { + protected: + PartDefinition(const PlayGround *pg, int nbOfCoresPerComp); + PartDefinition(const PartDefinition& other); + virtual ~PartDefinition(); + public: + std::vector< YACS::BASES::AutoRefCnt > partition(const std::vector< double >& wgs) const; + static YACS::BASES::AutoRefCnt BuildFrom(const PlayGround *pg, int nbOfCoresPerComp, const std::vector& coreIds); + const PlayGround *getPlayGround() const { return _pg; } + int getNbCoresPerCompo() const { return _nbOfCoresPerComp; } + void setNbCoresPerCompo(int newNbCores) { _nbOfCoresPerComp=newNbCores; } + int getSpaceSize() const { return _pg->getNumberOfCoresAvailable(); } + void stashPart(int nbCoresStashed, double weightOfRemain, YACS::BASES::AutoRefCnt& pdStashed, YACS::BASES::AutoRefCnt& pdRemain) const; + std::vector computeWorkerIdsCovered(int nbCoresPerComp) const; + virtual std::string printSelf() const = 0; + virtual std::vector getCoresOn() const = 0; + virtual PartDefinition *copy() const = 0; + virtual int getNumberOfCoresConsumed() const = 0; + private: + YACS::BASES::AutoConstRefCnt _pg; + int _nbOfCoresPerComp; + }; + + class YACSLIBENGINE_EXPORT ContigPartDefinition : public PartDefinition + { + public: + ContigPartDefinition(const PlayGround *pg, int nbOfCoresPerComp, int zeStart, int zeStop); + ContigPartDefinition(const ContigPartDefinition& other); + std::string printSelf() const; + std::vector getCoresOn() const; + int getStart() const { return _start; } + int getStop() const { return _stop; } + ContigPartDefinition *copy() const; + int getNumberOfCoresConsumed() const; + private: + ~ContigPartDefinition() { } + private: + int _start; + int _stop; + }; + + class YACSLIBENGINE_EXPORT NonContigPartDefinition : public PartDefinition + { + public: + NonContigPartDefinition(const PlayGround *pg, int nbOfCoresPerComp, const std::vector& ids); + NonContigPartDefinition(const ContigPartDefinition& other); + std::string printSelf() const; + std::vector getCoresOn() const; + std::vector getIDs() const { return _ids; } + NonContigPartDefinition *copy() const; + int getNumberOfCoresConsumed() const; + private: + void checkOKIds() const; + ~NonContigPartDefinition() { } + private: + std::vector _ids; + }; + + class AllPartDefinition : public PartDefinition + { + public: + AllPartDefinition(const PlayGround *pg, int nbOfCoresPerComp):PartDefinition(pg,nbOfCoresPerComp) { } + AllPartDefinition(const AllPartDefinition& other); + std::string printSelf() const; + std::vector getCoresOn() const; + AllPartDefinition *copy() const; + int getNumberOfCoresConsumed() const; + private: + ~AllPartDefinition() { } + }; + + class ForTestOmlyHPContCls + { + public: + +#ifndef SWIG + void setContainerType(const std::string& ct) { _container_type=ct; } + void setPD(YACS::BASES::AutoConstRefCnt pd) { _pd=pd; } + void setIDS(const std::vector& ids) { _ids=ids; } +#endif + std::string getContainerType() const { return _container_type; } + const PartDefinition *getPD() const { return _pd; } + std::vector getIDS() const; + private: + std::string _container_type; + YACS::BASES::AutoConstRefCnt _pd; + std::vector _ids; + }; + } +} + +#endif diff --git a/src/engine/RefCounter.cxx b/src/engine/RefCounter.cxx index 7c4c89279..c75e473fe 100644 --- a/src/engine/RefCounter.cxx +++ b/src/engine/RefCounter.cxx @@ -45,7 +45,7 @@ void RefCounter::incrRef() const _globalMutexForTS.unLock(); } -bool RefCounter::decrRef() +bool RefCounter::decrRef() const { _globalMutexForTS.lock(); #ifdef REFCNT diff --git a/src/engine/RefCounter.hxx b/src/engine/RefCounter.hxx index 989911561..90401e8c5 100644 --- a/src/engine/RefCounter.hxx +++ b/src/engine/RefCounter.hxx @@ -31,7 +31,7 @@ namespace YACS public: unsigned int getRefCnt() const { return _cnt; } void incrRef() const; - bool decrRef(); + bool decrRef() const; static unsigned int _totalCnt; protected: RefCounter(); diff --git a/src/engine/Runtime.hxx b/src/engine/Runtime.hxx index 91745b68d..749eaf232 100644 --- a/src/engine/Runtime.hxx +++ b/src/engine/Runtime.hxx @@ -70,7 +70,7 @@ namespace YACS public: virtual void init() { } virtual void fini() { } - + virtual std::vector< std::pair > getCatalogOfComputeNodes() const = 0; virtual Catalog* loadCatalog(const std::string& sourceKind,const std::string& path); virtual InlineFuncNode* createFuncNode(const std::string& kind,const std::string& name); virtual InlineNode* createScriptNode(const std::string& kind,const std::string& name); diff --git a/src/engine/SetOfPoints.cxx b/src/engine/SetOfPoints.cxx index 148930b5d..e869b88ca 100644 --- a/src/engine/SetOfPoints.cxx +++ b/src/engine/SetOfPoints.cxx @@ -86,3 +86,18 @@ int SetOfPoints::getMaxLevelOfParallelism() const { return _bp->getMaxLevelOfParallelism(); } + +double SetOfPoints::getWeightRegardingDPL() const +{ + return _bp->getWeightRegardingDPL(); +} + +void SetOfPoints::partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) const +{ + _bp->partitionRegardingDPL(pd,zeMap); +} + +AbstractPoint *SetOfPoints::getUniqueAndReleaseIt() const +{ + return _bp->getUniqueAndReleaseIt(); +} diff --git a/src/engine/SetOfPoints.hxx b/src/engine/SetOfPoints.hxx index b4fd278bf..d670ff692 100644 --- a/src/engine/SetOfPoints.hxx +++ b/src/engine/SetOfPoints.hxx @@ -21,7 +21,10 @@ #define __SETOFPOINTS_HXX__ #include "YACSlibEngineExport.hxx" +#include "PlayGround.hxx" +#include "AutoRefCnt.hxx" +#include #include #include @@ -32,7 +35,8 @@ namespace YACS class Node; class BagPoint; class AbstractPoint; - + class ComposedNode; + class YACSLIBENGINE_EXPORT SetOfPoints { public: @@ -43,6 +47,9 @@ namespace YACS AbstractPoint *findPointWithNode(Node *node); const std::list& getListOfPoints() const; int getMaxLevelOfParallelism() const; + double getWeightRegardingDPL() const; + void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) const; + AbstractPoint *getUniqueAndReleaseIt() const; private: BagPoint *_bp; }; diff --git a/src/engine/Switch.cxx b/src/engine/Switch.cxx index a65e8425f..5c0b7a2d8 100644 --- a/src/engine/Switch.cxx +++ b/src/engine/Switch.cxx @@ -408,6 +408,20 @@ int Switch::getMaxLevelOfParallelism() const return ret; } +double Switch::getWeightRegardingDPL() const +{ + double ret(0); + for(std::map< int , Node * >::const_iterator it=_mapOfNode.begin();it!=_mapOfNode.end();it++) + ret=std::max(ret,((*it).second)->getWeightRegardingDPL()); + return ret; +} + +void Switch::partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap) +{ + for(std::map< int , Node * >::const_iterator it=_mapOfNode.begin();it!=_mapOfNode.end();it++) + (*it).second->partitionRegardingDPL(pd,zeMap); +} + void Switch::edRemoveChild(Node *node) throw(YACS::Exception) { map< int , Node * >::iterator iter=_mapOfNode.begin(); diff --git a/src/engine/Switch.hxx b/src/engine/Switch.hxx index 1ae0a3e34..be5b1c16f 100644 --- a/src/engine/Switch.hxx +++ b/src/engine/Switch.hxx @@ -116,6 +116,8 @@ namespace YACS void writeDot(std::ostream &os) const; int getNumberOfInputPorts() const; int getMaxLevelOfParallelism() const; + double getWeightRegardingDPL() const; + void partitionRegardingDPL(const PartDefinition *pd, std::map >& zeMap); void edRemoveChild(Node *node) throw(Exception); std::list getSetOfInputPort() const; std::list getLocalInputPorts() const; diff --git a/src/engine/Test/RuntimeForEngineIntegrationTest.cxx b/src/engine/Test/RuntimeForEngineIntegrationTest.cxx index 89de9eb03..0c822a2bf 100644 --- a/src/engine/Test/RuntimeForEngineIntegrationTest.cxx +++ b/src/engine/Test/RuntimeForEngineIntegrationTest.cxx @@ -34,6 +34,11 @@ void RuntimeForEngineIntegrationTest::setRuntime() Runtime::_singleton = new RuntimeForEngineIntegrationTest; } +std::vector< std::pair > RuntimeForEngineIntegrationTest::getCatalogOfComputeNodes() const +{ + throw Exception("RuntimeForEngineIntegrationTest not implemented !"); +} + ElementaryNode* RuntimeForEngineIntegrationTest::createNode(const std::string& implementation, const std::string& name) throw (YACS::Exception) { if (implementation == ToyNode::MY_IMPL_NAME) diff --git a/src/engine/Test/RuntimeForEngineIntegrationTest.hxx b/src/engine/Test/RuntimeForEngineIntegrationTest.hxx index 77ec42bb9..cbfdbf1e6 100644 --- a/src/engine/Test/RuntimeForEngineIntegrationTest.hxx +++ b/src/engine/Test/RuntimeForEngineIntegrationTest.hxx @@ -30,6 +30,7 @@ namespace YACS { public: static void setRuntime(); + std::vector< std::pair > getCatalogOfComputeNodes() const; ElementaryNode* createNode(const std::string& implementation, const std::string& name) throw(Exception); InputPort* createInputPort(const std::string& name, const std::string& impl, Node * node, TypeCode * type); OutputPort* createOutputPort(const std::string& name, const std::string& impl, Node * node, TypeCode * type); diff --git a/src/engine/Test/RuntimeForEngineTest.cxx b/src/engine/Test/RuntimeForEngineTest.cxx index 5f6bd3797..53537a9f7 100644 --- a/src/engine/Test/RuntimeForEngineTest.cxx +++ b/src/engine/Test/RuntimeForEngineTest.cxx @@ -106,6 +106,11 @@ void RuntimeForEngineTest::setRuntime() Runtime::_singleton = new RuntimeForEngineTest; } +std::vector< std::pair > RuntimeForEngineTest::getCatalogOfComputeNodes() const +{ + throw Exception("RuntimeForEngineTest::getCatalogOfComputeNodes : not implemented !"); +} + ElementaryNode* RuntimeForEngineTest::createNode(const string& implementation, const string& name) throw(YACS::Exception) { return new TestElemNode(name); diff --git a/src/engine/Test/RuntimeForEngineTest.hxx b/src/engine/Test/RuntimeForEngineTest.hxx index 0495ab60a..d5e5a7ea3 100644 --- a/src/engine/Test/RuntimeForEngineTest.hxx +++ b/src/engine/Test/RuntimeForEngineTest.hxx @@ -66,6 +66,7 @@ namespace YACS { public: static void setRuntime(); + std::vector< std::pair > getCatalogOfComputeNodes() const; ElementaryNode* createNode(const std::string& implementation, const std::string& name) throw(Exception); InputPort* createInputPort(const std::string& name, const std::string& impl, Node * node, TypeCode * type); OutputPort* createOutputPort(const std::string& name, const std::string& impl, Node * node, TypeCode * type); diff --git a/src/engine_swig/CMakeLists.txt b/src/engine_swig/CMakeLists.txt index 4c525af2e..e882a14eb 100644 --- a/src/engine_swig/CMakeLists.txt +++ b/src/engine_swig/CMakeLists.txt @@ -123,3 +123,13 @@ ENDIF(WIN32) INSTALL(TARGETS ${SWIG_MODULE_pilot_REAL_NAME} ${SWIG_MODULE_pypilot_REAL_NAME} DESTINATION ${SALOME_INSTALL_PYTHON}) SALOME_INSTALL_SCRIPTS("${_swig_SCRIPTS}" ${SALOME_INSTALL_PYTHON}) + +### Test + +SALOME_GENERATE_TESTS_ENVIRONMENT(tests_env) +ADD_TEST(NAME PlayGround0 COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/testPlayGround0.py) +SET_TESTS_PROPERTIES(PlayGround0 PROPERTIES ENVIRONMENT "${tests_env}") +SET(LOCAL_TEST_DIR ${SALOME_YACS_INSTALL_TEST}/engine_swig) +SET(LOCAL_TEST_FILES testPlayGround0.py) +INSTALL(FILES ${LOCAL_TEST_FILES} DESTINATION ${LOCAL_TEST_DIR}) +INSTALL(FILES CTestTestfileInstall.cmake DESTINATION ${LOCAL_TEST_DIR} RENAME CTestTestfile.cmake) \ No newline at end of file diff --git a/src/engine_swig/CTestTestfileInstall.cmake b/src/engine_swig/CTestTestfileInstall.cmake new file mode 100644 index 000000000..745df03ff --- /dev/null +++ b/src/engine_swig/CTestTestfileInstall.cmake @@ -0,0 +1,24 @@ +# Copyright (C) 2015-2016 CEA/DEN, EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +IF(NOT WIN32) + SET(TEST_NAME ${COMPONENT_NAME}_PlayGround0) + ADD_TEST(${TEST_NAME} python testPlayGround0.py) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}") +ENDIF() diff --git a/src/engine_swig/PlayGround.i b/src/engine_swig/PlayGround.i new file mode 100644 index 000000000..e5f1e3e7e --- /dev/null +++ b/src/engine_swig/PlayGround.i @@ -0,0 +1,169 @@ +// Copyright (C) 2006-2016 CEA/DEN, EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// + +%template() std::pair< YACS::ENGINE::PartDefinition *, double>; +%template(vecppdd) std::vector< std::pair< YACS::ENGINE::PartDefinition *, double> >; + +%newobject YACS::ENGINE::PlayGround::copy; + +%{ +PyObject *convertPartDefinition(YACS::ENGINE::PartDefinition *pd) +{ + if(!pd) + return SWIG_NewPointerObj(SWIG_as_voidptr(pd),SWIGTYPE_p_YACS__ENGINE__PartDefinition, SWIG_POINTER_OWN | 0 ); + YACS::ENGINE::ContigPartDefinition *pd1(dynamic_cast(pd)); + if(pd1) + return SWIG_NewPointerObj(SWIG_as_voidptr(pd1),SWIGTYPE_p_YACS__ENGINE__ContigPartDefinition, SWIG_POINTER_OWN | 0 ); + YACS::ENGINE::NonContigPartDefinition *pd2(dynamic_cast(pd)); + if(pd2) + return SWIG_NewPointerObj(SWIG_as_voidptr(pd2),SWIGTYPE_p_YACS__ENGINE__NonContigPartDefinition, SWIG_POINTER_OWN | 0 ); + YACS::ENGINE::AllPartDefinition *pd3(dynamic_cast(pd)); + if(pd3) + return SWIG_NewPointerObj(SWIG_as_voidptr(pd3),SWIGTYPE_p_YACS__ENGINE__AllPartDefinition, SWIG_POINTER_OWN | 0 ); + return SWIG_NewPointerObj(SWIG_as_voidptr(pd),SWIGTYPE_p_YACS__ENGINE__PartDefinition, SWIG_POINTER_OWN | 0 ); + +} +%} + +%typemap(out) std::vector< YACS::BASES::AutoRefCnt > +{ + std::size_t sz($1.size()); + std::vector< YACS::BASES::AutoRefCnt >::iterator it($1.begin()); + $result = PyList_New($1.size()); + for (std::size_t i=0; i >& defOfRes); + std::string printSelf() const; + std::vector< std::pair > getData() const; + void setData(const std::vector< std::pair >& defOfRes); + int getNumberOfCoresAvailable() const; + int getMaxNumberOfContainersCanBeHostedWithoutOverlap(int nbCoresPerCont) const; + std::string deduceMachineFrom(int workerId, int nbProcPerNode) const; + %extend + { + std::string __str__() const + { + return self->printSelf(); + } + + std::vector< YACS::BASES::AutoRefCnt > partition(const std::vector< std::pair< YACS::ENGINE::PartDefinition *,double> >& parts) const + { + std::size_t sz(parts.size()); + std::vector< std::pair< const YACS::ENGINE::PartDefinition *,double> > partsCpp(sz); + for(std::size_t i=0;i(parts[i].first,parts[i].second); + return self->partition(partsCpp); + } + } + private: + ~PlayGround(); + }; + + class PartDefinition : public RefCounter + { + public: + virtual PartDefinition *copy() const; + virtual std::string printSelf() const; + virtual int getNumberOfCoresConsumed() const; + private: + PartDefinition(const PlayGround *pg, int nbOfCoresPerComp); + ~PartDefinition(); + }; + + class ContigPartDefinition : public PartDefinition + { + public: + ContigPartDefinition(const PlayGround *pg, int nbOfCoresPerComp, int zeStart, int zeStop); + int getStart() const; + int getStop() const; + %extend + { + std::string __str__() const + { + return self->printSelf(); + } + } + private: + ~ContigPartDefinition(); + }; + + class NonContigPartDefinition : public PartDefinition + { + public: + NonContigPartDefinition(const PlayGround *pg, int nbOfCoresPerComp, const std::vector& ids); + std::vector getIDs() const; + %extend + { + std::string __str__() const + { + return self->printSelf(); + } + } + private: + ~NonContigPartDefinition(); + }; + + class AllPartDefinition : public PartDefinition + { + public: + AllPartDefinition(const PlayGround *pg, int nbOfCoresPerComp); + %extend + { + std::string __str__() const + { + return self->printSelf(); + } + } + private: + ~AllPartDefinition(); + }; + + class PartDefinition; + + class ForTestOmlyHPContCls + { + public: + std::string getContainerType() const; + std::vector getIDS() const; + %extend + { + PyObject *getPD() const + { + const PartDefinition *ret(self->getPD()); + if(ret) + ret->incrRef(); + return convertPartDefinition(const_cast(ret)); + } + } + }; + } +} diff --git a/src/engine_swig/pilot.i b/src/engine_swig/pilot.i index 6b93d6614..98a7296db 100644 --- a/src/engine_swig/pilot.i +++ b/src/engine_swig/pilot.i @@ -58,7 +58,8 @@ #include "DeploymentTree.hxx" #include "ComponentInstance.hxx" #include "DataNode.hxx" - +#include "PlayGround.hxx" + using namespace YACS::ENGINE; %} @@ -104,6 +105,7 @@ using namespace YACS::ENGINE; %template() std::pair; %template() std::pair; %template() std::pair; +%template() std::pair< std::string, int >; //%template(TCmap) std::map; REFCOUNT_TEMPLATE(TCmap,YACS::ENGINE::TypeCode) %template(NODEmap) std::map; @@ -113,11 +115,13 @@ REFCOUNT_TEMPLATE(TCmap,YACS::ENGINE::TypeCode) REFCOUNT_TEMPLATE(CONTAINmap,YACS::ENGINE::Container) %template(strvec) std::vector; %template(uivec) std::vector; +%template(ivec) std::vector; %template(linksvec) std::vector< std::pair >; %template(linkvec) std::vector< std::pair >; %template(instreamlist) std::list; %template(outstreamlist) std::list; - +%template(vpsi) std::vector< std::pair< std::string, int > >; + %template() std::pair; %template(loadermap) std::map; %template() std::pair; @@ -349,6 +353,7 @@ EXCEPTION(YACS::ENGINE::ExecutorSwig::waitPause) // OptimizerAlgASync in module SALOMERuntime %ignore YACS::ENGINE::OptimizerAlgASync; %include +%include "PlayGround.i" %extend YACS::ENGINE::ConditionInputPort { diff --git a/src/engine_swig/testPlayGround0.py b/src/engine_swig/testPlayGround0.py new file mode 100644 index 000000000..3128e2126 --- /dev/null +++ b/src/engine_swig/testPlayGround0.py @@ -0,0 +1,76 @@ +# Copyright (C) 2006-2017 CEA/DEN, EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +import pilot +import unittest + +class TestPlayGround0(unittest.TestCase): + def test0(self): + pg=pilot.PlayGround([("a0",28),("a1",28),("a2",28)]) + pd=pilot.ContigPartDefinition(pg,4,0,3*28) + res=pg.partition([(pd,1.),(pd,1.)]) + assert(len(res)==2) + assert(isinstance(res[0],pilot.ContigPartDefinition)) + assert(isinstance(res[1],pilot.ContigPartDefinition)) + assert(res[0].getStart()==0 and res[0].getStop()==40) + assert(res[1].getStart()==40 and res[1].getStop()==84) + assert(sum([elt.getNumberOfCoresConsumed() for elt in res])==pg.getNumberOfCoresAvailable()) + pd2=pilot.AllPartDefinition(pg,4) + assert(pd2.getNumberOfCoresConsumed()==84) + res=pg.partition([(pd2,1.),(pd2,1.),(pd2,1.)]) + assert(len(res)==3) + assert(isinstance(res[0],pilot.ContigPartDefinition)) + assert(isinstance(res[1],pilot.ContigPartDefinition)) + assert(isinstance(res[2],pilot.ContigPartDefinition)) + assert(res[0].getStart()==0 and res[0].getStop()==28) + assert(res[1].getStart()==28 and res[1].getStop()==56) + assert(res[2].getStart()==56 and res[2].getStop()==84) + # + pg.setData([("a0",2),("a1",8),("a2",8)]) + res=pg.partition([(pilot.AllPartDefinition(pg,4),1.),(pilot.AllPartDefinition(pg,1),1.)]) + assert(len(res)==2) + assert(isinstance(res[0],pilot.ContigPartDefinition)) + assert(isinstance(res[1],pilot.NonContigPartDefinition)) + assert(res[0].getStart()==2 and res[0].getStop()==10) + assert(res[1].getIDs()==(0,1,10,11,12,13,14,15,16,17)) + pass + + def test1(self): + """ test focused on complicated cut due to lack of cores""" + pg=pilot.PlayGround([("a0",13)]) + pd=pilot.ContigPartDefinition(pg,4,0,13) + res=pg.partition([(pd,1.),(pd,1.)]) + assert(len(res)==2) + assert(isinstance(res[0],pilot.ContigPartDefinition) and isinstance(res[1],pilot.ContigPartDefinition)) + assert(res[0].getStart()==0 and res[0].getStop()==4) + assert(res[1].getStart()==4 and res[1].getStop()==12)# 1 core lost + # + pg=pilot.PlayGround([("a0",2),("a1",11)]) + pd=pilot.ContigPartDefinition(pg,4,0,13) + res=pg.partition([(pd,1.),(pd,1.)]) + assert(len(res)==2) + assert(isinstance(res[0],pilot.ContigPartDefinition) and isinstance(res[1],pilot.ContigPartDefinition)) + assert(res[0].getStart()==2 and res[0].getStop()==6) + assert(res[1].getStart()==6 and res[1].getStop()==10)# 5 cores lost + pass + + pass + +if __name__ == '__main__': + unittest.main() diff --git a/src/runtime/DistributedPythonNode.cxx b/src/runtime/DistributedPythonNode.cxx index edd891af9..8a2ccd7a7 100644 --- a/src/runtime/DistributedPythonNode.cxx +++ b/src/runtime/DistributedPythonNode.cxx @@ -113,13 +113,12 @@ void DistributedPythonNode::load() if(!_container) throw Exception("No container specified !"); SalomeContainer *containerCast0(dynamic_cast(_container)); - SalomeHPContainer *containerCast1(dynamic_cast(_container)); + SalomeHPContainerBase *containerCast1(dynamic_cast(_container)); if(containerCast0) objContainer=containerCast0->getContainerPtr(this); else if(containerCast1) { - YACS::BASES::AutoCppPtr tmpCont(SalomeContainerTmpForHP::BuildFrom(containerCast1,this)); - objContainer=tmpCont->getContainerPtr(this); + objContainer=containerCast1->getContainerPtr(this); } else throw Exception("Unrecognized type of container ! Salome one is expected !"); diff --git a/src/runtime/PythonNode.cxx b/src/runtime/PythonNode.cxx index b6b7bc47d..e1db34ebb 100644 --- a/src/runtime/PythonNode.cxx +++ b/src/runtime/PythonNode.cxx @@ -127,13 +127,12 @@ Engines::Container_var PythonEntry::commonRemoteLoadPart2(InlineNode *reqNode, b if(!container) throw Exception("No container specified !"); SalomeContainer *containerCast0(dynamic_cast(container)); - SalomeHPContainer *containerCast1(dynamic_cast(container)); + SalomeHPContainerBase *containerCast1(dynamic_cast(container)); if(containerCast0) objContainer=containerCast0->getContainerPtr(reqNode); else if(containerCast1) { - YACS::BASES::AutoCppPtr tmpCont(SalomeContainerTmpForHP::BuildFrom(containerCast1,reqNode)); - objContainer=tmpCont->getContainerPtr(reqNode); + objContainer=containerCast1->getContainerPtr(reqNode); } else throw Exception("Unrecognized type of container ! Salome one is expected for PythonNode/PyFuncNode !"); diff --git a/src/runtime/RuntimeSALOME.cxx b/src/runtime/RuntimeSALOME.cxx index 31985cf2f..39b0b795f 100644 --- a/src/runtime/RuntimeSALOME.cxx +++ b/src/runtime/RuntimeSALOME.cxx @@ -112,6 +112,12 @@ #ifdef SALOME_KERNEL #include "SALOME_NamingService.hxx" #include "SALOME_LifeCycleCORBA.hxx" +#include "SALOME_NamingService.hxx" +#include "SALOME_ResourcesManager.hxx" +#include "SALOME_ContainerManager.hxx" +#include "SALOMEconfig.h" +#include CORBA_CLIENT_HEADER(SALOME_ContainerManager) + #endif #include @@ -428,6 +434,44 @@ void RuntimeSALOME::fini() } } +std::vector< std::pair > RuntimeSALOME::getCatalogOfComputeNodes() const +{ + CORBA::ORB_ptr orb(getOrb()); + SALOME_NamingService namingService; + try + { + namingService.init_orb(orb); + } + catch(SALOME_Exception& e) + { + throw Exception("SalomeContainerToolsSpreadOverTheResDecorator::getParameters : Unable to contact the SALOME Naming Service"); + } + CORBA::Object_var obj(namingService.Resolve(SALOME_ResourcesManager::_ResourcesManagerNameInNS)); + if(CORBA::is_nil(obj)) + throw Exception("SalomeContainerToolsSpreadOverTheResDecorator::getParameters : Unable to access to the resource manager !"); + Engines::ResourcesManager_var resManager(Engines::ResourcesManager::_narrow(obj)); + if(CORBA::is_nil(resManager)) + throw Exception("SalomeContainerToolsSpreadOverTheResDecorator::getParameters : Internal error ! The entry attached to the res manager in NS does not have right type !"); + std::vector< std::pair > ret; + { + Engines::ResourceList *rl(0); + Engines::IntegerList *il(0); + resManager->ListAllAvailableResources(rl,il); + int sz(rl->length()); + if(il->length()!=sz) + throw Exception("SalomeContainerToolsSpreadOverTheResDecorator::getParameters : Internal error ! Invalid size !"); + ret.resize(sz); + for(int i=0;i(s,(*il)[i]); + } + delete rl; + delete il; + } + return ret; +} + std::string RuntimeSALOME::getVersion() const { #ifdef YACS_DEVELOPMENT @@ -1776,22 +1820,22 @@ InputPort* RuntimeSALOME::adapt(InputCppPort* source, // return result; // } -CORBA::ORB_ptr RuntimeSALOME::getOrb() +CORBA::ORB_ptr RuntimeSALOME::getOrb() const { return _orb; } -PyObject * RuntimeSALOME::getPyOrb() +PyObject * RuntimeSALOME::getPyOrb() const { return _pyorb; } -PyObject * RuntimeSALOME::getBuiltins() +PyObject * RuntimeSALOME::getBuiltins() const { return _bltins; } -DynamicAny::DynAnyFactory_ptr RuntimeSALOME::getDynFactory() +DynamicAny::DynAnyFactory_ptr RuntimeSALOME::getDynFactory() const { return _dynFactory; } diff --git a/src/runtime/RuntimeSALOME.hxx b/src/runtime/RuntimeSALOME.hxx index d9ba31c23..4479b3ded 100644 --- a/src/runtime/RuntimeSALOME.hxx +++ b/src/runtime/RuntimeSALOME.hxx @@ -98,7 +98,7 @@ namespace YACS virtual void init(long flags, int argc, char* argv[]); virtual void fini(); - + virtual std::vector< std::pair > getCatalogOfComputeNodes() const; virtual InputPort* createInputPort(const std::string& name, const std::string& impl, Node * node, @@ -245,10 +245,10 @@ namespace YACS virtual ~RuntimeSALOME(); - CORBA::ORB_ptr getOrb(); - PyObject * getPyOrb(); - PyObject * getBuiltins(); - DynamicAny::DynAnyFactory_ptr getDynFactory(); + CORBA::ORB_ptr getOrb() const; + PyObject * getPyOrb() const; + PyObject * getBuiltins() const; + DynamicAny::DynAnyFactory_ptr getDynFactory() const; omniORBpyAPI* getApi(); PyObject * get_omnipy(); diff --git a/src/runtime/SalomeContainerHelper.cxx b/src/runtime/SalomeContainerHelper.cxx index f35fe882a..b65e2bd46 100644 --- a/src/runtime/SalomeContainerHelper.cxx +++ b/src/runtime/SalomeContainerHelper.cxx @@ -81,8 +81,15 @@ void SalomeContainerMonoHelper::setContainer(const Task *askingNode, Engines::Co #endif } +bool SalomeContainerMonoHelper::isKernelContNull() const +{ + return CORBA::is_nil(_trueCont); +} + void SalomeContainerMonoHelper::shutdown() { + if(CORBA::is_nil(_trueCont)) + return ; try { DEBTRACE("shutdown SALOME container: " ); diff --git a/src/runtime/SalomeContainerHelper.hxx b/src/runtime/SalomeContainerHelper.hxx index 08ecb7c70..8fa363f5b 100644 --- a/src/runtime/SalomeContainerHelper.hxx +++ b/src/runtime/SalomeContainerHelper.hxx @@ -61,6 +61,7 @@ namespace YACS Engines::Container_var getContainer(const Task *askingNode) const; bool isAlreadyStarted(const Task *askingNode) const; void setContainer(const Task *askingNode, Engines::Container_var cont); + bool isKernelContNull() const; void shutdown(); private: ~SalomeContainerMonoHelper(); diff --git a/src/runtime/SalomeContainerTools.cxx b/src/runtime/SalomeContainerTools.cxx index f4d66286f..204187820 100644 --- a/src/runtime/SalomeContainerTools.cxx +++ b/src/runtime/SalomeContainerTools.cxx @@ -30,6 +30,7 @@ #include "ServiceNode.hxx" #include "ComponentInstance.hxx" #include "SalomeContainerHelper.hxx" +#include "SalomeHPContainerTools.hxx" #include "RuntimeSALOME.hxx" #include "Exception.hxx" @@ -42,23 +43,251 @@ using namespace YACS::ENGINE; -SalomeContainerTools::SalomeContainerTools() +/*! + * \param [in] compoNames + * \param [in,out] shutdownLevel + */ +void SalomeContainerToolsBase::Start(const std::vector& compoNames, SalomeContainerHelper *schelp, SalomeContainerToolsBase& sct, int& shutdownLevel, const Container *cont, const Task *askingNode) { - /* Init ContainerParameters */ - SALOME_LifeCycleCORBA::preSet(_params); + CORBA::ORB_ptr orb(getSALOMERuntime()->getOrb()); + SALOME_NamingService ns; + try + { + ns.init_orb(orb); + } + catch(SALOME_Exception& e) + { + throw Exception("SalomeContainer::start : Unable to contact the SALOME Naming Service"); + } + CORBA::Object_var obj(ns.Resolve(SALOME_ContainerManager::_ContainerManagerNameInNS)); + Engines::ContainerManager_var contManager(Engines::ContainerManager::_narrow(obj)); + + bool isEmptyName; + std::string str(sct.getNotNullContainerName(cont,askingNode,isEmptyName)); + DEBTRACE("SalomeContainer::start " << str <<";"<< sct.getHostName() <<";"<< schelp->getType()); + + // Finalize parameters with components found in the container + + for(std::vector::const_iterator iter=compoNames.begin();iter!=compoNames.end();iter++) + sct.addToComponentList(*iter); + + Engines::ContainerParameters myparams(sct.getParameters()); + { + std::string dftLauchMode(schelp->getDftLaunchMode()); + myparams.mode=CORBA::string_dup(dftLauchMode.c_str()); + } + + //If a container_name is given try to find an already existing container in naming service + //If not found start a new container with the given parameters + if (dynamic_cast(schelp) && !isEmptyName) + { + myparams.mode=CORBA::string_dup("getorstart"); + } + + if (isEmptyName) + { + shutdownLevel=1; + } + //sct.setContainerName(str); + SetContainerNameOf(myparams,str); + Engines::Container_var trueCont(Engines::Container::_nil()); + if(!isEmptyName && shutdownLevel==999) + { + //Make this only the first time start is called (_shutdownLevel==999) + //If the container is named, first try to get an existing container + //If there is an existing container use it and set the shutdown level to 3 + //If there is no existing container, try to launch a new one and set the shutdown level to 2 + myparams.mode="get"; + try + { + trueCont=contManager->GiveContainer(myparams); + } + catch( const SALOME::SALOME_Exception& ex ) + { + std::string msg="SalomeContainer::start : no existing container : "; + msg += '\n'; + msg += ex.details.text.in(); + DEBTRACE( msg ); + } + catch(...) + { + } + + if(!CORBA::is_nil(trueCont)) + { + shutdownLevel=3; + DEBTRACE( "container found: " << str << " " << _shutdownLevel ); + } + else + { + shutdownLevel=2; + myparams.mode="start"; + DEBTRACE( "container not found: " << str << " " << _shutdownLevel); + } + } + + if(CORBA::is_nil(trueCont)) + try + { + // --- GiveContainer is used in batch mode to retreive launched containers, + // and is equivalent to StartContainer when not in batch. + trueCont=contManager->GiveContainer(myparams); + } + catch( const SALOME::SALOME_Exception& ex ) + { + std::string msg="SalomeContainer::start : Unable to launch container in Salome : "; + msg += '\n'; + msg += ex.details.text.in(); + throw Exception(msg); + } + catch(CORBA::COMM_FAILURE&) + { + throw Exception("SalomeContainer::start : Unable to launch container in Salome : CORBA Comm failure detected"); + } + catch(CORBA::Exception&) + { + throw Exception("SalomeContainer::start : Unable to launch container in Salome : Unexpected CORBA failure detected"); + } + + if(CORBA::is_nil(trueCont)) + throw Exception("SalomeContainer::start : Unable to launch container in Salome. Check your CatalogResources.xml file"); + + schelp->setContainer(askingNode,trueCont); + + CORBA::String_var containerName(trueCont->name()),hostName(trueCont->getHostName()); + std::cerr << "SalomeContainer launched : " << containerName << " " << hostName << " " << trueCont->getPID() << std::endl; } -SalomeContainerTools::SalomeContainerTools(const SalomeContainerTools& other):_params(other._params),_propertyMap(other._propertyMap) +CORBA::Object_ptr SalomeContainerToolsBase::LoadComponent(SalomeContainerHelper *launchModeType, Container *cont, Task *askingNode) { + DEBTRACE("SalomeContainer::loadComponent "); + const ComponentInstance *inst(askingNode?askingNode->getComponent():0); + { + YACS::BASES::AutoLocker alck(cont);//To be sure + if(!cont->isAlreadyStarted(askingNode)) + cont->start(askingNode); + } + if(!inst) + throw Exception("SalomeContainerTools::LoadComponent : no instance of component in the task requesting for a load of its component !"); + CORBA::Object_ptr objComponent=CORBA::Object::_nil(); + { + YACS::BASES::AutoLocker alck(cont);//To be sure + std::string compoName(inst->getCompoName()); + Engines::Container_var container(launchModeType->getContainer(askingNode)); + + char *reason; + bool isLoadable(container->load_component_Library(compoName.c_str(), reason)); + if(isLoadable) + objComponent=CreateComponentInstance(cont,container,inst); + } + return objComponent; } -void SalomeContainerTools::clearProperties() +CORBA::Object_ptr SalomeContainerToolsBase::CreateComponentInstance(Container *cont, Engines::Container_ptr contPtr, const ComponentInstance *inst) +{ + if(!inst) + throw Exception("SalomeContainerTools::CreateComponentInstance : no instance of component in the task requesting for a load of its component !"); + char *reason(0); + std::string compoName(inst->getCompoName()); + CORBA::Object_ptr objComponent=CORBA::Object::_nil(); + int studyid(1); + Proc* p(cont->getProc()); + if(p) + { + std::string value(p->getProperty("DefaultStudyID")); + if(!value.empty()) + studyid= atoi(value.c_str()); + } + // prepare component instance properties + Engines::FieldsDict_var env(new Engines::FieldsDict); + std::map properties(inst->getProperties()); + if(p) + { + std::map procMap=p->getProperties(); + properties.insert(procMap.begin(),procMap.end()); + } + + std::map::const_iterator itm; + env->length(properties.size()); + int item=0; + for(itm = properties.begin(); itm != properties.end(); ++itm, item++) + { + DEBTRACE("envname="<first<<" envvalue="<< itm->second); + env[item].key= CORBA::string_dup(itm->first.c_str()); + env[item].value <<= itm->second.c_str(); + } + + objComponent=contPtr->create_component_instance_env(compoName.c_str(), studyid, env, reason); + if(CORBA::is_nil(objComponent)) + { + std::string text="Error while trying to create a new component: component '"+ compoName; + text=text+"' is not installed or it's a wrong name"; + text += '\n'; + text += reason; + CORBA::string_free(reason); + throw Exception(text); + } + return objComponent; +} + +std::string SalomeContainerToolsBase::GetPlacementId(const SalomeContainerHelper *launchModeType, const Container *cont, const Task *askingNode) +{ + if(cont->isAlreadyStarted(askingNode)) + { + Engines::Container_var container(launchModeType->getContainer(askingNode)); + const char *what="/"; + CORBA::String_var corbaStr(container->name()); + std::string ret(corbaStr); + + //Salome FOREVER ... + std::string::size_type i=ret.find_first_of(what,0); + i=ret.find_first_of(what, i==std::string::npos ? i:i+1); + if(i!=std::string::npos) + return ret.substr(i+1); + return ret; + } + else + return "Not placed yet !!!"; +} + +std::string SalomeContainerToolsBase::GetFullPlacementId(const SalomeContainerHelper *launchModeType, const Container *cont, const Task *askingNode) +{ + if(cont->isAlreadyStarted(askingNode)) + { + Engines::Container_var container(launchModeType->getContainer(askingNode)); + try + { + CORBA::String_var corbaStr(container->name()); + std::string ret(corbaStr); + return ret; + } + catch(...) + { + return "Unknown_placement"; + } + } + else + return "Not_placed_yet"; +} + +void SalomeContainerToolsBase::SetContainerNameOf(Engines::ContainerParameters& params, const std::string& name) +{ + params.container_name=CORBA::string_dup(name.c_str()); +} + + +///////////////////////////////// + +SalomeContainerToolsInter::SalomeContainerToolsInter(const SalomeContainerToolsInter& other):_propertyMap(other._propertyMap) +{ +} + +void SalomeContainerToolsInter::clearProperties() { _propertyMap.clear(); - _params=Engines::ContainerParameters(); } -std::string SalomeContainerTools::getProperty(const std::string& name) const +std::string SalomeContainerToolsInter::getProperty(const std::string& name) const { std::map::const_iterator it(_propertyMap.find(name)); if(it!=_propertyMap.end()) @@ -67,6 +296,69 @@ std::string SalomeContainerTools::getProperty(const std::string& name) const return std::string(); } +std::map SalomeContainerToolsInter::getResourceProperties(const std::string& name) const +{ + std::map properties; + + YACS::ENGINE::RuntimeSALOME* runTime = YACS::ENGINE::getSALOMERuntime(); + CORBA::ORB_ptr orb = runTime->getOrb(); + if (!orb) return properties; + SALOME_NamingService namingService(orb); + SALOME_LifeCycleCORBA lcc(&namingService); + CORBA::Object_var obj = namingService.Resolve(SALOME_ResourcesManager::_ResourcesManagerNameInNS); + if (CORBA::is_nil(obj)) + return properties; + Engines::ResourcesManager_var resManager = Engines::ResourcesManager::_narrow(obj); + if (CORBA::is_nil(resManager)) + return properties; + + std::ostringstream value; + Engines::ResourceDefinition_var resource_definition = resManager->GetResourceDefinition(name.c_str()); + properties["hostname"]=resource_definition->hostname.in(); + properties["OS"]=resource_definition->OS.in(); + value.str(""); value << resource_definition->mem_mb; + properties["mem_mb"]=value.str(); + value.str(""); value << resource_definition->cpu_clock; + properties["cpu_clock"]=value.str(); + value.str(""); value << resource_definition->nb_node; + properties["nb_node"]=value.str(); + value.str(""); value << resource_definition->nb_proc_per_node; + properties["nb_proc_per_node"]=value.str(); + /* + properties["component_list"]=""; + for(CORBA::ULong i=0; i < resource_definition->componentList.length(); i++) + { + if(i > 0) + properties["component_list"]=properties["component_list"]+","; + properties["component_list"]=properties["component_list"]+resource_definition->componentList[i].in(); + } + */ + return properties; +} + +//////////////////////// + +SalomeContainerTools::SalomeContainerTools() +{ + /* Init ContainerParameters */ + SALOME_LifeCycleCORBA::preSet(_params); +} + +SalomeContainerTools::SalomeContainerTools(const SalomeContainerTools& other):SalomeContainerToolsInter(other),_params(other._params) +{ +} + +int SalomeContainerTools::getNumberOfCoresPerWorker() const +{ + return _params.resource_params.nb_proc_per_node; +} + +void SalomeContainerTools::clearProperties() +{ + SalomeContainerToolsInter::clearProperties(); + _params=Engines::ContainerParameters(); +} + void SalomeContainerTools::setProperty(const std::string& name, const std::string& value) { //DEBTRACE("SalomeContainer::setProperty : " << name << " ; " << value); @@ -232,7 +524,12 @@ std::string SalomeContainerTools::getContainerName() const void SalomeContainerTools::setContainerName(const std::string& name) { - SetContainerNameOf(_params,name); + SetContainerNameOf(_params,name); +} + +std::string SalomeContainerTools::getHostName() const +{ + return std::string(_params.resource_params.hostname); } std::string SalomeContainerTools::getNotNullContainerName(const Container *contPtr, const Task *askingNode, bool& isEmpty) const @@ -257,279 +554,82 @@ std::string SalomeContainerTools::getNotNullContainerName(const Container *contP } } -std::string SalomeContainerTools::getHostName() const +////////////////////////// + +std::string SalomeContainerToolsDecoratorBase::getProperty(const std::string& name) const { - return std::string(_params.resource_params.hostname); + return _sct->getProperty(name); } -void SalomeContainerTools::SetContainerNameOf(Engines::ContainerParameters& params, const std::string& name) +void SalomeContainerToolsDecoratorBase::setProperty(const std::string& name, const std::string& value) { - params.container_name=CORBA::string_dup(name.c_str()); + _sct->setProperty(name,value); } -std::map SalomeContainerTools::getResourceProperties(const std::string& name) const +const std::map& SalomeContainerToolsDecoratorBase::getProperties() const { - std::map properties; - - YACS::ENGINE::RuntimeSALOME* runTime = YACS::ENGINE::getSALOMERuntime(); - CORBA::ORB_ptr orb = runTime->getOrb(); - if (!orb) return properties; - SALOME_NamingService namingService(orb); - SALOME_LifeCycleCORBA lcc(&namingService); - CORBA::Object_var obj = namingService.Resolve(SALOME_ResourcesManager::_ResourcesManagerNameInNS); - if (CORBA::is_nil(obj)) - return properties; - Engines::ResourcesManager_var resManager = Engines::ResourcesManager::_narrow(obj); - if (CORBA::is_nil(resManager)) - return properties; - - std::ostringstream value; - Engines::ResourceDefinition_var resource_definition = resManager->GetResourceDefinition(name.c_str()); - properties["hostname"]=resource_definition->hostname.in(); - properties["OS"]=resource_definition->OS.in(); - value.str(""); value << resource_definition->mem_mb; - properties["mem_mb"]=value.str(); - value.str(""); value << resource_definition->cpu_clock; - properties["cpu_clock"]=value.str(); - value.str(""); value << resource_definition->nb_node; - properties["nb_node"]=value.str(); - value.str(""); value << resource_definition->nb_proc_per_node; - properties["nb_proc_per_node"]=value.str(); - /* - properties["component_list"]=""; - for(CORBA::ULong i=0; i < resource_definition->componentList.length(); i++) - { - if(i > 0) - properties["component_list"]=properties["component_list"]+","; - properties["component_list"]=properties["component_list"]+resource_definition->componentList[i].in(); - } - */ - return properties; + return _sct->getProperties(); } -/*! - * \param [in] compoNames - * \param [in,out] shutdownLevel - */ -void SalomeContainerTools::Start(const std::vector& compoNames, SalomeContainerHelper *schelp, SalomeContainerTools& sct, int& shutdownLevel, const Container *cont, const Task *askingNode) +void SalomeContainerToolsDecoratorBase::clearProperties() { - CORBA::ORB_ptr orb(getSALOMERuntime()->getOrb()); - SALOME_NamingService ns; - try - { - ns.init_orb(orb); - } - catch(SALOME_Exception& e) - { - throw Exception("SalomeContainer::start : Unable to contact the SALOME Naming Service"); - } - CORBA::Object_var obj(ns.Resolve(SALOME_ContainerManager::_ContainerManagerNameInNS)); - Engines::ContainerManager_var contManager(Engines::ContainerManager::_narrow(obj)); - - bool isEmptyName; - std::string str(sct.getNotNullContainerName(cont,askingNode,isEmptyName)); - DEBTRACE("SalomeContainer::start " << str <<";"<< _sct.getHostName() <<";"<<_type); - - // Finalize parameters with components found in the container - - for(std::vector::const_iterator iter=compoNames.begin();iter!=compoNames.end();iter++) - sct.addToComponentList(*iter); - - Engines::ContainerParameters myparams(sct.getParameters()); - { - std::string dftLauchMode(schelp->getDftLaunchMode()); - myparams.mode=CORBA::string_dup(dftLauchMode.c_str()); - } - - //If a container_name is given try to find an already existing container in naming service - //If not found start a new container with the given parameters - if (dynamic_cast(schelp) && !isEmptyName) - { - myparams.mode=CORBA::string_dup("getorstart"); - } - - if (isEmptyName) - { - shutdownLevel=1; - } - //sct.setContainerName(str); - SetContainerNameOf(myparams,str); - Engines::Container_var trueCont(Engines::Container::_nil()); - if(!isEmptyName && shutdownLevel==999) - { - //Make this only the first time start is called (_shutdownLevel==999) - //If the container is named, first try to get an existing container - //If there is an existing container use it and set the shutdown level to 3 - //If there is no existing container, try to launch a new one and set the shutdown level to 2 - myparams.mode="get"; - try - { - trueCont=contManager->GiveContainer(myparams); - } - catch( const SALOME::SALOME_Exception& ex ) - { - std::string msg="SalomeContainer::start : no existing container : "; - msg += '\n'; - msg += ex.details.text.in(); - DEBTRACE( msg ); - } - catch(...) - { - } - - if(!CORBA::is_nil(trueCont)) - { - shutdownLevel=3; - DEBTRACE( "container found: " << str << " " << _shutdownLevel ); - } - else - { - shutdownLevel=2; - myparams.mode="start"; - DEBTRACE( "container not found: " << str << " " << _shutdownLevel); - } - } - - if(CORBA::is_nil(trueCont)) - try - { - // --- GiveContainer is used in batch mode to retreive launched containers, - // and is equivalent to StartContainer when not in batch. - trueCont=contManager->GiveContainer(myparams); - } - catch( const SALOME::SALOME_Exception& ex ) - { - std::string msg="SalomeContainer::start : Unable to launch container in Salome : "; - msg += '\n'; - msg += ex.details.text.in(); - throw Exception(msg); - } - catch(CORBA::COMM_FAILURE&) - { - throw Exception("SalomeContainer::start : Unable to launch container in Salome : CORBA Comm failure detected"); - } - catch(CORBA::Exception&) - { - throw Exception("SalomeContainer::start : Unable to launch container in Salome : Unexpected CORBA failure detected"); - } - - if(CORBA::is_nil(trueCont)) - throw Exception("SalomeContainer::start : Unable to launch container in Salome. Check your CatalogResources.xml file"); - - schelp->setContainer(askingNode,trueCont); + _sct->clearProperties(); +} - CORBA::String_var containerName(trueCont->name()),hostName(trueCont->getHostName()); - std::cerr << "SalomeContainer launched : " << containerName << " " << hostName << " " << trueCont->getPID() << std::endl; +std::map SalomeContainerToolsDecoratorBase::getResourceProperties(const std::string& name) const +{ + return _sct->getResourceProperties(name); } -CORBA::Object_ptr SalomeContainerTools::LoadComponent(SalomeContainerHelper *launchModeType, Container *cont, Task *askingNode) +void SalomeContainerToolsDecoratorBase::addToComponentList(const std::string& name) { - DEBTRACE("SalomeContainer::loadComponent "); - const ComponentInstance *inst(askingNode?askingNode->getComponent():0); - { - YACS::BASES::AutoLocker alck(cont);//To be sure - if(!cont->isAlreadyStarted(askingNode)) - cont->start(askingNode); - } - if(!inst) - throw Exception("SalomeContainerTools::LoadComponent : no instance of component in the task requesting for a load of its component !"); - CORBA::Object_ptr objComponent=CORBA::Object::_nil(); - { - YACS::BASES::AutoLocker alck(cont);//To be sure - std::string compoName(inst->getCompoName()); - Engines::Container_var container(launchModeType->getContainer(askingNode)); + _sct->addToComponentList(name); +} - char *reason; - bool isLoadable(container->load_component_Library(compoName.c_str(), reason)); - if(isLoadable) - objComponent=CreateComponentInstance(cont,container,inst); - } - return objComponent; +void SalomeContainerToolsDecoratorBase::addToResourceList(const std::string& name) +{ + _sct->addToResourceList(name); } -CORBA::Object_ptr SalomeContainerTools::CreateComponentInstance(Container *cont, Engines::Container_ptr contPtr, const ComponentInstance *inst) +Engines::ContainerParameters SalomeContainerToolsDecoratorBase::getParameters() const { - if(!inst) - throw Exception("SalomeContainerTools::CreateComponentInstance : no instance of component in the task requesting for a load of its component !"); - char *reason(0); - std::string compoName(inst->getCompoName()); - CORBA::Object_ptr objComponent=CORBA::Object::_nil(); - int studyid(1); - Proc* p(cont->getProc()); - if(p) - { - std::string value(p->getProperty("DefaultStudyID")); - if(!value.empty()) - studyid= atoi(value.c_str()); - } - // prepare component instance properties - Engines::FieldsDict_var env(new Engines::FieldsDict); - std::map properties(inst->getProperties()); - if(p) - { - std::map procMap=p->getProperties(); - properties.insert(procMap.begin(),procMap.end()); - } + return _sct->getParameters(); +} - std::map::const_iterator itm; - env->length(properties.size()); - int item=0; - for(itm = properties.begin(); itm != properties.end(); ++itm, item++) - { - DEBTRACE("envname="<first<<" envvalue="<< itm->second); - env[item].key= CORBA::string_dup(itm->first.c_str()); - env[item].value <<= itm->second.c_str(); - } +std::string SalomeContainerToolsDecoratorBase::getContainerName() const +{ + return _sct->getContainerName(); +} - objComponent=contPtr->create_component_instance_env(compoName.c_str(), studyid, env, reason); - if(CORBA::is_nil(objComponent)) - { - std::string text="Error while trying to create a new component: component '"+ compoName; - text=text+"' is not installed or it's a wrong name"; - text += '\n'; - text += reason; - CORBA::string_free(reason); - throw Exception(text); - } - return objComponent; +void SalomeContainerToolsDecoratorBase::setContainerName(const std::string& name) +{ + _sct->setContainerName(name); } -std::string SalomeContainerTools::GetPlacementId(const SalomeContainerHelper *launchModeType, const Container *cont, const Task *askingNode) +std::string SalomeContainerToolsDecoratorBase::getHostName() const { - if(cont->isAlreadyStarted(askingNode)) - { - Engines::Container_var container(launchModeType->getContainer(askingNode)); - const char *what="/"; - CORBA::String_var corbaStr(container->name()); - std::string ret(corbaStr); + return _sct->getHostName(); +} - //Salome FOREVER ... - std::string::size_type i=ret.find_first_of(what,0); - i=ret.find_first_of(what, i==std::string::npos ? i:i+1); - if(i!=std::string::npos) - return ret.substr(i+1); - return ret; - } - else - return "Not placed yet !!!"; + +std::string SalomeContainerToolsDecoratorBase::getNotNullContainerName(const Container *contPtr, const Task *askingNode, bool& isEmpty) const +{ + return _sct->getNotNullContainerName(contPtr,askingNode,isEmpty); } -std::string SalomeContainerTools::GetFullPlacementId(const SalomeContainerHelper *launchModeType, const Container *cont, const Task *askingNode) +////////////////////////////// + +Engines::ContainerParameters SalomeContainerToolsSpreadOverTheResDecorator::getParameters() const { - if(cont->isAlreadyStarted(askingNode)) - { - Engines::Container_var container(launchModeType->getContainer(askingNode)); - try - { - CORBA::String_var corbaStr(container->name()); - std::string ret(corbaStr); - return ret; - } - catch(...) - { - return "Unknown_placement"; - } - } - else - return "Not_placed_yet"; + Engines::ContainerParameters ret(getWorker()->getParameters()); + std::string st(ret.resource_params.hostname); + if(!st.empty()) + return ret; + int nbProcPerNode(ret.resource_params.nb_proc_per_node); + std::size_t iPos(_vh->locateTask(_node)),nPos(_vh->size()); + if(_vh->size()!=_pg->getNumberOfWorkers(nbProcPerNode)) + throw YACS::Exception("SalomeContainerToolsSpreadOverTheResDecorator::getParameters : Internal error !"); + std::string zeMachine(_pg->deduceMachineFrom(iPos,nbProcPerNode)); + ret.resource_params.hostname=CORBA::string_dup(zeMachine.c_str()); + return ret; } diff --git a/src/runtime/SalomeContainerTools.hxx b/src/runtime/SalomeContainerTools.hxx index 5955eee3f..793cf3488 100644 --- a/src/runtime/SalomeContainerTools.hxx +++ b/src/runtime/SalomeContainerTools.hxx @@ -37,34 +37,102 @@ namespace YACS class ComponentInstance; class SalomeContainerHelper; - class YACSRUNTIMESALOME_EXPORT SalomeContainerTools + class YACSRUNTIMESALOME_EXPORT SalomeContainerToolsBase + { + public: + virtual ~SalomeContainerToolsBase() { } + virtual std::string getProperty(const std::string& name) const = 0; + virtual void setProperty(const std::string& name, const std::string& value) = 0; + virtual const std::map& getProperties() const = 0; + virtual void clearProperties() = 0; + virtual std::map getResourceProperties(const std::string& name) const = 0; + virtual void addToComponentList(const std::string& name) = 0; + virtual void addToResourceList(const std::string& name) = 0; + virtual Engines::ContainerParameters getParameters() const = 0; + virtual std::string getContainerName() const = 0; + virtual void setContainerName(const std::string& name) = 0; + virtual std::string getHostName() const = 0; + virtual std::string getNotNullContainerName(const Container *contPtr, const Task *askingNode, bool& isEmpty) const = 0; + public: + static void Start(const std::vector& compoNames, SalomeContainerHelper *schelp, SalomeContainerToolsBase& sct, int& shutdownLevel, const Container *cont, const Task *askingNode); + static CORBA::Object_ptr LoadComponent(SalomeContainerHelper *launchModeType, Container *cont, Task *askingNode); + static CORBA::Object_ptr CreateComponentInstance(Container *cont, Engines::Container_ptr contPtr, const ComponentInstance *inst); + static std::string GetPlacementId(const SalomeContainerHelper *launchModeType, const Container *cont, const Task *askingNode); + static std::string GetFullPlacementId(const SalomeContainerHelper *launchModeType, const Container *cont, const Task *askingNode); + static void SetContainerNameOf(Engines::ContainerParameters& params, const std::string& name); + }; + + class YACSRUNTIMESALOME_EXPORT SalomeContainerToolsInter : public SalomeContainerToolsBase + { + public: + SalomeContainerToolsInter() { } + SalomeContainerToolsInter(const SalomeContainerToolsInter& other); + std::string getProperty(const std::string& name) const; + const std::map& getProperties() const { return _propertyMap; } + void clearProperties(); + std::map getResourceProperties(const std::string& name) const; + protected: + std::map _propertyMap; + }; + + class YACSRUNTIMESALOME_EXPORT SalomeContainerTools : public SalomeContainerToolsInter { public: SalomeContainerTools(); SalomeContainerTools(const SalomeContainerTools& other); + int getNumberOfCoresPerWorker() const; + public: + Engines::ContainerParameters getParameters() const { return _params; } + void clearProperties(); + void setProperty(const std::string& name, const std::string& value); + void addToComponentList(const std::string& name); + void addToResourceList(const std::string& name); + std::string getContainerName() const; + void setContainerName(const std::string& name); + std::string getHostName() const; + std::string getNotNullContainerName(const Container *contPtr, const Task *askingNode, bool& isEmpty) const; + protected: + Engines::ContainerParameters _params; + }; + + class PlayGround; + + class YACSRUNTIMESALOME_EXPORT SalomeContainerToolsDecoratorBase : public SalomeContainerToolsBase + { + protected: + SalomeContainerToolsDecoratorBase(SalomeContainerToolsBase *sct):_sct(sct) { } + SalomeContainerToolsBase *getWorker() { return _sct; } + const SalomeContainerToolsBase *getWorker() const { return _sct; } + public: std::string getProperty(const std::string& name) const; void setProperty(const std::string& name, const std::string& value); - const std::map& getProperties() const { return _propertyMap; } + const std::map& getProperties() const; void clearProperties(); std::map getResourceProperties(const std::string& name) const; void addToComponentList(const std::string& name); void addToResourceList(const std::string& name); - public: + Engines::ContainerParameters getParameters() const; std::string getContainerName() const; void setContainerName(const std::string& name); - std::string getNotNullContainerName(const Container *contPtr, const Task *askingNode, bool& isEmpty) const; std::string getHostName() const; - Engines::ContainerParameters getParameters() const { return _params; } - static void SetContainerNameOf(Engines::ContainerParameters& params, const std::string& name); + std::string getNotNullContainerName(const Container *contPtr, const Task *askingNode, bool& isEmpty) const; + private: + SalomeContainerToolsBase *_sct; + }; + + class SalomeHPContainerVectOfHelper; + + class YACSRUNTIMESALOME_EXPORT SalomeContainerToolsSpreadOverTheResDecorator : public SalomeContainerToolsDecoratorBase + { public: - static void Start(const std::vector& compoNames, SalomeContainerHelper *schelp, SalomeContainerTools& sct, int& shutdownLevel, const Container *cont, const Task *askingNode); - static CORBA::Object_ptr LoadComponent(SalomeContainerHelper *launchModeType, Container *cont, Task *askingNode); - static CORBA::Object_ptr CreateComponentInstance(Container *cont, Engines::Container_ptr contPtr, const ComponentInstance *inst); - static std::string GetPlacementId(const SalomeContainerHelper *launchModeType, const Container *cont, const Task *askingNode); - static std::string GetFullPlacementId(const SalomeContainerHelper *launchModeType, const Container *cont, const Task *askingNode); - protected: - std::map _propertyMap; - Engines::ContainerParameters _params; + SalomeContainerToolsSpreadOverTheResDecorator(SalomeContainerToolsBase *sct, const PlayGround *pg, const SalomeHPContainerVectOfHelper *vh, const Task *node):SalomeContainerToolsDecoratorBase(sct),_pg(pg),_vh(vh),_node(node) { } + Engines::ContainerParameters getParameters() const; + private: + static std::string DeduceMachineFrom(const std::vector< std::pair >& allResInfo, int iPos, int sz, int nbProcPerNode); + private: + const PlayGround *_pg; + const SalomeHPContainerVectOfHelper *_vh; + const Task *_node; }; } } diff --git a/src/runtime/SalomeHPContainer.cxx b/src/runtime/SalomeHPContainer.cxx index b7204fd3e..e057a5f1e 100644 --- a/src/runtime/SalomeHPContainer.cxx +++ b/src/runtime/SalomeHPContainer.cxx @@ -21,6 +21,7 @@ #include "SalomeHPComponent.hxx" #include "SalomeContainerTmpForHP.hxx" #include "AutoLocker.hxx" +#include "AutoRefCnt.hxx" #include @@ -28,108 +29,233 @@ using namespace YACS::ENGINE; const char SalomeHPContainer::KIND[]="HPSalome"; -SalomeHPContainer::SalomeHPContainer():_shutdownLevel(999) +SalomeHPContainerBase::SalomeHPContainerBase(SalomeHPContainerVectOfHelper *resShared):_launchModeType(resShared),_shutdownLevel(999) { } -SalomeHPContainer::SalomeHPContainer(const SalomeHPContainer& other):_componentNames(other._componentNames),_shutdownLevel(999),_sct(other._sct),_initScript(other._initScript) +SalomeHPContainerBase::SalomeHPContainerBase(const SalomeHPContainerBase& other):_shutdownLevel(999),_launchModeType(new SalomeHPContainerVectOfHelper),_initScript(other._initScript) { } -void SalomeHPContainer::setSizeOfPool(int sz) +SalomeHPContainer *SalomeHPContainerBase::getTheBoss() { - _launchModeType.resize(sz); + HomogeneousPoolContainer *ret(this); + while(ret->getDirectFather()) + ret=ret->getDirectFather(); + SalomeHPContainer *retC(dynamic_cast(ret)); + if(!retC) + throw Exception("SalomeHPContainerBase::getTheBoss : unexpected type of object !"); + return retC; } -int SalomeHPContainer::getSizeOfPool() const +const SalomeHPContainer *SalomeHPContainerBase::getTheBoss() const { - return _launchModeType.size(); + const HomogeneousPoolContainer *ret(this); + while(ret->getDirectFather()) + ret=ret->getDirectFather(); + const SalomeHPContainer *retC(dynamic_cast(ret)); + if(!retC) + throw Exception("SalomeHPContainerBase::getTheBoss : unexpected type of object !"); + return retC; } -std::size_t SalomeHPContainer::getNumberOfFreePlace() const +void SalomeHPContainerBase::startInternal(const Task *askingNode, SalomeContainerToolsBase& sct, const std::vector& compoNames) { - return _launchModeType.getNumberOfFreePlace(); + SalomeContainerMonoHelper *helper(_launchModeType->getHelperOfTaskThreadSafe(askingNode)); + SalomeContainerTools::Start(compoNames,helper,sct,_shutdownLevel,this,askingNode); } -void SalomeHPContainer::allocateFor(const std::vector& nodes) +void SalomeHPContainerBase::shutdown(int level) { - _launchModeType.allocateFor(nodes); + if(level < _shutdownLevel) + return; + _shutdownLevel=999; + _launchModeType->shutdown(); } -void SalomeHPContainer::release(const Task *node) +SalomeHPContainerBase::SalomeHPContainerBase(SalomeHPContainerVectOfHelper *resShared, bool isRefEaten):_launchModeType(resShared) { - _launchModeType.release(node); + if(!isRefEaten) + if(_launchModeType.isNotNull()) + _launchModeType->incrRef(); } -SalomeHPContainer::~SalomeHPContainer() +bool SalomeHPContainerBase::isAlreadyStarted(const Task *askingNode) const { + const SalomeContainerMonoHelper *helper(_launchModeType->getHelperOfTaskThreadSafe(askingNode)); + return helper->isAlreadyStarted(askingNode); } -void SalomeHPContainer::lock() +void SalomeHPContainerBase::release(const Task *node) { - _mutex.lock(); + _launchModeType->release(node); } -void SalomeHPContainer::unLock() +void SalomeHPContainerBase::lock() { - _mutex.unLock(); + _launchModeType->lock(); } -std::string SalomeHPContainer::getKind() const +void SalomeHPContainerBase::unLock() { - return KIND; + _launchModeType->unLock(); } -std::string SalomeHPContainer::getDiscreminantStrOfThis(const Task *askingNode) const +void SalomeHPContainerBase::setSizeOfPool(int sz) { - YACS::BASES::AutoCppPtr tmpCont(SalomeContainerTmpForHP::BuildFrom(this,askingNode)); - return tmpCont->getDiscreminantStrOfThis(askingNode); + _launchModeType->resize(sz); } -bool SalomeHPContainer::isAlreadyStarted(const Task *askingNode) const +int SalomeHPContainerBase::getSizeOfPool() const { - const SalomeContainerMonoHelper *helper(_launchModeType.getHelperOfTaskThreadSafe(this,askingNode)); - return helper->isAlreadyStarted(askingNode); + return _launchModeType->size(); } -void SalomeHPContainer::start(const Task *askingNode) throw(YACS::Exception) +void SalomeHPContainerBase::setProperty(const std::string& name,const std::string& value) { - SalomeContainerMonoHelper *helper(_launchModeType.getHelperOfTaskThreadSafe(this,askingNode)); - SalomeContainerTools::Start(_componentNames,helper,_sct,_shutdownLevel,this,askingNode); + if(name==AOC_ENTRY)//no sense to set it ! It is always true ! ignore it ! + return ; + else if(name==SIZE_OF_POOL_KEY) + { + std::istringstream iss(value); + int val(0); + iss >> val; + setSizeOfPool(val); + } + else if(name==INITIALIZE_SCRIPT_KEY) + { + _initScript=value; + } + else + getTheBoss()->getContainerInfo().setProperty(name,value); } -void SalomeHPContainer::shutdown(int level) +std::string SalomeHPContainerBase::getProperty(const std::string& name) const { - if(level < _shutdownLevel) - return; - _shutdownLevel=999; - for(std::size_t i=0;i<_launchModeType.size();i++) + if(name==AOC_ENTRY) + { + return std::string("1"); + } + else if(name==SIZE_OF_POOL_KEY) { - SalomeContainerMonoHelper *helper(_launchModeType.at(i)); - helper->shutdown(); + std::ostringstream oss; oss << getSizeOfPool(); + return oss.str(); } + else if(name==INITIALIZE_SCRIPT_KEY) + { + return _initScript; + } + else + return getTheBoss()->getContainerInfo().getProperty(name); } -std::string SalomeHPContainer::getPlacementId(const Task *askingNode) const +std::map SalomeHPContainerBase::getProperties() const +{ + std::map ret(getTheBoss()->getContainerInfo().getProperties()); + std::ostringstream oss; oss << getSizeOfPool(); + ret[SIZE_OF_POOL_KEY]=oss.str(); + if(!_initScript.empty()) + ret[INITIALIZE_SCRIPT_KEY]=_initScript; + return ret; +} + +void SalomeHPContainerBase::clearProperties() +{ + _initScript.clear(); + getTheBoss()->getContainerInfo().clearProperties(); +} + +std::string SalomeHPContainerBase::getPlacementId(const Task *askingNode) const { const SalomeContainerMonoHelper *helper(0); { - YACS::BASES::AutoLocker alckCont(const_cast(this)); - helper=_launchModeType.getHelperOfTask(askingNode); + YACS::BASES::AutoLocker alckCont(const_cast(this)); + helper=_launchModeType->getHelperOfTask(askingNode); } return SalomeContainerTools::GetPlacementId(helper,this,askingNode); } -std::string SalomeHPContainer::getFullPlacementId(const Task *askingNode) const +std::string SalomeHPContainerBase::getFullPlacementId(const Task *askingNode) const { const SalomeContainerMonoHelper *helper(0); { - YACS::BASES::AutoLocker alckCont(const_cast(this)); - helper=_launchModeType.getHelperOfTask(askingNode); + YACS::BASES::AutoLocker alckCont(const_cast(this)); + helper=_launchModeType->getHelperOfTask(askingNode); } return SalomeContainerTools::GetFullPlacementId(helper,this,askingNode); } +std::map SalomeHPContainerBase::getResourceProperties(const std::string& name) const +{ + return getTheBoss()->getResourceProperties(name); +} + +void SalomeHPContainerBase::addComponentName(const std::string& name) +{ + getTheBoss()->addComponentNameSpe(name); +} + +void SalomeHPContainerBase::checkCapabilityToDealWith(const ComponentInstance *inst) const throw(YACS::Exception) +{ + getTheBoss()->checkCapabilityToDealWith(inst); +} + +YACS::BASES::AutoRefCnt SalomeHPContainerBase::decorate(YACS::BASES::AutoConstRefCnt pd) +{ + YACS::BASES::AutoRefCnt ret(new SalomeHPContainerShared(pd,_launchModeType,this)); + return ret; +} + +Engines::Container_var SalomeHPContainerBase::getContainerPtr(const Task *askingNode) const +{ + const SalomeContainerMonoHelper *helper(0); + { + YACS::BASES::AutoLocker alck(const_cast(this)); + helper=_launchModeType->getHelperOfTask(askingNode); + } + return helper->getContainer(NULL); +} + +//////////////// + +SalomeHPContainer::SalomeHPContainer():SalomeHPContainerBase(new SalomeHPContainerVectOfHelper) +{ +} + +SalomeHPContainer::SalomeHPContainer(const SalomeHPContainer& other):SalomeHPContainerBase(other),_sct(other._sct),_componentNames(other._componentNames) +{ +} + +std::size_t SalomeHPContainer::getNumberOfFreePlace() const +{ + return _launchModeType->getNumberOfFreePlace(); +} + +void SalomeHPContainer::allocateFor(const std::vector& nodes) +{ + _launchModeType->allocateFor(nodes); +} + +SalomeHPContainer::~SalomeHPContainer() +{ +} + +std::string SalomeHPContainer::getKind() const +{ + return KIND; +} + +std::string SalomeHPContainer::getDiscreminantStrOfThis(const Task *askingNode) const +{ + YACS::BASES::AutoCppPtr tmpCont(SalomeContainerTmpForHP::BuildFrom(this,askingNode)); + return tmpCont->getDiscreminantStrOfThis(askingNode); +} + +void SalomeHPContainer::start(const Task *askingNode) throw(Exception) +{ + startInternal(askingNode,_sct,_componentNames); +} + /*! * It is not a bug here ! clone for homogeneous container is not supposed to be copied ! */ @@ -144,72 +270,90 @@ Container *SalomeHPContainer::cloneAlways() const return new SalomeHPContainer(*this); } -void SalomeHPContainer::setProperty(const std::string& name,const std::string& value) +int SalomeHPContainer::getNumberOfCoresPerWorker() const { - if(name==AOC_ENTRY)//no sense to set it ! It is always true ! ignore it ! - return ; - else if(name==SIZE_OF_POOL_KEY) - { - std::istringstream iss(value); - int val(0); - iss >> val; - setSizeOfPool(val); - } - else if(name==INITIALIZE_SCRIPT_KEY) - { - _initScript=value; - } - else - _sct.setProperty(name,value); + return _sct.getNumberOfCoresPerWorker(); } -std::string SalomeHPContainer::getProperty(const std::string& name) const +std::map SalomeHPContainer::getResourcePropertiesSpe(const std::string& name) const { - if(name==AOC_ENTRY) - { - return std::string("1"); - } - else if(name==SIZE_OF_POOL_KEY) - { - std::ostringstream oss; oss << getSizeOfPool(); - return oss.str(); - } - else if(name==INITIALIZE_SCRIPT_KEY) - { - return _initScript; - } - else - return _sct.getProperty(name); + return _sct.getResourceProperties(name); } -void SalomeHPContainer::clearProperties() +void SalomeHPContainer::addComponentNameSpe(const std::string& name) { - _initScript.clear(); - _sct.clearProperties(); + _componentNames.push_back(name); } -void SalomeHPContainer::addComponentName(const std::string& name) +void SalomeHPContainer::checkCapabilityToDealWithSpe(const ComponentInstance *inst) const throw(YACS::Exception) { - _componentNames.push_back(name); + if(inst->getKind()!=SalomeHPComponent::KIND) + throw Exception("SalomeHPContainer::checkCapabilityToDealWithSpe : SalomeContainer is not able to deal with this type of ComponentInstance."); } -std::map SalomeHPContainer::getProperties() const +void SalomeHPContainer::forYourTestsOnly(ForTestOmlyHPContCls *data) const { - std::map ret(_sct.getProperties()); - std::ostringstream oss; oss << getSizeOfPool(); - ret[SIZE_OF_POOL_KEY]=oss.str(); - if(!_initScript.empty()) - ret[INITIALIZE_SCRIPT_KEY]=_initScript; - return ret; + data->setContainerType("HPContainer"); } -std::map SalomeHPContainer::getResourceProperties(const std::string& name) const +////////////////////////////////// + +SalomeHPContainerShared::SalomeHPContainerShared(YACS::BASES::AutoConstRefCnt pd, SalomeHPContainerVectOfHelper *resShared, SalomeHPContainerBase *directFather):SalomeHPContainerBase(resShared,false),_pd(pd) { - return _sct.getResourceProperties(name); + if(!directFather) + throw Exception("SalomeHPContainerShared : NULL pointer not allowed !"); + _directFather.takeRef(directFather); } -void SalomeHPContainer::checkCapabilityToDealWith(const ComponentInstance *inst) const throw(YACS::Exception) +std::string SalomeHPContainerShared::getKind() const { - if(inst->getKind()!=SalomeHPComponent::KIND) - throw Exception("SalomeHPContainer::checkCapabilityToDealWith : SalomeContainer is not able to deal with this type of ComponentInstance."); + return SalomeHPContainer::KIND; } + +void SalomeHPContainerShared::prepareMaskForExecution() const +{ + _idsOfKernelContainers=_pd->computeWorkerIdsCovered(getNumberOfCoresPerWorker()); +} + +/*! + * It is not a bug here ! clone for homogeneous container is not supposed to be copied ! + */ +Container *SalomeHPContainerShared::clone() const +{ + incrRef(); + return const_cast(this); +} + +Container *SalomeHPContainerShared::cloneAlways() const +{ + throw Exception("SalomeHPContainerShared::cloneAlways : you are not supposed to be in this situation ! This type of container has only existence during execution !"); +} + +void SalomeHPContainerShared::start(const Task *askingNode) throw(Exception) +{ + SalomeContainerToolsSpreadOverTheResDecorator sct(&getTheBoss()->getContainerInfo(),_pd->getPlayGround(),_launchModeType,askingNode); + startInternal(askingNode,sct,getTheBoss()->getComponentNames()); +} + +void SalomeHPContainerShared::allocateFor(const std::vector& nodes) +{ + _launchModeType->allocateForAmong(_idsOfKernelContainers,nodes); +} + +std::size_t SalomeHPContainerShared::getNumberOfFreePlace() const +{ + return _launchModeType->getNumberOfFreePlaceAmong(_idsOfKernelContainers); +} + +void SalomeHPContainerShared::forYourTestsOnly(ForTestOmlyHPContCls *data) const +{ + data->setContainerType("HPContainerShared"); + data->setPD(_pd); + data->setIDS(_idsOfKernelContainers); +} + +/* + * SalomeHPContainerVectOfHelper is an holder of vector of SalomeContainerMonoHelper (holding itself a Kernel Container) + * SalomeContainerTools is a Engines::ContainerParameters holder. It is the data keeper for GiveContainer invokation. + * + */ diff --git a/src/runtime/SalomeHPContainer.hxx b/src/runtime/SalomeHPContainer.hxx index f97bf8dfa..80001ad83 100644 --- a/src/runtime/SalomeHPContainer.hxx +++ b/src/runtime/SalomeHPContainer.hxx @@ -25,7 +25,11 @@ #include "SalomeContainerHelper.hxx" #include "SalomeContainerTools.hxx" #include "SalomeHPContainerTools.hxx" +#include "PlayGround.hxx" + #include "Mutex.hxx" +#include "AutoRefCnt.hxx" + #include #include #include @@ -39,49 +43,82 @@ namespace YACS { class Task; class SalomeComponent; - - class YACSRUNTIMESALOME_EXPORT SalomeHPContainer : public HomogeneousPoolContainer + class SalomeHPContainer; + + class YACSRUNTIMESALOME_EXPORT SalomeHPContainerBase : public HomogeneousPoolContainer { + protected: + SalomeHPContainerBase(SalomeHPContainerVectOfHelper *resShared); + SalomeHPContainerBase(SalomeHPContainerVectOfHelper *resShared, bool isRefEaten); + SalomeHPContainerBase(const SalomeHPContainerBase& other); + void startInternal(const Task *askingNode, SalomeContainerToolsBase& sct, const std::vector& compoNames); + SalomeHPContainer *getTheBoss(); + const SalomeHPContainer *getTheBoss() const; public: - SalomeHPContainer(); - SalomeHPContainer(const SalomeHPContainer& other); - //HP specific part - void setSizeOfPool(int sz); - int getSizeOfPool() const; - std::size_t getNumberOfFreePlace() const; - void allocateFor(const std::vector& nodes); void release(const Task *node); + // + bool isAlreadyStarted(const Task *askingNode) const; + void shutdown(int level); //! For thread safety for concurrent load operation on same Container. void lock(); //! For thread safety for concurrent load operation on same Container. void unLock(); + void setSizeOfPool(int sz); + int getSizeOfPool() const; + void setProperty(const std::string& name,const std::string& value); + std::string getProperty(const std::string& name) const; + std::map getProperties() const; + void clearProperties(); + std::string getPlacementId(const Task *askingNode) const; + std::string getFullPlacementId(const Task *askingNode) const; + std::map getResourceProperties(const std::string& name) const; + void addComponentName(const std::string& name); + void checkCapabilityToDealWith(const ComponentInstance *inst) const throw(YACS::Exception); + Engines::Container_var getContainerPtr(const Task *askingNode) const; + public: + int getShutdownLev() const { return _shutdownLevel; } + YACS::BASES::AutoRefCnt decorate(YACS::BASES::AutoConstRefCnt pd); +#ifndef SWIG + const SalomeHPContainerVectOfHelper *getDirectAccessToVecOfCont() const { return _launchModeType; } +#endif + protected: + int _shutdownLevel; + YACS::BASES::AutoRefCnt _launchModeType; + std::string _initScript; + }; + + class YACSRUNTIMESALOME_EXPORT SalomeHPContainer : public SalomeHPContainerBase + { + public: + SalomeHPContainer(); + SalomeHPContainer(const SalomeHPContainer& other); + //HP specific part + std::size_t getNumberOfFreePlace() const; + void allocateFor(const std::vector& nodes); // std::string getKind() const; std::string getDiscreminantStrOfThis(const Task *askingNode) const; - bool isAlreadyStarted(const Task *askingNode) const; void start(const Task *askingNode) throw(Exception); - void shutdown(int level); - std::string getPlacementId(const Task *askingNode) const; - std::string getFullPlacementId(const Task *askingNode) const; Container *clone() const; Container *cloneAlways() const; - void setProperty(const std::string& name,const std::string& value); - std::string getProperty(const std::string& name) const; - void clearProperties(); - void addComponentName(const std::string& name); - std::map getProperties() const; - std::map getResourceProperties(const std::string& name) const; - void checkCapabilityToDealWith(const ComponentInstance *inst) const throw(YACS::Exception); + int getNumberOfCoresPerWorker() const; + //! do nothing. because no special actions to do. Only for decorators + void prepareMaskForExecution() const { } // #ifndef SWIG - std::size_t locateTask(const Task *askingNode) const { return _launchModeType.locateTask(askingNode); } + void forYourTestsOnly(ForTestOmlyHPContCls *data) const; + std::map getResourcePropertiesSpe(const std::string& name) const; + void addComponentNameSpe(const std::string& name); + void checkCapabilityToDealWithSpe(const ComponentInstance *inst) const throw(YACS::Exception); + std::size_t locateTask(const Task *askingNode) const { return _launchModeType->locateTask(askingNode); } const SalomeContainerTools &getContainerInfo() const { return _sct; } + SalomeContainerTools &getContainerInfo() { return _sct; } std::vector getComponentNames() const { return _componentNames; } - int getShutdownLev() const { return _shutdownLevel; } - SalomeContainerMonoHelper *getHelperOfTask(const Task *node) { return _launchModeType.getHelperOfTask(node); } - const SalomeContainerMonoHelper *getHelperOfTask(const Task *node) const { return _launchModeType.getHelperOfTask(node); } + SalomeContainerMonoHelper *getHelperOfTask(const Task *node) { return _launchModeType->getHelperOfTask(node); } + const SalomeContainerMonoHelper *getHelperOfTask(const Task *node) const { return _launchModeType->getHelperOfTask(node); } // - YACS::BASES::Mutex& getLocker() { return _mutex; } + HomogeneousPoolContainer *getDirectFather() { return NULL; } + const HomogeneousPoolContainer *getDirectFather() const { return NULL; } #endif public: static const char KIND[]; @@ -90,14 +127,35 @@ namespace YACS ~SalomeHPContainer(); #endif protected: - int _shutdownLevel; SalomeContainerTools _sct; - YACS::BASES::Mutex _mutex; std::vector _componentNames; + }; + +#ifndef SWIG + class YACSRUNTIMESALOME_EXPORT SalomeHPContainerShared : public SalomeHPContainerBase + { + public: + SalomeHPContainerShared(YACS::BASES::AutoConstRefCnt pd, SalomeHPContainerVectOfHelper *resShared, SalomeHPContainerBase *directFather); + HomogeneousPoolContainer *getDirectFather() { return _directFather; } + const HomogeneousPoolContainer *getDirectFather() const { return _directFather; } + int getNumberOfCoresPerWorker() const { return _directFather->getNumberOfCoresPerWorker(); } + void prepareMaskForExecution() const; + void forYourTestsOnly(ForTestOmlyHPContCls *data) const; + public: + std::string getKind() const; + void start(const Task *askingNode) throw(Exception); + Container *clone() const; + Container *cloneAlways() const; // - SalomeHPContainerVectOfHelper _launchModeType; - std::string _initScript; + void allocateFor(const std::vector& nodes); + std::size_t getNumberOfFreePlace() const; + private: + YACS::BASES::AutoRefCnt _directFather; + YACS::BASES::AutoConstRefCnt _pd; + //! ids in _launchModeType covered by _pd. + mutable std::vector _idsOfKernelContainers; }; +#endif } } diff --git a/src/runtime/SalomeHPContainerTools.cxx b/src/runtime/SalomeHPContainerTools.cxx index 4e8f971e1..bb9ae661d 100644 --- a/src/runtime/SalomeHPContainerTools.cxx +++ b/src/runtime/SalomeHPContainerTools.cxx @@ -44,6 +44,15 @@ std::size_t SalomeHPContainerVectOfHelper::getNumberOfFreePlace() const return std::count(_whichOccupied.begin(),_whichOccupied.end(),false); } +std::size_t SalomeHPContainerVectOfHelper::getNumberOfFreePlaceAmong(const std::vector& idsOfKernelContainers) const +{ + std::size_t ret; + for(std::vector::const_iterator it=idsOfKernelContainers.begin();it!=idsOfKernelContainers.end();it++) + if(!_whichOccupied[*it]) + ret++; + return ret; +} + void SalomeHPContainerVectOfHelper::allocateFor(const std::vector& nodes) { for(std::vector::const_iterator it=nodes.begin();it!=nodes.end();it++) @@ -60,6 +69,27 @@ void SalomeHPContainerVectOfHelper::allocateFor(const std::vector& } } +void SalomeHPContainerVectOfHelper::allocateForAmong(const std::vector& idsOfKernelContainers, const std::vector& nodes) +{ + for(std::vector::const_iterator it=nodes.begin();it!=nodes.end();it++) + { + if(!(*it)) + continue; + if(_currentlyWorking.find(*it)!=_currentlyWorking.end()) + throw Exception("Searching 2 to allocate for a ServiceNode instance already declared as allocated !"); + std::size_t it2(std::numeric_limits::max()); + for(std::vector::const_iterator it=idsOfKernelContainers.begin();it!=idsOfKernelContainers.end();it++) + if(!_whichOccupied[*it]) + { + it2=*it; + break; + } + if(it2==std::numeric_limits::max()) + throw Exception("All 2 ressources are already occupied ! You are expected to wait for released resources !"); + _currentlyWorking[*it]=it2; _whichOccupied[it2]=true; + } +} + void SalomeHPContainerVectOfHelper::release(const Task *node) { if(!node) @@ -81,9 +111,9 @@ std::size_t SalomeHPContainerVectOfHelper::locateTask(const Task *node) const return ret; } -const SalomeContainerMonoHelper *SalomeHPContainerVectOfHelper::getHelperOfTaskThreadSafe(const SalomeHPContainer *cont, const Task *node) const +const SalomeContainerMonoHelper *SalomeHPContainerVectOfHelper::getHelperOfTaskThreadSafe(const Task *node) const { - YACS::BASES::AutoLocker alck(const_cast(cont)); + YACS::BASES::AutoLocker alck(const_cast(this)); return _launchModeType[locateTask(node)]; } @@ -92,9 +122,9 @@ const SalomeContainerMonoHelper *SalomeHPContainerVectOfHelper::getHelperOfTask( return _launchModeType[locateTask(node)]; } -SalomeContainerMonoHelper *SalomeHPContainerVectOfHelper::getHelperOfTaskThreadSafe(SalomeHPContainer *cont, const Task *node) +SalomeContainerMonoHelper *SalomeHPContainerVectOfHelper::getHelperOfTaskThreadSafe(const Task *node) { - YACS::BASES::AutoLocker alck(cont); + YACS::BASES::AutoLocker alck(this); return _launchModeType[locateTask(node)]; } @@ -118,3 +148,21 @@ void SalomeHPContainerVectOfHelper::checkPosInVec(std::size_t pos) const if(pos<0 || pos>=_launchModeType.size()) throw Exception("The task has been found, but its id is not in the correct range ! resize of of container size during run ?"); } + +void SalomeHPContainerVectOfHelper::shutdown() +{ + for(std::vector< BASES::AutoRefCnt >::iterator it=_launchModeType.begin();it!=_launchModeType.end();it++) + if((*it).isNotNull()) + if(!(*it)->isKernelContNull()) + (*it)->shutdown(); +} + +void SalomeHPContainerVectOfHelper::lock() +{ + _mutex.lock(); +} + +void SalomeHPContainerVectOfHelper::unLock() +{ + _mutex.unLock(); +} diff --git a/src/runtime/SalomeHPContainerTools.hxx b/src/runtime/SalomeHPContainerTools.hxx index 895fa6405..4b080d57e 100644 --- a/src/runtime/SalomeHPContainerTools.hxx +++ b/src/runtime/SalomeHPContainerTools.hxx @@ -23,6 +23,9 @@ #include "YACSRuntimeSALOMEExport.hxx" #include "SalomeContainerHelper.hxx" +#include "RefCounter.hxx" + +#include "Mutex.hxx" #include "AutoRefCnt.hxx" #include @@ -36,25 +39,34 @@ namespace YACS class SalomeComponent; class SalomeHPContainer; class SalomeContainerMonoHelper; - class SalomeHPContainerVectOfHelper + class SalomeHPContainerVectOfHelper : public RefCounter { public: std::size_t size() const { return _launchModeType.size(); } void resize(std::size_t sz); std::size_t getNumberOfFreePlace() const; + std::size_t getNumberOfFreePlaceAmong(const std::vector& idsOfKernelContainers) const; void allocateFor(const std::vector& nodes); + void allocateForAmong(const std::vector& idsOfKernelContainers, const std::vector& nodes); void release(const Task *node); std::size_t locateTask(const Task *node) const; const SalomeContainerMonoHelper *at(std::size_t pos) const { checkPosInVec(pos); return _launchModeType[pos]; } SalomeContainerMonoHelper *at(std::size_t pos) { checkPosInVec(pos); return _launchModeType[pos]; } - const SalomeContainerMonoHelper *getHelperOfTaskThreadSafe(const SalomeHPContainer *cont, const Task *node) const; + const SalomeContainerMonoHelper *getHelperOfTaskThreadSafe(const Task *node) const; const SalomeContainerMonoHelper *getHelperOfTask(const Task *node) const; - SalomeContainerMonoHelper *getHelperOfTaskThreadSafe(SalomeHPContainer *cont, const Task *node); + SalomeContainerMonoHelper *getHelperOfTaskThreadSafe(const Task *node); SalomeContainerMonoHelper *getHelperOfTask(const Task *node); + void shutdown(); + //! For thread safety for concurrent load operation on same Container. + void lock(); + //! For thread safety for concurrent load operation on same Container. + void unLock(); private: + ~SalomeHPContainerVectOfHelper() { } void checkNoCurrentWork() const; void checkPosInVec(std::size_t pos) const; private: + YACS::BASES::Mutex _mutex; std::vector _whichOccupied; std::vector< BASES::AutoRefCnt > _launchModeType; std::map _currentlyWorking; diff --git a/src/yacsloader_swig/Test/CMakeLists.txt b/src/yacsloader_swig/Test/CMakeLists.txt index f079c81d2..5493e1449 100644 --- a/src/yacsloader_swig/Test/CMakeLists.txt +++ b/src/yacsloader_swig/Test/CMakeLists.txt @@ -29,6 +29,9 @@ IF(NOT WIN32) SALOME_GENERATE_TESTS_ENVIRONMENT(tests_env) ADD_TEST(NAME StdAloneYacsLoaderTest1 COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/StdAloneYacsLoaderTest1.py) SET_TESTS_PROPERTIES(StdAloneYacsLoaderTest1 PROPERTIES ENVIRONMENT "${tests_env}") + + ADD_TEST(NAME HPDecorator COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/testHPDecorator.py) + SET_TESTS_PROPERTIES(HPDecorator PROPERTIES ENVIRONMENT "${tests_env}") # For salome test SET(LOCAL_TEST_DIR ${SALOME_YACS_INSTALL_TEST}/yacsloader_swig) @@ -41,6 +44,7 @@ IF(NOT WIN32) testResume.py testSave.py testSaveLoadRun.py + testHPDecorator.py optim_plugin.py testValidationChecks.py testProgress.py diff --git a/src/yacsloader_swig/Test/CTestTestfileInstall.cmake b/src/yacsloader_swig/Test/CTestTestfileInstall.cmake index 6c4d0699d..662167a4e 100644 --- a/src/yacsloader_swig/Test/CTestTestfileInstall.cmake +++ b/src/yacsloader_swig/Test/CTestTestfileInstall.cmake @@ -26,6 +26,12 @@ IF(NOT WIN32) SET(TEST_NAME ${COMPONENT_NAME}_StdAloneYacsLoaderTest1) ADD_TEST(${TEST_NAME} python ${SALOME_TEST_DRIVER} ${TIMEOUT} python StdAloneYacsLoaderTest1.py) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES + LABELS "${COMPONENT_NAME}" + ) + + SET(TEST_NAME ${COMPONENT_NAME}_HPDecorator) + ADD_TEST(${TEST_NAME} python ${SALOME_TEST_DRIVER} ${TIMEOUT} python testHPDecorator.py) SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES LABELS "${COMPONENT_NAME}" ) diff --git a/src/yacsloader_swig/Test/testHPDecorator.py b/src/yacsloader_swig/Test/testHPDecorator.py new file mode 100644 index 000000000..789d1d45a --- /dev/null +++ b/src/yacsloader_swig/Test/testHPDecorator.py @@ -0,0 +1,293 @@ +# Copyright (C) 2006-2016 CEA/DEN, EDF R&D +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +# + +import pilot +import SALOMERuntime +import unittest + +class TestHPDecortator(unittest.TestCase): + def test0(self): + """ First test coming from the big boss.""" + SALOMERuntime.RuntimeSALOME.setRuntime() + r=SALOMERuntime.getSALOMERuntime() + pg=pilot.PlayGround() + pg.loadFromKernelCatalog() + assert(len(pg.getData())!=0) + p=r.createProc("p0") + td=p.createType("double","double") + tdd=p.createSequenceTc("seqdouble","seqdouble",td) + hp1=p.createContainer("HP1","HPSalome") + hp4=p.createContainer("HP4","HPSalome") + # + n0=r.createScriptNode("Salome","n0") + n0.setExecutionMode("remote") + out0_0=n0.edAddOutputPort("o1",tdd) + n0.setScript("""o1=[float(i)+0.1 for i in range(1000)]""") + n0.setContainer(hp4) + p.edAddChild(n0) + # + n1_0=r.createForEachLoop("n1_0",td) + p.edAddChild(n1_0) + p.edAddCFLink(n0,n1_0) + p.edAddLink(out0_0,n1_0.edGetSeqOfSamplesPort()) + ## + n1_0_sc=r.createScriptNode("Salome","n1_0_sc") + n1_0.edAddChild(n1_0_sc) + n1_0_sc.setExecutionMode("remote") + n1_0_sc.setScript("""2*i1""") + i1_0_sc=n1_0_sc.edAddInputPort("i1",td) + p.edAddLink(n1_0.edGetSamplePort(),i1_0_sc) + n1_0_sc.setContainer(hp4) + ## + # + n1_1=r.createForEachLoop("n1_1",td) + p.edAddChild(n1_1) + p.edAddCFLink(n0,n1_1) + p.edAddLink(out0_0,n1_1.edGetSeqOfSamplesPort()) + ## + n1_1_sc=r.createScriptNode("Salome","n1_1_sc") + n1_1.edAddChild(n1_1_sc) + n1_1_sc.setExecutionMode("remote") + n1_1_sc.setScript("""3*i1""") + i1_1_sc=n1_1_sc.edAddInputPort("i1",td) + p.edAddLink(n1_1.edGetSamplePort(),i1_1_sc) + n1_1_sc.setContainer(hp1) + ## + hp1.setProperty("nb_proc_per_node","1") + hp4.setProperty("nb_proc_per_node","4") + pg.setData([("m0",8),("m1",8),("m2",8),("m3",8)]) # virtual machine with 32 cores spread over 4 nodes + assert(n1_0.getWeight()==1.) + assert(n1_1.getWeight()==1.) + p.fitToPlayGround(pg)########### ZE CALL + fyto=pilot.ForTestOmlyHPContCls() + assert(hp4.getSizeOfPool()==8)# 32/4 + n1_0_sc.getContainer().forYourTestsOnly(fyto) + assert(fyto.getContainerType()=="HPContainerShared") + pd=fyto.getPD() + assert(isinstance(pd,pilot.ContigPartDefinition)) + assert(pd.getStart()==0 and pd.getStop()==16) + assert(fyto.getIDS()==(0,1,2,3)) + # + assert(hp1.getSizeOfPool()==32)# 32/1 + fyto=pilot.ForTestOmlyHPContCls() + n1_1_sc.getContainer().forYourTestsOnly(fyto) + assert(fyto.getContainerType()=="HPContainerShared") + pd=fyto.getPD() + assert(isinstance(pd,pilot.ContigPartDefinition)) + assert(pd.getStart()==16 and pd.getStop()==32) + assert(fyto.getIDS()==(16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31)) + assert(n1_0.edGetNbOfBranchesPort().getPyObj()==4) + assert(n1_1.edGetNbOfBranchesPort().getPyObj()==16) + # + fyto=pilot.ForTestOmlyHPContCls() + n0.getContainer().forYourTestsOnly(fyto) + assert(fyto.getContainerType()=="HPContainerShared") + pd=fyto.getPD() + assert(isinstance(pd,pilot.AllPartDefinition)) + assert(list(fyto.getIDS())==range(8)) + ############################# + # Change weight of ForEach # + ############################# + n1_0.setWeight(2) + p.fitToPlayGround(pg)########### ZE CALL + assert(hp4.getSizeOfPool()==8)# 32/4 + n1_0_sc.getContainer().forYourTestsOnly(fyto) + assert(fyto.getContainerType()=="HPContainerShared") + pd=fyto.getPD() + assert(isinstance(pd,pilot.ContigPartDefinition)) + assert(pd.getStart()==0 and pd.getStop()==21) + assert(fyto.getIDS()==(0,1,2,3,4)) + assert(hp1.getSizeOfPool()==32)# 32/1 + fyto=pilot.ForTestOmlyHPContCls() + n1_1_sc.getContainer().forYourTestsOnly(fyto) + assert(fyto.getContainerType()=="HPContainerShared") + pd=fyto.getPD() + assert(isinstance(pd,pilot.ContigPartDefinition)) + assert(pd.getStart()==21 and pd.getStop()==32) + assert(fyto.getIDS()==(21,22,23,24,25,26,27,28,29,30,31)) + assert(n1_0.edGetNbOfBranchesPort().getPyObj()==6) + assert(n1_1.edGetNbOfBranchesPort().getPyObj()==11) + # + fyto=pilot.ForTestOmlyHPContCls() + n0.getContainer().forYourTestsOnly(fyto) + assert(fyto.getContainerType()=="HPContainerShared") + pd=fyto.getPD() + assert(isinstance(pd,pilot.AllPartDefinition)) + assert(list(fyto.getIDS())==range(8)) + pass + + def test1(self): + """ Second test as test0 + script node in // with 2 FE""" + SALOMERuntime.RuntimeSALOME.setRuntime() + r=SALOMERuntime.getSALOMERuntime() + pg=pilot.PlayGround() + pg.loadFromKernelCatalog() + assert(len(pg.getData())!=0) + p=r.createProc("p0") + td=p.createType("double","double") + tdd=p.createSequenceTc("seqdouble","seqdouble",td) + hp1=p.createContainer("HP1","HPSalome") + hp4=p.createContainer("HP4","HPSalome") + # + n0=r.createScriptNode("Salome","n0") + n0.setExecutionMode("remote") + out0_0=n0.edAddOutputPort("o1",tdd) + n0.setScript("""o1=[float(i)+0.1 for i in range(1000)]""") + n0.setContainer(hp4) + p.edAddChild(n0) + # + n1_0=r.createForEachLoop("n1_0",td) + p.edAddChild(n1_0) + p.edAddCFLink(n0,n1_0) + p.edAddLink(out0_0,n1_0.edGetSeqOfSamplesPort()) + ## + n1_0_sc=r.createScriptNode("Salome","n1_0_sc") + n1_0.edAddChild(n1_0_sc) + n1_0_sc.setExecutionMode("remote") + n1_0_sc.setScript("""2*i1""") + i1_0_sc=n1_0_sc.edAddInputPort("i1",td) + p.edAddLink(n1_0.edGetSamplePort(),i1_0_sc) + n1_0_sc.setContainer(hp4) + ## + # + n1_1=r.createForEachLoop("n1_1",td) + p.edAddChild(n1_1) + p.edAddCFLink(n0,n1_1) + p.edAddLink(out0_0,n1_1.edGetSeqOfSamplesPort()) + ## + n1_1_sc=r.createScriptNode("Salome","n1_1_sc") + n1_1.edAddChild(n1_1_sc) + n1_1_sc.setExecutionMode("remote") + n1_1_sc.setScript("""3*i1""") + i1_1_sc=n1_1_sc.edAddInputPort("i1",td) + p.edAddLink(n1_1.edGetSamplePort(),i1_1_sc) + n1_1_sc.setContainer(hp1) + # + n1_2=r.createScriptNode("Salome","n1_2") + p.edAddChild(n1_2) + n1_2.setExecutionMode("remote") + n1_2.setContainer(hp4) + n1_2.setScript("""my_container""") + p.edAddCFLink(n0,n1_2) + ## + hp1.setProperty("nb_proc_per_node","1") + hp4.setProperty("nb_proc_per_node","4") + pg.setData([("m0",8),("m1",8),("m2",8),("m3",8)]) # virtual machine with 32 cores spread over 4 nodes + assert(n1_0.getWeight()==1.) + assert(n1_1.getWeight()==1.) + p.fitToPlayGround(pg)########### ZE CALL + assert(hp4.getSizeOfPool()==8)# 32/4 + fyto=pilot.ForTestOmlyHPContCls() + n1_0_sc.getContainer().forYourTestsOnly(fyto) + assert(fyto.getContainerType()=="HPContainerShared") + pd=fyto.getPD() + assert(isinstance(pd,pilot.ContigPartDefinition)) + assert(pd.getStart()==0 and pd.getStop()==16) + assert(fyto.getIDS()==(0,1,2,3)) + # + assert(hp1.getSizeOfPool()==32)# 32/1 + fyto=pilot.ForTestOmlyHPContCls() + n1_1_sc.getContainer().forYourTestsOnly(fyto) + assert(fyto.getContainerType()=="HPContainerShared") + pd=fyto.getPD() + assert(isinstance(pd,pilot.ContigPartDefinition)) + assert(pd.getStart()==16 and pd.getStop()==32) + assert(fyto.getIDS()==(16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31)) + assert(n1_0.edGetNbOfBranchesPort().getPyObj()==4) + assert(n1_1.edGetNbOfBranchesPort().getPyObj()==16) + # + fyto=pilot.ForTestOmlyHPContCls() + n0.getContainer().forYourTestsOnly(fyto) + assert(fyto.getContainerType()=="HPContainerShared") + pd=fyto.getPD() + assert(isinstance(pd,pilot.AllPartDefinition)) + assert(list(fyto.getIDS())==range(8)) + fyto=pilot.ForTestOmlyHPContCls() + n1_2.getContainer().forYourTestsOnly(fyto) + assert(fyto.getContainerType()=="HPContainerShared") + pd=fyto.getPD() + assert(isinstance(pd,pilot.AllPartDefinition)) + assert(list(fyto.getIDS())==range(8)) + pass + + def test2(self): + """ This test is not launched here because it requires 2 machines in catalog""" + m1="dsp0764200" + m2="dsp0764412" + SALOMERuntime.RuntimeSALOME.setRuntime() + r=SALOMERuntime.getSALOMERuntime() + pg=pilot.PlayGround() + pg.loadFromKernelCatalog() + assert(len(pg.getData())!=0) + p=r.createProc("p0") + td=p.createType("double","double") + tdd=p.createSequenceTc("seqdouble","seqdouble",td) + hp1=p.createContainer("HP1","HPSalome") + # + n0=r.createScriptNode("Salome","n0") + n0.setExecutionMode("remote") + out0_0=n0.edAddOutputPort("o1",tdd) + n0.setScript("""o1=[float(i)+0.1 for i in range(1000)]""") + n0.setContainer(hp1) + p.edAddChild(n0) + ## + n1_0_sc=r.createScriptNode("Salome","n1_0_sc") + p.edAddChild(n1_0_sc) + p.edAddCFLink(n0,n1_0_sc) + n1_0_sc.setExecutionMode("remote") + n1_0_sc.setScript("""assert(my_container.getHostName()=="%s")"""%m1) + n1_0_sc.setContainer(hp1) + ## + n1_1_sc=r.createScriptNode("Salome","n1_1_sc") + p.edAddChild(n1_1_sc) + p.edAddCFLink(n0,n1_1_sc) + n1_1_sc.setExecutionMode("remote") + n1_1_sc.setScript("""assert(my_container.getHostName()=="%s")"""%m2) + n1_1_sc.setContainer(hp1) + ## + hp1.setProperty("nb_proc_per_node","1") + pg.setData([(m1,8),(m2,8)]) + p.fitToPlayGround(pg)########### ZE CALL + assert(hp1.getSizeOfPool()==16)# 16/1 + fyto=pilot.ForTestOmlyHPContCls() + n1_0_sc.getContainer().forYourTestsOnly(fyto) + assert(fyto.getContainerType()=="HPContainerShared") + pd=fyto.getPD() + assert(isinstance(pd,pilot.ContigPartDefinition)) + assert(pd.getStart()==0 and pd.getStop()==8) + assert(fyto.getIDS()==(0,1,2,3,4,5,6,7)) + # + fyto=pilot.ForTestOmlyHPContCls() + n1_1_sc.getContainer().forYourTestsOnly(fyto) + assert(fyto.getContainerType()=="HPContainerShared") + pd=fyto.getPD() + assert(isinstance(pd,pilot.ContigPartDefinition)) + assert(pd.getStart()==8 and pd.getStop()==16) + assert(fyto.getIDS()==(8,9,10,11,12,13,14,15)) + # + exe=pilot.ExecutorSwig() + assert(p.getState()==pilot.READY) + exe.RunW(p,0) + assert(p.getState()==pilot.DONE) + pass + + pass + +if __name__ == '__main__': + unittest.main() -- 2.30.2