From 8c070ba64a78e74545039c67963794a5b3fab2fc Mon Sep 17 00:00:00 2001 From: Anthony Geay Date: Tue, 31 Mar 2020 17:16:09 +0200 Subject: [PATCH] Some tuning to make it work --- src/engine/Bloc_impl.cxx | 2 +- src/engine/Executor.cxx | 29 +++++++++++++++++++-------- src/engine/PlayGround.cxx | 26 ++++++++++++++++++++++-- src/engine/PlayGround.hxx | 2 ++ src/yacsloader_swig/Test/testFEDyn.py | 6 +++--- 5 files changed, 51 insertions(+), 14 deletions(-) diff --git a/src/engine/Bloc_impl.cxx b/src/engine/Bloc_impl.cxx index 0fe584cdb..8110ba54d 100644 --- a/src/engine/Bloc_impl.cxx +++ b/src/engine/Bloc_impl.cxx @@ -119,7 +119,7 @@ void Bloc::propagePlayGround(const PlayGround *pg) void visitBloc(Bloc *node) { node->ComposedNode::accept(this); } void visitElementaryNode(ElementaryNode *node) { } void visitForEachLoop(ForEachLoop *node) { throw YACS::Exception(MSG); } - void visitForEachLoopDyn(ForEachLoopDyn *node) { } + void visitForEachLoopDyn(ForEachLoopDyn *node) { node->ComposedNode::accept(this); } void visitOptimizerLoop(OptimizerLoop *node) { throw YACS::Exception(MSG); } void visitDynParaLoop(DynParaLoop *node) { throw YACS::Exception(MSG); } void visitForLoop(ForLoop *node) { throw YACS::Exception(MSG); } diff --git a/src/engine/Executor.cxx b/src/engine/Executor.cxx index de7909f6f..e4a76ec5f 100644 --- a/src/engine/Executor.cxx +++ b/src/engine/Executor.cxx @@ -1338,6 +1338,20 @@ void Executor::sendEvent(const std::string& event) disp->dispatch(_root,event); } +struct HPCCompare +{ + bool operator()(HomogeneousPoolContainer * lhs, HomogeneousPoolContainer * rhs) const + { + if(!lhs && !rhs) + return false; + if(!lhs) + return true; + if(!rhs) + return false; + return lhs->getNumberOfCoresPerWorker() < rhs->getNumberOfCoresPerWorker(); + } +}; + /*! * This method takes in input a list of tasks and selects from that lists a part of it considering only the containers. * If tasks have no container instance subclass of HomogeneousPoolContainer this method will let the \a tsks untouched. @@ -1346,32 +1360,31 @@ void Executor::sendEvent(const std::string& event) */ void Executor::FilterTasksConsideringContainers(std::vector& tsks) { - std::map > m; - for(std::vector::const_iterator it=tsks.begin();it!=tsks.end();it++) + std::map, HPCCompare > m; + for(auto cur : tsks) { - Task *cur(*it); if(!cur) continue; Container *cont(cur->getContainer()); if(!cont) { - m[(HomogeneousPoolContainer *)NULL].push_back(cur); + m[nullptr].push_back(cur); continue; } HomogeneousPoolContainer *contC(dynamic_cast(cont)); if(!contC) { - m[(HomogeneousPoolContainer *)NULL].push_back(cur); + m[nullptr].push_back(cur); continue; } m[contC].push_back(cur); } // std::vector ret; - for(std::map >::const_iterator it=m.begin();it!=m.end();it++) + for(auto it : m) { - HomogeneousPoolContainer *curhpc((*it).first); - const std::vector& curtsks((*it).second); + HomogeneousPoolContainer *curhpc(it.first); + const std::vector& curtsks(it.second); if(!curhpc) { ret.insert(ret.end(),curtsks.begin(),curtsks.end()); diff --git a/src/engine/PlayGround.cxx b/src/engine/PlayGround.cxx index fb526134f..633473327 100644 --- a/src/engine/PlayGround.cxx +++ b/src/engine/PlayGround.cxx @@ -25,6 +25,7 @@ #include #include #include +#include #include using namespace YACS::ENGINE; @@ -86,6 +87,18 @@ std::size_t Resource::getNumberOfWorkers(int nbCoresPerCont) const return static_cast(this->nbCores())/static_cast(nbCoresPerCont); } +void Resource::printSelf(std::ostream& oss) const +{ + oss << this->name() << " (" << this->nbCores() << ") : "; + for(auto it : this->_occupied) + { + if(it) + oss << "1"; + else + oss << "0"; + } +} + std::string PlayGround::printSelf() const { std::ostringstream oss; @@ -182,7 +195,7 @@ std::vector PlayGround::allocateFor(std::size_t nbOfPlacesToTake, i { std::vector ret; std::size_t nbOfPlacesToTakeCpy(nbOfPlacesToTake),offset(0); - for(auto res : _data) + for(const auto& res : _data) { std::vector contIdsInRes(res.allocateFor(nbOfPlacesToTakeCpy,nbCoresPerCont)); std::for_each(contIdsInRes.begin(),contIdsInRes.end(),[offset](std::size_t& val) { val += offset; }); @@ -197,7 +210,7 @@ std::vector PlayGround::allocateFor(std::size_t nbOfPlacesToTake, i void PlayGround::release(std::size_t workerId, int nbCoresPerCont) const { std::size_t offset(0); - for(auto res : _data) + for(const auto& res : _data) { std::size_t nbOfWorker(static_cast(res.nbCores()/nbCoresPerCont)); std::size_t minId(offset),maxId(offset+nbOfWorker); @@ -209,6 +222,15 @@ void PlayGround::release(std::size_t workerId, int nbCoresPerCont) const } } +void PlayGround::printMe() const +{ + for(auto it : _data) + { + it.printSelf(std::cout); + std::cout << std::endl; + } +} + std::vector PlayGround::BuildVectOfIdsFromVecBool(const std::vector& v) { std::size_t sz(std::count(v.begin(),v.end(),true)),i(0); diff --git a/src/engine/PlayGround.hxx b/src/engine/PlayGround.hxx index 0d7d54dc0..1e198d2e2 100644 --- a/src/engine/PlayGround.hxx +++ b/src/engine/PlayGround.hxx @@ -48,6 +48,7 @@ namespace YACS std::vector allocateFor(std::size_t& nbOfPlacesToTake, int nbCoresPerCont) const; void release(std::size_t workerId, int nbCoresPerCont) const; std::size_t getNumberOfWorkers(int nbCoresPerCont) const; + void printSelf(std::ostream& oss) const; private: std::string _name; int _nbCores; @@ -98,6 +99,7 @@ namespace YACS std::vector allocateFor(std::size_t nbOfPlacesToTake, int nbCoresPerCont) const; void release(std::size_t workerId, int nbCoresPerCont) const; std::mutex& getLocker() const { return _locker; } + void printMe() const; private: std::vector< std::pair > bigToTiny(const std::vector< std::pair > &weights, std::map &saveOrder) const; std::vector< std::vector > backToOriginalOrder(const std::vector< std::vector > &disorderVec, const std::map &saveOrder) const; diff --git a/src/yacsloader_swig/Test/testFEDyn.py b/src/yacsloader_swig/Test/testFEDyn.py index 6f16c0c99..9f3a642cc 100644 --- a/src/yacsloader_swig/Test/testFEDyn.py +++ b/src/yacsloader_swig/Test/testFEDyn.py @@ -16,7 +16,7 @@ cont.setProperty("nb_proc_per_node","1") # pg=pilot.PlayGround() #pg.loadFromKernelCatalog() -pg.setData([("localhost",8)]) +pg.setData([("localhost",4)]) # #cont.setSizeOfPool(4) cont.setProperty("name","localhost") @@ -80,13 +80,13 @@ p=l.load(fname) p.saveSchema(fname2) p.init() p.propagePlayGround(pg) -"""ex=pilot.ExecutorSwig() +ex=pilot.ExecutorSwig() assert(p.getState()==pilot.READY) stt = datetime.datetime.now() ex.RunW(p,0) print(str(datetime.datetime.now()-stt)) assert(p.getState()==pilot.DONE) o4=p.getChildByName("n2").getOutputPort("o4") -assert(o4.getPyObj()==[0, 5, 10, 15, 20, 25, 30, 35])""" +assert(o4.getPyObj()==[0, 5, 10, 15, 20, 25, 30, 35]) # Ideal ForEachDyn time = 22 s # ForEach time = 40 s""" \ No newline at end of file -- 2.39.2