From 886b9d4f15888becea63405c231ddc2c85e43be0 Mon Sep 17 00:00:00 2001 From: Anthony Geay Date: Fri, 6 Mar 2020 23:23:34 +0100 Subject: [PATCH] WIP --- src/ParaMEDMEM/CommInterface.hxx | 25 ++++++++++++---- src/ParaMEDMEM/ParaUMesh.cxx | 49 +++++++++++++++++++++----------- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/src/ParaMEDMEM/CommInterface.hxx b/src/ParaMEDMEM/CommInterface.hxx index e76f22e6e..76692c86e 100644 --- a/src/ParaMEDMEM/CommInterface.hxx +++ b/src/ParaMEDMEM/CommInterface.hxx @@ -17,12 +17,13 @@ // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com // -#ifndef __COMMINTERFACE_HXX__ -#define __COMMINTERFACE_HXX__ +#pragma once + +#include "ParaIdType.hxx" #include -#include "ParaIdType.hxx" +#include namespace MEDCoupling { @@ -89,7 +90,21 @@ namespace MEDCoupling int reduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm) const { return MPI_Reduce(sendbuf, recvbuf, count, datatype, op, root, comm); } int allReduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) const { return MPI_Allreduce(sendbuf, recvbuf, count, datatype, op, comm); } + public: + static std::unique_ptr ToIntArray(const std::unique_ptr& arr, std::size_t size) + { + std::unique_ptr ret(new int[size]); + std::copy(arr.get(),arr.get()+size,ret.get()); + } + static std::unique_ptr ComputeOffset(const std::unique_ptr& counts, std::size_t sizeOfCounts) + { + std::unique_ptr ret(new int[sizeOfCounts]); + ret[0] = 0; + for(std::size_t i = 1 ; i < sizeOfCounts ; ++i) + { + ret[i] = ret[i-1] + counts[i-1]; + } + return ret; + } }; } - -#endif /*COMMINTERFACE_HXX_*/ diff --git a/src/ParaMEDMEM/ParaUMesh.cxx b/src/ParaMEDMEM/ParaUMesh.cxx index e1d873d8f..1db9288dc 100644 --- a/src/ParaMEDMEM/ParaUMesh.cxx +++ b/src/ParaMEDMEM/ParaUMesh.cxx @@ -24,6 +24,7 @@ #include "MPIProcessorGroup.hxx" #include "Topology.hxx" #include "BlockTopology.hxx" +#include "CommInterface.hxx" #include "MEDCouplingMemArray.hxx" #include "mpi.h" @@ -56,25 +57,41 @@ MCAuto ParaUMesh::getCellIdsLyingOnNodes(DataArrayIdType *globa int rk,size; MPI_Comm_rank(comm,&rk); MPI_Comm_size(comm,&size); - std::unique_ptr nbOfElems(new long[size]); + std::unique_ptr nbOfElems(new long[size]),nbOfElems2(new long[size]); long nbOfNodeIdsLoc(FromIdType(globalNodeIds->getNumberOfTuples())); MPI_Allgather(&nbOfNodeIdsLoc,1,MPI_LONG,nbOfElems.get(),size,MPI_LONG,comm); long nbOfNodeIdsSum(std::accumulate(nbOfElems.get(),nbOfElems.get()+size,0L)); - std::unique_ptr allGlobalNodeIds(new int[nbOfNodeIdsSum]); - MPI_Allgather(globalNodeIds->begin(),1,MPI_INT,allGlobalNodeIds.get(),size,MPI_INT,comm); - // compute for each proc the cell contribution among _mesh cells - mcIdType offset(0); - for(int curRk = 0 ; curRk < size ; ++curRk) + std::vector< MCAuto > tabs(size); { - MCAuto globalNodeIdsOfCurProc(DataArrayIdType::New()); - globalNodeIdsOfCurProc->useArray(allGlobalNodeIds.get()+offset,false,DeallocType::CPP_DEALLOC,nbOfElems[curRk],1); - offset += nbOfElems[curRk]; - MCAuto globalNodeIdsCaptured(_node_global->buildIntersection(globalNodeIdsOfCurProc)); - MCAuto localNodeIdsToLocate(_node_global->findIdForEach(globalNodeIdsCaptured->begin(),globalNodeIdsCaptured->end())); - MCAuto localCellCaptured(_mesh->getCellIdsLyingOnNodes(localNodeIdsToLocate->begin(),localNodeIdsToLocate->end(),false)); - MCAuto localCellCapturedGlob(_cell_global->selectByTupleIdSafe(localCellCaptured->begin(),localCellCaptured->end())); - + std::unique_ptr allGlobalNodeIds(new int[nbOfNodeIdsSum]); + MPI_Allgather(globalNodeIds->begin(),1,MPI_INT,allGlobalNodeIds.get(),1,MPI_INT,comm); + // compute for each proc the cell contribution among _mesh cells + long offset(0); + for(int curRk = 0 ; curRk < size ; ++curRk) + { + MCAuto globalNodeIdsOfCurProc(DataArrayIdType::New()); + globalNodeIdsOfCurProc->useArray(allGlobalNodeIds.get()+offset,false,DeallocType::CPP_DEALLOC,nbOfElems[curRk],1); + offset += nbOfElems[curRk]; + MCAuto globalNodeIdsCaptured(_node_global->buildIntersection(globalNodeIdsOfCurProc)); + MCAuto localNodeIdsToLocate(_node_global->findIdForEach(globalNodeIdsCaptured->begin(),globalNodeIdsCaptured->end())); + MCAuto localCellCaptured(_mesh->getCellIdsLyingOnNodes(localNodeIdsToLocate->begin(),localNodeIdsToLocate->end(),false)); + MCAuto localCellCapturedGlob(_cell_global->selectByTupleIdSafe(localCellCaptured->begin(),localCellCaptured->end())); + nbOfElems[curRk] = FromIdType(localCellCapturedGlob->getNumberOfTuples()); + tabs[curRk] = localCellCapturedGlob; + } } - // - MCAuto local; + std::vector tabss(tabs.begin(),tabs.end()); + MCAuto cells(DataArrayIdType::Aggregate(tabss)); + MPI_Alltoall(nbOfElems.get(),1,MPI_LONG,nbOfElems2.get(),1,MPI_LONG,comm); + long nbOfCellIdsSum(std::accumulate(nbOfElems2.get(),nbOfElems2.get()+size,0L)); + MCAuto cellIdsFromProcs(DataArrayIdType::New()); + cellIdsFromProcs->alloc(nbOfCellIdsSum,1); + { + std::unique_ptr nbOfElemsInt( CommInterface::ToIntArray(nbOfElems,size) ),nbOfElemsOutInt( CommInterface::ToIntArray(nbOfElems2,size) ); + std::unique_ptr offsetsIn( CommInterface::ComputeOffset(nbOfElemsInt,size) ), offsetsOut( CommInterface::ComputeOffset(CommInterface::ToIntArray(nbOfElems2,size),size) ); + MPI_Alltoallv(cells->begin(),nbOfElemsInt.get(),offsetsIn.get(),MPI_INT, + cellIdsFromProcs->getPointer(),nbOfElemsOutInt.get(),offsetsOut.get(),MPI_INT,comm); + } + cellIdsFromProcs->sort(); + return cellIdsFromProcs; } -- 2.39.2