From 89a8619ecf39c522b8c2e3a80bae948934710741 Mon Sep 17 00:00:00 2001 From: Anthony Geay Date: Fri, 17 Apr 2020 23:49:57 +0200 Subject: [PATCH] WIP --- src/ParaMEDMEM/CMakeLists.txt | 2 + src/ParaMEDMEM/CommInterface.cxx | 56 +++++++++++++++++++++++++++- src/ParaMEDMEM/CommInterface.hxx | 7 +++- src/ParaMEDMEM/ParaSkyLineArray.cxx | 54 +++++++++++++++++++++++++++ src/ParaMEDMEM/ParaSkyLineArray.hxx | 47 +++++++++++++++++++++++ src/ParaMEDMEM/ParaUMesh.cxx | 58 ----------------------------- 6 files changed, 162 insertions(+), 62 deletions(-) create mode 100644 src/ParaMEDMEM/ParaSkyLineArray.cxx create mode 100644 src/ParaMEDMEM/ParaSkyLineArray.hxx diff --git a/src/ParaMEDMEM/CMakeLists.txt b/src/ParaMEDMEM/CMakeLists.txt index 5a7c28443..6871cd11c 100644 --- a/src/ParaMEDMEM/CMakeLists.txt +++ b/src/ParaMEDMEM/CMakeLists.txt @@ -40,7 +40,9 @@ SET(paramedmem_SOURCES ProcessorGroup.cxx MPIProcessorGroup.cxx ParaMESH.cxx + CommInterface.cxx ParaUMesh.cxx + ParaSkyLineArray.cxx ComponentTopology.cxx MPIAccess.cxx InterpolationMatrix.cxx diff --git a/src/ParaMEDMEM/CommInterface.cxx b/src/ParaMEDMEM/CommInterface.cxx index d5cf6d550..96c85fd9e 100644 --- a/src/ParaMEDMEM/CommInterface.cxx +++ b/src/ParaMEDMEM/CommInterface.cxx @@ -19,6 +19,8 @@ #include "CommInterface.hxx" +#include + namespace MEDCoupling { /*! \anchor CommInterface-det @@ -56,11 +58,61 @@ namespace MEDCoupling \endverbatim */ - CommInterface::CommInterface() + /*! + * Generalized AllGather collective communication. + * This method send input \a array to all procs. + */ + void CommInterface::allGatherArrays(MPI_Comm comm, const DataArrayIdType *array, std::unique_ptr& result, std::unique_ptr& resultIndex) const { + int size; + this->commSize(comm,&size); + std::unique_ptr nbOfElems(new mcIdType[size]); + mcIdType nbOfCellsRequested(array->getNumberOfTuples()); + this->allGather(&nbOfCellsRequested,1,MPI_ID_TYPE,nbOfElems.get(),1,MPI_ID_TYPE,comm); + mcIdType nbOfCellIdsSum(std::accumulate(nbOfElems.get(),nbOfElems.get()+size,0)); + result.reset(new mcIdType[nbOfCellIdsSum]); + std::unique_ptr nbOfElemsInt( CommInterface::ToIntArray(nbOfElems,size) ); + std::unique_ptr offsetsIn( CommInterface::ComputeOffset(nbOfElemsInt,size) ); + this->allGatherV(array->begin(),nbOfCellsRequested,MPI_ID_TYPE,result.get(),nbOfElemsInt.get(),offsetsIn.get(),MPI_ID_TYPE,comm); + resultIndex = std::move(nbOfElems); } - CommInterface::~CommInterface() + /*! + * Generalized AllToAll collective communication. + */ + void CommInterface::allToAllArrays(MPI_Comm comm, const std::vector< MCAuto >& arrays, std::vector< MCAuto >& arraysOut) const { + int size; + this->commSize(comm,&size); + if( arrays.size() != ToSizeT(size) ) + throw INTERP_KERNEL::Exception("AllToAllArrays : internal error ! Invalid size of input array."); + std::vector< const DataArrayIdType *> arraysBis(size); + { + std::vector< const DataArrayIdType *>::iterator itArraysBis(arraysBis.begin()); + std::for_each(arrays.begin(),arrays.end(),[&itArraysBis](MCAuto elt) { *itArraysBis++=elt; }); + } + std::unique_ptr nbOfElems2(new mcIdType[size]),nbOfElems3(new mcIdType[size]); + for(int curRk = 0 ; curRk < size ; ++curRk) + { + nbOfElems3[curRk] = arrays[curRk]->getNumberOfTuples(); + } + this->allToAll(nbOfElems3.get(),1,MPI_ID_TYPE,nbOfElems2.get(),1,MPI_ID_TYPE,comm); + mcIdType nbOfCellIdsSum(std::accumulate(nbOfElems2.get(),nbOfElems2.get()+size,0)); + MCAuto cellIdsFromProcs(DataArrayIdType::New()); + cellIdsFromProcs->alloc(nbOfCellIdsSum,1); + std::unique_ptr nbOfElemsInt( CommInterface::ToIntArray(nbOfElems3,size) ),nbOfElemsOutInt( CommInterface::ToIntArray(nbOfElems2,size) ); + std::unique_ptr offsetsIn( CommInterface::ComputeOffset(nbOfElemsInt,size) ), offsetsOut( CommInterface::ComputeOffset(nbOfElemsOutInt,size) ); + { + MCAuto arraysAcc(DataArrayIdType::Aggregate(arraysBis)); + this->allToAllV(arraysAcc->begin(),nbOfElemsInt.get(),offsetsIn.get(),MPI_ID_TYPE, + cellIdsFromProcs->getPointer(),nbOfElemsOutInt.get(),offsetsOut.get(),MPI_ID_TYPE,comm); + } + std::unique_ptr offsetsOutIdType( CommInterface::ComputeOffset(nbOfElems2,size) ); + // build output arraysOut by spliting cellIdsFromProcs into parts + arraysOut.resize(size); + for(int curRk = 0 ; curRk < size ; ++curRk) + { + arraysOut[curRk] = DataArrayIdType::NewFromArray(cellIdsFromProcs->begin()+offsetsOutIdType[curRk],cellIdsFromProcs->begin()+offsetsOutIdType[curRk]+nbOfElems2[curRk]); + } } } diff --git a/src/ParaMEDMEM/CommInterface.hxx b/src/ParaMEDMEM/CommInterface.hxx index 96471f670..8b2e7aa8c 100644 --- a/src/ParaMEDMEM/CommInterface.hxx +++ b/src/ParaMEDMEM/CommInterface.hxx @@ -32,8 +32,8 @@ namespace MEDCoupling class CommInterface { public: - CommInterface(){} - virtual ~CommInterface(){} + CommInterface() { } + virtual ~CommInterface() { } int worldSize() const { int size; MPI_Comm_size(MPI_COMM_WORLD, &size); @@ -94,6 +94,9 @@ namespace MEDCoupling MPI_Op op, int root, MPI_Comm comm) const { return MPI_Reduce(sendbuf, recvbuf, count, datatype, op, root, comm); } int allReduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) const { return MPI_Allreduce(sendbuf, recvbuf, count, datatype, op, comm); } public: + void allGatherArrays(MPI_Comm comm, const DataArrayIdType *array, std::unique_ptr& result, std::unique_ptr& resultIndex) const; + void allToAllArrays(MPI_Comm comm, const std::vector< MCAuto >& arrays, std::vector< MCAuto >& arraysOut) const; + public: /*! * \a counts is expected to be an array of array length. This method returns an array of split array. diff --git a/src/ParaMEDMEM/ParaSkyLineArray.cxx b/src/ParaMEDMEM/ParaSkyLineArray.cxx new file mode 100644 index 000000000..9b39c082d --- /dev/null +++ b/src/ParaMEDMEM/ParaSkyLineArray.cxx @@ -0,0 +1,54 @@ +// +// Copyright (C) 2020 CEA/DEN, EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +// Author : Anthony Geay (EDF R&D) + +#include "ParaSkyLineArray.hxx" +#include "ProcessorGroup.hxx" +#include "MPIProcessorGroup.hxx" +#include "Topology.hxx" +#include "BlockTopology.hxx" +#include "CommInterface.hxx" +#include "MEDCouplingMemArray.hxx" + +#include "mpi.h" + +#include +#include +#include +#include +#include + +using namespace std; +using namespace MEDCoupling; + +ParaSkyLineArray::ParaSkyLineArray(MEDCouplingSkyLineArray *ska, DataArrayIdType *globalIds) +{ + _ska.takeRef(ska); + _global_ids.takeRef(globalIds); + _ska.checkNotNull(); + _global_ids.checkNotNull(); + if(_ska->getNumberOf() != _global_ids->getNumberOfTuples()) + throw INTERP_KERNEL::Exception("ParaSkyLineArray constructor : mismatch between # globalIds and len of indices in SkyLineArray."); +} + +MCAuto ParaSkyLineArray::equiRedistribute(mcIdType nbOfEntities) const +{ + //TODO +} \ No newline at end of file diff --git a/src/ParaMEDMEM/ParaSkyLineArray.hxx b/src/ParaMEDMEM/ParaSkyLineArray.hxx new file mode 100644 index 000000000..968a9d6aa --- /dev/null +++ b/src/ParaMEDMEM/ParaSkyLineArray.hxx @@ -0,0 +1,47 @@ +// Copyright (C) 2020 CEA/DEN, EDF R&D +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com +// +// Author : Anthony Geay (EDF R&D) + +#pragma once + +#include "MEDCouplingSkyLineArray.hxx" +#include "ProcessorGroup.hxx" +#include "MEDCouplingMemArray.hxx" + +#include +#include + +namespace MEDCoupling +{ + /*! + * Parallel representation of a SkyLineArray + * + * This class is very specific to the requirement of parallel code computations. + */ + class ParaSkyLineArray + { + public: + ParaSkyLineArray(MEDCouplingSkyLineArray *ska, DataArrayIdType *globalIds); + MCAuto equiRedistribute(mcIdType nbOfEntities) const; + virtual ~ParaSkyLineArray() { } + private: + MCAuto _ska; + MCAuto _global_ids; + }; +} diff --git a/src/ParaMEDMEM/ParaUMesh.cxx b/src/ParaMEDMEM/ParaUMesh.cxx index 0b844e96c..5a44f74b4 100644 --- a/src/ParaMEDMEM/ParaUMesh.cxx +++ b/src/ParaMEDMEM/ParaUMesh.cxx @@ -117,64 +117,6 @@ MCAuto ParaUMesh::getCellIdsLyingOnNodes(const DataArrayIdType return cellIdsFromProcs; } -/*! - * Generalized AllGather collective communication. - * This method send input \a array to all procs. - */ -void AllGatherArrays(const CommInterface& ci, MPI_Comm comm, const DataArrayIdType *array, std::unique_ptr& result, std::unique_ptr& resultIndex) -{ - int size; - ci.commSize(comm,&size); - std::unique_ptr nbOfElems(new mcIdType[size]); - mcIdType nbOfCellsRequested(array->getNumberOfTuples()); - ci.allGather(&nbOfCellsRequested,1,MPI_ID_TYPE,nbOfElems.get(),1,MPI_ID_TYPE,comm); - mcIdType nbOfCellIdsSum(std::accumulate(nbOfElems.get(),nbOfElems.get()+size,0)); - result.reset(new mcIdType[nbOfCellIdsSum]); - std::unique_ptr nbOfElemsInt( CommInterface::ToIntArray(nbOfElems,size) ); - std::unique_ptr offsetsIn( CommInterface::ComputeOffset(nbOfElemsInt,size) ); - ci.allGatherV(array->begin(),nbOfCellsRequested,MPI_ID_TYPE,result.get(),nbOfElemsInt.get(),offsetsIn.get(),MPI_ID_TYPE,comm); - resultIndex = std::move(nbOfElems); -} - -/*! - * Generalized AllToAll collective communication. - */ -void AllToAllArrays(const CommInterface& ci, MPI_Comm comm, const std::vector< MCAuto >& arrays, std::vector< MCAuto >& arraysOut) -{ - int size; - ci.commSize(comm,&size); - if( arrays.size() != ToSizeT(size) ) - throw INTERP_KERNEL::Exception("AllToAllArrays : internal error ! Invalid size of input array."); - std::vector< const DataArrayIdType *> arraysBis(size); - { - std::vector< const DataArrayIdType *>::iterator itArraysBis(arraysBis.begin()); - std::for_each(arrays.begin(),arrays.end(),[&itArraysBis](MCAuto elt) { *itArraysBis++=elt; }); - } - std::unique_ptr nbOfElems2(new mcIdType[size]),nbOfElems3(new mcIdType[size]); - for(int curRk = 0 ; curRk < size ; ++curRk) - { - nbOfElems3[curRk] = arrays[curRk]->getNumberOfTuples(); - } - ci.allToAll(nbOfElems3.get(),1,MPI_ID_TYPE,nbOfElems2.get(),1,MPI_ID_TYPE,comm); - mcIdType nbOfCellIdsSum(std::accumulate(nbOfElems2.get(),nbOfElems2.get()+size,0)); - MCAuto cellIdsFromProcs(DataArrayIdType::New()); - cellIdsFromProcs->alloc(nbOfCellIdsSum,1); - std::unique_ptr nbOfElemsInt( CommInterface::ToIntArray(nbOfElems3,size) ),nbOfElemsOutInt( CommInterface::ToIntArray(nbOfElems2,size) ); - std::unique_ptr offsetsIn( CommInterface::ComputeOffset(nbOfElemsInt,size) ), offsetsOut( CommInterface::ComputeOffset(nbOfElemsOutInt,size) ); - { - MCAuto arraysAcc(DataArrayIdType::Aggregate(arraysBis)); - ci.allToAllV(arraysAcc->begin(),nbOfElemsInt.get(),offsetsIn.get(),MPI_ID_TYPE, - cellIdsFromProcs->getPointer(),nbOfElemsOutInt.get(),offsetsOut.get(),MPI_ID_TYPE,comm); - } - std::unique_ptr offsetsOutIdType( CommInterface::ComputeOffset(nbOfElems2,size) ); - // build output arraysOut by spliting cellIdsFromProcs into parts - arraysOut.resize(size); - for(int curRk = 0 ; curRk < size ; ++curRk) - { - arraysOut[curRk] = DataArrayIdType::NewFromArray(cellIdsFromProcs->begin()+offsetsOutIdType[curRk],cellIdsFromProcs->begin()+offsetsOutIdType[curRk]+nbOfElems2[curRk]); - } -} - /*! */ MCAuto ParaUMesh::redistributeCells(const DataArrayIdType *globalCellIds) const -- 2.39.2