1 // Copyright (C) 2020-2021 CEA/DEN, EDF R&D
3 // This library is free software; you can redistribute it and/or
4 // modify it under the terms of the GNU Lesser General Public
5 // License as published by the Free Software Foundation; either
6 // version 2.1 of the License, or (at your option) any later version.
8 // This library is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 // Lesser General Public License for more details.
13 // You should have received a copy of the GNU Lesser General Public
14 // License along with this library; if not, write to the Free Software
15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
19 // Author : Anthony Geay (EDF R&D)
23 #include "MEDCouplingUMesh.hxx"
24 #include "ProcessorGroup.hxx"
25 #include "MEDCouplingMemArray.hxx"
33 * Parallel representation of an unstructured mesh.
35 * This class is very specific to the requirement of parallel code computations.
37 class ParaUMesh : public RefCountObject
40 static ParaUMesh *New(MEDCouplingUMesh *mesh, DataArrayIdType *globalCellIds, DataArrayIdType *globalNodeIds);
41 MCAuto<DataArrayIdType> getCellIdsLyingOnNodes(const DataArrayIdType *globalNodeIds, bool fullyIn) const;
42 ParaUMesh *redistributeCells(const DataArrayIdType *globalCellIds) const;
43 DataArrayDouble *redistributeCellField(const DataArrayIdType *globalCellIds, const DataArrayDouble *fieldValueToRed) const;
44 DataArrayIdType *redistributeCellField(const DataArrayIdType *globalCellIds, const DataArrayIdType *fieldValueToRed) const;
45 DataArrayDouble *redistributeNodeField(const DataArrayIdType *globalCellIds, const DataArrayDouble *fieldValueToRed) const;
46 DataArrayIdType *redistributeNodeField(const DataArrayIdType *globalCellIds, const DataArrayIdType *fieldValueToRed) const;
47 MEDCouplingUMesh *getMesh() { return _mesh; }
48 DataArrayIdType *getGlobalCellIds() { return _cell_global; }
49 DataArrayIdType *getGlobalNodeIds() { return _node_global; }
51 virtual ~ParaUMesh() { }
52 ParaUMesh(MEDCouplingUMesh *mesh, DataArrayIdType *globalCellIds, DataArrayIdType *globalNodeIds);
53 std::string getClassName() const override { return "ParaUMesh"; }
54 std::size_t getHeapMemorySizeWithoutChildren() const override;
55 std::vector<const BigMemoryObject *> getDirectChildrenWithNull() const override;
57 MCAuto<MEDCouplingUMesh> _mesh;
58 MCAuto<DataArrayIdType> _cell_global;
59 MCAuto<DataArrayIdType> _node_global;
61 MCAuto<DataArrayIdType> getCellIdsLyingOnNodesFalse(const DataArrayIdType *globalNodeIds) const;
62 MCAuto<DataArrayIdType> getCellIdsLyingOnNodesTrue(const DataArrayIdType *globalNodeIds) const;
64 typename Traits<T>::ArrayType *redistributeCellFieldT(const DataArrayIdType *globalCellIds, const typename Traits<T>::ArrayType *fieldValueToRed) const
66 using DataArrayT = typename Traits<T>::ArrayType;
67 MPI_Comm comm(MPI_COMM_WORLD);
69 if( _cell_global->getNumberOfTuples() != fieldValueToRed->getNumberOfTuples() )
70 throw INTERP_KERNEL::Exception("PAraUMesh::redistributeCellFieldT : invalid input length of array !");
71 std::unique_ptr<mcIdType[]> allGlobalCellIds,allGlobalCellIdsIndex;
72 int size(ci.allGatherArrays(comm,globalCellIds,allGlobalCellIds,allGlobalCellIdsIndex));
73 // Prepare ParaUMesh parts to be sent : compute for each proc the contribution of current rank.
74 std::vector< MCAuto<DataArrayIdType> > globalCellIdsToBeSent(size);
75 std::vector< MCAuto<DataArrayT> > fieldToBeSent(size);
76 for(int curRk = 0 ; curRk < size ; ++curRk)
78 mcIdType offset(allGlobalCellIdsIndex[curRk]);
79 MCAuto<DataArrayIdType> globalCellIdsOfCurProc(DataArrayIdType::New());
80 globalCellIdsOfCurProc->useArray(allGlobalCellIds.get()+offset,false,DeallocType::CPP_DEALLOC,allGlobalCellIdsIndex[curRk+1]-offset,1);
81 // the key call is here : compute for rank curRk the cells to be sent
82 MCAuto<DataArrayIdType> globalCellIdsCaptured(_cell_global->buildIntersection(globalCellIdsOfCurProc));// OK for the global cellIds
83 MCAuto<DataArrayIdType> localCellIdsCaptured(_cell_global->findIdForEach(globalCellIdsCaptured->begin(),globalCellIdsCaptured->end()));
84 globalCellIdsToBeSent[curRk] = globalCellIdsCaptured;
85 fieldToBeSent[curRk] = fieldValueToRed->selectByTupleIdSafe(localCellIdsCaptured->begin(),localCellIdsCaptured->end());
88 std::vector< MCAuto<DataArrayIdType> > globalCellIdsReceived;
89 ci.allToAllArrays(comm,globalCellIdsToBeSent,globalCellIdsReceived);
90 std::vector< MCAuto<DataArrayT> > fieldValueReceived;
91 ci.allToAllArrays(comm,fieldToBeSent,fieldValueReceived);
92 // use globalCellIdsReceived to reorganize everything
93 MCAuto<DataArrayIdType> aggregatedCellIds( DataArrayIdType::Aggregate(FromVecAutoToVecOfConst<DataArrayIdType>(globalCellIdsReceived)) );
94 MCAuto<DataArrayIdType> aggregatedCellIdsSorted(aggregatedCellIds->copySorted());
95 MCAuto<DataArrayIdType> idsIntoAggregatedIds(DataArrayIdType::FindPermutationFromFirstToSecondDuplicate(aggregatedCellIdsSorted,aggregatedCellIds));
96 MCAuto<DataArrayIdType> cellIdsOfSameNodeIds(aggregatedCellIdsSorted->indexOfSameConsecutiveValueGroups());
97 MCAuto<DataArrayIdType> n2o_cells(idsIntoAggregatedIds->selectByTupleIdSafe(cellIdsOfSameNodeIds->begin(),cellIdsOfSameNodeIds->end()-1));//new == new ordering so that global cell ids are sorted . old == coarse ordering implied by the aggregation
99 MCAuto<DataArrayT> fieldAggregated(DataArrayT::Aggregate(FromVecAutoToVecOfConst<DataArrayT>(fieldValueReceived)));
100 MCAuto<DataArrayT> ret(fieldAggregated->selectByTupleIdSafe(n2o_cells->begin(),n2o_cells->end()));
105 typename Traits<T>::ArrayType *redistributeNodeFieldT(const DataArrayIdType *globalCellIds, const typename Traits<T>::ArrayType *fieldValueToRed) const
107 using DataArrayT = typename Traits<T>::ArrayType;
108 MPI_Comm comm(MPI_COMM_WORLD);
110 if( _node_global->getNumberOfTuples() != fieldValueToRed->getNumberOfTuples() )
111 throw INTERP_KERNEL::Exception("PAraUMesh::redistributeNodeFieldT : invalid input length of array !");
112 std::unique_ptr<mcIdType[]> allGlobalCellIds,allGlobalCellIdsIndex;
113 int size(ci.allGatherArrays(comm,globalCellIds,allGlobalCellIds,allGlobalCellIdsIndex));
114 // Prepare ParaUMesh parts to be sent : compute for each proc the contribution of current rank.
115 std::vector< MCAuto<DataArrayIdType> > globalNodeIdsToBeSent(size);
116 std::vector< MCAuto<DataArrayT> > fieldToBeSent(size);
117 for(int curRk = 0 ; curRk < size ; ++curRk)
119 mcIdType offset(allGlobalCellIdsIndex[curRk]);
120 MCAuto<DataArrayIdType> globalCellIdsOfCurProc(DataArrayIdType::New());
121 globalCellIdsOfCurProc->useArray(allGlobalCellIds.get()+offset,false,DeallocType::CPP_DEALLOC,allGlobalCellIdsIndex[curRk+1]-offset,1);
122 // the key call is here : compute for rank curRk the cells to be sent
123 MCAuto<DataArrayIdType> globalCellIdsCaptured(_cell_global->buildIntersection(globalCellIdsOfCurProc));// OK for the global cellIds
124 MCAuto<DataArrayIdType> localCellIdsCaptured(_cell_global->findIdForEach(globalCellIdsCaptured->begin(),globalCellIdsCaptured->end()));
125 MCAuto<MEDCouplingUMesh> meshPart(_mesh->buildPartOfMySelf(localCellIdsCaptured->begin(),localCellIdsCaptured->end(),true));
126 MCAuto<DataArrayIdType> o2n(meshPart->zipCoordsTraducer());// OK for the mesh
127 MCAuto<DataArrayIdType> n2o(o2n->invertArrayO2N2N2O(meshPart->getNumberOfNodes()));
128 MCAuto<DataArrayIdType> globalNodeIdsPart(_node_global->selectByTupleIdSafe(n2o->begin(),n2o->end())); // OK for the global nodeIds
129 globalNodeIdsToBeSent[curRk] = globalNodeIdsPart;
130 fieldToBeSent[curRk] = fieldValueToRed->selectByTupleIdSafe(n2o->begin(),n2o->end());
133 std::vector< MCAuto<DataArrayIdType> > globalNodeIdsReceived;
134 ci.allToAllArrays(comm,globalNodeIdsToBeSent,globalNodeIdsReceived);
135 std::vector< MCAuto<DataArrayT> > fieldValueReceived;
136 ci.allToAllArrays(comm,fieldToBeSent,fieldValueReceived);
137 // firstly deal with nodes.
138 MCAuto<DataArrayIdType> aggregatedNodeIds( DataArrayIdType::Aggregate(FromVecAutoToVecOfConst<DataArrayIdType>(globalNodeIdsReceived)) );
139 MCAuto<DataArrayIdType> aggregatedNodeIdsSorted(aggregatedNodeIds->copySorted());
140 MCAuto<DataArrayIdType> nodeIdsIntoAggregatedIds(DataArrayIdType::FindPermutationFromFirstToSecondDuplicate(aggregatedNodeIdsSorted,aggregatedNodeIds));
141 MCAuto<DataArrayIdType> idxOfSameNodeIds(aggregatedNodeIdsSorted->indexOfSameConsecutiveValueGroups());
142 MCAuto<DataArrayIdType> n2o_nodes(nodeIdsIntoAggregatedIds->selectByTupleIdSafe(idxOfSameNodeIds->begin(),idxOfSameNodeIds->end()-1));//new == new ordering so that global node ids are sorted . old == coarse ordering implied by the aggregation
144 MCAuto<DataArrayT> fieldAggregated(DataArrayT::Aggregate(FromVecAutoToVecOfConst<DataArrayT>(fieldValueReceived)));
145 MCAuto<DataArrayT> ret(fieldAggregated->selectByTupleIdSafe(n2o_nodes->begin(),n2o_nodes->end()));