1 // Copyright (C) 2007-2020 CEA/DEN, EDF R&D
3 // This library is free software; you can redistribute it and/or
4 // modify it under the terms of the GNU Lesser General Public
5 // License as published by the Free Software Foundation; either
6 // version 2.1 of the License, or (at your option) any later version.
8 // This library is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 // Lesser General Public License for more details.
13 // You should have received a copy of the GNU Lesser General Public
14 // License along with this library; if not, write to the Free Software
15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
21 #include "CommInterface.hxx"
22 #include "Topology.hxx"
23 #include "BlockTopology.hxx"
24 #include "ComponentTopology.hxx"
25 #include "ParaFIELD.hxx"
26 #include "MPIProcessorGroup.hxx"
27 #include "StructuredCoincidentDEC.hxx"
28 #include "InterpKernelUtilities.hxx"
37 StructuredCoincidentDEC::StructuredCoincidentDEC():_topo_source(0),_topo_target(0),
38 _send_counts(0),_recv_counts(0),
39 _send_displs(0),_recv_displs(0),
40 _recv_buffer(0),_send_buffer(0)
45 StructuredCoincidentDEC::~StructuredCoincidentDEC()
47 delete [] _send_buffer;
48 delete [] _recv_buffer;
49 delete []_send_displs;
50 delete [] _recv_displs;
51 delete [] _send_counts;
52 delete [] _recv_counts;
53 if (! _source_group->containsMyRank())
55 if(!_target_group->containsMyRank())
59 StructuredCoincidentDEC::StructuredCoincidentDEC(ProcessorGroup& local_group, ProcessorGroup& distant_group):
60 DisjointDEC(local_group,distant_group),
61 _topo_source(0),_topo_target(0),
62 _send_counts(0),_recv_counts(0),
63 _send_displs(0),_recv_displs(0),
64 _recv_buffer(0),_send_buffer(0)
68 /*! Synchronization process for exchanging topologies
70 void StructuredCoincidentDEC::synchronizeTopology()
72 if (_source_group->containsMyRank())
73 _topo_source = dynamic_cast<BlockTopology*>(_local_field->getTopology());
74 if (_target_group->containsMyRank())
75 _topo_target = dynamic_cast<BlockTopology*>(_local_field->getTopology());
77 // Transmitting source topology to target code
78 broadcastTopology(_topo_source,1000);
79 // Transmitting target topology to source code
80 broadcastTopology(_topo_target,2000);
81 if (_topo_source->getNbElements() != _topo_target->getNbElements())
82 throw INTERP_KERNEL::Exception("Incompatible dimensions for target and source topologies");
86 /*! Creates the arrays necessary for the data transfer
87 * and fills the send array with the values of the
90 void StructuredCoincidentDEC::prepareSourceDE()
92 ////////////////////////////////////
93 //Step 1 : _buffer array creation
95 if (!_topo_source->getProcGroup()->containsMyRank())
97 MPIProcessorGroup* group=new MPIProcessorGroup(_topo_source->getProcGroup()->getCommInterface());
99 int myranksource = _topo_source->getProcGroup()->myRank();
101 vector <mcIdType>* target_arrays=new vector<mcIdType>[_topo_target->getProcGroup()->size()];
103 //cout<<" topotarget size"<< _topo_target->getProcGroup()->size()<<endl;
105 mcIdType nb_local = _topo_source-> getNbLocalElements();
106 for (mcIdType ielem=0; ielem< nb_local ; ielem++)
108 // cout <<"source local :"<<myranksource<<","<<ielem<<endl;
109 mcIdType global = _topo_source->localToGlobal(make_pair(myranksource, ielem));
110 // cout << "global "<<global<<endl;
111 pair<int,mcIdType> target_local =_topo_target->globalToLocal(global);
112 // cout << "target local : "<<target_local.first<<","<<target_local.second<<endl;
113 target_arrays[target_local.first].push_back(target_local.second);
116 std::size_t union_size=group->size();
118 _send_counts=new int[union_size];
119 _send_displs=new int[union_size];
120 _recv_counts=new int[union_size];
121 _recv_displs=new int[union_size];
123 for (std::size_t i=0; i< union_size; i++)
131 for (int iproc=0; iproc < _topo_target->getProcGroup()->size(); iproc++)
133 //converts the rank in target to the rank in union communicator
134 int unionrank=group->translateRank(_topo_target->getProcGroup(),iproc);
135 _send_counts[unionrank]=(int)target_arrays[iproc].size();
138 for (int iproc=1; iproc<group->size();iproc++)
139 _send_displs[iproc]=_send_displs[iproc-1]+_send_counts[iproc-1];
141 _send_buffer = new double [nb_local ];
143 /////////////////////////////////////////////////////////////
144 //Step 2 : filling the _buffers with the source field values
146 int* counter=new int [_topo_target->getProcGroup()->size()];
148 for (int i=1; i<_topo_target->getProcGroup()->size(); i++)
149 counter[i]=counter[i-1]+(int)target_arrays[i-1].size();
152 const double* value = _local_field->getField()->getArray()->getPointer();
153 //cout << "Nb local " << nb_local<<endl;
154 for (int ielem=0; ielem<nb_local ; ielem++)
156 mcIdType global = _topo_source->localToGlobal(make_pair(myranksource, ielem));
157 pair<int,mcIdType> target_local =_topo_target->globalToLocal(global);
158 //cout <<"global : "<< global<<" local :"<<target_local.first<<" "<<target_local.second;
159 //cout <<"counter[]"<<counter[target_local.first]<<endl;
160 _send_buffer[counter[target_local.first]++]=value[ielem];
163 delete[] target_arrays;
169 * Creates the buffers for receiving the fields on the target side
171 void StructuredCoincidentDEC::prepareTargetDE()
173 if (!_topo_target->getProcGroup()->containsMyRank())
175 MPIProcessorGroup* group=new MPIProcessorGroup(_topo_source->getProcGroup()->getCommInterface());
177 int myranktarget = _topo_target->getProcGroup()->myRank();
179 vector < vector <mcIdType> > source_arrays(_topo_source->getProcGroup()->size());
180 mcIdType nb_local = _topo_target-> getNbLocalElements();
181 for (mcIdType ielem=0; ielem< nb_local ; ielem++)
183 // cout <<"TS target local :"<<myranktarget<<","<<ielem<<endl;
184 mcIdType global = _topo_target->localToGlobal(make_pair(myranktarget, ielem));
185 //cout << "TS global "<<global<<endl;
186 pair<int,mcIdType> source_local =_topo_source->globalToLocal(global);
187 // cout << "TS source local : "<<source_local.first<<","<<source_local.second<<endl;
188 source_arrays[source_local.first].push_back(source_local.second);
190 std::size_t union_size=group->size();
191 _recv_counts=new int[union_size];
192 _recv_displs=new int[union_size];
193 _send_counts=new int[union_size];
194 _send_displs=new int[union_size];
196 for (std::size_t i=0; i< union_size; i++)
202 for (int iproc=0; iproc < _topo_source->getProcGroup()->size(); iproc++)
204 //converts the rank in target to the rank in union communicator
205 int unionrank=group->translateRank(_topo_source->getProcGroup(),iproc);
206 _recv_counts[unionrank]=(int)source_arrays[iproc].size();
208 for (std::size_t i=1; i<union_size; i++)
209 _recv_displs[i]=_recv_displs[i-1]+_recv_counts[i-1];
210 _recv_buffer=new double[nb_local];
217 * Synchronizing a topology so that all the
218 * group possesses it.
220 * \param topo Topology that is transmitted. It is read on processes where it already exists, and it is created and filled on others.
221 * \param tag Communication tag associated with this operation.
223 void StructuredCoincidentDEC::broadcastTopology(BlockTopology*& topo, int tag)
227 mcIdType* serializer=0;
230 MPIProcessorGroup* group=new MPIProcessorGroup(*_comm_interface);
232 // The master proc creates a send buffer containing
233 // a serialized topology
236 if (topo!=0 && topo->getProcGroup()->myRank()==0)
238 MESSAGE ("Master rank");
239 topo->serialize(serializer, size);
240 rank_master = group->translateRank(topo->getProcGroup(),0);
241 MESSAGE("Master rank world number is "<<rank_master);
242 MESSAGE("World Size is "<<group->size());
243 for (int i=0; i< group->size(); i++)
246 _comm_interface->send(&rank_master,1,MPI_INT, i,tag+i,*(group->getComm()));
251 MESSAGE(" rank "<<group->myRank()<< " waiting ...");
252 _comm_interface->recv(&rank_master, 1,MPI_INT, MPI_ANY_SOURCE, tag+group->myRank(), *(group->getComm()),&status);
253 MESSAGE(" rank "<<group->myRank()<< "received master rank"<<rank_master);
255 // The topology is broadcasted to all processors in the group
256 _comm_interface->broadcast(&size, 1,MPI_ID_TYPE,rank_master,*(group->getComm()));
258 mcIdType* buffer=new mcIdType[size];
259 if (topo!=0 && topo->getProcGroup()->myRank()==0)
260 copy(serializer, serializer+size, buffer);
261 _comm_interface->broadcast(buffer,(int)size,MPI_ID_TYPE,rank_master,*(group->getComm()));
263 // Processors which did not possess the source topology
266 BlockTopology* topotemp=new BlockTopology();
267 topotemp->unserialize(buffer, *_comm_interface);
278 MESSAGE (" rank "<<group->myRank()<< " unserialize is over");
284 void StructuredCoincidentDEC::recvData()
286 //MPI_COMM_WORLD is used instead of group because there is no
287 //mechanism for creating the union group yet
289 for (int i=0; i< 4; i++)
290 cout << _recv_counts[i]<<" ";
292 for (int i=0; i< 4; i++)
293 cout << _recv_displs[i]<<" ";
296 cout<<"start AllToAll"<<endl;
297 MPI_Comm comm = *(dynamic_cast<MPIProcessorGroup*>(_union_group)->getComm());
298 _comm_interface->allToAllV(_send_buffer, _send_counts, _send_displs, MPI_DOUBLE,
299 _recv_buffer, _recv_counts, _recv_displs, MPI_DOUBLE,comm);
300 cout<<"end AllToAll"<<endl;
302 mcIdType nb_local = _topo_target->getNbLocalElements();
303 //double* value=new double[nb_local];
304 double* value=const_cast<double*>(_local_field->getField()->getArray()->getPointer());
306 int myranktarget=_topo_target->getProcGroup()->myRank();
307 vector<int> counters(_topo_source->getProcGroup()->size());
309 for (int i=0; i<_topo_source->getProcGroup()->size()-1; i++)
311 MPIProcessorGroup* group=new MPIProcessorGroup(*_comm_interface);
312 int worldrank=group->translateRank(_topo_source->getProcGroup(),i);
313 counters[i+1]=counters[i]+_recv_counts[worldrank];
317 for (mcIdType ielem=0; ielem<nb_local ; ielem++)
319 mcIdType global = _topo_target->localToGlobal(make_pair(myranktarget, ielem));
320 pair<int,mcIdType> source_local =_topo_source->globalToLocal(global);
321 value[ielem]=_recv_buffer[counters[source_local.first]++];
325 //_local_field->getField()->setValue(value);
328 void StructuredCoincidentDEC::sendData()
330 MESSAGE ("sendData");
331 for (int i=0; i< 4; i++)
332 cout << _send_counts[i]<<" ";
334 for (int i=0; i< 4; i++)
335 cout << _send_displs[i]<<" ";
337 cout <<"start AllToAll"<<endl;
338 MPI_Comm comm = *(dynamic_cast<MPIProcessorGroup*>(_union_group)->getComm());
339 _comm_interface->allToAllV(_send_buffer, _send_counts, _send_displs, MPI_DOUBLE,
340 _recv_buffer, _recv_counts, _recv_displs, MPI_DOUBLE,comm);
341 cout<<"end AllToAll"<<endl;
344 /*! Prepares a DEC for data exchange
346 This method broadcasts the topologies from source to target
347 so that the target side can analyse from which processors it
348 is expected to receive data.
351 void StructuredCoincidentDEC::synchronize()
353 if (_source_group->containsMyRank())
355 synchronizeTopology();
358 else if (_target_group->containsMyRank())
360 synchronizeTopology();