1 // Copyright (C) 2007-2008 CEA/DEN, EDF R&D
3 // This library is free software; you can redistribute it and/or
4 // modify it under the terms of the GNU Lesser General Public
5 // License as published by the Free Software Foundation; either
6 // version 2.1 of the License.
8 // This library is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 // Lesser General Public License for more details.
13 // You should have received a copy of the GNU Lesser General Public
14 // License along with this library; if not, write to the Free Software
15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
20 #include "CommInterface.hxx"
21 #include "Topology.hxx"
22 #include "BlockTopology.hxx"
23 #include "ComponentTopology.hxx"
24 #include "ParaFIELD.hxx"
25 #include "MPIProcessorGroup.hxx"
26 #include "StructuredCoincidentDEC.hxx"
27 #include "InterpKernelUtilities.hxx"
36 /*! \defgroup structuredcoincidentdec StructuredCoincidentDEC
38 This class is meant for remapping fields that have identical
39 supports with different parallel topologies. It can be used to couple
40 together multiphysics codes that operate on the same domain
41 with different partitionings, which can be useful if one of
42 the computation is much faster than the other. It can also be used
43 to couple together codes that share an interface that was generated
44 in the same manner (with identical global ids).
45 Also, this DEC can be used for fields that have component topologies,
46 i.e., components that are scattered over several processors.
48 The remapping between the two supports is based on identity of global
49 ids, instead of geometrical considerations as it is the case for
50 NonCoincidentDEC and IntersectionDEC. Therefore, this DEC must not be used
51 for coincident meshes that do not have the same numbering.
53 As all the other DECs, its use is made of two phases :
54 - a setup phase during whih the topologies are exchanged so that
55 the target side knows from which processors it should expect
57 - a send/recv phase during which the field data is actually transferred.
59 This example illustrates the sending of a field with
63 StructuredCoincidentDEC dec(groupA, groupB);
64 dec.attachLocalField(field);
66 if (groupA.containsMyRank())
68 else if (groupB.containsMyRank())
73 Creating a ParaFIELD to be attached to the DEC is exactly the same as for
74 other DECs in the case when the remapping concerns similar meshes
75 that only have different partitionings. In the case when the
76 fields have also different component topologies, creating the ParaFIELD
77 requires some more effort. See \ref parafield section for more details.
81 StructuredCoincidentDEC::StructuredCoincidentDEC():_topo_source(0),_topo_target(0),
82 _recv_buffer(0),_send_buffer(0),
83 _recv_counts(0),_send_counts(0),
84 _recv_displs(0),_send_displs(0)
89 StructuredCoincidentDEC::~StructuredCoincidentDEC()
91 delete[] _send_buffer;
92 delete[] _recv_buffer;
94 delete[] _recv_displs;
95 delete[] _send_counts;
96 delete[] _recv_counts;
97 if (! _source_group->containsMyRank())
99 if(!_target_group->containsMyRank())
104 \addtogroup structuredcoincidentdec
107 StructuredCoincidentDEC::StructuredCoincidentDEC(ProcessorGroup& local_group, ProcessorGroup& distant_group):DEC(local_group,distant_group),_topo_source(0),_topo_target(0),_recv_buffer(0),_send_buffer(0)
111 /*! Synchronization process for exchanging topologies
113 void StructuredCoincidentDEC::synchronizeTopology()
115 if (_source_group->containsMyRank())
116 _topo_source = dynamic_cast<BlockTopology*>(_local_field->getTopology());
117 if (_target_group->containsMyRank())
118 _topo_target = dynamic_cast<BlockTopology*>(_local_field->getTopology());
120 // Transmitting source topology to target code
121 broadcastTopology(_topo_source,1000);
122 // Transmitting target topology to source code
123 broadcastTopology(_topo_target,2000);
124 if (_topo_source->getNbElements() != _topo_target->getNbElements())
125 throw INTERP_KERNEL::Exception("Incompatible dimensions for target and source topologies");
129 /*! Creates the arrays necessary for the data transfer
130 * and fills the send array with the values of the
133 void StructuredCoincidentDEC::prepareSourceDE()
135 ////////////////////////////////////
136 //Step 1 : _buffer array creation
138 if (!_topo_source->getProcGroup()->containsMyRank())
140 MPIProcessorGroup* group=new MPIProcessorGroup(_topo_source->getProcGroup()->getCommInterface());
142 int myranksource = _topo_source->getProcGroup()->myRank();
144 vector <int>* target_arrays=new vector<int>[_topo_target->getProcGroup()->size()];
146 //cout<<" topotarget size"<< _topo_target->getProcGroup()->size()<<endl;
148 int nb_local = _topo_source-> getNbLocalElements();
149 for (int ielem=0; ielem< nb_local ; ielem++)
151 // cout <<"source local :"<<myranksource<<","<<ielem<<endl;
152 int global = _topo_source->localToGlobal(make_pair(myranksource, ielem));
153 // cout << "global "<<global<<endl;
154 pair<int,int> target_local =_topo_target->globalToLocal(global);
155 // cout << "target local : "<<target_local.first<<","<<target_local.second<<endl;
156 target_arrays[target_local.first].push_back(target_local.second);
159 int union_size=group->size();
161 _send_counts=new int[union_size];
162 _send_displs=new int[union_size];
163 _recv_counts=new int[union_size];
164 _recv_displs=new int[union_size];
166 for (int i=0; i< union_size; i++)
174 for (int iproc=0; iproc < _topo_target->getProcGroup()->size(); iproc++)
176 //converts the rank in target to the rank in union communicator
177 int unionrank=group->translateRank(_topo_target->getProcGroup(),iproc);
178 _send_counts[unionrank]=target_arrays[iproc].size();
181 for (int iproc=1; iproc<group->size();iproc++)
182 _send_displs[iproc]=_send_displs[iproc-1]+_send_counts[iproc-1];
184 _send_buffer = new double [nb_local ];
186 /////////////////////////////////////////////////////////////
187 //Step 2 : filling the _buffers with the source field values
189 int* counter=new int [_topo_target->getProcGroup()->size()];
191 for (int i=1; i<_topo_target->getProcGroup()->size(); i++)
192 counter[i]=counter[i-1]+target_arrays[i-1].size();
195 const double* value = _local_field->getField()->getArray()->getPointer();
196 //cout << "Nb local " << nb_local<<endl;
197 for (int ielem=0; ielem<nb_local ; ielem++)
199 int global = _topo_source->localToGlobal(make_pair(myranksource, ielem));
200 pair<int,int> target_local =_topo_target->globalToLocal(global);
201 //cout <<"global : "<< global<<" local :"<<target_local.first<<" "<<target_local.second;
202 //cout <<"counter[]"<<counter[target_local.first]<<endl;
203 _send_buffer[counter[target_local.first]++]=value[ielem];
206 delete[] target_arrays;
212 * Creates the _buffers for receiving the fields on the target side
214 void StructuredCoincidentDEC::prepareTargetDE()
216 if (!_topo_target->getProcGroup()->containsMyRank())
218 MPIProcessorGroup* group=new MPIProcessorGroup(_topo_source->getProcGroup()->getCommInterface());
220 int myranktarget = _topo_target->getProcGroup()->myRank();
222 vector < vector <int> > source_arrays(_topo_source->getProcGroup()->size());
223 int nb_local = _topo_target-> getNbLocalElements();
224 for (int ielem=0; ielem< nb_local ; ielem++)
226 // cout <<"TS target local :"<<myranktarget<<","<<ielem<<endl;
227 int global = _topo_target->localToGlobal(make_pair(myranktarget, ielem));
228 //cout << "TS global "<<global<<endl;
229 pair<int,int> source_local =_topo_source->globalToLocal(global);
230 // cout << "TS source local : "<<source_local.first<<","<<source_local.second<<endl;
231 source_arrays[source_local.first].push_back(source_local.second);
233 int union_size=group->size();
234 _recv_counts=new int[union_size];
235 _recv_displs=new int[union_size];
236 _send_counts=new int[union_size];
237 _send_displs=new int[union_size];
239 for (int i=0; i< union_size; i++)
245 for (int iproc=0; iproc < _topo_source->getProcGroup()->size(); iproc++)
247 //converts the rank in target to the rank in union communicator
248 int unionrank=group->translateRank(_topo_source->getProcGroup(),iproc);
249 _recv_counts[unionrank]=source_arrays[iproc].size();
251 for (int i=1; i<union_size; i++)
252 _recv_displs[i]=_recv_displs[i-1]+_recv_counts[i-1];
253 _recv_buffer=new double[nb_local];
260 * Synchronizing a topology so that all the
261 * group possesses it.
263 * \param topo Topology that is transmitted. It is read on processes where it already exists, and it is created and filled on others.
264 * \param tag Communication tag associated with this operation.
266 void StructuredCoincidentDEC::broadcastTopology(BlockTopology*& topo, int tag)
273 MPIProcessorGroup* group=new MPIProcessorGroup(*_comm_interface);
275 // The master proc creates a send buffer containing
276 // a serialized topology
279 if (topo!=0 && topo->getProcGroup()->myRank()==0)
281 MESSAGE ("Master rank");
282 topo->serialize(serializer, size);
283 rank_master = group->translateRank(topo->getProcGroup(),0);
284 MESSAGE("Master rank world number is "<<rank_master);
285 MESSAGE("World Size is "<<group->size());
286 for (int i=0; i< group->size(); i++)
289 _comm_interface->send(&rank_master,1,MPI_INT, i,tag+i,*(group->getComm()));
294 MESSAGE(" rank "<<group->myRank()<< " waiting ...");
295 _comm_interface->recv(&rank_master, 1,MPI_INT, MPI_ANY_SOURCE, tag+group->myRank(), *(group->getComm()),&status);
296 MESSAGE(" rank "<<group->myRank()<< "received master rank"<<rank_master);
298 // The topology is broadcasted to all processsors in the group
299 _comm_interface->broadcast(&size, 1,MPI_INT,rank_master,*(group->getComm()));
301 int* buffer=new int[size];
302 if (topo!=0 && topo->getProcGroup()->myRank()==0)
303 copy(serializer, serializer+size, buffer);
304 _comm_interface->broadcast(buffer,size,MPI_INT,rank_master,*(group->getComm()));
306 // Processors which did not possess the source topology
309 BlockTopology* topotemp=new BlockTopology();
310 topotemp->unserialize(buffer, *_comm_interface);
321 MESSAGE (" rank "<<group->myRank()<< " unserialize is over");
327 void StructuredCoincidentDEC::recvData()
329 //MPI_COMM_WORLD is used instead of group because there is no
330 //mechanism for creating the union group yet
332 for (int i=0; i< 4; i++)
333 cout << _recv_counts[i]<<" ";
335 for (int i=0; i< 4; i++)
336 cout << _recv_displs[i]<<" ";
339 cout<<"start AllToAll"<<endl;
340 MPI_Comm comm = *(dynamic_cast<MPIProcessorGroup*>(_union_group)->getComm());
341 _comm_interface->allToAllV(_send_buffer, _send_counts, _send_displs, MPI_DOUBLE,
342 _recv_buffer, _recv_counts, _recv_displs, MPI_DOUBLE,comm);
343 cout<<"end AllToAll"<<endl;
345 int nb_local = _topo_target->getNbLocalElements();
346 //double* value=new double[nb_local];
347 double* value=const_cast<double*>(_local_field->getField()->getArray()->getPointer());
349 int myranktarget=_topo_target->getProcGroup()->myRank();
350 vector<int> counters(_topo_source->getProcGroup()->size());
352 for (int i=0; i<_topo_source->getProcGroup()->size()-1; i++)
354 MPIProcessorGroup* group=new MPIProcessorGroup(*_comm_interface);
355 int worldrank=group->translateRank(_topo_source->getProcGroup(),i);
356 counters[i+1]=counters[i]+_recv_counts[worldrank];
360 for (int ielem=0; ielem<nb_local ; ielem++)
362 int global = _topo_target->localToGlobal(make_pair(myranktarget, ielem));
363 pair<int,int> source_local =_topo_source->globalToLocal(global);
364 value[ielem]=_recv_buffer[counters[source_local.first]++];
368 //_local_field->getField()->setValue(value);
371 void StructuredCoincidentDEC::sendData()
373 MESSAGE ("sendData");
374 for (int i=0; i< 4; i++)
375 cout << _send_counts[i]<<" ";
377 for (int i=0; i< 4; i++)
378 cout << _send_displs[i]<<" ";
380 cout <<"start AllToAll"<<endl;
381 MPI_Comm comm = *(dynamic_cast<MPIProcessorGroup*>(_union_group)->getComm());
382 _comm_interface->allToAllV(_send_buffer, _send_counts, _send_displs, MPI_DOUBLE,
383 _recv_buffer, _recv_counts, _recv_displs, MPI_DOUBLE,comm);
384 cout<<"end AllToAll"<<endl;
387 /*! Prepares a DEC for data exchange
389 This method broadcasts the topologies from source to target
390 so that the target side can analyse from which processors it
391 is expected to receive data.
394 void StructuredCoincidentDEC::synchronize()
396 if (_source_group->containsMyRank())
398 synchronizeTopology();
401 else if (_target_group->containsMyRank())
403 synchronizeTopology();