1 // Copyright (C) 2007-2016 CEA/DEN, EDF R&D
3 // This library is free software; you can redistribute it and/or
4 // modify it under the terms of the GNU Lesser General Public
5 // License as published by the Free Software Foundation; either
6 // version 2.1 of the License, or (at your option) any later version.
8 // This library is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 // Lesser General Public License for more details.
13 // You should have received a copy of the GNU Lesser General Public
14 // License along with this library; if not, write to the Free Software
15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
20 #ifndef __MPIACCESS_HXX__
21 #define __MPIACCESS_HXX__
23 #include "CommInterface.hxx"
24 #include "ProcessorGroup.hxx"
25 #include "MPIProcessorGroup.hxx"
41 static MPI_Request mpirequestnull = MPI_REQUEST_NULL ;
42 enum _MessageIdent { _message_unknown, _message_time, _message_int, _message_double } ;
52 bool MPIAsynchronous ;
54 MPI_Datatype MPIDatatype ;
55 MPI_Request MPIRequest ;
56 MPI_Status *MPIStatus ;
60 MPIAccess(MPIProcessorGroup * ProcessorGroup, int BaseTag=0, int MaxTag=0) ;
61 virtual ~MPIAccess() ;
63 void trace( bool trace = true ) ;
65 void deleteRequest( int RequestId ) ;
66 void deleteRequests(int size , int *ArrayOfSendRequests ) ;
68 int sendMPITag(int destrank) ;
69 int recvMPITag(int sourcerank) ;
71 int sendRequestIdsSize() ;
72 int sendRequestIds(int size, int *ArrayOfSendRequests) ;
73 int recvRequestIdsSize() ;
74 int recvRequestIds(int size, int *ArrayOfRecvRequests) ;
76 int sendRequestIdsSize(int destrank) ;
77 int sendRequestIds(int destrank, int size, int *ArrayOfSendRequests) ;
78 int recvRequestIdsSize(int sourcerank) ;
79 int recvRequestIds(int sourcerank, int size, int *ArrayOfRecvRequests) ;
81 int send(void* buffer, int count, MPI_Datatype datatype, int target,
83 int ISend(void* buffer, int count, MPI_Datatype datatype, int target,
85 int recv(void* buffer, int count, MPI_Datatype datatype, int source,
86 int &RequestId, int *OutCount=NULL) ;
87 int IRecv(void* buffer, int count, MPI_Datatype datatype, int source,
89 int sendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest,
90 int &SendRequestId, void* recvbuf, int recvcount,
91 MPI_Datatype recvtype, int source,
92 int &RecvRequestId, int *OutCount=NULL) ;
93 int ISendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest,
94 int &SendRequestId, void* recvbuf, int recvcount,
95 MPI_Datatype recvtype, int source, int &RecvRequestId) ;
97 int wait(int RequestId) ;
98 int test(int RequestId, int &flag) ;
99 int waitAny(int count, int *array_of_RequestIds, int &RequestId) ;
100 int testAny(int count, int *array_of_RequestIds, int &RequestId, int &flag) ;
101 int waitAll(int count, int *array_of_RequestIds) ;
102 int testAll(int count, int *array_of_RequestIds, int &flag) ;
103 int waitSome(int count, int *array_of_RequestIds, int outcount,
104 int *outarray_of_RequestIds) ;
105 int testSome(int count, int *array_of_RequestIds, int outcounts,
106 int *outarray_of_RequestIds) ;
107 int probe(int FromSource, int &source, int &MPITag, MPI_Datatype &datatype,
109 int IProbe(int FromSource, int &source, int &MPITag, MPI_Datatype &datatype,
110 int &outcount, int &flag) ;
111 int cancel( int RecvRequestId, int &flag ) ;
112 int cancel( int source, int MPITag, MPI_Datatype datatype, int outcount,
116 int errorString(int errorcode, char *string, int *resultlen) const ;
117 int status(int RequestId, int &source, int &tag, int &error, int &outcount,
118 bool keepRequestStruct=false) ;
119 int requestFree( MPI_Request *request ) ;
123 MPI_Datatype timeType() const ;
124 bool isTimeMessage( int MPITag ) const ;
125 MPI_Aint timeExtent() const ;
126 MPI_Aint intExtent() const ;
127 MPI_Aint doubleExtent() const ;
128 MPI_Aint extent( MPI_Datatype datatype ) const ;
130 int MPITag( int RequestId ) ;
131 int MPITarget( int RequestId ) ;
132 bool MPIIsRecv( int RequestId ) ;
133 bool MPIAsynchronous( int RequestId ) ;
134 bool MPICompleted( int RequestId ) ;
135 MPI_Datatype MPIDatatype( int RequestId ) ;
136 int MPIOutCount( int RequestId ) ;
139 int newRequest( MPI_Datatype datatype, int tag , int destsourcerank ,
140 bool fromsourcerank , bool asynchronous ) ;
141 int newSendTag( MPI_Datatype datatype, int destrank , int method ,
142 bool asynchronous, int &RequestId ) ;
143 int newRecvTag( MPI_Datatype datatype, int sourcerank , int method ,
144 bool asynchronous, int &RequestId ) ;
145 int incrTag( int prevtag ) ;
146 int valTag( int tag, int method ) ;
148 void deleteSendRecvRequest( int RequestId ) ;
150 void deleteStatus( int RequestId ) ;
152 MPI_Request *MPIRequest( int RequestId ) ;
153 MPI_Status *MPIStatus( int RequestId ) ;
154 void setMPICompleted( int RequestId , bool completed ) ;
155 void setMPIOutCount( int RequestId , int outcount ) ;
156 void clearMPIStatus( int RequestId ) ;
158 _MessageIdent methodId( MPI_Datatype datatype ) const ;
159 MPI_Datatype datatype( _MessageIdent aMethodIdent ) const ;
161 const CommInterface &_comm_interface ;
162 const MPI_Comm* _intra_communicator ;
163 MPIProcessorGroup * _processor_group ;
164 int _processor_group_size ;
170 int * _send_request ;
171 int * _recv_request ;
172 std::vector< std::list< int > > _send_requests ;
173 std::vector< std::list< int > > _recv_requests ;
176 int * _send_MPI_tag ;
177 int * _recv_MPI_Tag ;
178 MPI_Datatype _MPI_TIME ;
179 static const int MODULO_TAG=10;
180 std::map< int , RequestStruct * > _map_of_request_struct ;
184 inline void MPIAccess::trace( bool atrace )
189 // Delete the structure Request corresponding to RequestId identifier after
190 // the deletion of the structures MPI_Request * and MPI_Status *
191 // remove it from _MapOfRequestStruct (erase)
192 inline void MPIAccess::deleteRequest( int RequestId )
194 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
195 if ( aRequestStruct )
198 std::cout << "MPIAccess::DeleteRequest" << _my_rank << "( " << RequestId << " ) "
199 << aRequestStruct << " MPIRequest " << aRequestStruct->MPIRequest
200 << " MPIIsRecv " << aRequestStruct->MPIIsRecv << std::endl ;
201 if ( _map_of_request_struct[RequestId]->MPIRequest != MPI_REQUEST_NULL )
202 requestFree( &_map_of_request_struct[RequestId]->MPIRequest ) ;
203 deleteSendRecvRequest( RequestId ) ;
204 deleteStatus( RequestId ) ;
205 _map_of_request_struct.erase( RequestId ) ;
206 delete aRequestStruct ;
211 std::cout << "MPIAccess::DeleteRequest" << _my_rank << "( " << RequestId
212 << " ) Request not found" << std::endl ;
216 // Delete all requests of the array ArrayOfSendRequests
217 inline void MPIAccess::deleteRequests(int size , int *ArrayOfSendRequests )
219 for (int i = 0 ; i < size ; i++ )
220 deleteRequest( ArrayOfSendRequests[i] ) ;
223 // Returns the last MPITag of the destination rank destrank
224 inline int MPIAccess::sendMPITag(int destrank)
226 return _send_MPI_tag[destrank] ;
229 // Returns the last MPITag of the source rank sourcerank
230 inline int MPIAccess::recvMPITag(int sourcerank)
232 return _recv_MPI_Tag[sourcerank] ;
235 // Returns the number of all SendRequestIds matching a destination rank. It may be
236 // used to allocate ArrayOfSendRequests for the call to SendRequestIds
237 inline int MPIAccess::sendRequestIdsSize(int destrank)
239 return _send_requests[destrank].size() ;
242 // Returns the number of all RecvRequestIds matching a source rank. It may be
243 // used to allocate ArrayOfRecvRequests for the call to RecvRequestIds
244 inline int MPIAccess::recvRequestIdsSize(int sourcerank)
246 return _recv_requests[sourcerank].size() ;
249 // Returns the MPI_Datatype (registered in MPI in the constructor with
250 // MPI_Type_struct and MPI_Type_commit) for TimeMessages
251 inline MPI_Datatype MPIAccess::timeType() const
256 // Returns true if the tag MPITag corresponds to a TimeMessage
257 inline bool MPIAccess::isTimeMessage( int aMPITag ) const
259 return ((aMPITag%MODULO_TAG) == _message_time) ;
262 // Returns the MPI size of the MPI_Datatype datatype
263 inline MPI_Aint MPIAccess::extent( MPI_Datatype adatatype ) const
265 if ( adatatype == _MPI_TIME )
266 return timeExtent() ;
267 if ( adatatype == MPI_INT )
269 if ( adatatype == MPI_DOUBLE )
270 return doubleExtent() ;
274 // Returns the MPITag of the request corresponding to RequestId identifier
275 inline int MPIAccess::MPITag( int RequestId )
277 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
278 if ( aRequestStruct )
279 return aRequestStruct->MPITag ;
283 // Returns the MPITarget of the request corresponding to RequestId identifier
284 inline int MPIAccess::MPITarget( int RequestId )
286 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
287 if ( aRequestStruct )
288 return aRequestStruct->MPITarget ;
292 // Returns true if the request corresponding to RequestId identifier was [I]Recv
293 inline bool MPIAccess::MPIIsRecv( int RequestId )
295 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
296 if ( aRequestStruct )
297 return aRequestStruct->MPIIsRecv ;
301 // Returns true if the request corresponding to RequestId identifier was asynchronous
302 inline bool MPIAccess::MPIAsynchronous( int RequestId )
304 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
305 if ( aRequestStruct )
306 return aRequestStruct->MPIAsynchronous ;
310 // Returns true if the request corresponding to RequestId identifier was completed
311 inline bool MPIAccess::MPICompleted( int RequestId )
313 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
314 if ( aRequestStruct )
315 return aRequestStruct->MPICompleted;
319 // Returns the MPI_datatype of the request corresponding to RequestId identifier
320 inline MPI_Datatype MPIAccess::MPIDatatype( int RequestId )
322 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
323 if ( aRequestStruct )
324 return aRequestStruct->MPIDatatype;
325 return MPI_DATATYPE_NULL;
328 // Returns the size of the receiving message of the request corresponding to
329 // RequestId identifier
330 inline int MPIAccess::MPIOutCount( int RequestId )
332 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
333 if ( aRequestStruct )
334 return aRequestStruct->MPIOutCount;
338 // Increments the previous tag value (cyclically)
339 // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
340 inline int MPIAccess::incrTag( int prevtag )
343 if ( (prevtag % MODULO_TAG) == _message_time )
344 tag = ((prevtag/MODULO_TAG)*MODULO_TAG);
346 tag = ((prevtag/MODULO_TAG + 1)*MODULO_TAG);
347 if ( tag > _max_MPI_tag )
348 tag = _base_MPI_tag ;
352 // Returns the MPITag with the method-type field
353 // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
354 inline int MPIAccess::valTag( int tag, int method )
356 return ((tag/MODULO_TAG)*MODULO_TAG) + method;
359 // Remove a Request identifier from the list _RecvRequests/_SendRequests for
360 // the corresponding target.
361 inline void MPIAccess::deleteSendRecvRequest( int RequestId )
364 std::cout << "MPIAccess::DeleteSendRecvRequest" << _my_rank
365 << "( " << RequestId << " ) " << std::endl ;
366 if ( MPIIsRecv( RequestId ) )
367 _recv_requests[ MPITarget( RequestId ) ].remove( RequestId );
369 _send_requests[ MPITarget( RequestId ) ].remove( RequestId );
372 // Delete the MPI structure MPI_status * of a ReaquestId
373 inline void MPIAccess::deleteStatus( int RequestId )
375 if ( _map_of_request_struct[RequestId]->MPIStatus != NULL )
377 delete _map_of_request_struct[RequestId]->MPIStatus ;
378 clearMPIStatus( RequestId ) ;
382 // Returns the MPI structure MPI_request * of a RequestId
383 inline MPI_Request * MPIAccess::MPIRequest( int RequestId )
385 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
386 if ( aRequestStruct )
387 return &aRequestStruct->MPIRequest;
388 return &mpirequestnull ;
391 // Returns the MPI structure MPI_status * of a RequestId
392 inline MPI_Status * MPIAccess::MPIStatus( int RequestId )
394 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ];
395 if ( aRequestStruct )
396 return aRequestStruct->MPIStatus;
400 // Set the MPICompleted field of the structure Request corresponding to RequestId
401 // identifier with the value completed
402 inline void MPIAccess::setMPICompleted( int RequestId , bool completed )
404 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
405 if ( aRequestStruct )
406 aRequestStruct->MPICompleted = completed;
409 // Set the MPIOutCount field of the structure Request corresponding to RequestId
410 // identifier with the value outcount
411 inline void MPIAccess::setMPIOutCount( int RequestId , int outcount )
413 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
414 if ( aRequestStruct )
415 aRequestStruct->MPIOutCount = outcount;
418 // Nullify the MPIStatusfield of the structure Request corresponding to RequestId
420 inline void MPIAccess::clearMPIStatus( int RequestId )
422 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
423 if ( aRequestStruct )
424 aRequestStruct->MPIStatus = NULL ;
427 // Returns the _MessageIdent enum value corresponding to the MPI_Datatype datatype
428 // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
429 inline _MessageIdent MPIAccess::methodId( MPI_Datatype adatatype ) const
431 _MessageIdent aMethodIdent ;
432 if ( adatatype == _MPI_TIME )
433 aMethodIdent = _message_time;
434 else if ( adatatype == MPI_INT )
435 aMethodIdent = _message_int ;
436 else if ( adatatype == MPI_DOUBLE )
437 aMethodIdent = _message_double ;
439 aMethodIdent = _message_unknown ;
440 return aMethodIdent ;
443 // Returns the MPI_Datatype corresponding to the _MessageIdent enum aMethodIdent
444 inline MPI_Datatype MPIAccess::datatype( _MessageIdent aMethodIdent ) const
446 MPI_Datatype aDataType ;
447 switch( aMethodIdent )
450 aDataType = _MPI_TIME ;
453 aDataType = MPI_INT ;
455 case _message_double :
456 aDataType = MPI_DOUBLE ;
459 aDataType = (MPI_Datatype) -1 ;
465 std::ostream & operator<< (std::ostream &,const _MessageIdent &);
467 std::ostream & operator<< (std::ostream &,const TimeMessage &);