1 // Copyright (C) 2007-2013 CEA/DEN, EDF R&D
3 // This library is free software; you can redistribute it and/or
4 // modify it under the terms of the GNU Lesser General Public
5 // License as published by the Free Software Foundation; either
6 // version 2.1 of the License.
8 // This library is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 // Lesser General Public License for more details.
13 // You should have received a copy of the GNU Lesser General Public
14 // License along with this library; if not, write to the Free Software
15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
20 #ifndef __MPIACCESS_HXX__
21 #define __MPIACCESS_HXX__
23 #include "CommInterface.hxx"
24 #include "ProcessorGroup.hxx"
25 #include "MPIProcessorGroup.hxx"
41 static MPI_Request mpirequestnull = MPI_REQUEST_NULL ;
42 enum _MessageIdent { _message_unknown, _message_time, _message_int, _message_double } ;
52 bool MPIAsynchronous ;
54 MPI_Datatype MPIDatatype ;
55 MPI_Request MPIRequest ;
56 MPI_Status *MPIStatus ;
60 MPIAccess(MPIProcessorGroup * ProcessorGroup, int BaseTag=0, int MaxTag=0) ;
61 virtual ~MPIAccess() ;
63 void trace( bool trace = true ) ;
65 void deleteRequest( int RequestId ) ;
66 void deleteRequests(int size , int *ArrayOfSendRequests ) ;
68 int sendMPITag(int destrank) ;
69 int recvMPITag(int sourcerank) ;
71 int sendRequestIdsSize() ;
72 int sendRequestIds(int size, int *ArrayOfSendRequests) ;
73 int recvRequestIdsSize() ;
74 int recvRequestIds(int size, int *ArrayOfRecvRequests) ;
76 int sendRequestIdsSize(int destrank) ;
77 int sendRequestIds(int destrank, int size, int *ArrayOfSendRequests) ;
78 int recvRequestIdsSize(int sourcerank) ;
79 int recvRequestIds(int sourcerank, int size, int *ArrayOfRecvRequests) ;
81 int send(void* buffer, int count, MPI_Datatype datatype, int target,
83 int ISend(void* buffer, int count, MPI_Datatype datatype, int target,
85 int recv(void* buffer, int count, MPI_Datatype datatype, int source,
86 int &RequestId, int *OutCount=NULL) ;
87 int IRecv(void* buffer, int count, MPI_Datatype datatype, int source,
89 int sendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest,
90 int &SendRequestId, void* recvbuf, int recvcount,
91 MPI_Datatype recvtype, int source,
92 int &RecvRequestId, int *OutCount=NULL) ;
93 int ISendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest,
94 int &SendRequestId, void* recvbuf, int recvcount,
95 MPI_Datatype recvtype, int source, int &RecvRequestId) ;
97 int wait(int RequestId) ;
98 int test(int RequestId, int &flag) ;
99 int waitAny(int count, int *array_of_RequestIds, int &RequestId) ;
100 int testAny(int count, int *array_of_RequestIds, int &RequestId, int &flag) ;
101 int waitAll(int count, int *array_of_RequestIds) ;
102 int testAll(int count, int *array_of_RequestIds, int &flag) ;
103 int waitSome(int count, int *array_of_RequestIds, int outcount,
104 int *outarray_of_RequestIds) ;
105 int testSome(int count, int *array_of_RequestIds, int outcounts,
106 int *outarray_of_RequestIds) ;
107 int probe(int FromSource, int &source, int &MPITag, MPI_Datatype &datatype,
109 int IProbe(int FromSource, int &source, int &MPITag, MPI_Datatype &datatype,
110 int &outcount, int &flag) ;
111 int cancel( int RecvRequestId, int &flag ) ;
112 int cancel( int source, int MPITag, MPI_Datatype datatype, int outcount,
116 int errorString(int errorcode, char *string, int *resultlen) const ;
117 int status(int RequestId, int &source, int &tag, int &error, int &outcount,
118 bool keepRequestStruct=false) ;
119 int requestFree( MPI_Request *request ) ;
123 MPI_Datatype timeType() const ;
124 bool isTimeMessage( int MPITag ) const ;
125 MPI_Aint timeExtent() const ;
126 MPI_Aint intExtent() const ;
127 MPI_Aint doubleExtent() const ;
128 MPI_Aint extent( MPI_Datatype datatype ) const ;
130 int MPITag( int RequestId ) ;
131 int MPITarget( int RequestId ) ;
132 bool MPIIsRecv( int RequestId ) ;
133 bool MPIAsynchronous( int RequestId ) ;
134 bool MPICompleted( int RequestId ) ;
135 MPI_Datatype MPIDatatype( int RequestId ) ;
136 int MPIOutCount( int RequestId ) ;
139 int newRequest( MPI_Datatype datatype, int tag , int destsourcerank ,
140 bool fromsourcerank , bool asynchronous ) ;
141 int newSendTag( MPI_Datatype datatype, int destrank , int method ,
142 bool asynchronous, int &RequestId ) ;
143 int newRecvTag( MPI_Datatype datatype, int sourcerank , int method ,
144 bool asynchronous, int &RequestId ) ;
145 int incrTag( int prevtag ) ;
146 int valTag( int tag, int method ) ;
148 void deleteSendRecvRequest( int RequestId ) ;
150 void deleteStatus( int RequestId ) ;
152 MPI_Request *MPIRequest( int RequestId ) ;
153 MPI_Status *MPIStatus( int RequestId ) ;
154 void setMPICompleted( int RequestId , bool completed ) ;
155 void setMPIOutCount( int RequestId , int outcount ) ;
156 void clearMPIStatus( int RequestId ) ;
158 _MessageIdent methodId( MPI_Datatype datatype ) const ;
159 MPI_Datatype datatype( _MessageIdent aMethodIdent ) const ;
161 const CommInterface &_comm_interface ;
162 const MPI_Comm* _intra_communicator ;
163 MPIProcessorGroup * _processor_group ;
164 int _processor_group_size ;
170 int * _send_request ;
171 int * _recv_request ;
172 std::vector< std::list< int > > _send_requests ;
173 std::vector< std::list< int > > _recv_requests ;
176 int * _send_MPI_tag ;
177 int * _recv_MPI_Tag ;
178 MPI_Datatype _MPI_TIME ;
179 static const int MODULO_TAG=10;
180 std::map< int , RequestStruct * > _map_of_request_struct ;
184 inline void MPIAccess::trace( bool trace )
189 // Delete the structure Request corresponding to RequestId identifier after
190 // the deletion of the structures MPI_Request * and MPI_Status *
191 // remove it from _MapOfRequestStruct (erase)
192 inline void MPIAccess::deleteRequest( int RequestId )
194 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
195 if ( aRequestStruct )
198 std::cout << "MPIAccess::DeleteRequest" << _my_rank << "( " << RequestId << " ) "
199 << aRequestStruct << " MPIRequest " << aRequestStruct->MPIRequest
200 << " MPIIsRecv " << aRequestStruct->MPIIsRecv << std::endl ;
201 if ( _map_of_request_struct[RequestId]->MPIRequest != MPI_REQUEST_NULL )
202 requestFree( &_map_of_request_struct[RequestId]->MPIRequest ) ;
203 deleteSendRecvRequest( RequestId ) ;
204 deleteStatus( RequestId ) ;
205 _map_of_request_struct.erase( RequestId ) ;
206 delete aRequestStruct ;
211 std::cout << "MPIAccess::DeleteRequest" << _my_rank << "( " << RequestId
212 << " ) Request not found" << std::endl ;
216 // Delete all requests of the array ArrayOfSendRequests
217 inline void MPIAccess::deleteRequests(int size , int *ArrayOfSendRequests )
219 for (int i = 0 ; i < size ; i++ )
220 deleteRequest( ArrayOfSendRequests[i] ) ;
223 // Returns the last MPITag of the destination rank destrank
224 inline int MPIAccess::sendMPITag(int destrank)
226 return _send_MPI_tag[destrank] ;
229 // Returns the last MPITag of the source rank sourcerank
230 inline int MPIAccess::recvMPITag(int sourcerank)
232 return _recv_MPI_Tag[sourcerank] ;
235 // Returns the number of all SendRequestIds matching a destination rank. It may be
236 // used to allocate ArrayOfSendRequests for the call to SendRequestIds
237 inline int MPIAccess::sendRequestIdsSize(int destrank)
239 return _send_requests[destrank].size() ;
242 // Returns the number of all RecvRequestIds matching a source rank. It may be
243 // used to allocate ArrayOfRecvRequests for the call to RecvRequestIds
244 inline int MPIAccess::recvRequestIdsSize(int sourcerank)
246 return _recv_requests[sourcerank].size() ;
249 // Returns the MPI_Datatype (registered in MPI in the constructor with
250 // MPI_Type_struct and MPI_Type_commit) for TimeMessages
251 inline MPI_Datatype MPIAccess::timeType() const
256 // Returns true if the tag MPITag corresponds to a TimeMessage
257 inline bool MPIAccess::isTimeMessage( int MPITag ) const
259 return ((MPITag%MODULO_TAG) == _message_time) ;
262 // Returns the MPI size of a TimeMessage
263 inline MPI_Aint MPIAccess::timeExtent() const
266 MPI_Type_extent( _MPI_TIME , &extent ) ;
270 // Returns the MPI size of a MPI_INT
271 inline MPI_Aint MPIAccess::intExtent() const
274 MPI_Type_extent( MPI_INT , &extent ) ;
278 // Returns the MPI size of a MPI_DOUBLE
279 inline MPI_Aint MPIAccess::doubleExtent() const
282 MPI_Type_extent( MPI_DOUBLE , &extent ) ;
286 // Returns the MPI size of the MPI_Datatype datatype
287 inline MPI_Aint MPIAccess::extent( MPI_Datatype datatype ) const
289 if ( datatype == _MPI_TIME )
290 return timeExtent() ;
291 if ( datatype == MPI_INT )
293 if ( datatype == MPI_DOUBLE )
294 return doubleExtent() ;
298 // Returns the MPITag of the request corresponding to RequestId identifier
299 inline int MPIAccess::MPITag( int RequestId )
301 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
302 if ( aRequestStruct )
303 return aRequestStruct->MPITag ;
307 // Returns the MPITarget of the request corresponding to RequestId identifier
308 inline int MPIAccess::MPITarget( int RequestId )
310 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
311 if ( aRequestStruct )
312 return aRequestStruct->MPITarget ;
316 // Returns true if the request corresponding to RequestId identifier was [I]Recv
317 inline bool MPIAccess::MPIIsRecv( int RequestId )
319 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
320 if ( aRequestStruct )
321 return aRequestStruct->MPIIsRecv ;
325 // Returns true if the request corresponding to RequestId identifier was asynchronous
326 inline bool MPIAccess::MPIAsynchronous( int RequestId )
328 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
329 if ( aRequestStruct )
330 return aRequestStruct->MPIAsynchronous ;
334 // Returns true if the request corresponding to RequestId identifier was completed
335 inline bool MPIAccess::MPICompleted( int RequestId )
337 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
338 if ( aRequestStruct )
339 return aRequestStruct->MPICompleted;
343 // Returns the MPI_datatype of the request corresponding to RequestId identifier
344 inline MPI_Datatype MPIAccess::MPIDatatype( int RequestId )
346 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
347 if ( aRequestStruct )
348 return aRequestStruct->MPIDatatype;
349 return (MPI_Datatype ) NULL ;
352 // Returns the size of the receiving message of the request corresponding to
353 // RequestId identifier
354 inline int MPIAccess::MPIOutCount( int RequestId )
356 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
357 if ( aRequestStruct )
358 return aRequestStruct->MPIOutCount;
362 // Increments the previous tag value (cyclically)
363 // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
364 inline int MPIAccess::incrTag( int prevtag )
367 if ( (prevtag % MODULO_TAG) == _message_time )
368 tag = ((prevtag/MODULO_TAG)*MODULO_TAG);
370 tag = ((prevtag/MODULO_TAG + 1)*MODULO_TAG);
371 if ( tag > _max_MPI_tag )
372 tag = _base_MPI_tag ;
376 // Returns the MPITag with the method-type field
377 // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
378 inline int MPIAccess::valTag( int tag, int method )
380 return ((tag/MODULO_TAG)*MODULO_TAG) + method;
383 // Remove a Request identifier from the list _RecvRequests/_SendRequests for
384 // the corresponding target.
385 inline void MPIAccess::deleteSendRecvRequest( int RequestId )
388 std::cout << "MPIAccess::DeleteSendRecvRequest" << _my_rank
389 << "( " << RequestId << " ) " << std::endl ;
390 if ( MPIIsRecv( RequestId ) )
391 _recv_requests[ MPITarget( RequestId ) ].remove( RequestId );
393 _send_requests[ MPITarget( RequestId ) ].remove( RequestId );
396 // Delete the MPI structure MPI_status * of a ReaquestId
397 inline void MPIAccess::deleteStatus( int RequestId )
399 if ( _map_of_request_struct[RequestId]->MPIStatus != NULL )
401 delete _map_of_request_struct[RequestId]->MPIStatus ;
402 clearMPIStatus( RequestId ) ;
406 // Returns the MPI structure MPI_request * of a RequestId
407 inline MPI_Request * MPIAccess::MPIRequest( int RequestId )
409 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
410 if ( aRequestStruct )
411 return &aRequestStruct->MPIRequest;
412 return &mpirequestnull ;
415 // Returns the MPI structure MPI_status * of a RequestId
416 inline MPI_Status * MPIAccess::MPIStatus( int RequestId )
418 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ];
419 if ( aRequestStruct )
420 return aRequestStruct->MPIStatus;
424 // Set the MPICompleted field of the structure Request corresponding to RequestId
425 // identifier with the value completed
426 inline void MPIAccess::setMPICompleted( int RequestId , bool completed )
428 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
429 if ( aRequestStruct )
430 aRequestStruct->MPICompleted = completed;
433 // Set the MPIOutCount field of the structure Request corresponding to RequestId
434 // identifier with the value outcount
435 inline void MPIAccess::setMPIOutCount( int RequestId , int outcount )
437 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
438 if ( aRequestStruct )
439 aRequestStruct->MPIOutCount = outcount;
442 // Nullify the MPIStatusfield of the structure Request corresponding to RequestId
444 inline void MPIAccess::clearMPIStatus( int RequestId )
446 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
447 if ( aRequestStruct )
448 aRequestStruct->MPIStatus = NULL ;
451 // Returns the _MessageIdent enum value corresponding to the MPI_Datatype datatype
452 // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
453 inline _MessageIdent MPIAccess::methodId( MPI_Datatype datatype ) const
455 _MessageIdent aMethodIdent ;
456 if ( datatype == _MPI_TIME )
457 aMethodIdent = _message_time;
458 else if ( datatype == MPI_INT )
459 aMethodIdent = _message_int ;
460 else if ( datatype == MPI_DOUBLE )
461 aMethodIdent = _message_double ;
463 aMethodIdent = _message_unknown ;
464 return aMethodIdent ;
467 // Returns the MPI_Datatype corresponding to the _MessageIdent enum aMethodIdent
468 inline MPI_Datatype MPIAccess::datatype( _MessageIdent aMethodIdent ) const
470 MPI_Datatype aDataType ;
471 switch( aMethodIdent )
474 aDataType = _MPI_TIME ;
477 aDataType = MPI_INT ;
479 case _message_double :
480 aDataType = MPI_DOUBLE ;
483 aDataType = (MPI_Datatype) -1 ;
489 std::ostream & operator<< (std::ostream &,const _MessageIdent &);
491 std::ostream & operator<< (std::ostream &,const TimeMessage &);