1 // Copyright (C) 2007-2008 CEA/DEN, EDF R&D
3 // This library is free software; you can redistribute it and/or
4 // modify it under the terms of the GNU Lesser General Public
5 // License as published by the Free Software Foundation; either
6 // version 2.1 of the License.
8 // This library is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 // Lesser General Public License for more details.
13 // You should have received a copy of the GNU Lesser General Public
14 // License along with this library; if not, write to the Free Software
15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
19 #ifndef __MPIACCESS_HXX__
20 #define __MPIACCESS_HXX__
22 #include "CommInterface.hxx"
23 #include "ProcessorGroup.hxx"
24 #include "MPIProcessorGroup.hxx"
40 static MPI_Request mpirequestnull = MPI_REQUEST_NULL ;
41 enum _MessageIdent { _message_unknown, _message_time, _message_int, _message_double } ;
51 bool MPIAsynchronous ;
53 MPI_Datatype MPIDatatype ;
54 MPI_Request MPIRequest ;
55 MPI_Status *MPIStatus ;
59 MPIAccess(MPIProcessorGroup * ProcessorGroup, int BaseTag=0, int MaxTag=0) ;
60 virtual ~MPIAccess() ;
62 void trace( bool trace = true ) ;
64 void deleteRequest( int RequestId ) ;
65 void deleteRequests(int size , int *ArrayOfSendRequests ) ;
67 int sendMPITag(int destrank) ;
68 int recvMPITag(int sourcerank) ;
70 int sendRequestIdsSize() ;
71 int sendRequestIds(int size, int *ArrayOfSendRequests) ;
72 int recvRequestIdsSize() ;
73 int recvRequestIds(int size, int *ArrayOfRecvRequests) ;
75 int sendRequestIdsSize(int destrank) ;
76 int sendRequestIds(int destrank, int size, int *ArrayOfSendRequests) ;
77 int recvRequestIdsSize(int sourcerank) ;
78 int recvRequestIds(int sourcerank, int size, int *ArrayOfRecvRequests) ;
80 int send(void* buffer, int count, MPI_Datatype datatype, int target,
82 int ISend(void* buffer, int count, MPI_Datatype datatype, int target,
84 int recv(void* buffer, int count, MPI_Datatype datatype, int source,
85 int &RequestId, int *OutCount=NULL) ;
86 int IRecv(void* buffer, int count, MPI_Datatype datatype, int source,
88 int sendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest,
89 int &SendRequestId, void* recvbuf, int recvcount,
90 MPI_Datatype recvtype, int source,
91 int &RecvRequestId, int *OutCount=NULL) ;
92 int ISendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest,
93 int &SendRequestId, void* recvbuf, int recvcount,
94 MPI_Datatype recvtype, int source, int &RecvRequestId) ;
96 int wait(int RequestId) ;
97 int test(int RequestId, int &flag) ;
98 int waitAny(int count, int *array_of_RequestIds, int &RequestId) ;
99 int testAny(int count, int *array_of_RequestIds, int &RequestId, int &flag) ;
100 int waitAll(int count, int *array_of_RequestIds) ;
101 int testAll(int count, int *array_of_RequestIds, int &flag) ;
102 int waitSome(int count, int *array_of_RequestIds, int outcount,
103 int *outarray_of_RequestIds) ;
104 int testSome(int count, int *array_of_RequestIds, int outcounts,
105 int *outarray_of_RequestIds) ;
106 int probe(int FromSource, int &source, int &MPITag, MPI_Datatype &datatype,
108 int IProbe(int FromSource, int &source, int &MPITag, MPI_Datatype &datatype,
109 int &outcount, int &flag) ;
110 int cancel( int RecvRequestId, int &flag ) ;
111 int cancel( int source, int MPITag, MPI_Datatype datatype, int outcount,
115 int errorString(int errorcode, char *string, int *resultlen) const ;
116 int status(int RequestId, int &source, int &tag, int &error, int &outcount,
117 bool keepRequestStruct=false) ;
118 int requestFree( MPI_Request *request ) ;
122 MPI_Datatype timeType() const ;
123 bool isTimeMessage( int MPITag ) const ;
124 MPI_Aint timeExtent() const ;
125 MPI_Aint intExtent() const ;
126 MPI_Aint doubleExtent() const ;
127 MPI_Aint extent( MPI_Datatype datatype ) const ;
129 int MPITag( int RequestId ) ;
130 int MPITarget( int RequestId ) ;
131 bool MPIIsRecv( int RequestId ) ;
132 bool MPIAsynchronous( int RequestId ) ;
133 bool MPICompleted( int RequestId ) ;
134 MPI_Datatype MPIDatatype( int RequestId ) ;
135 int MPIOutCount( int RequestId ) ;
138 int newRequest( MPI_Datatype datatype, int tag , int destsourcerank ,
139 bool fromsourcerank , bool asynchronous ) ;
140 int newSendTag( MPI_Datatype datatype, int destrank , int method ,
141 bool asynchronous, int &RequestId ) ;
142 int newRecvTag( MPI_Datatype datatype, int sourcerank , int method ,
143 bool asynchronous, int &RequestId ) ;
144 int incrTag( int prevtag ) ;
145 int valTag( int tag, int method ) ;
147 void deleteSendRecvRequest( int RequestId ) ;
149 void deleteStatus( int RequestId ) ;
151 MPI_Request *MPIRequest( int RequestId ) ;
152 MPI_Status *MPIStatus( int RequestId ) ;
153 void setMPICompleted( int RequestId , bool completed ) ;
154 void setMPIOutCount( int RequestId , int outcount ) ;
155 void clearMPIStatus( int RequestId ) ;
157 _MessageIdent methodId( MPI_Datatype datatype ) const ;
158 MPI_Datatype datatype( _MessageIdent aMethodIdent ) const ;
160 const CommInterface &_comm_interface ;
161 const MPI_Comm* _intra_communicator ;
162 MPIProcessorGroup * _processor_group ;
163 int _processor_group_size ;
169 int * _send_request ;
170 int * _recv_request ;
171 std::vector< std::list< int > > _send_requests ;
172 std::vector< std::list< int > > _recv_requests ;
175 int * _send_MPI_tag ;
176 int * _recv_MPI_Tag ;
177 MPI_Datatype _MPI_TIME ;
178 static const int MODULO_TAG=10;
179 std::map< int , RequestStruct * > _map_of_request_struct ;
183 inline void MPIAccess::trace( bool trace )
188 // Delete the structure Request corresponding to RequestId identifier after
189 // the deletion of the structures MPI_Request * and MPI_Status *
190 // remove it from _MapOfRequestStruct (erase)
191 inline void MPIAccess::deleteRequest( int RequestId )
193 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
194 if ( aRequestStruct )
197 std::cout << "MPIAccess::DeleteRequest" << _my_rank << "( " << RequestId << " ) "
198 << aRequestStruct << " MPIRequest " << aRequestStruct->MPIRequest
199 << " MPIIsRecv " << aRequestStruct->MPIIsRecv << std::endl ;
200 if ( _map_of_request_struct[RequestId]->MPIRequest != MPI_REQUEST_NULL )
201 requestFree( &_map_of_request_struct[RequestId]->MPIRequest ) ;
202 deleteSendRecvRequest( RequestId ) ;
203 deleteStatus( RequestId ) ;
204 _map_of_request_struct.erase( RequestId ) ;
205 delete aRequestStruct ;
210 std::cout << "MPIAccess::DeleteRequest" << _my_rank << "( " << RequestId
211 << " ) Request not found" << std::endl ;
215 // Delete all requests of the array ArrayOfSendRequests
216 inline void MPIAccess::deleteRequests(int size , int *ArrayOfSendRequests )
218 for (int i = 0 ; i < size ; i++ )
219 deleteRequest( ArrayOfSendRequests[i] ) ;
222 // Returns the last MPITag of the destination rank destrank
223 inline int MPIAccess::sendMPITag(int destrank)
225 return _send_MPI_tag[destrank] ;
228 // Returns the last MPITag of the source rank sourcerank
229 inline int MPIAccess::recvMPITag(int sourcerank)
231 return _recv_MPI_Tag[sourcerank] ;
234 // Returns the number of all SendRequestIds matching a destination rank. It may be
235 // used to allocate ArrayOfSendRequests for the call to SendRequestIds
236 inline int MPIAccess::sendRequestIdsSize(int destrank)
238 return _send_requests[destrank].size() ;
241 // Returns the number of all RecvRequestIds matching a source rank. It may be
242 // used to allocate ArrayOfRecvRequests for the call to RecvRequestIds
243 inline int MPIAccess::recvRequestIdsSize(int sourcerank)
245 return _recv_requests[sourcerank].size() ;
248 // Returns the MPI_Datatype (registered in MPI in the constructor with
249 // MPI_Type_struct and MPI_Type_commit) for TimeMessages
250 inline MPI_Datatype MPIAccess::timeType() const
255 // Returns true if the tag MPITag corresponds to a TimeMessage
256 inline bool MPIAccess::isTimeMessage( int MPITag ) const
258 return ((MPITag%MODULO_TAG) == _message_time) ;
261 // Returns the MPI size of a TimeMessage
262 inline MPI_Aint MPIAccess::timeExtent() const
265 MPI_Type_extent( _MPI_TIME , &extent ) ;
269 // Returns the MPI size of a MPI_INT
270 inline MPI_Aint MPIAccess::intExtent() const
273 MPI_Type_extent( MPI_INT , &extent ) ;
277 // Returns the MPI size of a MPI_DOUBLE
278 inline MPI_Aint MPIAccess::doubleExtent() const
281 MPI_Type_extent( MPI_DOUBLE , &extent ) ;
285 // Returns the MPI size of the MPI_Datatype datatype
286 inline MPI_Aint MPIAccess::extent( MPI_Datatype datatype ) const
288 if ( datatype == _MPI_TIME )
289 return timeExtent() ;
290 if ( datatype == MPI_INT )
292 if ( datatype == MPI_DOUBLE )
293 return doubleExtent() ;
297 // Returns the MPITag of the request corresponding to RequestId identifier
298 inline int MPIAccess::MPITag( int RequestId )
300 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
301 if ( aRequestStruct )
302 return aRequestStruct->MPITag ;
306 // Returns the MPITarget of the request corresponding to RequestId identifier
307 inline int MPIAccess::MPITarget( int RequestId )
309 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
310 if ( aRequestStruct )
311 return aRequestStruct->MPITarget ;
315 // Returns true if the request corresponding to RequestId identifier was [I]Recv
316 inline bool MPIAccess::MPIIsRecv( int RequestId )
318 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
319 if ( aRequestStruct )
320 return aRequestStruct->MPIIsRecv ;
324 // Returns true if the request corresponding to RequestId identifier was asynchronous
325 inline bool MPIAccess::MPIAsynchronous( int RequestId )
327 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
328 if ( aRequestStruct )
329 return aRequestStruct->MPIAsynchronous ;
333 // Returns true if the request corresponding to RequestId identifier was completed
334 inline bool MPIAccess::MPICompleted( int RequestId )
336 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
337 if ( aRequestStruct )
338 return aRequestStruct->MPICompleted;
342 // Returns the MPI_datatype of the request corresponding to RequestId identifier
343 inline MPI_Datatype MPIAccess::MPIDatatype( int RequestId )
345 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
346 if ( aRequestStruct )
347 return aRequestStruct->MPIDatatype;
348 return (MPI_Datatype ) NULL ;
351 // Returns the size of the receiving message of the request corresponding to
352 // RequestId identifier
353 inline int MPIAccess::MPIOutCount( int RequestId )
355 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
356 if ( aRequestStruct )
357 return aRequestStruct->MPIOutCount;
361 // Increments the previous tag value (cyclically)
362 // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
363 inline int MPIAccess::incrTag( int prevtag )
366 if ( (prevtag % MODULO_TAG) == _message_time )
367 tag = ((prevtag/MODULO_TAG)*MODULO_TAG);
369 tag = ((prevtag/MODULO_TAG + 1)*MODULO_TAG);
370 if ( tag > _max_MPI_tag )
371 tag = _base_MPI_tag ;
375 // Returns the MPITag with the method-type field
376 // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
377 inline int MPIAccess::valTag( int tag, int method )
379 return ((tag/MODULO_TAG)*MODULO_TAG) + method;
382 // Remove a Request identifier from the list _RecvRequests/_SendRequests for
383 // the corresponding target.
384 inline void MPIAccess::deleteSendRecvRequest( int RequestId )
387 std::cout << "MPIAccess::DeleteSendRecvRequest" << _my_rank
388 << "( " << RequestId << " ) " << std::endl ;
389 if ( MPIIsRecv( RequestId ) )
390 _recv_requests[ MPITarget( RequestId ) ].remove( RequestId );
392 _send_requests[ MPITarget( RequestId ) ].remove( RequestId );
395 // Delete the MPI structure MPI_status * of a ReaquestId
396 inline void MPIAccess::deleteStatus( int RequestId )
398 if ( _map_of_request_struct[RequestId]->MPIStatus != NULL )
400 delete _map_of_request_struct[RequestId]->MPIStatus ;
401 clearMPIStatus( RequestId ) ;
405 // Returns the MPI structure MPI_request * of a RequestId
406 inline MPI_Request * MPIAccess::MPIRequest( int RequestId )
408 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
409 if ( aRequestStruct )
410 return &aRequestStruct->MPIRequest;
411 return &mpirequestnull ;
414 // Returns the MPI structure MPI_status * of a RequestId
415 inline MPI_Status * MPIAccess::MPIStatus( int RequestId )
417 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ];
418 if ( aRequestStruct )
419 return aRequestStruct->MPIStatus;
423 // Set the MPICompleted field of the structure Request corresponding to RequestId
424 // identifier with the value completed
425 inline void MPIAccess::setMPICompleted( int RequestId , bool completed )
427 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
428 if ( aRequestStruct )
429 aRequestStruct->MPICompleted = completed;
432 // Set the MPIOutCount field of the structure Request corresponding to RequestId
433 // identifier with the value outcount
434 inline void MPIAccess::setMPIOutCount( int RequestId , int outcount )
436 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
437 if ( aRequestStruct )
438 aRequestStruct->MPIOutCount = outcount;
441 // Nullify the MPIStatusfield of the structure Request corresponding to RequestId
443 inline void MPIAccess::clearMPIStatus( int RequestId )
445 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
446 if ( aRequestStruct )
447 aRequestStruct->MPIStatus = NULL ;
450 // Returns the _MessageIdent enum value corresponding to the MPI_Datatype datatype
451 // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
452 inline _MessageIdent MPIAccess::methodId( MPI_Datatype datatype ) const
454 _MessageIdent aMethodIdent ;
455 if ( datatype == _MPI_TIME )
456 aMethodIdent = _message_time;
457 else if ( datatype == MPI_INT )
458 aMethodIdent = _message_int ;
459 else if ( datatype == MPI_DOUBLE )
460 aMethodIdent = _message_double ;
462 aMethodIdent = _message_unknown ;
463 return aMethodIdent ;
466 // Returns the MPI_Datatype corresponding to the _MessageIdent enum aMethodIdent
467 inline MPI_Datatype MPIAccess::datatype( _MessageIdent aMethodIdent ) const
469 MPI_Datatype aDataType ;
470 switch( aMethodIdent )
473 aDataType = _MPI_TIME ;
476 aDataType = MPI_INT ;
478 case _message_double :
479 aDataType = MPI_DOUBLE ;
482 aDataType = (MPI_Datatype) -1 ;
488 std::ostream & operator<< (std::ostream &,const _MessageIdent &);
490 std::ostream & operator<< (std::ostream &,const TimeMessage &);