1 // Copyright (C) 2007-2024 CEA, EDF
3 // This library is free software; you can redistribute it and/or
4 // modify it under the terms of the GNU Lesser General Public
5 // License as published by the Free Software Foundation; either
6 // version 2.1 of the License, or (at your option) any later version.
8 // This library is distributed in the hope that it will be useful,
9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 // Lesser General Public License for more details.
13 // You should have received a copy of the GNU Lesser General Public
14 // License along with this library; if not, write to the Free Software
15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
20 #ifndef __MPIACCESS_HXX__
21 #define __MPIACCESS_HXX__
23 #include "CommInterface.hxx"
24 #include "ProcessorGroup.hxx"
25 #include "MPIProcessorGroup.hxx"
41 static MPI_Request mpirequestnull = MPI_REQUEST_NULL ;
42 enum _MessageIdent { _message_unknown, _message_time, _message_int, _message_double } ;
52 bool MPIAsynchronous ;
54 MPI_Datatype MPIDatatype ;
55 MPI_Request MPIRequest ;
56 MPI_Status *MPIStatus ;
60 MPIAccess(MPIProcessorGroup * ProcessorGroup, int BaseTag=0, int MaxTag=0) ;
61 virtual ~MPIAccess() ;
63 void trace( bool trace = true ) ;
65 void deleteRequest( int RequestId ) ;
66 void deleteRequests(int size , int *ArrayOfSendRequests ) ;
68 int sendMPITag(int destrank) ;
69 int recvMPITag(int sourcerank) ;
71 int sendRequestIdsSize() ;
72 int sendRequestIds(int size, int *ArrayOfSendRequests) ;
73 int recvRequestIdsSize() ;
74 int recvRequestIds(int size, int *ArrayOfRecvRequests) ;
76 int sendRequestIdsSize(int destrank) ;
77 int sendRequestIds(int destrank, int size, int *ArrayOfSendRequests) ;
78 int recvRequestIdsSize(int sourcerank) ;
79 int recvRequestIds(int sourcerank, int size, int *ArrayOfRecvRequests) ;
81 int send(void* buffer, int count, MPI_Datatype datatype, int target,
83 int ISend(void* buffer, int count, MPI_Datatype datatype, int target,
85 int recv(void* buffer, int count, MPI_Datatype datatype, int source,
86 int &RequestId, int *OutCount=NULL) ;
87 int IRecv(void* buffer, int count, MPI_Datatype datatype, int source,
89 int sendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest,
90 int &SendRequestId, void* recvbuf, int recvcount,
91 MPI_Datatype recvtype, int source,
92 int &RecvRequestId, int *OutCount=NULL) ;
93 int ISendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest,
94 int &SendRequestId, void* recvbuf, int recvcount,
95 MPI_Datatype recvtype, int source, int &RecvRequestId) ;
97 int wait(int RequestId) ;
98 int test(int RequestId, int &flag) ;
99 int waitAny(int count, int *array_of_RequestIds, int &RequestId) ;
100 int testAny(int count, int *array_of_RequestIds, int &RequestId, int &flag) ;
101 int waitAll(int count, int *array_of_RequestIds) ;
102 int testAll(int count, int *array_of_RequestIds, int &flag) ;
103 int waitSome(int count, int *array_of_RequestIds, int outcount,
104 int *outarray_of_RequestIds) ;
105 int testSome(int count, int *array_of_RequestIds, int outcounts,
106 int *outarray_of_RequestIds) ;
107 int probe(int FromSource, int &source, int &MPITag, MPI_Datatype &datatype,
109 int IProbe(int FromSource, int &source, int &MPITag, MPI_Datatype &datatype,
110 int &outcount, int &flag) ;
111 int cancel( int RecvRequestId, int &flag ) ;
112 int cancel( int source, int MPITag, MPI_Datatype datatype, int outcount,
116 int errorString(int errorcode, char *string, int *resultlen) const ;
117 int status(int RequestId, int &source, int &tag, int &error, int &outcount,
118 bool keepRequestStruct=false) ;
119 int requestFree( MPI_Request *request ) ;
123 MPI_Datatype timeType() const ;
124 bool isTimeMessage( int MPITag ) const ;
125 MPI_Aint timeExtent() const ;
126 MPI_Aint intExtent() const ;
127 MPI_Aint longExtent() const ;
128 MPI_Aint doubleExtent() const ;
129 MPI_Aint extent( MPI_Datatype datatype ) const ;
131 int MPITag( int RequestId ) ;
132 int MPITarget( int RequestId ) ;
133 bool MPIIsRecv( int RequestId ) ;
134 bool MPIAsynchronous( int RequestId ) ;
135 bool MPICompleted( int RequestId ) ;
136 MPI_Datatype MPIDatatype( int RequestId ) ;
137 int MPIOutCount( int RequestId ) ;
140 int newRequest( MPI_Datatype datatype, int tag , int destsourcerank ,
141 bool fromsourcerank , bool asynchronous ) ;
142 int newSendTag( MPI_Datatype datatype, int destrank , int method ,
143 bool asynchronous, int &RequestId ) ;
144 int newRecvTag( MPI_Datatype datatype, int sourcerank , int method ,
145 bool asynchronous, int &RequestId ) ;
146 int incrTag( int prevtag ) ;
147 int valTag( int tag, int method ) ;
149 void deleteSendRecvRequest( int RequestId ) ;
151 void deleteStatus( int RequestId ) ;
153 MPI_Request *MPIRequest( int RequestId ) ;
154 MPI_Status *MPIStatus( int RequestId ) ;
155 void setMPICompleted( int RequestId , bool completed ) ;
156 void setMPIOutCount( int RequestId , int outcount ) ;
157 void clearMPIStatus( int RequestId ) ;
159 _MessageIdent methodId( MPI_Datatype datatype ) const ;
160 MPI_Datatype datatype( _MessageIdent aMethodIdent ) const ;
162 const CommInterface &_comm_interface ;
163 const MPI_Comm* _intra_communicator ;
164 MPIProcessorGroup * _processor_group ;
165 int _processor_group_size ;
171 int * _send_request ;
172 int * _recv_request ;
173 std::vector< std::list< int > > _send_requests ;
174 std::vector< std::list< int > > _recv_requests ;
177 int * _send_MPI_tag ;
178 int * _recv_MPI_Tag ;
179 MPI_Datatype _MPI_TIME ;
180 static const int MODULO_TAG=10;
181 std::map< int , RequestStruct * > _map_of_request_struct ;
185 inline void MPIAccess::trace( bool atrace )
190 // Delete the structure Request corresponding to RequestId identifier after
191 // the deletion of the structures MPI_Request * and MPI_Status *
192 // remove it from _MapOfRequestStruct (erase)
193 inline void MPIAccess::deleteRequest( int RequestId )
195 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
196 if ( aRequestStruct )
199 std::cout << "MPIAccess::DeleteRequest" << _my_rank << "( " << RequestId << " ) "
200 << aRequestStruct << " MPIRequest " << aRequestStruct->MPIRequest
201 << " MPIIsRecv " << aRequestStruct->MPIIsRecv << std::endl ;
202 if ( _map_of_request_struct[RequestId]->MPIRequest != MPI_REQUEST_NULL )
203 requestFree( &_map_of_request_struct[RequestId]->MPIRequest ) ;
204 deleteSendRecvRequest( RequestId ) ;
205 deleteStatus( RequestId ) ;
206 _map_of_request_struct.erase( RequestId ) ;
207 delete aRequestStruct ;
212 std::cout << "MPIAccess::DeleteRequest" << _my_rank << "( " << RequestId
213 << " ) Request not found" << std::endl ;
217 // Delete all requests of the array ArrayOfSendRequests
218 inline void MPIAccess::deleteRequests(int size , int *ArrayOfSendRequests )
220 for (int i = 0 ; i < size ; i++ )
221 deleteRequest( ArrayOfSendRequests[i] ) ;
224 // Returns the last MPITag of the destination rank destrank
225 inline int MPIAccess::sendMPITag(int destrank)
227 return _send_MPI_tag[destrank] ;
230 // Returns the last MPITag of the source rank sourcerank
231 inline int MPIAccess::recvMPITag(int sourcerank)
233 return _recv_MPI_Tag[sourcerank] ;
236 // Returns the number of all SendRequestIds matching a destination rank. It may be
237 // used to allocate ArrayOfSendRequests for the call to SendRequestIds
238 inline int MPIAccess::sendRequestIdsSize(int destrank)
240 return (int)_send_requests[destrank].size() ;
243 // Returns the number of all RecvRequestIds matching a source rank. It may be
244 // used to allocate ArrayOfRecvRequests for the call to RecvRequestIds
245 inline int MPIAccess::recvRequestIdsSize(int sourcerank)
247 return (int)_recv_requests[sourcerank].size() ;
250 // Returns the MPI_Datatype (registered in MPI in the constructor with
251 // MPI_Type_struct and MPI_Type_commit) for TimeMessages
252 inline MPI_Datatype MPIAccess::timeType() const
257 // Returns true if the tag MPITag corresponds to a TimeMessage
258 inline bool MPIAccess::isTimeMessage( int aMPITag ) const
260 return ((aMPITag%MODULO_TAG) == _message_time) ;
263 // Returns the MPI size of the MPI_Datatype datatype
264 inline MPI_Aint MPIAccess::extent( MPI_Datatype adatatype ) const
266 if ( adatatype == _MPI_TIME )
267 return timeExtent() ;
268 if ( adatatype == MPI_INT )
270 if ( adatatype == MPI_LONG )
271 return longExtent() ;
272 if ( adatatype == MPI_DOUBLE )
273 return doubleExtent() ;
277 // Returns the MPITag of the request corresponding to RequestId identifier
278 inline int MPIAccess::MPITag( int RequestId )
280 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
281 if ( aRequestStruct )
282 return aRequestStruct->MPITag ;
286 // Returns the MPITarget of the request corresponding to RequestId identifier
287 inline int MPIAccess::MPITarget( int RequestId )
289 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
290 if ( aRequestStruct )
291 return aRequestStruct->MPITarget ;
295 // Returns true if the request corresponding to RequestId identifier was [I]Recv
296 inline bool MPIAccess::MPIIsRecv( int RequestId )
298 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
299 if ( aRequestStruct )
300 return aRequestStruct->MPIIsRecv ;
304 // Returns true if the request corresponding to RequestId identifier was asynchronous
305 inline bool MPIAccess::MPIAsynchronous( int RequestId )
307 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
308 if ( aRequestStruct )
309 return aRequestStruct->MPIAsynchronous ;
313 // Returns true if the request corresponding to RequestId identifier was completed
314 inline bool MPIAccess::MPICompleted( int RequestId )
316 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
317 if ( aRequestStruct )
318 return aRequestStruct->MPICompleted;
322 // Returns the MPI_datatype of the request corresponding to RequestId identifier
323 inline MPI_Datatype MPIAccess::MPIDatatype( int RequestId )
325 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
326 if ( aRequestStruct )
327 return aRequestStruct->MPIDatatype;
328 return MPI_DATATYPE_NULL;
331 // Returns the size of the receiving message of the request corresponding to
332 // RequestId identifier
333 inline int MPIAccess::MPIOutCount( int RequestId )
335 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
336 if ( aRequestStruct )
337 return aRequestStruct->MPIOutCount;
341 // Increments the previous tag value (cyclically)
342 // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
343 inline int MPIAccess::incrTag( int prevtag )
346 if ( (prevtag % MODULO_TAG) == _message_time )
347 tag = ((prevtag/MODULO_TAG)*MODULO_TAG);
349 tag = ((prevtag/MODULO_TAG + 1)*MODULO_TAG);
350 if ( tag > _max_MPI_tag )
351 tag = _base_MPI_tag ;
355 // Returns the MPITag with the method-type field
356 // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
357 inline int MPIAccess::valTag( int tag, int method )
359 return ((tag/MODULO_TAG)*MODULO_TAG) + method;
362 // Remove a Request identifier from the list _RecvRequests/_SendRequests for
363 // the corresponding target.
364 inline void MPIAccess::deleteSendRecvRequest( int RequestId )
367 std::cout << "MPIAccess::DeleteSendRecvRequest" << _my_rank
368 << "( " << RequestId << " ) " << std::endl ;
369 if ( MPIIsRecv( RequestId ) )
370 _recv_requests[ MPITarget( RequestId ) ].remove( RequestId );
372 _send_requests[ MPITarget( RequestId ) ].remove( RequestId );
375 // Delete the MPI structure MPI_status * of a ReaquestId
376 inline void MPIAccess::deleteStatus( int RequestId )
378 if ( _map_of_request_struct[RequestId]->MPIStatus != NULL )
380 delete _map_of_request_struct[RequestId]->MPIStatus ;
381 clearMPIStatus( RequestId ) ;
385 // Returns the MPI structure MPI_request * of a RequestId
386 inline MPI_Request * MPIAccess::MPIRequest( int RequestId )
388 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
389 if ( aRequestStruct )
390 return &aRequestStruct->MPIRequest;
391 return &mpirequestnull ;
394 // Returns the MPI structure MPI_status * of a RequestId
395 inline MPI_Status * MPIAccess::MPIStatus( int RequestId )
397 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ];
398 if ( aRequestStruct )
399 return aRequestStruct->MPIStatus;
403 // Set the MPICompleted field of the structure Request corresponding to RequestId
404 // identifier with the value completed
405 inline void MPIAccess::setMPICompleted( int RequestId , bool completed )
407 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
408 if ( aRequestStruct )
409 aRequestStruct->MPICompleted = completed;
412 // Set the MPIOutCount field of the structure Request corresponding to RequestId
413 // identifier with the value outcount
414 inline void MPIAccess::setMPIOutCount( int RequestId , int outcount )
416 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
417 if ( aRequestStruct )
418 aRequestStruct->MPIOutCount = outcount;
421 // Nullify the MPIStatusfield of the structure Request corresponding to RequestId
423 inline void MPIAccess::clearMPIStatus( int RequestId )
425 struct RequestStruct *aRequestStruct = _map_of_request_struct[ RequestId ] ;
426 if ( aRequestStruct )
427 aRequestStruct->MPIStatus = NULL ;
430 // Returns the _MessageIdent enum value corresponding to the MPI_Datatype datatype
431 // Look at MPIAccess::NewSendTag/NewRecvTag in MPIAccess.cxx
432 inline _MessageIdent MPIAccess::methodId( MPI_Datatype adatatype ) const
434 _MessageIdent aMethodIdent ;
435 if ( adatatype == _MPI_TIME )
436 aMethodIdent = _message_time;
437 else if ( adatatype == MPI_INT )
438 aMethodIdent = _message_int ;
439 else if ( adatatype == MPI_DOUBLE )
440 aMethodIdent = _message_double ;
442 aMethodIdent = _message_unknown ;
443 return aMethodIdent ;
446 // Returns the MPI_Datatype corresponding to the _MessageIdent enum aMethodIdent
447 inline MPI_Datatype MPIAccess::datatype( _MessageIdent aMethodIdent ) const
449 MPI_Datatype aDataType ;
450 switch( aMethodIdent )
453 aDataType = _MPI_TIME ;
456 aDataType = MPI_INT ;
458 case _message_double :
459 aDataType = MPI_DOUBLE ;
462 aDataType = (MPI_Datatype) -1 ;
468 std::ostream & operator<< (std::ostream &,const _MessageIdent &);
470 std::ostream & operator<< (std::ostream &,const TimeMessage &);