Debugs for CheckFinalRecv

diff --git a/src/ParaMEDMEM/MPI_AccessDEC.cxx b/src/ParaMEDMEM/MPI_AccessDEC.cxx
index 83bdf8da67224fa0bf3dc1993d91578baca45f6e..f38b6a4eed11d19de5ab43ab216ee7cd626456bd 100644 (file)
--- a/src/ParaMEDMEM/MPI_AccessDEC.cxx
+++ b/src/ParaMEDMEM/MPI_AccessDEC.cxx
@@ -41,6 +41,16 @@ MPI_AccessDEC::MPI_AccessDEC( const ProcessorGroup& local_group,
   _Asynchronous = Asynchronous ;
   _TimeMessages = new vector< vector< TimeMessage > > ;
   _TimeMessages->resize( _GroupSize ) ;
+  _OutOfTime = new vector< bool > ;
+  _OutOfTime->resize( _GroupSize ) ;
+  _DataMessagesRecvCount = new vector< int > ;
+  _DataMessagesRecvCount->resize( _GroupSize ) ;
+  for ( i = 0 ; i < _GroupSize ; i++ ) {
+     (*_OutOfTime)[i] = false ;
+     (*_DataMessagesRecvCount)[i] = 0 ;
+  }
+  _DataMessagesType = new vector< MPI_Datatype > ;
+  _DataMessagesType->resize( _GroupSize ) ;
   _DataMessages = new vector< vector< void * > > ;
   _DataMessages->resize( _GroupSize ) ;
   _TimeInterpolator = NULL ;
@@ -57,6 +67,12 @@ MPI_AccessDEC::~MPI_AccessDEC() {
     delete _TimeInterpolator ;
   if ( _TimeMessages )
     delete _TimeMessages ;
+  if ( _OutOfTime )
+    delete _OutOfTime ;
+  if ( _DataMessagesRecvCount )
+    delete _DataMessagesRecvCount ;
+  if ( _DataMessagesType )
+    delete _DataMessagesType ;
   if ( _DataMessages )
     delete _DataMessages ;
   if ( _MapOfSendBuffers )
@@ -263,7 +279,7 @@ int MPI_AccessDEC::AllToAll( void* sendbuf, int sendcount, MPI_Datatype sendtype
   int SendRequestId ;
   int RecvRequestId ;
   SendBuffStruct * aSendDataStruct = NULL ;
-  cout << "AllToAll" << _MyRank << " sendbuf " << sendbuf << " recvbuf " << recvbuf << endl ;
+//  cout << "AllToAll" << _MyRank << " sendbuf " << sendbuf << " recvbuf " << recvbuf << endl ;
   if ( _Asynchronous && sendbuf ) {
     aSendDataStruct = new SendBuffStruct ;
     aSendDataStruct->SendBuffer = sendbuf ;
@@ -282,7 +298,7 @@ int MPI_AccessDEC::AllToAll( void* sendbuf, int sendcount, MPI_Datatype sendtype
      recvoffset += recvcount ;
   }
   if ( !_Asynchronous && sendbuf ) {
-    cout << "AllToAll" << _MyRank << " free of sendbuf " << sendbuf << endl ;
+//    cout << "AllToAll" << _MyRank << " free of sendbuf " << sendbuf << endl ;
 //    free( sendbuf ) ;
     if ( sendtype == MPI_INT ) {
       delete [] (int *) sendbuf ;
@@ -312,8 +328,8 @@ int MPI_AccessDEC::AllToAllv( void* sendbuf, int* sendcounts, int* sdispls,
   int SendRequestId ;
   int RecvRequestId ;
   SendBuffStruct * aSendDataStruct = NULL ;
-  cout << "AllToAllv" << _MyRank << " sendbuf " << sendbuf << " recvbuf " << recvbuf
-       << endl ;
+//  cout << "AllToAllv" << _MyRank << " sendbuf " << sendbuf << " recvbuf " << recvbuf
+//       << endl ;
   if ( _Asynchronous && sendbuf ) {
     aSendDataStruct = new SendBuffStruct ;
     aSendDataStruct->SendBuffer = sendbuf ;
@@ -332,7 +348,7 @@ int MPI_AccessDEC::AllToAllv( void* sendbuf, int* sendcounts, int* sdispls,
      }
   }
   if ( !_Asynchronous && sendbuf ) {
-    cout << "AllToAllv" << _MyRank << " free of sendbuf " << sendbuf << endl ;
+//    cout << "AllToAllv" << _MyRank << " free of sendbuf " << sendbuf << endl ;
 //    free( sendbuf ) ;
     if ( sendtype == MPI_INT ) {
       delete [] (int *) sendbuf ;
@@ -371,8 +387,8 @@ int MPI_AccessDEC::AllToAllTime( void* sendbuf, int sendcount , MPI_Datatype sen
 //DoSend : Time + SendBuff
   SendBuffStruct * aSendTimeStruct = NULL ;
   SendBuffStruct * aSendDataStruct = NULL ;
-  cout << "AllToAllTime" << _MyRank << " sendbuf " << sendbuf << " recvbuf " << recvbuf
-       << endl ;
+//  cout << "AllToAllTime" << _MyRank << " sendbuf " << sendbuf << " recvbuf " << recvbuf
+//       << endl ;
   if ( sendbuf && sendcount ) {
     TimeMessage * aSendTimeMessage = new TimeMessage ;
     if ( _Asynchronous ) {
@@ -407,8 +423,8 @@ int MPI_AccessDEC::AllToAllTime( void* sendbuf, int sendcount , MPI_Datatype sen
        sendoffset += sendcount ;
     }
     if ( !_Asynchronous ) {
-      cout << "SynchronousAllToAllTime" << _MyRank << " free of SendTimeMessage & sendbuf "
-           << sendbuf << endl ;
+//      cout << "SynchronousAllToAllTime" << _MyRank << " free of SendTimeMessage & sendbuf "
+//           << sendbuf << endl ;
       delete aSendTimeMessage ;
 //      free( sendbuf ) ;
       if ( sendtype == MPI_INT ) {
@@ -426,7 +442,7 @@ int MPI_AccessDEC::AllToAllTime( void* sendbuf, int sendcount , MPI_Datatype sen
        int recvsize = recvcount*_MPIAccess->Extent( recvtype ) ;
 //       bool OutOfTime ;
 //       CheckTime( recvcount , recvsize , recvtype , target , OutOfTime) ;
-       CheckTime( recvcount , recvsize , recvtype , target ) ;
+       CheckTime( recvcount , recvtype , target , false ) ;
        //cout << "AllToAllTime" << _MyRank << " memcpy to recvbuf " << recvbuf
        //     << "+target" << target << "*" << recvsize << " bytes" << endl ;
 //===========================================================================
@@ -434,7 +450,7 @@ int MPI_AccessDEC::AllToAllTime( void* sendbuf, int sendcount , MPI_Datatype sen
 //===========================================================================
        if ( _TimeInterpolator && (*_TimeMessages)[target][0].time != -1 ) {
 //       if ( _TimeInterpolator && !OutOfTime ) {
-           if ( _OutOfTime ) {
+           if ( (*_OutOfTime)[target] ) {
              cout << " =====================================================" << endl
                   << "Recv" << _MyRank << " <-- target " << target << " t0 "
                   << (*_TimeMessages)[target][0].time << " < t1 "
@@ -488,8 +504,8 @@ int MPI_AccessDEC::AllToAllvTime( void* sendbuf, int* sendcounts, int* sdispls,
 //DoSend : Time + SendBuff
   SendBuffStruct * aSendTimeStruct = NULL ;
   SendBuffStruct * aSendDataStruct = NULL ;
-  cout << "AllToAllvTime" << _MyRank << " sendbuf " << sendbuf << " recvbuf " << recvbuf
-       << endl ;
+//  cout << "AllToAllvTime" << _MyRank << " sendbuf " << sendbuf << " recvbuf " << recvbuf
+//       << endl ;
   if ( sendbuf ) {
     TimeMessage * aSendTimeMessage = new TimeMessage ;
     if ( _Asynchronous ) {
@@ -571,7 +587,7 @@ int MPI_AccessDEC::AllToAllvTime( void* sendbuf, int* sendcounts, int* sdispls,
          int recvsize = recvcounts[target]*_MPIAccess->Extent( recvtype ) ;
 //         bool OutOfTime ;
 //         CheckTime( recvcounts[target] , recvsize , recvtype , target , OutOfTime ) ;
-         CheckTime( recvcounts[target] , recvsize , recvtype , target ) ;
+         CheckTime( recvcounts[target] , recvtype , target , false ) ;
          //cout << "AllToAllvTime" << _MyRank << " memcpy to recvbuf " << recvbuf
          //     << "+target" << target << "*" << recvsize << " bytes" << endl ;
 //===========================================================================
@@ -579,7 +595,7 @@ int MPI_AccessDEC::AllToAllvTime( void* sendbuf, int* sendcounts, int* sdispls,
 //===========================================================================
          if ( _TimeInterpolator && (*_TimeMessages)[target][0].time != -1 ) {
 //         if ( _TimeInterpolator && !OutOfTime ) {
-             if ( _OutOfTime ) {
+             if ( (*_OutOfTime)[target] ) {
                cout << " =====================================================" << endl
                     << "Recv" << _MyRank << " <-- target " << target << " t0 "
                     << (*_TimeMessages)[target][0].time << " < t1 "
@@ -630,12 +646,12 @@ int MPI_AccessDEC::AllToAllvTime( void* sendbuf, int* sendcounts, int* sdispls,
   return sts ;
 }
 
-int MPI_AccessDEC::CheckTime( int recvcount , int recvsize , MPI_Datatype recvtype ,
-                              int target ) {
+int MPI_AccessDEC::CheckTime( int recvcount , MPI_Datatype recvtype , int target ,
+                              bool UntilEnd ) {
 //                              int target , bool &OutOfTime ) {
   //cout << "CheckTime" << _MyRank << " time " << _t << " deltatime " << _dt << endl ;
 
-  int sts ;
+  int sts = MPI_SUCCESS ;
   int RecvTimeRequestId ;
   int RecvDataRequestId ;
 //Pour l'instant on cherche _TimeMessages[target][0] < _t <= _TimeMessages[target][1]
@@ -643,8 +659,9 @@ int MPI_AccessDEC::CheckTime( int recvcount , int recvsize , MPI_Datatype recvty
 //TODO : it is assumed actually that we have only 1 timestep before and after
 //       instead of _nStepBefore and _nStepAfter ...
 //===========================================================================
-  _DataMessagesType = recvtype ;
-  _OutOfTime = false ;
+  (*_DataMessagesRecvCount)[target] = recvcount ;
+  (*_DataMessagesType)[target] = recvtype ;
+//  (*_OutOfTime)[target] = false ;
 //Actually we need 1 timestep before and after
 //  if ( _steptime <= 1 ) {
   if ( (*_TimeMessages)[target][1].time == -1 ) {
@@ -661,11 +678,9 @@ int MPI_AccessDEC::CheckTime( int recvcount , int recvsize , MPI_Datatype recvty
     }
     sts = Recv( (*_DataMessages)[target][1] , recvcount , recvtype , target ,
                 RecvDataRequestId ) ;
-//    _OutOfTime = true ;
   }
-//  if ( _steptime > 0 ) {
   else {
-    while ( _t > (*_TimeMessages)[target][1].time &&
+    while ( ( _t > (*_TimeMessages)[target][1].time || UntilEnd ) &&
             (*_TimeMessages)[target][1].deltatime != 0 ) {
          //cout << "CheckTime" << _MyRank << " TimeMessage target " << target << " _t "
          //     << _t << " > " << (*_TimeMessages)[target][1].time << " et "
@@ -674,7 +689,6 @@ int MPI_AccessDEC::CheckTime( int recvcount , int recvsize , MPI_Datatype recvty
          (*_TimeMessages)[target][0] = (*_TimeMessages)[target][1] ;
          sts = Recv( &(*_TimeMessages)[target][1] , 1 , _MPIAccess->TimeType() ,
                      target , RecvTimeRequestId ) ;
-//         free( (*_DataMessages)[target][0] ) ;
          if ( recvtype == MPI_INT ) {
            delete [] (int *) (*_DataMessages)[target][0] ;
          }
@@ -682,7 +696,6 @@ int MPI_AccessDEC::CheckTime( int recvcount , int recvsize , MPI_Datatype recvty
            delete [] (double *) (*_DataMessages)[target][0] ;
          }
          (*_DataMessages)[target][0] = (*_DataMessages)[target][1] ;
-//         (*_DataMessages)[target][1] = malloc( recvsize ) ;
          if ( recvtype == MPI_INT ) {
            (*_DataMessages)[target][1] = new int[recvcount] ;
          }
@@ -698,7 +711,7 @@ int MPI_AccessDEC::CheckTime( int recvcount , int recvsize , MPI_Datatype recvty
 //Ok
     }
     else {
-      _OutOfTime = true ;
+      (*_OutOfTime)[target] = true ;
     }
   }
 
@@ -712,17 +725,27 @@ int MPI_AccessDEC::CheckTime( int recvcount , int recvsize , MPI_Datatype recvty
   //cout << endl ;
   return sts ;
 }
-
 int MPI_AccessDEC::CheckSent(bool WithWait) {
   int sts = MPI_SUCCESS ;
   int flag = WithWait ;
   int size = _MPIAccess->SendRequestIdsSize() ;
   int * ArrayOfSendRequests = new int[ size ] ;
   int nSendRequest = _MPIAccess->SendRequestIds( size , ArrayOfSendRequests ) ;
-  //cout << "CheckSent" << _MyRank << " nSendRequest " << nSendRequest << " :" << endl ;
+  bool SendTrace = false ;
+//  if ( nSendRequest > 2*_GroupSize || (_t > 1800 && _MyRank== 0) ) {
+//    SendTrace = true ;
+//    _MPIAccess->Trace() ;
+//    if ( nSendRequest > 3*_GroupSize ) {
+//      WithWait = true ;
+//    }
+//  }
+//  cout << "CheckSent" << _MyRank << " nSendRequest " << nSendRequest << " SendTrace "
+//       << SendTrace << " WithWait " << WithWait << " :" << endl ;
   int i ;
   for ( i = 0 ; i < nSendRequest ; i++ ) {
      if ( WithWait ) {
+       cout << "CheckSent" << _MyRank << " " << i << "./" << nSendRequest
+            << " SendRequestId " << ArrayOfSendRequests[i] << " Wait :" << endl ;
        sts = _MPIAccess->Wait( ArrayOfSendRequests[i] ) ;
      }
      else {
@@ -730,38 +753,42 @@ int MPI_AccessDEC::CheckSent(bool WithWait) {
      }
      if ( flag ) {
        _MPIAccess->DeleteRequest( ArrayOfSendRequests[i] ) ;
-//       cout << "CheckSent" << _MyRank << " " << i << "./" << nSendRequest
-//            << " SendRequestId " << ArrayOfSendRequests[i]
-//            << " flag " << flag << " SendBuffStruct/SendBuffer "
-//            << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ] << "/"
-//            << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer
-//            << " Counter " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter
-//            << " TimeBuf " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->TimeBuf
-//            << endl ;
+       if ( SendTrace ) {
+         cout << "CheckSent" << _MyRank << " " << i << "./" << nSendRequest
+              << " SendRequestId " << ArrayOfSendRequests[i]
+              << " flag " << flag
+              << " Counter " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter
+              << " DataType " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->DataType
+              << endl ;
+       }
        (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter -= 1 ;
-//       if ( (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->TimeBuf ) {
-//         cout << "CheckTimeSent" << _MyRank << " Request " ;
-//       }
-//       else {
-//         cout << "CheckDataSent" << _MyRank << " Request " ;
-//       }
-//       cout << ArrayOfSendRequests[i]
-//            << " _MapOfSendBuffers->SendBuffer "
-//            << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer
-//            << " Counter " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter
-//            << endl ;
+       if ( SendTrace ) {
+         if ( (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->DataType == 
+              _MPIAccess->TimeType() ) {
+           cout << "CheckTimeSent" << _MyRank << " Request " ;
+         }
+         else {
+           cout << "CheckDataSent" << _MyRank << " Request " ;
+         }
+         cout << ArrayOfSendRequests[i]
+              << " _MapOfSendBuffers->SendBuffer "
+              << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer
+              << " Counter " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter
+              << endl ;
+       }
        if ( (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter  == 0 ) {
-//         cout << "CheckSent" << _MyRank << " SendRequestId " << ArrayOfSendRequests[i]
-//              << " Counter " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter
-//              << " flag " << flag << " SendBuffer "
-//              << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer
-//              << " deleted. Erase in _MapOfSendBuffers :" << endl ;
+         if ( SendTrace ) {
+           cout << "CheckSent" << _MyRank << " SendRequestId " << ArrayOfSendRequests[i]
+                << " Counter " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter
+                << " flag " << flag << " SendBuffer "
+                << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer
+                << " deleted. Erase in _MapOfSendBuffers :" << endl ;
+         }
          if ( (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->DataType ==
               _MPIAccess->TimeType() ) {
            delete (TimeMessage * ) (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer ;
          }
          else {
-//           free( (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer ) ;
            if ( (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->DataType == MPI_INT ) {
              delete [] (int *) (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer ;
            }
@@ -771,40 +798,57 @@ int MPI_AccessDEC::CheckSent(bool WithWait) {
          }
          delete (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ] ;
        }
-//       cout << "CheckSent" << _MyRank << " Erase in _MapOfSendBuffers SendRequestId "
-//            << ArrayOfSendRequests[i] << endl ;
+       if ( SendTrace ) {
+         cout << "CheckSent" << _MyRank << " Erase in _MapOfSendBuffers SendRequestId "
+              << ArrayOfSendRequests[i] << endl ;
+       }
        (*_MapOfSendBuffers).erase( ArrayOfSendRequests[i] ) ;
      }
+     else if ( SendTrace ) {
+       cout << "CheckSent" << _MyRank << " " << i << "./" << nSendRequest
+            << " SendRequestId " << ArrayOfSendRequests[i]
+            << " flag " << flag
+            << " Counter " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter
+            << " DataType " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->DataType
+            << endl ;
+     }
+  }
+  if ( SendTrace ) {
+    _MPIAccess->Check() ;
   }
   delete [] ArrayOfSendRequests ;
   return sts ;
 }
 
 int MPI_AccessDEC::CheckFinalRecv() {
+  int sts = MPI_SUCCESS ;
   if ( _TimeInterpolator ) {
     int target ;
     for ( target = 0 ; target < _GroupSize ; target++ ) {
        if ( (*_DataMessages)[target][0] != NULL ) {
-         if ( _DataMessagesType == MPI_INT ) {
+         sts = CheckTime( (*_DataMessagesRecvCount)[target] , (*_DataMessagesType)[target] ,
+                          target , true ) ;
+         if ( (*_DataMessagesType)[target] == MPI_INT ) {
            delete [] (int *) (*_DataMessages)[target][0] ;
          }
          else {
            delete [] (double *) (*_DataMessages)[target][0] ;
          }
          (*_DataMessages)[target][0] = NULL ;
-       }
-       if ( (*_DataMessages)[target][1] != NULL ) {
-         if ( _DataMessagesType == MPI_INT ) {
-           delete [] (int *) (*_DataMessages)[target][1] ;
-         }
-         else {
-           delete [] (double *) (*_DataMessages)[target][1] ;
+         if ( (*_DataMessages)[target][1] != NULL ) {
+           if ( (*_DataMessagesType)[target] == MPI_INT ) {
+             delete [] (int *) (*_DataMessages)[target][1] ;
+           }
+           else {
+             delete [] (double *) (*_DataMessages)[target][1] ;
+           }
+           (*_DataMessages)[target][1] = NULL ;
          }
-         (*_DataMessages)[target][1] = NULL ;
        }
      }
   }
-  return _MPIAccess->CancelAll() ;
+//  return _MPIAccess->CancelAll() ;
+  return sts ;
 }
 
 ostream & operator<< (ostream & f ,const TimeInterpolationMethod & interpolationmethod ) {

diff --git a/src/ParaMEDMEM/MPI_AccessDEC.hxx b/src/ParaMEDMEM/MPI_AccessDEC.hxx
index 6d27f9dd0af79d61626199cd84b710580402fab0..ebf20dd1e722e5a84c3c88f71c926e1302889e8f 100644 (file)
--- a/src/ParaMEDMEM/MPI_AccessDEC.hxx
+++ b/src/ParaMEDMEM/MPI_AccessDEC.hxx
@@ -30,7 +30,7 @@ namespace ParaMEDMEM {
 
       void SetTime( double t ) ;
       void SetTime( double t , double dt ) ;
-      bool OutOfTime() ;
+      bool OutOfTime( int target ) ;
 
       int Send( void* sendbuf, int sendcount , MPI_Datatype sendtype , int target ) ;
       int Recv( void* recvbuf, int recvcount , MPI_Datatype recvtype , int target ) ;
@@ -52,7 +52,7 @@ namespace ParaMEDMEM {
                          MPI_Datatype recvtype ) ;
 //      int CheckTime( int recvcount , int recvsize , MPI_Datatype recvtype , int target ,
 //                     bool &OutOfTime ) ;
-      int CheckTime( int recvcount , int recvsize , MPI_Datatype recvtype , int target ) ;
+      int CheckTime( int recvcount , MPI_Datatype recvtype , int target , bool UntilEnd ) ;
       int CheckSent(bool WithWait=false) ;
       int CheckFinalSent() {
           return CheckSent( true ) ; } ;
@@ -88,14 +88,16 @@ namespace ParaMEDMEM {
       int                    _GroupSize ;
       MPI_Access           * _MPIAccess ;
 
+// Current time and deltatime of current process
       double                 _t ;
       double                 _dt ;
-      bool                   _OutOfTime ;
 
 // TimeMessages from each target _TimeMessages[target][Step] : TimeMessage
       vector< vector< TimeMessage > > *_TimeMessages ;
 // Corresponding DataMessages from each target _DataMessages[target][~TimeStep]
-      MPI_Datatype               _DataMessagesType ;
+      vector< bool >             * _OutOfTime ;
+      vector< int >              * _DataMessagesRecvCount ;
+      vector< MPI_Datatype >     * _DataMessagesType ;
       vector< vector< void * > > *_DataMessages ;
 
       typedef struct { void * SendBuffer ;
@@ -118,8 +120,8 @@ namespace ParaMEDMEM {
          _t = t ; _dt = -1 ; } ;
   inline void MPI_AccessDEC::SetTime( double t , double dt ) {
          _t = t ; _dt = dt ; } ;
-  inline bool MPI_AccessDEC::OutOfTime() {
-         return _OutOfTime ; } ;
+  inline bool MPI_AccessDEC::OutOfTime( int target ) {
+         return (*_OutOfTime)[target] ; } ;
 
   inline int MPI_AccessDEC::Send( void* sendbuf, int sendcount , MPI_Datatype sendtype ,
                                   int target ) {