From dfdb92cbac7ada95d8df762a06279aab0a3706b8 Mon Sep 17 00:00:00 2001 From: rahuel Date: Tue, 15 Jan 2008 16:06:49 +0000 Subject: [PATCH] Asynchronisme MPI --- src/ParaMEDMEM/CommInterface.hxx | 154 +++- src/ParaMEDMEM/ElementLocator.cxx | 8 +- src/ParaMEDMEM/InterpolationMatrix.cxx | 8 + src/ParaMEDMEM/InterpolationMatrix.hxx | 6 +- src/ParaMEDMEM/IntersectionDEC.cxx | 25 +- src/ParaMEDMEM/IntersectionDEC.hxx | 9 +- src/ParaMEDMEM/LinearTimeInterpolator.cxx | 41 + src/ParaMEDMEM/LinearTimeInterpolator.hxx | 31 + src/ParaMEDMEM/MPI_Access.cxx | 828 ++++++++++++++++++ src/ParaMEDMEM/MPI_Access.hxx | 395 +++++++++ src/ParaMEDMEM/MPI_AccessDEC.cxx | 824 +++++++++++++++++ src/ParaMEDMEM/MPI_AccessDEC.hxx | 201 +++++ src/ParaMEDMEM/Makefile.in | 8 + src/ParaMEDMEM/MxN_Mapping.cxx | 176 ++-- src/ParaMEDMEM/MxN_Mapping.hxx | 11 +- src/ParaMEDMEM/Test/ParaMEDMEMTest.hxx | 19 +- .../Test/ParaMEDMEMTest_IntersectionDEC.cxx | 115 ++- src/ParaMEDMEM/TimeInterpolator.cxx | 18 + src/ParaMEDMEM/TimeInterpolator.hxx | 38 + src/ParaMEDMEM/test_IntersectionDEC.cxx | 2 +- 20 files changed, 2766 insertions(+), 151 deletions(-) create mode 100644 src/ParaMEDMEM/LinearTimeInterpolator.cxx create mode 100644 src/ParaMEDMEM/LinearTimeInterpolator.hxx create mode 100644 src/ParaMEDMEM/MPI_Access.cxx create mode 100644 src/ParaMEDMEM/MPI_Access.hxx create mode 100644 src/ParaMEDMEM/MPI_AccessDEC.cxx create mode 100644 src/ParaMEDMEM/MPI_AccessDEC.hxx create mode 100644 src/ParaMEDMEM/TimeInterpolator.cxx create mode 100644 src/ParaMEDMEM/TimeInterpolator.hxx diff --git a/src/ParaMEDMEM/CommInterface.hxx b/src/ParaMEDMEM/CommInterface.hxx index 840639935..e9f3afd0b 100644 --- a/src/ParaMEDMEM/CommInterface.hxx +++ b/src/ParaMEDMEM/CommInterface.hxx @@ -10,55 +10,115 @@ class CommInterface public: CommInterface(){} virtual ~CommInterface(){} - int commSize(MPI_Comm comm, int* size) const { return MPI_Comm_size(comm,size);} - int commRank(MPI_Comm comm, int* rank) const { return MPI_Comm_rank(comm,rank);} - int commGroup(MPI_Comm comm, MPI_Group* group) const - {return MPI_Comm_group(comm, group);} - int groupIncl(MPI_Group group, int size, int* ranks, MPI_Group* group_output) const - {return MPI_Group_incl(group, size, ranks, group_output);} - int commCreate(MPI_Comm comm, MPI_Group group, MPI_Comm* comm_output) const - {return MPI_Comm_create(comm,group,comm_output);} - int groupFree(MPI_Group* group) const {return MPI_Group_free(group);} + int worldSize() const { + int size; + MPI_Comm_size(MPI_COMM_WORLD, &size); + return size;} + int commSize(MPI_Comm comm, int* size) const { + return MPI_Comm_size(comm,size);} + int commRank(MPI_Comm comm, int* rank) const { + return MPI_Comm_rank(comm,rank);} + int commGroup(MPI_Comm comm, MPI_Group* group) const { + return MPI_Comm_group(comm, group);} + int groupIncl(MPI_Group group, int size, int* ranks, + MPI_Group* group_output) const { + return MPI_Group_incl(group, size, ranks, group_output);} + int commCreate(MPI_Comm comm, MPI_Group group, MPI_Comm* comm_output) const { + return MPI_Comm_create(comm,group,comm_output);} + int groupFree(MPI_Group* group) const { + return MPI_Group_free(group);} int commFree(MPI_Comm* comm) const {return MPI_Comm_free(comm);} - int broadcast(void* buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm)const - {return MPI_Bcast(buffer, count, datatype, root, comm);} - int send(void* buffer, int count, MPI_Datatype datatype, int target, int tag, MPI_Comm comm) const - {return MPI_Send(buffer,count, datatype, target, tag, comm);} - int recv(void* buffer, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status* status) const - {return MPI_Recv(buffer,count, datatype, source, tag, comm, status);} - int allToAllV(void* sendbuf, int* sendcounts, int* senddispls, MPI_Datatype sendtype, - void* recvbuf, int* recvcounts, int* recvdispls, MPI_Datatype recvtype, - MPI_Comm comm) const - {return MPI_Alltoallv(sendbuf, sendcounts, senddispls, sendtype, - recvbuf, recvcounts, recvdispls, recvtype, - comm);} - int allToAll(void* sendbuf, int sendcount, MPI_Datatype sendtype, - void* recvbuf, int recvcount, MPI_Datatype recvtype, - MPI_Comm comm) const - { - return MPI_Alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); - } + + int send(void* buffer, int count, MPI_Datatype datatype, int target, + int tag, MPI_Comm comm) const { + return MPI_Send(buffer,count, datatype, target, tag, comm);} + int recv(void* buffer, int count, MPI_Datatype datatype, int source, + int tag, MPI_Comm comm, MPI_Status* status) const { + return MPI_Recv(buffer,count, datatype, source, tag, comm, status);} + int sendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int sendtag, void* recvbuf, int recvcount, + MPI_Datatype recvtype, int source, int recvtag, MPI_Comm comm, + MPI_Status* status) { + return MPI_Sendrecv(sendbuf, sendcount, sendtype, dest, sendtag, + recvbuf, recvcount, recvtype, source, recvtag, + comm,status); } + + int Isend(void* buffer, int count, MPI_Datatype datatype, int target, + int tag, MPI_Comm comm, MPI_Request *request) const { + return MPI_Isend(buffer,count, datatype, target, tag, comm, request);} + int Irecv(void* buffer, int count, MPI_Datatype datatype, int source, + int tag, MPI_Comm comm, MPI_Request* request) const { + return MPI_Irecv(buffer,count, datatype, source, tag, comm, request);} + + int wait(MPI_Request *request, MPI_Status *status) const { + return MPI_Wait(request, status) ;} + int test(MPI_Request *request, int *flag, MPI_Status *status) const { + return MPI_Test(request, flag, status) ; } + int request_free(MPI_Request *request) const { + return MPI_Request_free(request) ; } + int waitany(int count, MPI_Request *array_of_requests, int *index, + MPI_Status *status) const { + return MPI_Waitany(count, array_of_requests, index, status) ; } + int testany(int count, MPI_Request *array_of_requests, int *index, + int *flag, MPI_Status *status) const { + return MPI_Testany(count, array_of_requests, index, flag, status) ; } + int waitall(int count, MPI_Request *array_of_requests, + MPI_Status *array_of_status) const { + return MPI_Waitall(count, array_of_requests, array_of_status) ;} + int testall(int count, MPI_Request *array_of_requests, int *flag, + MPI_Status *array_of_status) const { + return MPI_Testall(count, array_of_requests, flag, array_of_status) ; } + int waitsome(int incount, MPI_Request *array_of_requests,int *outcount, + int *array_of_indices, MPI_Status *array_of_status) const { + return MPI_Waitsome(incount, array_of_requests, outcount, + array_of_indices, array_of_status) ;} + int testsome(int incount, MPI_Request *array_of_requests, int *outcount, + int *array_of_indices, MPI_Status *array_of_status) const { + return MPI_Testsome(incount, array_of_requests, outcount, + array_of_indices, array_of_status) ; } + int probe(int source, int tag, MPI_Comm comm, MPI_Status *status) const { + return MPI_Probe(source, tag, comm, status) ; } + int Iprobe(int source, int tag, MPI_Comm comm, int *flag, + MPI_Status *status) const { + return MPI_Iprobe(source, tag, comm, flag, status) ; } + int cancel(MPI_Request *request) const { + return MPI_Cancel(request) ; } + int test_cancelled(MPI_Status *status, int *flag) const { + return MPI_Test_cancelled(status, flag) ; } + int barrier(MPI_Comm comm) const { + return MPI_Barrier(comm) ; } + int error_string(int errorcode, char *string, int *resultlen) const { + return MPI_Error_string(errorcode, string, resultlen) ; } + int get_count(MPI_Status *status, MPI_Datatype datatype, int *count) const { + return MPI_Get_count(status, datatype, count) ; } + + int broadcast(void* buffer, int count, MPI_Datatype datatype, int root, + MPI_Comm comm) const { + return MPI_Bcast(buffer, count, datatype, root, comm);} int allGather(void* sendbuf, int sendcount, MPI_Datatype sendtype, - void* recvbuf, int recvcount, MPI_Datatype recvtype, - MPI_Comm comm) const - {return MPI_Allgather(sendbuf,sendcount, sendtype, recvbuf, recvcount, recvtype, comm); - } - int sendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, - int dest, int sendtag, void* recvbuf, int recvcount, - MPI_Datatype recvtype, int source, int recvtag, MPI_Comm comm, - MPI_Status* status) - { - return - MPI_Sendrecv(sendbuf, sendcount, sendtype, dest, sendtag, recvbuf, - recvcount, recvtype, source, recvtag, comm,status); - } - int worldSize() const {int size; MPI_Comm_size(MPI_COMM_WORLD, &size); return size;} - int reduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, - int root, MPI_Comm comm) const - {return MPI_Reduce(sendbuf, recvbuf, count, datatype, op, root, comm);} - int allReduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, - MPI_Comm comm) const - {return MPI_Allreduce(sendbuf, recvbuf, count, datatype, op, comm);} + void* recvbuf, int recvcount, MPI_Datatype recvtype, + MPI_Comm comm) const { + return MPI_Allgather(sendbuf,sendcount, sendtype, + recvbuf, recvcount, recvtype, comm); } + int allToAll(void* sendbuf, int sendcount, MPI_Datatype sendtype, + void* recvbuf, int recvcount, MPI_Datatype recvtype, + MPI_Comm comm) const { + return MPI_Alltoall(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, comm); } + int allToAllV(void* sendbuf, int* sendcounts, int* senddispls, + MPI_Datatype sendtype, void* recvbuf, int* recvcounts, + int* recvdispls, MPI_Datatype recvtype, + MPI_Comm comm) const { + return MPI_Alltoallv(sendbuf, sendcounts, senddispls, sendtype, + recvbuf, recvcounts, recvdispls, recvtype, + comm);} + + int reduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, + MPI_Op op, int root, MPI_Comm comm) const { + return MPI_Reduce(sendbuf, recvbuf, count, datatype, op, root, comm);} + int allReduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, + MPI_Op op, MPI_Comm comm) const { + return MPI_Allreduce(sendbuf, recvbuf, count, datatype, op, comm);} }; } diff --git a/src/ParaMEDMEM/ElementLocator.cxx b/src/ParaMEDMEM/ElementLocator.cxx index 10bb3d5c1..ddcdda268 100644 --- a/src/ParaMEDMEM/ElementLocator.cxx +++ b/src/ParaMEDMEM/ElementLocator.cxx @@ -11,6 +11,8 @@ #include "MEDMEM_Meshing.hxx" #include +using namespace std; + namespace ParaMEDMEM { @@ -23,7 +25,8 @@ _local_group(*mesh.getBlockTopology()->getProcGroup()), { _union_group = _local_group.fuse(distant_group); _computeBoundingBoxes(); - registerOption(&_adjustment_eps,"BoundingBoxAdjustement",0.1); +//JR registerOption(&_adjustment_eps,"BoundingBoxAdjustement",0.1); + registerOption(&_adjustment_eps,"BoundingBoxAdjustment",0.1); } ElementLocator::ElementLocator(const ParaSUPPORT& support, const ProcessorGroup& distant_group) @@ -31,7 +34,8 @@ ElementLocator::ElementLocator(const ParaSUPPORT& support, const ProcessorGroup& _distant_group(distant_group), _union_group(_local_group.fuse(distant_group)) { - registerOption(&_adjustment_eps,"BoundingBoxAdjustement",0.1); +// registerOption(&_adjustment_eps,"BoundingBoxAdjustement",0.1); + registerOption(&_adjustment_eps,"BoundingBoxAdjustment",0.1); throw ("Element Locator SUPPORT constructor not implemented yet"); } diff --git a/src/ParaMEDMEM/InterpolationMatrix.cxx b/src/ParaMEDMEM/InterpolationMatrix.cxx index efd24ff3e..af0785ef4 100644 --- a/src/ParaMEDMEM/InterpolationMatrix.cxx +++ b/src/ParaMEDMEM/InterpolationMatrix.cxx @@ -211,6 +211,14 @@ void InterpolationMatrix::multiply(MEDMEM::FIELD& field) const target_value[i*nbcomp+icomp] /= _target_volume[i]; } + if (_target_group.containsMyRank()) + { + int nbelems=field.getSupport()->getNumberOfElements(MED_EN::MED_ALL_ELEMENTS); + int nbcomp = field.getNumberOfComponents(); + double* value = const_cast (field.getValue()); + for (int i=0; i&)const; void prepare(); int getNbRows() const {return _row_offsets.size();} - + void setAllToAllMethod(const AllToAllMethod& method) + { _mapping.setAllToAllMethod(method);} + MPI_AccessDEC* getAccessDEC(){return _mapping.getAccessDEC();} + private: MEDMEM::FIELD* InterpolationMatrix::getSupportVolumes(const MEDMEM::SUPPORT& field); diff --git a/src/ParaMEDMEM/IntersectionDEC.cxx b/src/ParaMEDMEM/IntersectionDEC.cxx index 2eed43952..ab3791b16 100644 --- a/src/ParaMEDMEM/IntersectionDEC.cxx +++ b/src/ParaMEDMEM/IntersectionDEC.cxx @@ -61,7 +61,7 @@ In the P0-P0 case, this matrix is a plain rectangular matrix with coefficients e - * \section intersectiondec_options Options +/* \section intersectiondec_options Options * On top of \ref dec_options, options supported by IntersectionDEC objects are * * @@ -89,10 +89,13 @@ The constructor must be called synchronously on all processors of both processor */ IntersectionDEC::IntersectionDEC(ProcessorGroup& local_group, ProcessorGroup& distant_group): - DEC(local_group, distant_group),_interpolation_matrix(0) + DEC(local_group, distant_group),_interpolation_matrix(0),_asynchronous(false),_timeinterpolationmethod(WithoutTimeInterp),_allToAllMethod(Native) { registerOption(&_method,string("Method"),string("P0")); registerOption(&_bb_adjustment, "BoundingBoxAdjustment",0.1); + registerOption(&_asynchronous, "Asynchronous",false); + registerOption(&_timeinterpolationmethod,"TimeInterpolation",WithoutTimeInterp); + registerOption(&_allToAllMethod,"AllToAllMethod",Native); } IntersectionDEC::~IntersectionDEC() @@ -119,6 +122,9 @@ void IntersectionDEC::synchronize() const ParaMEDMEM::ParaMESH* para_mesh = _local_field->getSupport()->getMesh(); cout <<"size of Interpolation Matrix"<setAllToAllMethod(_allToAllMethod); + _interpolation_matrix->getAccessDEC()->Asynchronous( _asynchronous ) ; + _interpolation_matrix->getAccessDEC()->SetTimeInterpolator( _timeinterpolationmethod ) ; //setting up the communication DEC on both sides if (_source_group->containsMyRank()) @@ -193,6 +199,8 @@ Receives the data whether the processor is on the working side or on the lazy si */ void IntersectionDEC::recvData() { + +// _interpolation_matrix->setAllToAllMethod(_allToAllMethod); if (_source_group->containsMyRank()) _interpolation_matrix->transposeMultiply(*_local_field->getField()); else if (_target_group->containsMyRank()) @@ -215,6 +223,11 @@ void IntersectionDEC::recvData() } +void IntersectionDEC::recvData( double time ) +{ + _interpolation_matrix->getAccessDEC()->SetTime(time); + recvData() ; +} /*! Sends the data whether the processor is on the working side or on the lazy side. @@ -222,8 +235,10 @@ It must match a recvData() call on the other side. */ void IntersectionDEC::sendData() { +// _interpolation_matrix->setAllToAllMethod(_allToAllMethod); if (_source_group->containsMyRank()) { + _interpolation_matrix->multiply(*_local_field->getField()); if (_forced_renormalization_flag) renormalizeTargetField(); @@ -241,6 +256,12 @@ void IntersectionDEC::sendData() _interpolation_matrix->transposeMultiply(*_local_field->getField()); } +void IntersectionDEC::sendData( double time , double deltatime ) +{ + _interpolation_matrix->getAccessDEC()->SetTime(time,deltatime); + sendData() ; +} + /*! @} */ } diff --git a/src/ParaMEDMEM/IntersectionDEC.hxx b/src/ParaMEDMEM/IntersectionDEC.hxx index 88b490e3d..90817549d 100644 --- a/src/ParaMEDMEM/IntersectionDEC.hxx +++ b/src/ParaMEDMEM/IntersectionDEC.hxx @@ -2,12 +2,13 @@ #define INTERSECTIONDEC_HXX_ #include "MEDMEM_OptionManager.hxx" +#include "MPI_AccessDEC.hxx" namespace ParaMEDMEM { class DEC; class InterpolationMatrix; - + class IntersectionDEC:public DEC { public: @@ -19,8 +20,10 @@ namespace ParaMEDMEM void synchronize(); void recvData(); + void recvData( double time ); void sendData(); + void sendData( double time , double deltatime ); void prepareSourceDE(){}; void prepareTargetDE(){}; @@ -42,6 +45,10 @@ namespace ParaMEDMEM InterpolationMatrix* _interpolation_matrix; double _bb_adjustment; + + bool _asynchronous; + TimeInterpolationMethod _timeinterpolationmethod ; + AllToAllMethod _allToAllMethod; }; } diff --git a/src/ParaMEDMEM/LinearTimeInterpolator.cxx b/src/ParaMEDMEM/LinearTimeInterpolator.cxx new file mode 100644 index 000000000..36e4123a3 --- /dev/null +++ b/src/ParaMEDMEM/LinearTimeInterpolator.cxx @@ -0,0 +1,41 @@ + +#include "LinearTimeInterpolator.hxx" + +using namespace std; + +namespace ParaMEDMEM { + +LinearTimeInterpolator::LinearTimeInterpolator( double InterpPrecision, int nStepBefore, + int nStepAfter ): + TimeInterpolator( InterpPrecision, nStepBefore, nStepAfter ) { +} + +LinearTimeInterpolator::~LinearTimeInterpolator() { +} + +void LinearTimeInterpolator::DoInterp( double time0, double time1, double time, + int recvcount , int nbuff0, int nbuff1, + int **recvbuff0, int **recvbuff1, int *result ) { + int i ; + for ( i = 0 ; i < recvcount ; i++ ) { + result[i] = (int ) ((recvbuff0[0][i]*(time1 - time) + recvbuff1[0][i]*(time - time0))/(time1 - time0) + _InterpPrecision) ; + //cout << "DoInterpint time " << time << " time0 " << time0 << " time1 " << time1 + // << " recvbuff0 " << recvbuff0[0][i] << " recvbuff1 " << recvbuff1[0][i] + // << " --> " << result[i] << endl ; + } +} + +void LinearTimeInterpolator::DoInterp( double time0, double time1, double time, + int recvcount , int nbuff0, int nbuff1, + double **recvbuff0, double **recvbuff1, + double *result ) { + int i ; + for ( i = 0 ; i < recvcount ; i++ ) { + result[i] = (recvbuff0[0][i]*(time1 - time) + recvbuff1[0][i]*(time - time0))/(time1 - time0) ; + //cout << "DoInterpdouble time " << time << " time0 " << time0 << " time1 " << time1 + // << " recvbuff0 " << recvbuff0[0][i] << " recvbuff1 " << recvbuff1[0][i] + // << " --> " << result[i] << endl ; + } +} + +} diff --git a/src/ParaMEDMEM/LinearTimeInterpolator.hxx b/src/ParaMEDMEM/LinearTimeInterpolator.hxx new file mode 100644 index 000000000..523fb06fc --- /dev/null +++ b/src/ParaMEDMEM/LinearTimeInterpolator.hxx @@ -0,0 +1,31 @@ +#ifndef LINEARTIMEINTERPOLATOR_HXX_ +#define LINEARTIMEINTERPOLATOR_HXX_ + +#include +#include + +#include "TimeInterpolator.hxx" + +namespace ParaMEDMEM { + + class DEC; + + class LinearTimeInterpolator:public TimeInterpolator { + + public: + LinearTimeInterpolator( double InterpPrecision=0, int nStepBefore=1, + int nStepAfter=1 ) ; + virtual ~LinearTimeInterpolator(); + + void DoInterp( double time0, double time1, double time, int recvcount, + int nbuff0, int nbuff1, + int **recvbuff0, int **recvbuff1, int *result ) ; + void DoInterp( double time0, double time1, double time, int recvcount, + int nbuff0, int nbuff1, + double **recvbuff0, double **recvbuff1, double *result ) ; + + private : + }; +} + +#endif diff --git a/src/ParaMEDMEM/MPI_Access.cxx b/src/ParaMEDMEM/MPI_Access.cxx new file mode 100644 index 000000000..c4cf555fc --- /dev/null +++ b/src/ParaMEDMEM/MPI_Access.cxx @@ -0,0 +1,828 @@ +#include + +#include "MPI_Access.hxx" +#include "MEDMEM_Exception.hxx" + +using namespace std; + +namespace ParaMEDMEM +{ +/*! \defgroup mpi_access MPI_Access + Class \a MPI_Access is the gateway to the MPI library. + It is a helper class that gathers the calls to the MPI + library that are made in the ParaMEDMEM library. This gathering + allows easier gathering of information about the communication + in the library. With MPI_Access, tags are managed automatically + and asynchronous operations are easier. + +It is typically called after the MPI_Init() call in a program. It is afterwards passed as a parameter to the constructors of ParaMEDMEM objects so that they access the MPI library via the MPI_Access. + +As an example, the following code initializes a processor group made of the zero processor. + +\verbatim +#include "MPI_Access.hxx" +#include "ProcessorGroup.hxx" + +int main(int argc, char** argv) +{ + //initialization + MPI_Init(&argc, &argv); + ParaMEDMEM::CommInterface comm_interface; + + //setting up a processor group with proc 0 + set procs; + procs.insert(0); + ParaMEDMEM::ProcessorGroup group(procs, comm_interface); + + ParaMEDMEM::MPI_Access mpi_access(group); + + //cleanup + MPI_Finalize(); +} +\endverbatim +*/ + + + /*! Creates a MPI_Access that is based on the processors included in \a ProcessorGroup. +This routine may be called for easier use of MPI API. + +\param ProcessorGroup MPIProcessorGroup object giving access to group management +\param BaseTag and MaxTag define the range of tags to be used. +Tags are managed by MPI_Access. They are cyclically incremented. +When there is a Send or a Receive operation there is a new RequestId tag returned +to the caller. That RequestId may be used to manage the operation Wait, Check of +status etc... The MPITag internally managed by MPI_Access is used as "tag" argument +in MPI call. + */ + +MPI_Access::MPI_Access(MPIProcessorGroup * ProcessorGroup, int BaseTag, int MaxTag) : + _CommInterface( ProcessorGroup->getCommInterface() ) , + _IntraCommunicator( ProcessorGroup->getComm() ) { + int mpitagub ; + int flag ; +//MPI_Attr_get does not run with _IntraCommunicator ??? + //MPI_Attr_get(*_IntraCommunicator,MPI_TAG_UB,&mpitagub,&flag) ; + MPI_Attr_get(MPI_COMM_WORLD,MPI_TAG_UB,&mpitagub,&flag) ; + if ( BaseTag != 0 ) { + BaseTag = (BaseTag/ModuloTag)*ModuloTag ; + } + if ( MaxTag == 0 ) { + MaxTag = (mpitagub/ModuloTag-1)*ModuloTag ; + } + MPI_Comm_rank( *_IntraCommunicator, &_MyRank ) ; + cout << "MPI_Access::MPI_Access" << _MyRank << " this " << this << " BaseTag " << BaseTag + << " MaxTag " << MaxTag << " mpitagub " << mpitagub << " (minimum 32767) " + << " flag " << flag << endl ; + if ( !flag | (BaseTag < 0) | (BaseTag >= MaxTag) | (MaxTag > mpitagub) ) + throw MEDMEM::MEDEXCEPTION("wrong call to MPI_Access constructor"); + + _ProcessorGroup = ProcessorGroup ; + _ProcessorGroupSize = _ProcessorGroup->size() ; + _Trace = false ; + + cout << "MPI_Access::MPI_Access" << _MyRank << " _ProcessorGroupSize " + << _ProcessorGroupSize << endl ; + + _BaseRequest = -1 ; + _MaxRequest = 4294967295 ; + _Request = _BaseRequest ; + + _BaseMPITag = BaseTag ; + _MaxMPITag = MaxTag ; + + _SendRequest = new int[ _ProcessorGroupSize ] ; + _RecvRequest = new int[ _ProcessorGroupSize ] ; + + _SendRequests.resize( _ProcessorGroupSize ) ; + _RecvRequests.resize( _ProcessorGroupSize ) ; + + _SendMPITag = new int[ _ProcessorGroupSize ] ; + _RecvMPITag = new int[ _ProcessorGroupSize ] ; + int i ; + for (i = 0 ; i < _ProcessorGroupSize ; i++ ) { + _SendRequest[ i ] = _MaxRequest ; + _RecvRequest[ i ] = _MaxRequest ; + _SendRequests[ i ].resize(0) ; + _RecvRequests[ i ].resize(0) ; + _SendMPITag[ i ] = _MaxMPITag ; + _RecvMPITag[ i ] = _MaxMPITag ; + } + MPI_Datatype array_of_types[3] ; + array_of_types[0] = MPI_DOUBLE ; + array_of_types[1] = MPI_DOUBLE ; + array_of_types[2] = MPI_INT ; + int array_of_blocklengths[3] ; + array_of_blocklengths[0] = 1 ; + array_of_blocklengths[1] = 1 ; + array_of_blocklengths[2] = 1 ; + MPI_Aint array_of_displacements[3] ; + array_of_displacements[0] = 0 ; + array_of_displacements[1] = sizeof(double) ; + array_of_displacements[2] = 2*sizeof(double) ; + MPI_Type_struct(3, array_of_blocklengths, array_of_displacements, + array_of_types, &_MPI_TIME) ; + MPI_Type_commit(&_MPI_TIME) ; +} + +MPI_Access::~MPI_Access() { + cout << "MPI_Access::~MPI_Access" << _MyRank << " this " << this << endl ; + delete [] _SendRequest ; + delete [] _RecvRequest ; + delete [] _SendMPITag ; + delete [] _RecvMPITag ; + cout << "End of MPI_Access::~MPI_Access" << _MyRank << " this " << this << endl ; +} + +int MPI_Access::NewRequest( MPI_Datatype datatype, int tag , int destsourcerank , + bool fromsourcerank , bool asynchronous ) { + RequestStruct *mpiaccessstruct = new RequestStruct; + mpiaccessstruct->MPITag = tag ; + mpiaccessstruct->MPIDatatype = datatype ; + mpiaccessstruct->MPITarget = destsourcerank ; + mpiaccessstruct->MPIIsRecv = fromsourcerank ; + MPI_Status *aStatus = new MPI_Status ; + mpiaccessstruct->MPIStatus = aStatus ; + mpiaccessstruct->MPIAsynchronous = asynchronous ; + mpiaccessstruct->MPICompleted = !asynchronous ; + mpiaccessstruct->MPIOutCount = -1 ; + if ( !asynchronous ) { + mpiaccessstruct->MPIRequest = MPI_REQUEST_NULL ; + mpiaccessstruct->MPIStatus->MPI_SOURCE = destsourcerank ; + mpiaccessstruct->MPIStatus->MPI_TAG = tag ; + mpiaccessstruct->MPIStatus->MPI_ERROR = MPI_SUCCESS ; + } + //cout << "NewRequest old _Request" << _MyRank << " " << _Request + // << " _MaxRequest " << _MaxRequest << endl ; + if ( _Request == _MaxRequest ) { + _Request = _BaseRequest ; + } + _Request += 1 ; + _MapOfRequestStruct[_Request] = mpiaccessstruct ; + if ( fromsourcerank ) { + _RecvRequest[ destsourcerank ] = _Request ; + } + else { + _SendRequest[ destsourcerank ] = _Request ; + } + if ( _Trace ) + cout << "NewRequest" << _MyRank << "( " << _Request << " ) " + << mpiaccessstruct << endl ; + return _Request ; +} + +int MPI_Access::NewSendTag( MPI_Datatype datatype, int destrank , int method , + bool asynchronous, int &RequestId ) { + int tag ; + if ( method == _MessageTime ) { + tag = method ; + } + else { + tag = IncrTag( _SendMPITag[destrank] ) ; + tag = ValTag( tag, method ) ; + _SendMPITag[ destrank ] = tag ; + } + RequestId = NewRequest( datatype, tag, destrank , false , asynchronous ) ; + _SendRequest[ destrank ] = RequestId ; + _SendRequests[ destrank ].push_back( RequestId ) ; + return tag ; +} + +int MPI_Access::NewRecvTag( MPI_Datatype datatype, int sourcerank , int method , + bool asynchronous, int &RequestId ) { + int tag ; + if ( method == _MessageTime ) { + tag = method ; + } + else { + tag = IncrTag( _RecvMPITag[sourcerank] ) ; + tag = ValTag( tag, method ) ; + _RecvMPITag[ sourcerank ] = tag ; + } + RequestId = NewRequest( datatype, tag , sourcerank , true , asynchronous ) ; + _RecvRequest[ sourcerank ] = RequestId ; + _RecvRequests[ sourcerank ].push_back( RequestId ) ; + return tag ; +} + +int MPI_Access::SendRequestIdsSize() { + int size = 0 ; + int i ; + for (i = 0 ; i < _ProcessorGroupSize ; i++ ) { + size += _SendRequests[ i ].size() ; + } + return size ; +} + +int MPI_Access::SendRequestIds(int size, int *ArrayOfSendRequests) { + int destrank ; + int i = 0 ; + for ( destrank = 0 ; destrank < _ProcessorGroupSize ; destrank++ ) { + list< int >::const_iterator iter ; + for (iter = _SendRequests[ destrank ].begin() ; iter != _SendRequests[destrank].end() ; iter++ ) { + ArrayOfSendRequests[i++] = *iter ; + } + } + return i ; +} + +int MPI_Access::RecvRequestIdsSize() { + int size = 0 ; + int i ; + for (i = 0 ; i < _ProcessorGroupSize ; i++ ) { + size += _RecvRequests[ i ].size() ; + } + return size ; +} + +int MPI_Access::RecvRequestIds(int size, int *ArrayOfRecvRequests) { + int sourcerank ; + int i = 0 ; + for ( sourcerank = 0 ; sourcerank < _ProcessorGroupSize ; sourcerank++ ) { + list< int >::const_iterator iter ; + for (iter = _RecvRequests[ sourcerank ].begin() ; iter != _RecvRequests[sourcerank].end() ; iter++ ) { + ArrayOfRecvRequests[i++] = *iter ; + } + } + return i ; +} + +int MPI_Access::SendRequestIds(int destrank, int size, int *ArrayOfSendRequests) { + if (size < _SendRequests[destrank].size() ) throw MEDMEM::MEDEXCEPTION("wrong call to MPI_Access::SendRequestIds"); + int i = 0 ; + list< int >::const_iterator iter ; + for (iter = _SendRequests[ destrank ].begin() ; iter != _SendRequests[destrank].end() ; iter++ ) { + ArrayOfSendRequests[i++] = *iter ; + } + return _SendRequests[destrank].size() ; +} + +int MPI_Access::RecvRequestIds(int sourcerank, int size, int *ArrayOfRecvRequests) { + if (size < _RecvRequests[sourcerank].size() ) throw MEDMEM::MEDEXCEPTION("wrong call to MPI_Access::RecvRequestIds"); + int i = 0 ; + list< int >::const_iterator iter ; + _RecvRequests[ sourcerank ] ; + for (iter = _RecvRequests[ sourcerank ].begin() ; iter != _RecvRequests[sourcerank].end() ; iter++ ) { + ArrayOfRecvRequests[i++] = *iter ; + } + return _RecvRequests[sourcerank].size() ; +} + +int MPI_Access::Send(void* buffer, int count, MPI_Datatype datatype, int target, + int &RequestId) { + int sts = MPI_SUCCESS ; + if ( count ) { + _MessageIdent aMethodIdent = MethodId( datatype ) ; + int MPItag = NewSendTag( datatype, target , aMethodIdent , false , RequestId ) ; + if ( aMethodIdent == _MessageTime ) { + TimeMessage *aTimeMsg = (TimeMessage *) buffer ; + aTimeMsg->tag = MPItag ; + } + DeleteRequest( RequestId ) ; + sts = _CommInterface.send(buffer, count, datatype, target, MPItag, + *_IntraCommunicator ) ; + } + return sts ; +} + +int MPI_Access::Recv(void* buffer, int count, MPI_Datatype datatype, int source, + int &RequestId, int *OutCount) { + int sts = MPI_SUCCESS ; + if ( count ) { + _MessageIdent aMethodIdent = MethodId( datatype ) ; + int MPItag = NewRecvTag( datatype, source , aMethodIdent , false , RequestId ) ; + sts = _CommInterface.recv(buffer, count, datatype, source, MPItag, + *_IntraCommunicator , MPIStatus( RequestId ) ) ; + int outcount = 0 ; + if ( sts == MPI_SUCCESS ) { + MPI_Datatype datatype = MPIDatatype( RequestId ) ; + _CommInterface.get_count(MPIStatus( RequestId ), datatype, &outcount ) ; + SetMPIOutCount( RequestId , outcount ) ; + SetMPICompleted( RequestId , true ) ; + DeleteStatus( RequestId ) ; + } + if ( OutCount != NULL ) { + *OutCount = outcount ; + } + DeleteRequest( RequestId ) ; + } + return sts ; +} + +int MPI_Access::ISend(void* buffer, int count, MPI_Datatype datatype, int target, + int &RequestId) { + int sts = MPI_SUCCESS ; + if ( count ) { + _MessageIdent aMethodIdent = MethodId( datatype ) ; + int MPItag = NewSendTag( datatype, target , aMethodIdent , true , RequestId ) ; + if ( aMethodIdent == _MessageTime ) { + TimeMessage *aTimeMsg = (TimeMessage *) buffer ; + aTimeMsg->tag = MPItag ; + } + MPI_Request *aSendRequest = MPIRequest( RequestId ) ; + sts = _CommInterface.Isend(buffer, count, datatype, target, MPItag, + *_IntraCommunicator , aSendRequest) ; + } + return sts ; +} + +int MPI_Access::IRecv(void* buffer, int count, MPI_Datatype datatype, int source, + int &RequestId) { + int sts = MPI_SUCCESS ; + if ( count ) { + _MessageIdent aMethodIdent = MethodId( datatype ) ; + int MPItag = NewRecvTag( datatype, source , aMethodIdent , true , RequestId ) ; + MPI_Request *aRecvRequest = MPIRequest( RequestId ) ; + sts = _CommInterface.Irecv(buffer, count, datatype, source, MPItag, + *_IntraCommunicator , aRecvRequest) ; + } + return sts ; +} + +int MPI_Access::SendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int &SendRequestId, + void* recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int &RecvRequestId, int *OutCount) { + int sts = MPI_SUCCESS ; + SendRequestId = -1 ; + RecvRequestId = -1 ; + if ( recvcount ) { + sts = IRecv(recvbuf, recvcount, recvtype, source, RecvRequestId) ; + } + int outcount = -1 ; + if ( _Trace ) + cout << "MPI_Access::SendRecv" << _MyRank << " IRecv RecvRequestId " + << RecvRequestId << endl ; + if ( sts == MPI_SUCCESS ) { + if ( sendcount ) { + sts = Send(sendbuf, sendcount, sendtype, dest, SendRequestId) ; + } + if ( _Trace ) + cout << "MPI_Access::SendRecv" << _MyRank << " Send SendRequestId " + << SendRequestId << endl ; + if ( sts == MPI_SUCCESS && recvcount ) { + sts = Wait( RecvRequestId ) ; + outcount = MPIOutCount( RecvRequestId ) ; + if ( _Trace ) + cout << "MPI_Access::SendRecv" << _MyRank << " IRecv RecvRequestId " + << RecvRequestId << " outcount " << outcount << endl ; + } + } + if ( OutCount != NULL ) { + *OutCount = outcount ; + if ( _Trace ) + cout << "MPI_Access::SendRecv" << _MyRank << " *OutCount = " << *OutCount + << endl ; + } + DeleteRequest( RecvRequestId ) ; + return sts ; +} + +int MPI_Access::ISendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int &SendRequestId, + void* recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int &RecvRequestId) { + int sts = MPI_SUCCESS ; + SendRequestId = -1 ; + RecvRequestId = -1 ; + if ( recvcount ) { + sts = IRecv(recvbuf, recvcount, recvtype, source, RecvRequestId) ; + } + if ( sts == MPI_SUCCESS ) { + if ( sendcount ) { + sts = ISend(sendbuf, sendcount, sendtype, dest, SendRequestId) ; + } + } + return sts ; +} + +int MPI_Access::Wait( int RequestId ) { + int status = MPI_SUCCESS ; + if ( !MPICompleted( RequestId ) ) { + if ( *MPIRequest( RequestId ) != MPI_REQUEST_NULL ) { + if ( _Trace ) + cout << "MPI_Access::Wait" << _MyRank << " -> wait( " << RequestId + << " ) MPIRequest " << MPIRequest( RequestId ) << " MPIStatus " + << MPIStatus( RequestId ) << " MPITag " << MPITag( RequestId ) + << " MPIIsRecv " << MPIIsRecv( RequestId ) << endl ; + status = _CommInterface.wait(MPIRequest( RequestId ), MPIStatus( RequestId )) ; + } + else { + if ( _Trace ) + cout << "MPI_Access::Wait" << _MyRank << " MPIRequest == MPI_REQUEST_NULL" + << endl ; + } + SetMPICompleted( RequestId , true ) ; + if ( MPIIsRecv( RequestId ) && MPIStatus( RequestId ) ) { + MPI_Datatype datatype = MPIDatatype( RequestId ) ; + int outcount ; + status = _CommInterface.get_count(MPIStatus( RequestId ), datatype, + &outcount ) ; + if ( status == MPI_SUCCESS ) { + SetMPIOutCount( RequestId , outcount ) ; + DeleteStatus( RequestId ) ; + if ( _Trace ) + cout << "MPI_Access::Wait" << _MyRank << " RequestId " << RequestId + << "MPIIsRecv " << MPIIsRecv( RequestId ) << " outcount " << outcount + << endl ; + } + else { + if ( _Trace ) + cout << "MPI_Access::Wait" << _MyRank << " MPIIsRecv " + << MPIIsRecv( RequestId ) << " outcount " << outcount << endl ; + } + } + else { + if ( _Trace ) + cout << "MPI_Access::Wait" << _MyRank << " MPIIsRecv " << MPIIsRecv( RequestId ) + << " MPIOutCount " << MPIOutCount( RequestId ) << endl ; + } + } + if ( _Trace ) + cout << "MPI_Access::Wait" << _MyRank << " RequestId " << RequestId + << " Request " << MPIRequest( RequestId ) + << " Status " << MPIStatus( RequestId ) << " MPICompleted " + << MPICompleted( RequestId ) << " MPIOutCount " << MPIOutCount( RequestId ) + << endl ; + return status ; +} + +int MPI_Access::Test(int RequestId, int &flag) { + int status = MPI_SUCCESS ; + flag = MPICompleted( RequestId ) ; + if ( _Trace ) + cout << "MPI_Access::Test" << _MyRank << " flag " << flag ; + if ( MPIIsRecv( RequestId ) ) { + if ( _Trace ) + cout << " Recv" ; + } + else { + if ( _Trace ) + cout << " Send" ; + } + if ( _Trace ) + cout << "Request" << RequestId << " " << MPIRequest( RequestId ) + << " Status " << MPIStatus( RequestId ) << endl ; + if ( !flag ) { + if ( *MPIRequest( RequestId ) != MPI_REQUEST_NULL ) { + if ( _Trace ) + cout << "MPI_Access::Test" << _MyRank << " -> test( " << RequestId + << " ) MPIRequest " << MPIRequest( RequestId ) + << " MPIStatus " << MPIStatus( RequestId ) + << " MPITag " << MPITag( RequestId ) + << " MPIIsRecv " << MPIIsRecv( RequestId ) << endl ; + status = _CommInterface.test(MPIRequest( RequestId ), &flag, + MPIStatus( RequestId )) ; + } + else { + if ( _Trace ) + cout << "MPI_Access::Test" << _MyRank << " MPIRequest == MPI_REQUEST_NULL" + << endl ; + } + if ( flag ) { + SetMPICompleted( RequestId , true ) ; + if ( MPIIsRecv( RequestId ) && MPIStatus( RequestId ) ) { + int outcount ; + MPI_Datatype datatype = MPIDatatype( RequestId ) ; + status = _CommInterface.get_count( MPIStatus( RequestId ), datatype, + &outcount ) ; + if ( status == MPI_SUCCESS ) { + SetMPIOutCount( RequestId , outcount ) ; + DeleteStatus( RequestId ) ; + if ( _Trace ) + cout << "MPI_Access::Test" << _MyRank << " MPIIsRecv " + << MPIIsRecv( RequestId ) << " outcount " << outcount << endl ; + } + else { + if ( _Trace ) + cout << "MPI_Access::Test" << _MyRank << " MPIIsRecv " + << MPIIsRecv( RequestId ) << " outcount " << outcount << endl ; + } + } + else { + if ( _Trace ) + cout << "MPI_Access::Test" << _MyRank << " MPIIsRecv " + << MPIIsRecv( RequestId ) << " MPIOutCount " + << MPIOutCount( RequestId ) << endl ; + } + } + } + if ( _Trace ) + cout << "MPI_Access::Test" << _MyRank << " RequestId " << RequestId + << " flag " << flag << " MPICompleted " << MPICompleted( RequestId ) + << " MPIOutCount " << MPIOutCount( RequestId ) << endl ; + return status ; +} + +int MPI_Access::WaitAny(int count, int *array_of_RequestIds, int &RequestId) { + int status ; + return status ; +} + +int MPI_Access::TestAny(int count, int *array_of_RequestIds, int &RequestId, + int &flag) { + int status ; + return status ; +} + +int MPI_Access::WaitAll(int count, int *array_of_RequestIds) { + if ( _Trace ) + cout << "WaitAll" << _MyRank << " : count " << count << endl ; + int status ; + int retstatus = MPI_SUCCESS ; + int i ; + for ( i = 0 ; i < count ; i++ ) { + if ( _Trace ) + cout << "WaitAll" << _MyRank << " " << i << " -> Wait( " + << array_of_RequestIds[i] << " )" << endl ; + status = Wait( array_of_RequestIds[i] ) ; + if ( status != MPI_SUCCESS ) { + retstatus = status ; + } + } + if ( _Trace ) + cout << "EndWaitAll" << _MyRank << endl ; + return retstatus ; +} + +int MPI_Access::TestAll(int count, int *array_of_RequestIds, int &flag) { + if ( _Trace ) + cout << "TestAll" << _MyRank << " : count " << count << endl ; + int status ; + int retstatus = MPI_SUCCESS ; + bool retflag = true ; + int i ; + for ( i = 0 ; i < count ; i++ ) { + status = Test( array_of_RequestIds[i] , flag ) ; + retflag = retflag && (flag != 0) ; + if ( status != MPI_SUCCESS ) { + retstatus = status ; + } + } + flag = retflag ; + if ( _Trace ) + cout << "EndTestAll" << _MyRank << endl ; + return retstatus ; +} + +int MPI_Access::WaitSome(int count, int *array_of_RequestIds, int outcount, + int *outarray_of_RequestIds) { + int status ; + return status ; +} + +int MPI_Access::TestSome(int count, int *array_of_RequestIds, int outcounts, + int *outarray_of_RequestIds) { + int status ; + return status ; +} + +int MPI_Access::Probe(int FromSource, int &source, int &MPITag, + MPI_Datatype &datatype, int &outcount) { + MPI_Status aMPIStatus ; + int sts = _CommInterface.probe( FromSource, MPI_ANY_TAG, + *_IntraCommunicator , &aMPIStatus ) ; + if ( sts == MPI_SUCCESS ) { + source = aMPIStatus.MPI_SOURCE ; + MPITag = aMPIStatus.MPI_TAG ; + int MethodId = (MPITag % ModuloTag) ; + datatype = Datatype( (ParaMEDMEM::_MessageIdent) MethodId ) ; + _CommInterface.get_count(&aMPIStatus, datatype, &outcount ) ; + if ( _Trace ) + cout << "MPI_Access::Probe" << _MyRank << " FromSource " << FromSource + << " source " << source << " MPITag " << MPITag << " MethodId " + << MethodId << " datatype " << datatype << " outcount " << outcount + << endl ; + } + else { + source = -1 ; + MPITag = -1 ; + outcount = -1 ; + } + return sts ; +} + +int MPI_Access::IProbe(int FromSource, int &source, int &MPITag, + MPI_Datatype &datatype, int &outcount, int &flag) { + MPI_Status aMPIStatus ; + int sts = _CommInterface.Iprobe( FromSource, MPI_ANY_TAG, + *_IntraCommunicator , &flag, + &aMPIStatus ) ; + if ( sts == MPI_SUCCESS && flag ) { + source = aMPIStatus.MPI_SOURCE ; + MPITag = aMPIStatus.MPI_TAG ; + int MethodId = (MPITag % ModuloTag) ; + datatype = Datatype( (ParaMEDMEM::_MessageIdent) MethodId ) ; + _CommInterface.get_count(&aMPIStatus, datatype, &outcount ) ; + if ( _Trace ) + cout << "MPI_Access::IProbe" << _MyRank << " FromSource " << FromSource + << " source " << source << " MPITag " << MPITag << " MethodId " + << MethodId << " datatype " << datatype << " outcount " << outcount + << " flag " << flag << endl ; + } + else { + source = -1 ; + MPITag = -1 ; + outcount = -1 ; + } + return sts ; +} + +int MPI_Access::Cancel( int RecvRequestId, int &flag ) { + flag = 0 ; + int sts = _CommInterface.cancel( MPIRequest( RecvRequestId ) ) ; + if ( sts == MPI_SUCCESS ) { + sts = _CommInterface.wait( MPIRequest( RecvRequestId ) , + MPIStatus( RecvRequestId ) ) ; + if ( sts == MPI_SUCCESS ) { + sts = _CommInterface.test_cancelled( MPIStatus( RecvRequestId ) , &flag ) ; + } + } + return sts ; +} + +int MPI_Access::Cancel( int source, int theMPITag, MPI_Datatype datatype, + int outcount, int &flag ) { + int sts ; + MPI_Aint extent ; + flag = 0 ; + sts = MPI_Type_extent( datatype , &extent ) ; + if ( sts == MPI_SUCCESS ) { + void * recvbuf = malloc( extent*outcount ) ; + MPI_Request aRecvRequest ; + if ( _Trace ) + cout << "MPI_Access::Cancel" << _MyRank << " Irecv extent " << extent + << " datatype " << datatype << " source " << source << " theMPITag " + << theMPITag << endl ; + sts = _CommInterface.Irecv( recvbuf, outcount, datatype, source, theMPITag, + *_IntraCommunicator , &aRecvRequest ) ; + if ( sts == MPI_SUCCESS ) { + sts = _CommInterface.cancel( &aRecvRequest ) ; + if ( _Trace ) + cout << "MPI_Access::Cancel" << _MyRank << " theMPITag " << theMPITag + << " cancel done" << endl ; + if ( sts == MPI_SUCCESS ) { + MPI_Status aStatus ; + if ( _Trace ) + cout << "MPI_Access::Cancel" << _MyRank << " wait" << endl ; + sts = _CommInterface.wait( &aRecvRequest , &aStatus ) ; + if ( sts == MPI_SUCCESS ) { + if ( _Trace ) + cout << "MPI_Access::Cancel" << _MyRank << " test_cancelled" << endl ; + sts = _CommInterface.test_cancelled( &aStatus , &flag ) ; + } + } + } + if ( _Trace && datatype == TimeType() ) + cout << "MPI_Access::Cancel" << _MyRank << " time " + << ((TimeMessage *) recvbuf)->time << " " + << ((TimeMessage *) recvbuf)->deltatime << endl ; + free( recvbuf ) ; + } + if ( _Trace ) + cout << "MPI_Access::Cancel" << _MyRank << " flag " << flag << endl ; + return sts ; +} + +int MPI_Access::CancelAll() { + int sts = MPI_SUCCESS ; + int target ; + int source ; + int MPITag ; + MPI_Datatype datatype ; + int outcount ; + int flag ; + for ( target = 0 ; target < _ProcessorGroupSize ; target++ ) { + sts = IProbe(target, source, MPITag, datatype, outcount, flag) ; + if ( sts == MPI_SUCCESS && flag ) { + sts = Cancel(source, MPITag, datatype, outcount, flag) ; + if ( _Trace ) + cout << "MPI_Access::CancelAll" << _MyRank << " source " << source + << " MPITag " << MPITag << " datatype " << datatype + << " outcount " << outcount << " Cancel flag " << flag << endl ; + if ( sts != MPI_SUCCESS ) { + break ; + } + } + else if ( sts != MPI_SUCCESS ) { + break ; + } + } + return sts ; +} + +int MPI_Access::Barrier() { + int status = _CommInterface.barrier( *_IntraCommunicator ) ; + return status ; +} + +int MPI_Access::Error_String(int errorcode, char *string, int *resultlen) const { + return _CommInterface.error_string( errorcode, string, resultlen) ; +} + +int MPI_Access::Status(int RequestId, int &source, int &tag, int &error, + int &outcount, bool keepRequestStruct) { + MPI_Status *status = MPIStatus( RequestId ) ; + if ( _Trace ) + cout << "MPI_Access::Status" << _MyRank << " RequestId " << RequestId + << " status " << status << endl ; + if ( status != NULL && MPIAsynchronous( RequestId ) && + MPICompleted( RequestId ) ) { + if ( MPIIsRecv( RequestId ) ) { + source = status->MPI_SOURCE ; + tag = status->MPI_TAG ; + error = status->MPI_ERROR ; + MPI_Datatype datatype = MPIDatatype( RequestId ) ; + _CommInterface.get_count(status, datatype, &outcount ) ; + if ( _Trace ) + cout << "MPI_Access::Status" << _MyRank << " RequestId " << RequestId + << " status " << status << " outcount " << outcount << endl ; + SetMPIOutCount( RequestId , outcount ) ; + } + else { + source = MPITarget( RequestId ) ; + tag = MPITag( RequestId ) ; + error = 0 ; + outcount = MPIOutCount( RequestId ) ; + } + if ( !keepRequestStruct ) { + DeleteRequest( RequestId ) ; + } + return MPI_SUCCESS ; + } + else { + source = MPITarget( RequestId ) ; + tag = MPITag( RequestId ) ; + error = 0 ; + outcount = MPIOutCount( RequestId ) ; + } + return MPI_SUCCESS ; +} + +int MPI_Access::Request_Free( MPI_Request *request ) { + return _CommInterface.request_free( request ) ; +} + +void MPI_Access::Check() const { + int i = 0 ; + map< int , RequestStruct * >::const_iterator MapOfRequestStructiterator ; + cout << "MPI_Access::Check" << _MyRank << "_MapOfRequestStructSize " + << _MapOfRequestStruct.size() << endl ; + for ( MapOfRequestStructiterator = _MapOfRequestStruct.begin() ; + MapOfRequestStructiterator != _MapOfRequestStruct.end() ; + MapOfRequestStructiterator++ ) { + cout << " Check" << _MyRank << " " << i << ". Request" + << MapOfRequestStructiterator->first << "-->" ; + if ( MapOfRequestStructiterator->second == NULL ) { + cout << " MapOfRequestStructiterator->second == NULL" << endl ; + } + else { + if ( (MapOfRequestStructiterator->second)->MPIAsynchronous ) { + cout << "I" ; + } + if ( (MapOfRequestStructiterator->second)->MPIIsRecv ) { + cout << "Recv from " ; + } + else { + cout << "Send to " ; + } + cout << (MapOfRequestStructiterator->second)->MPITarget + << " MPITag " << (MapOfRequestStructiterator->second)->MPITag + << " DataType " << (MapOfRequestStructiterator->second)->MPIDatatype + << " Request " << (MapOfRequestStructiterator->second)->MPIRequest + << " Status " << (MapOfRequestStructiterator->second)->MPIStatus + << " Completed " << (MapOfRequestStructiterator->second)->MPICompleted + << endl ; + } + i++ ; + } + cout << "EndCheck" << _MyRank << endl ; +} + +ostream & operator<< (ostream & f ,const TimeMessage & aTimeMsg ) { + f << " time " << aTimeMsg.time << " deltatime " << aTimeMsg.deltatime + << " tag " << aTimeMsg.tag ; + + return f; +} + +ostream & operator<< (ostream & f ,const _MessageIdent & methodtype ) { + switch (methodtype) { + case _MessageTime : + f << " MethodTime "; + break; + case _MessageInt : + f << " MPI_INT "; + break; + case _MessageDouble : + f << " MPI_DOUBLE "; + break; + default : + f << " UnknownMethodType "; + break; + } + + return f; +} + +} + diff --git a/src/ParaMEDMEM/MPI_Access.hxx b/src/ParaMEDMEM/MPI_Access.hxx new file mode 100644 index 000000000..e78b7f202 --- /dev/null +++ b/src/ParaMEDMEM/MPI_Access.hxx @@ -0,0 +1,395 @@ +#ifndef MPI_ACCESS_HXX_ +#define MPI_ACCESS_HXX_ + +#include "CommInterface.hxx" +#include "ProcessorGroup.hxx" +#include "MPIProcessorGroup.hxx" + +#include +#include +#include + +namespace ParaMEDMEM +{ +typedef struct { double time ; + double deltatime ; + int tag ; } TimeMessage ; + +static MPI_Request mpirequestnull = MPI_REQUEST_NULL ; +enum _MessageIdent { _MessageUnknown, _MessageTime, _MessageInt, _MessageDouble } ; + +class MPI_Access { + public: +#define ModuloTag 10 + MPI_Access(MPIProcessorGroup * ProcessorGroup, int BaseTag=0, + int MaxTag=0) ; + virtual ~MPI_Access() ; + + void Trace( bool trace = true ) ; + + void DeleteRequest( int RequestId ) ; + void DeleteRequests(int size , int *ArrayOfSendRequests ) ; + + int SendMPITag(int destrank) ; + int RecvMPITag(int sourcerank) ; + + int SendRequestIdsSize() ; + int SendRequestIds(int size, int *ArrayOfSendRequests) ; + int RecvRequestIdsSize() ; + int RecvRequestIds(int size, int *ArrayOfRecvRequests) ; + + int SendRequestIdsSize(int destrank) ; + int SendRequestIds(int destrank, int size, int *ArrayOfSendRequests) ; + int RecvRequestIdsSize(int sourcerank) ; + int RecvRequestIds(int sourcerank, int size, int *ArrayOfRecvRequests) ; + + int Send(void* buffer, int count, MPI_Datatype datatype, int target, + int &RequestId) ; + int ISend(void* buffer, int count, MPI_Datatype datatype, int target, + int &RequestId) ; + int Recv(void* buffer, int count, MPI_Datatype datatype, int source, + int &RequestId, int *OutCount=NULL) ; + int IRecv(void* buffer, int count, MPI_Datatype datatype, int source, + int &RequestId) ; + int SendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest, + int &SendRequestId, void* recvbuf, int recvcount, + MPI_Datatype recvtype, int source, + int &RecvRequestId, int *OutCount=NULL) ; + int ISendRecv(void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest, + int &SendRequestId, void* recvbuf, int recvcount, + MPI_Datatype recvtype, int source, int &RecvRequestId) ; + + int Wait(int RequestId) ; + int Test(int RequestId, int &flag) ; + int WaitAny(int count, int *array_of_RequestIds, int &RequestId) ; + int TestAny(int count, int *array_of_RequestIds, int &RequestId, int &flag) ; + int WaitAll(int count, int *array_of_RequestIds) ; + int TestAll(int count, int *array_of_RequestIds, int &flag) ; + int WaitSome(int count, int *array_of_RequestIds, int outcount, + int *outarray_of_RequestIds) ; + int TestSome(int count, int *array_of_RequestIds, int outcounts, + int *outarray_of_RequestIds) ; + int Probe(int FromSource, int &source, int &MPITag, MPI_Datatype &datatype, + int &outcount) ; + int IProbe(int FromSource, int &source, int &MPITag, MPI_Datatype &datatype, + int &outcount, int &flag) ; + int Cancel( int RecvRequestId, int &flag ) ; + int Cancel( int source, int MPITag, MPI_Datatype datatype, int outcount, + int &flag ) ; + int CancelAll() ; + int Barrier() ; + int Error_String(int errorcode, char *string, int *resultlen) const ; + int Status(int RequestId, int &source, int &tag, int &error, int &outcount, + bool keepRequestStruct=false) ; + int Request_Free( MPI_Request *request ) ; + + void Check() const ; + + MPI_Datatype TimeType() const ; + bool IsTimeMessage( int MPITag ) const ; + MPI_Aint TimeExtent() const ; + MPI_Aint IntExtent() const ; + MPI_Aint DoubleExtent() const ; + MPI_Aint Extent( MPI_Datatype datatype ) const ; + + int MPITag( int RequestId ) ; + int MPITarget( int RequestId ) ; + bool MPIIsRecv( int RequestId ) ; + bool MPIAsynchronous( int RequestId ) ; + bool MPICompleted( int RequestId ) ; + MPI_Datatype MPIDatatype( int RequestId ) ; + int MPIOutCount( int RequestId ) ; + + private: + int NewRequest( MPI_Datatype datatype, int tag , int destsourcerank , + bool fromsourcerank , bool asynchronous ) ; + int NewSendTag( MPI_Datatype datatype, int destrank , int method , + bool asynchronous, int &RequestId ) ; + int NewRecvTag( MPI_Datatype datatype, int sourcerank , int method , + bool asynchronous, int &RequestId ) ; + int IncrTag( int prevtag ) ; + int ValTag( int tag, int method ) ; + + void DeleteSendRecvRequest( int RequestId ) ; + + void DeleteStatus( int RequestId ) ; + + MPI_Request *MPIRequest( int RequestId ) ; + MPI_Status *MPIStatus( int RequestId ) ; + void SetMPICompleted( int RequestId , bool completed ) ; + void SetMPIOutCount( int RequestId , int outcount ) ; + void ClearMPIStatus( int RequestId ) ; + + _MessageIdent MethodId( MPI_Datatype datatype ) const ; + MPI_Datatype Datatype( _MessageIdent aMethodIdent ) const ; + + const CommInterface &_CommInterface ; + const MPI_Comm* _IntraCommunicator ; + MPIProcessorGroup * _ProcessorGroup ; + int _ProcessorGroupSize ; + int _MyRank ; + bool _Trace ; + + int _BaseRequest ; + int _MaxRequest ; + int _Request ; + int * _SendRequest ; + int * _RecvRequest ; + vector< list< int > > _SendRequests ; + vector< list< int > > _RecvRequests ; + + int _BaseMPITag ; + int _MaxMPITag ; + int * _SendMPITag ; + int * _RecvMPITag ; + + MPI_Datatype _MPI_TIME ; + + struct RequestStruct { int MPITarget ; + bool MPIIsRecv ; + int MPITag ; + bool MPIAsynchronous ; + bool MPICompleted ; + MPI_Datatype MPIDatatype ; + MPI_Request MPIRequest ; + MPI_Status *MPIStatus ; + int MPIOutCount ; } ; + map< int , RequestStruct * > _MapOfRequestStruct ; + +}; + + inline void MPI_Access::Trace( bool trace ) { + _Trace = trace ; + } + + inline void MPI_Access::DeleteRequest( int RequestId ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + if ( aRequestStruct ) { + if ( _Trace ) + cout << "MPI_Access::DeleteRequest" << _MyRank << "( " << RequestId << " ) " + << aRequestStruct << " MPIRequest " << aRequestStruct->MPIRequest + << " MPIIsRecv " << aRequestStruct->MPIIsRecv << endl ; + if ( _MapOfRequestStruct[RequestId]->MPIRequest != MPI_REQUEST_NULL ) { + Request_Free( &_MapOfRequestStruct[RequestId]->MPIRequest ) ; + } + DeleteSendRecvRequest( RequestId ) ; + DeleteStatus( RequestId ) ; + _MapOfRequestStruct.erase( RequestId ) ; + delete aRequestStruct ; + } + else { + if ( _Trace ) + cout << "MPI_Access::DeleteRequest" << _MyRank << "( " << RequestId + << " ) Request not found" << endl ; + } + } + inline void MPI_Access::DeleteRequests(int size , int *ArrayOfSendRequests ) { + int i ; + for ( i = 0 ; i < size ; i++ ) { + DeleteRequest( ArrayOfSendRequests[i] ) ; + } + } + + inline int MPI_Access::SendMPITag(int destrank) { + return _SendMPITag[destrank] ; + } + inline int MPI_Access::RecvMPITag(int sourcerank) { + return _RecvMPITag[sourcerank] ; + } + + inline int MPI_Access::SendRequestIdsSize(int destrank) { + return _SendRequests[destrank].size() ; + } + inline int MPI_Access::RecvRequestIdsSize(int sourcerank) { + return _RecvRequests[sourcerank].size() ; + } + inline MPI_Datatype MPI_Access::TimeType() const { + return _MPI_TIME ; + } + inline bool MPI_Access::IsTimeMessage( int MPITag ) const { + return ((MPITag%ModuloTag) == _MessageTime) ; } ; + inline MPI_Aint MPI_Access::TimeExtent() const { + MPI_Aint extent ; + MPI_Type_extent( _MPI_TIME , &extent ) ; + return extent ; + } + inline MPI_Aint MPI_Access::IntExtent() const { + MPI_Aint extent ; + MPI_Type_extent( MPI_INT , &extent ) ; + return extent ; + } + inline MPI_Aint MPI_Access::DoubleExtent() const { + MPI_Aint extent ; + MPI_Type_extent( MPI_DOUBLE , &extent ) ; + return extent ; + } + inline MPI_Aint MPI_Access::Extent( MPI_Datatype datatype ) const { + if ( datatype == _MPI_TIME ) + return TimeExtent() ; + if ( datatype == MPI_INT ) + return IntExtent() ; + if ( datatype == MPI_DOUBLE ) + return DoubleExtent() ; + return 0 ; + } + + inline int MPI_Access::MPITag( int RequestId ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + if ( aRequestStruct ) { + return aRequestStruct->MPITag ; + } + return -1 ; + } + inline int MPI_Access::MPITarget( int RequestId ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + if ( aRequestStruct ) { + return aRequestStruct->MPITarget ; + } + return -1 ; + } + inline bool MPI_Access::MPIIsRecv( int RequestId ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + if ( aRequestStruct ) { + return aRequestStruct->MPIIsRecv ; + } + return false ; + } + inline bool MPI_Access::MPIAsynchronous( int RequestId ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + if ( aRequestStruct ) { + return aRequestStruct->MPIAsynchronous ; + } + return false ; + } + inline bool MPI_Access::MPICompleted( int RequestId ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + if ( aRequestStruct ) { + return aRequestStruct->MPICompleted ; + } + return true ; + } + inline MPI_Datatype MPI_Access::MPIDatatype( int RequestId ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + if ( aRequestStruct ) { + return aRequestStruct->MPIDatatype ; ; + } + return (MPI_Datatype ) NULL ; + } + inline int MPI_Access::MPIOutCount( int RequestId ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + if ( aRequestStruct ) { + return aRequestStruct->MPIOutCount ; + } + return 0 ; + } + + inline int MPI_Access::IncrTag( int prevtag ) { + int tag = ((prevtag/ModuloTag + 1)*ModuloTag) ; + if ( tag > _MaxMPITag ) + tag = _BaseMPITag ; + return tag ; + } + inline int MPI_Access::ValTag( int tag, int method ) { + return ((tag/ModuloTag)*ModuloTag) + method ; + } + + inline void MPI_Access::DeleteSendRecvRequest( int RequestId ) { + if ( MPIIsRecv( RequestId ) ) { + _RecvRequests[ MPITarget( RequestId ) ].remove( RequestId ) ; + } + else { + _SendRequests[ MPITarget( RequestId ) ].remove( RequestId ) ; + } + } + + inline void MPI_Access::DeleteStatus( int RequestId ) { + if ( _MapOfRequestStruct[RequestId]->MPIStatus != NULL ) { + delete _MapOfRequestStruct[RequestId]->MPIStatus ; + ClearMPIStatus( RequestId ) ; + } + } + + inline MPI_Request * MPI_Access::MPIRequest( int RequestId ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + //cout << "MPIRequest" << _MyRank << "(" << RequestId + // << " ) aRequestStruct->MPIRequest " << aRequestStruct->MPIRequest + // << endl ; + if ( aRequestStruct ) { + return &aRequestStruct->MPIRequest ; + } + return &mpirequestnull ; + } + inline MPI_Status * MPI_Access::MPIStatus( int RequestId ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + //cout << "MPIStatus" << _MyRank << "(" << RequestId + // << " ) aRequestStruct->MPIStatus " << aRequestStruct->MPIStatus + // << endl ; + if ( aRequestStruct ) { + return aRequestStruct->MPIStatus ; ; + } + return NULL ; + } + inline void MPI_Access::SetMPICompleted( int RequestId , bool completed ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + if ( aRequestStruct ) { + aRequestStruct->MPICompleted = completed ; + //if ( completed ) { + // DeleteRequest( RequestId ) ; + //} + } + } + inline void MPI_Access::SetMPIOutCount( int RequestId , int outcount ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + if ( aRequestStruct ) { + aRequestStruct->MPIOutCount = outcount ; + } + } + inline void MPI_Access::ClearMPIStatus( int RequestId ) { + struct RequestStruct *aRequestStruct = _MapOfRequestStruct[ RequestId ] ; + if ( aRequestStruct ) { + aRequestStruct->MPIStatus = NULL ; + } + } + + inline _MessageIdent MPI_Access::MethodId( MPI_Datatype datatype ) const { + _MessageIdent aMethodIdent ; + if ( datatype == _MPI_TIME ) { + aMethodIdent = _MessageTime ; + } + else if ( datatype == MPI_INT ) { + aMethodIdent = _MessageInt ; + } + else if ( datatype == MPI_DOUBLE ) { + aMethodIdent = _MessageDouble ; + } + else { + aMethodIdent = _MessageUnknown ; + } + return aMethodIdent ; + } + inline MPI_Datatype MPI_Access::Datatype( _MessageIdent aMethodIdent ) const { + MPI_Datatype aDataType ; + switch( aMethodIdent ) { + case _MessageTime : + aDataType = _MPI_TIME ; + break ; + case _MessageInt : + aDataType = MPI_INT ; + break ; + case _MessageDouble : + aDataType = MPI_DOUBLE ; + break ; + default : + aDataType = (MPI_Datatype) -1 ; + break ; + } + return aDataType ; + } + +ostream & operator<< (ostream &,const _MessageIdent &); + +ostream & operator<< (ostream &,const TimeMessage &); + +} + +#endif /*MPI_ACCESS_HXX_*/ diff --git a/src/ParaMEDMEM/MPI_AccessDEC.cxx b/src/ParaMEDMEM/MPI_AccessDEC.cxx new file mode 100644 index 000000000..374418835 --- /dev/null +++ b/src/ParaMEDMEM/MPI_AccessDEC.cxx @@ -0,0 +1,824 @@ + +#include "MPI_AccessDEC.hxx" + +using namespace std; + +namespace ParaMEDMEM { + +/*! +This constructor creates an MPI_AccessDEC which has \a local_group as a working side +and \a distant_group as an idle side. +The constructor must be called synchronously on all processors of both processor groups. + +\param local_group working side ProcessorGroup +\param distant_group lazy side ProcessorGroup +\param Asynchronous Communication mode (default asynchronous) +\param nStepBefore Number of Time step needed for the interpolation before current time +\param nStepAfter Number of Time step needed for the interpolation after current time + +*/ +//MPI_AccessDEC::MPI_AccessDEC( ProcessorGroup& local_group, +// ProcessorGroup& distant_group, +// TimeInterpolator * aTimeInterpolator, +// bool Asynchronous ) { +MPI_AccessDEC::MPI_AccessDEC( const ProcessorGroup& local_group, + const ProcessorGroup& distant_group, + bool Asynchronous ) { + _local_group = &local_group ; + _distant_group = &distant_group ; + ProcessorGroup * union_group = _local_group->fuse( *_distant_group ) ; + int i ; + std::set procs; + for ( i = 0 ; i < union_group->size() ; i++ ) { + procs.insert(i) ; + } + _MPI_union_group = new ParaMEDMEM::MPIProcessorGroup( + union_group->getCommInterface() , procs); + delete union_group ; + _MyRank = _MPI_union_group->myRank() ; + _GroupSize = _MPI_union_group->size() ; + _MPIAccess = new MPI_Access( _MPI_union_group ) ; + _Asynchronous = Asynchronous ; + _TimeMessages = new vector< vector< TimeMessage > > ; + _TimeMessages->resize( _GroupSize ) ; + _DataMessages = new vector< vector< void * > > ; + _DataMessages->resize( _GroupSize ) ; + _TimeInterpolator = NULL ; + _MapOfSendBuffers = new map< int , SendBuffStruct * > ; + cout << "MPI_AccessDEC" << _MyRank << " Asynchronous " << _Asynchronous << endl ; +} + +MPI_AccessDEC::~MPI_AccessDEC() { + CheckFinalSent() ; + CheckFinalRecv() ; + delete _MPI_union_group ; + delete _MPIAccess ; + if ( _TimeInterpolator ) + delete _TimeInterpolator ; + if ( _TimeMessages ) + delete _TimeMessages ; + if ( _DataMessages ) + delete _DataMessages ; + if ( _MapOfSendBuffers ) + delete _MapOfSendBuffers ; +} + +void MPI_AccessDEC::SetTimeInterpolator( TimeInterpolationMethod aTimeInterp , + double InterpPrecision, int nStepBefore, + int nStepAfter ) { + cout << "MPI_AccessDEC::SetTimeInterpolator" << _MyRank << " Asynchronous " + << _Asynchronous << " TimeInterpolationMethod " << aTimeInterp + << " InterpPrecision " << InterpPrecision << " nStepBefore " << nStepBefore + << " nStepAfter " << nStepAfter << endl ; + if ( _TimeInterpolator ) + delete _TimeInterpolator ; + switch ( aTimeInterp ) { + case WithoutTimeInterp : + _TimeInterpolator = NULL ; + _nStepBefore = 0 ; + _nStepAfter = 0 ; + break ; + case LinearTimeInterp : + _TimeInterpolator = new LinearTimeInterpolator( InterpPrecision , nStepBefore , + nStepAfter ) ; + _nStepBefore = nStepBefore ; + _nStepAfter = nStepAfter ; + int i ; + for ( i = 0 ; i < _GroupSize ; i++ ) { + (*_TimeMessages)[ i ].resize( _nStepBefore + _nStepAfter ) ; + (*_DataMessages)[ i ].resize( _nStepBefore + _nStepAfter ) ; + int j ; + for ( j = 0 ; j < _nStepBefore + _nStepAfter ; j++ ) { + (*_TimeMessages)[ i ][ j ].time = -1 ; + (*_TimeMessages)[ i ][ j ].deltatime = -1 ; + (*_DataMessages)[ i ][ j ] = NULL ; + } + } + break ; + } +} + +/*! +Send sendcount datas from sendbuf[offset] with type sendtype to target of IntraCommunicator +(Internal Protected method) + +Returns the request identifier SendRequestId + +*/ +int MPI_AccessDEC::Send( void* sendbuf, int sendcount , int offset , + MPI_Datatype sendtype , int target , int &SendRequestId ) { + int sts ; + if ( _Asynchronous ) { + if ( sendtype == MPI_INT ) { + sts = _MPIAccess->ISend( &((int *) sendbuf)[offset] , sendcount , sendtype , + target , SendRequestId ) ; + } + else { + sts = _MPIAccess->ISend( &((double *) sendbuf)[offset] , sendcount , sendtype , + target , SendRequestId ) ; + } + } + else { + if ( sendtype == MPI_INT ) { + sts = _MPIAccess->Send( &((int *) sendbuf)[offset] , sendcount , sendtype , + target , SendRequestId ) ; + } + else { + sts = _MPIAccess->Send( &((double *) sendbuf)[offset] , sendcount , sendtype , + target , SendRequestId ) ; + } + } + return sts ; +} + +/*! +Receive recvcount datas to recvbuf[offset] with type recvtype from target of IntraCommunicator +(Internal Protected method) + +Returns the request identifier RecvRequestId + +*/ +int MPI_AccessDEC::Recv( void* recvbuf, int recvcount , int offset , + MPI_Datatype recvtype , int target , int &RecvRequestId ) { + int sts ; + if ( _Asynchronous ) { + if ( recvtype == MPI_INT ) { + sts = _MPIAccess->IRecv( &((int *) recvbuf)[offset] , recvcount , recvtype , + target , RecvRequestId ) ; + } + else { + sts = _MPIAccess->IRecv( &((double *) recvbuf)[offset] , recvcount , recvtype , + target , RecvRequestId ) ; + } + } + else { + if ( recvtype == MPI_INT ) { + sts = _MPIAccess->Recv( &((int *) recvbuf)[offset] , recvcount , recvtype , + target , RecvRequestId ) ; + } + else { + sts = _MPIAccess->Recv( &((double *) recvbuf)[offset] , recvcount , recvtype , + target , RecvRequestId ) ; + } + } + return sts ; +} + +/*! +Send sendcount datas from sendbuf[offset] with type sendtype to target of IntraCommunicator +Receive recvcount datas to recvbuf[offset] with type recvtype from target of IntraCommunicator +(Internal Protected method) + +Returns the request identifier SendRequestId +Returns the request identifier RecvRequestId + +*/ +int MPI_AccessDEC::SendRecv( void* sendbuf, int sendcount , int sendoffset , + MPI_Datatype sendtype , + void* recvbuf, int recvcount , int recvoffset , + MPI_Datatype recvtype , int target , + int &SendRequestId , int &RecvRequestId ) { + int sts ; + if ( _Asynchronous ) { + if ( sendtype == MPI_INT ) { + if ( recvtype == MPI_INT ) { + sts = _MPIAccess->ISendRecv( &((int *) sendbuf)[sendoffset] , sendcount , + sendtype , target , SendRequestId , + &((int *) recvbuf)[recvoffset] , recvcount , + recvtype , target , RecvRequestId ) ; + } + else { + sts = _MPIAccess->ISendRecv( &((int *) sendbuf)[sendoffset] , sendcount , + sendtype , target , SendRequestId , + &((double *) recvbuf)[recvoffset] , + recvcount , recvtype , target , RecvRequestId ) ; + } + } + else { + if ( recvtype == MPI_INT ) { + sts = _MPIAccess->ISendRecv( &((double *) sendbuf)[sendoffset] , sendcount , + sendtype , target , SendRequestId , + &((int *) recvbuf)[recvoffset] , + recvcount , recvtype , target , RecvRequestId ) ; + } + else { + sts = _MPIAccess->ISendRecv( &((double *) sendbuf)[sendoffset] , sendcount , + sendtype , target , SendRequestId , + &((double *) recvbuf)[recvoffset] , + recvcount , recvtype , target , RecvRequestId ) ; + } + } + } + else { + if ( sendtype == MPI_INT ) { + if ( recvtype == MPI_INT ) { + sts = _MPIAccess->SendRecv( &((int *) sendbuf)[sendoffset] , sendcount , + sendtype , target , SendRequestId , + &((int *) recvbuf)[recvoffset] , recvcount , + recvtype , target , RecvRequestId ) ; + } + else { + sts = _MPIAccess->SendRecv( &((int *) sendbuf)[sendoffset] , sendcount , + sendtype , target , SendRequestId , + &((double *) recvbuf)[recvoffset] , + recvcount , recvtype , target , RecvRequestId ) ; + } + } + else { + if ( recvtype == MPI_INT ) { + sts = _MPIAccess->SendRecv( &((double *) sendbuf)[sendoffset] , sendcount , + sendtype , target , SendRequestId , + &((int *) recvbuf)[recvoffset] , + recvcount , recvtype , target , RecvRequestId ) ; + } + else { + cout << "SendRecv" << _MyRank << " target " << target << " sendbuf " + << &((double *) sendbuf)[sendoffset] << " sendcount " << sendcount + << " recvbuf " << &((double *) recvbuf)[recvoffset] << " recvcount " + << recvcount << endl ; + sts = _MPIAccess->SendRecv( &((double *) sendbuf)[sendoffset] , sendcount , + sendtype , target , SendRequestId , + &((double *) recvbuf)[recvoffset] , + recvcount , recvtype , target , RecvRequestId ) ; + } + } + } + return sts ; +} + +/*! +Send sendcount datas from sendbuf[offset] with type sendtype to all targets of IntraCommunicator +Receive recvcount datas to recvbuf[offset] with type recvtype from all targets of IntraCommunicator + +*/ +int MPI_AccessDEC::AllToAll( void* sendbuf, int sendcount, MPI_Datatype sendtype , + void* recvbuf, int recvcount, MPI_Datatype recvtype ) { + if ( _TimeInterpolator ) { + return AllToAllTime( sendbuf, sendcount, sendtype , recvbuf, recvcount, recvtype ) ; + } + int sts ; + int target ; + int sendoffset = 0 ; + int recvoffset = 0 ; + int SendRequestId ; + int RecvRequestId ; + SendBuffStruct * aSendDataStruct = NULL ; + cout << "AllToAll" << _MyRank << " sendbuf " << sendbuf << " recvbuf " << recvbuf << endl ; + if ( _Asynchronous && sendbuf ) { + aSendDataStruct = new SendBuffStruct ; + aSendDataStruct->SendBuffer = sendbuf ; + aSendDataStruct->Counter = 0 ; + aSendDataStruct->DataType = sendtype ; + } + for ( target = 0 ; target < _GroupSize ; target++ ) { + sts = SendRecv( sendbuf , sendcount , sendoffset , sendtype , + recvbuf , recvcount , recvoffset , recvtype , + target , SendRequestId , RecvRequestId ) ; + if ( _Asynchronous && sendbuf && sendcount ) { + aSendDataStruct->Counter += 1 ; + (*_MapOfSendBuffers)[ SendRequestId ] = aSendDataStruct ; + } + sendoffset += sendcount ; + recvoffset += recvcount ; + } + if ( !_Asynchronous && sendbuf ) { + cout << "AllToAll" << _MyRank << " free of sendbuf " << sendbuf << endl ; +// free( sendbuf ) ; + if ( sendtype == MPI_INT ) { + delete [] (int *) sendbuf ; + } + else { + delete [] (double *) sendbuf ; + } + } + return sts ; +} + +/*! +Send sendcounts[target] datas from sendbuf[sdispls[target]] with type sendtype to all targets of IntraCommunicator +Receive recvcounts[target] datas to recvbuf[rdispls[target]] with type recvtype from all targets of IntraCommunicator + +*/ +int MPI_AccessDEC::AllToAllv( void* sendbuf, int* sendcounts, int* sdispls, + MPI_Datatype sendtype , + void* recvbuf, int* recvcounts, int* rdispls, + MPI_Datatype recvtype ) { + if ( _TimeInterpolator ) { + return AllToAllvTime( sendbuf, sendcounts, sdispls, sendtype , + recvbuf, recvcounts, rdispls, recvtype ) ; + } + int sts ; + int target ; + int SendRequestId ; + int RecvRequestId ; + SendBuffStruct * aSendDataStruct = NULL ; + cout << "AllToAllv" << _MyRank << " sendbuf " << sendbuf << " recvbuf " << recvbuf + << endl ; + if ( _Asynchronous && sendbuf ) { + aSendDataStruct = new SendBuffStruct ; + aSendDataStruct->SendBuffer = sendbuf ; + aSendDataStruct->Counter = 0 ; + aSendDataStruct->DataType = sendtype ; + } + for ( target = 0 ; target < _GroupSize ; target++ ) { + if ( sendcounts[target] || recvcounts[target] ) { + sts = SendRecv( sendbuf , sendcounts[target] , sdispls[target] , sendtype , + recvbuf , recvcounts[target] , rdispls[target] , recvtype , + target , SendRequestId , RecvRequestId ) ; + if ( _Asynchronous && sendbuf && sendcounts[target]) { + aSendDataStruct->Counter += 1 ; + (*_MapOfSendBuffers)[ SendRequestId ] = aSendDataStruct ; + } + } + } + if ( !_Asynchronous && sendbuf ) { + cout << "AllToAllv" << _MyRank << " free of sendbuf " << sendbuf << endl ; +// free( sendbuf ) ; + if ( sendtype == MPI_INT ) { + delete [] (int *) sendbuf ; + } + else { + delete [] (double *) sendbuf ; + } + } + return sts ; +} + +/*! +Send a TimeMessage to all targets of IntraCommunicator +Receive the TimeMessages from targets of IntraCommunicator if necessary. + +Send sendcount datas from sendbuf[offset] with type sendtype to all targets of IntraCommunicator +Returns recvcount datas to recvbuf[offset] with type recvtype after an interpolation +with datas received from all targets of IntraCommunicator. + +*/ +int MPI_AccessDEC::AllToAllTime( void* sendbuf, int sendcount , MPI_Datatype sendtype , + void* recvbuf, int recvcount , MPI_Datatype recvtype ) { + int sts ; + int target ; + int sendoffset = 0 ; + int SendTimeRequestId ; + int SendRequestId ; + + if ( _TimeInterpolator == NULL ) { + return MPI_ERR_OTHER ; + } + +//Free of SendBuffers + CheckSent() ; + +//DoSend : Time + SendBuff + SendBuffStruct * aSendTimeStruct = NULL ; + SendBuffStruct * aSendDataStruct = NULL ; + cout << "AllToAllTime" << _MyRank << " sendbuf " << sendbuf << " recvbuf " << recvbuf + << endl ; + if ( sendbuf && sendcount ) { + TimeMessage * aSendTimeMessage = new TimeMessage ; + if ( _Asynchronous ) { + aSendTimeStruct = new SendBuffStruct ; + aSendTimeStruct->SendBuffer = aSendTimeMessage ; + aSendTimeStruct->Counter = 0 ; + aSendTimeStruct->DataType = _MPIAccess->TimeType() ; + aSendDataStruct = new SendBuffStruct ; + aSendDataStruct->SendBuffer = sendbuf ; + aSendDataStruct->Counter = 0 ; + aSendDataStruct->DataType = sendtype ; + } + aSendTimeMessage->time = _t ; + aSendTimeMessage->deltatime = _dt ; + for ( target = 0 ; target < _GroupSize ; target++ ) { + sts = Send( aSendTimeMessage , 1 , 0 , _MPIAccess->TimeType() , target , + SendTimeRequestId ) ; +// cout << "Send" << _MyRank << " --> target " << target << " _t " << _t +// << " SendTimeRequestId " << SendTimeRequestId << endl ; +// << " aSendTimeStruct/aSendTimeMessage " << aSendTimeStruct << "/" +// << aSendTimeMessage << endl ; + sts = Send( sendbuf , sendcount , sendoffset , sendtype , target , SendRequestId ) ; +// cout << "Send" << _MyRank << " --> target " << target << " SendRequestId " +// << SendRequestId << " aSendDataStruct/sendbuf " << aSendDataStruct << "/" +// << sendbuf << endl ; + if ( _Asynchronous ) { + aSendTimeStruct->Counter += 1 ; + (*_MapOfSendBuffers)[ SendTimeRequestId ] = aSendTimeStruct ; + aSendDataStruct->Counter += 1 ; + (*_MapOfSendBuffers)[ SendRequestId ] = aSendDataStruct ; + } + sendoffset += sendcount ; + } + if ( !_Asynchronous ) { + cout << "SynchronousAllToAllTime" << _MyRank << " free of SendTimeMessage & sendbuf " + << sendbuf << endl ; + delete aSendTimeMessage ; +// free( sendbuf ) ; + if ( sendtype == MPI_INT ) { + delete [] (int *) sendbuf ; + } + else { + delete [] (double *) sendbuf ; + } + } + } + +//CheckTime + DoRecv + if ( recvbuf && recvcount ) { + for ( target = 0 ; target < _GroupSize ; target++ ) { + int recvsize = recvcount*_MPIAccess->Extent( recvtype ) ; +// bool OutOfTime ; +// CheckTime( recvcount , recvsize , recvtype , target , OutOfTime) ; + CheckTime( recvcount , recvsize , recvtype , target ) ; + //cout << "AllToAllTime" << _MyRank << " memcpy to recvbuf " << recvbuf + // << "+target" << target << "*" << recvsize << " bytes" << endl ; +//=========================================================================== +//TODO : it is assumed actually that we have only 1 timestep before nad after +//=========================================================================== + if ( _TimeInterpolator && (*_TimeMessages)[target][0].time != -1 ) { +// if ( _TimeInterpolator && !OutOfTime ) { + if ( _OutOfTime ) { + cout << " =====================================================" << endl + << "Recv" << _MyRank << " <-- target " << target << " t0 " + << (*_TimeMessages)[target][0].time << " < t1 " + << (*_TimeMessages)[target][1].time << " < t* " << _t << endl + << " =====================================================" << endl ; + } + if ( recvtype == MPI_INT ) { + _TimeInterpolator->DoInterp( (*_TimeMessages)[target][0].time, + (*_TimeMessages)[target][1].time, _t, recvcount , + _nStepBefore, _nStepAfter, + (int **) &(*_DataMessages)[target][0], + (int **) &(*_DataMessages)[target][1], + &((int *)recvbuf)[target*recvcount] ) ; + } + else { + _TimeInterpolator->DoInterp( (*_TimeMessages)[target][0].time, + (*_TimeMessages)[target][1].time, _t, recvcount , + _nStepBefore, _nStepAfter, + (double **) &(*_DataMessages)[target][0], + (double **) &(*_DataMessages)[target][1], + &((double *)recvbuf)[target*recvcount] ) ; + } + } + else { + char * buffdest = (char *) recvbuf ; + char * buffsrc = (char *) (*_DataMessages)[target][1] ; + memcpy( &buffdest[target*recvsize] , buffsrc , recvsize ) ; + } + } + } + + return sts ; +} + +int MPI_AccessDEC::AllToAllvTime( void* sendbuf, int* sendcounts, int* sdispls, + MPI_Datatype sendtype , + void* recvbuf, int* recvcounts, int* rdispls, + MPI_Datatype recvtype ) { + int sts ; + int target ; + int SendTimeRequestId ; + int SendRequestId ; + + if ( _TimeInterpolator == NULL ) { + return MPI_ERR_OTHER ; + } + +//Free of SendBuffers + CheckSent() ; + +//DoSend : Time + SendBuff + SendBuffStruct * aSendTimeStruct = NULL ; + SendBuffStruct * aSendDataStruct = NULL ; + cout << "AllToAllvTime" << _MyRank << " sendbuf " << sendbuf << " recvbuf " << recvbuf + << endl ; + if ( sendbuf ) { + TimeMessage * aSendTimeMessage = new TimeMessage ; + if ( _Asynchronous ) { + aSendTimeStruct = new SendBuffStruct ; + aSendTimeStruct->SendBuffer = aSendTimeMessage ; + aSendTimeStruct->Counter = 0 ; + aSendTimeStruct->DataType = _MPIAccess->TimeType() ; + aSendDataStruct = new SendBuffStruct ; + aSendDataStruct->SendBuffer = sendbuf ; + aSendDataStruct->Counter = 0 ; + aSendDataStruct->DataType = sendtype ; + } + aSendTimeMessage->time = _t ; + aSendTimeMessage->deltatime = _dt ; + for ( target = 0 ; target < _GroupSize ; target++ ) { + if ( sendcounts[target] ) { + sts = Send( aSendTimeMessage , 1 , 0 , _MPIAccess->TimeType() , target , + SendTimeRequestId ) ; +// cout << "Send" << _MyRank << " --> target " << target << " _t " << _t +// << " SendTimeRequestId " << SendTimeRequestId << endl ; +// << " aSendTimeStruct/aSendTimeMessage " << aSendTimeStruct << "/" +// << aSendTimeMessage << endl ; + //int i ; + //cout << "Send" << _MyRank << " --> target " << target + // << "sendcount" << sendcounts[target] << endl ; + //for ( i = 0 ; i < sendcounts[target] ; i++ ) { + // cout << " " << ((double *) sendbuf)[sdispls[target]+i] ; + //} + //cout << endl ; + sts = Send( sendbuf , sendcounts[target] , sdispls[target] , sendtype , target , + SendRequestId ) ; + //cout << "Send" << _MyRank << " --> target " << target << " SendRequestId " + // << SendRequestId << " aSendDataStruct/sendbuf " << aSendDataStruct << "/" + // << sendbuf << " sendcount " << sendcounts[target] << " sdispls " + // << sdispls[target] << endl ; + //int i ; + //cout << "Send" << _MyRank << " --> target " << target << " count " + // << sendcounts[target] << endl ; + //for ( i = 0 ; i < sendcounts[target] ; i++ ) { + // cout << " " << ((int *)sendbuf)[sdispls[target]+i] ; + //} + //cout << endl ; + if ( _Asynchronous ) { + aSendTimeStruct->Counter += 1 ; + (*_MapOfSendBuffers)[ SendTimeRequestId ] = aSendTimeStruct ; + //cout << "TimeSent" << _MyRank << " Request " << SendTimeRequestId + // << " _MapOfSendBuffers->SendBuffer " + // << (*_MapOfSendBuffers)[ SendTimeRequestId ]->SendBuffer + // << " Counter " << (*_MapOfSendBuffers)[ SendTimeRequestId ]->Counter + // << endl ; + aSendDataStruct->Counter += 1 ; + (*_MapOfSendBuffers)[ SendRequestId ] = aSendDataStruct ; + //cout << "DataSent" << _MyRank << " Request " << SendRequestId + // << " _MapOfSendBuffers->SendBuffer " + // << (*_MapOfSendBuffers)[ SendRequestId ]->SendBuffer + // << " Counter " << (*_MapOfSendBuffers)[ SendRequestId ]->Counter + // << endl ; + } + } + } + if ( !_Asynchronous ) { + //cout << "SynchronousAllToAllv" << _MyRank << " free of SendTimeMessage & sendbuf " + // << sendbuf << endl ; + delete aSendTimeMessage ; +// free( sendbuf ) ; + if ( sendtype == MPI_INT ) { + delete [] (int *) sendbuf ; + } + else { + delete [] (double *) sendbuf ; + } + } + } + +//CheckTime + DoRecv + if ( recvbuf ) { + for ( target = 0 ; target < _GroupSize ; target++ ) { + if ( recvcounts[target] ) { + int recvsize = recvcounts[target]*_MPIAccess->Extent( recvtype ) ; +// bool OutOfTime ; +// CheckTime( recvcounts[target] , recvsize , recvtype , target , OutOfTime ) ; + CheckTime( recvcounts[target] , recvsize , recvtype , target ) ; + //cout << "AllToAllvTime" << _MyRank << " memcpy to recvbuf " << recvbuf + // << "+target" << target << "*" << recvsize << " bytes" << endl ; +//=========================================================================== +//TODO : it is assumed actually that we have only 1 timestep before nad after +//=========================================================================== + if ( _TimeInterpolator && (*_TimeMessages)[target][0].time != -1 ) { +// if ( _TimeInterpolator && !OutOfTime ) { + if ( _OutOfTime ) { + cout << " =====================================================" << endl + << "Recv" << _MyRank << " <-- target " << target << " t0 " + << (*_TimeMessages)[target][0].time << " < t1 " + << (*_TimeMessages)[target][1].time << " < t* " << _t << endl + << " =====================================================" << endl ; + } + if ( recvtype == MPI_INT ) { + _TimeInterpolator->DoInterp( (*_TimeMessages)[target][0].time, + (*_TimeMessages)[target][1].time, _t, + recvcounts[target] , _nStepBefore, _nStepAfter, + (int **) &(*_DataMessages)[target][0], + (int **) &(*_DataMessages)[target][1], + &((int *)recvbuf)[rdispls[target]] ) ; + } + else { + _TimeInterpolator->DoInterp( (*_TimeMessages)[target][0].time, + (*_TimeMessages)[target][1].time, _t, + recvcounts[target] , _nStepBefore, _nStepAfter, + (double **) &(*_DataMessages)[target][0], + (double **) &(*_DataMessages)[target][1], + &((double *)recvbuf)[rdispls[target]] ) ; + } + } + else { + char * buffdest = (char *) recvbuf ; + char * buffsrc = (char *) (*_DataMessages)[target][1] ; + memcpy( &buffdest[rdispls[target]*_MPIAccess->Extent( recvtype )] , buffsrc , + recvsize ) ; + } + //int i ; + //cout << "Recv" << _MyRank << " --> target " << target << " recvbuf " << recvbuf + // << " recvbuf " << recvbuf << " &recvbuf[rdispls[target]] " + // << (&((int *) recvbuf)[rdispls[target]]) << endl ; + //for ( i = 0 ; i < recvcounts[target] ; i++ ) { + // cout << " " << ((int *) (*_DataMessages)[target][1])[i] ; + //} + //cout << endl ; + //cout << "Recv" << _MyRank << " <-- target " << target << " count " + // << recvcounts[target] << endl ; + //for ( i = 0 ; i < recvcounts[target] ; i++ ) { + // cout << " " << (&((int *) recvbuf)[rdispls[target]])[i] ; + //} + //cout << endl ; + } + } + } + + return sts ; +} + +int MPI_AccessDEC::CheckTime( int recvcount , int recvsize , MPI_Datatype recvtype , + int target ) { +// int target , bool &OutOfTime ) { + //cout << "CheckTime" << _MyRank << " time " << _t << " deltatime " << _dt << endl ; + + int sts ; + int RecvTimeRequestId ; + int RecvDataRequestId ; +//Pour l'instant on cherche _TimeMessages[target][0] < _t <= _TimeMessages[target][1] +//=========================================================================== +//TODO : it is assumed actually that we have only 1 timestep before and after +// instead of _nStepBefore and _nStepAfter ... +//=========================================================================== + _DataMessagesType = recvtype ; + _OutOfTime = false ; +//Actually we need 1 timestep before and after +// if ( _steptime <= 1 ) { + if ( (*_TimeMessages)[target][1].time == -1 ) { + (*_TimeMessages)[target][0] = (*_TimeMessages)[target][1] ; + sts = Recv( &(*_TimeMessages)[target][1] , 1 , _MPIAccess->TimeType() , + target , RecvTimeRequestId ) ; + (*_DataMessages)[target][0] = (*_DataMessages)[target][1] ; +// (*_DataMessages)[target][1] = malloc( recvsize ) ; + if ( recvtype == MPI_INT ) { + (*_DataMessages)[target][1] = new int[recvcount] ; + } + else { + (*_DataMessages)[target][1] = new double[recvcount] ; + } + sts = Recv( (*_DataMessages)[target][1] , recvcount , recvtype , target , + RecvDataRequestId ) ; +// _OutOfTime = true ; + } +// if ( _steptime > 0 ) { + else { + while ( _t > (*_TimeMessages)[target][1].time && + (*_TimeMessages)[target][1].deltatime != 0 ) { + //cout << "CheckTime" << _MyRank << " TimeMessage target " << target << " _t " + // << _t << " > " << (*_TimeMessages)[target][1].time << " et " + // << (*_TimeMessages)[target][1].deltatime + // << " != 0 ==> Recv" << endl ; + (*_TimeMessages)[target][0] = (*_TimeMessages)[target][1] ; + sts = Recv( &(*_TimeMessages)[target][1] , 1 , _MPIAccess->TimeType() , + target , RecvTimeRequestId ) ; +// free( (*_DataMessages)[target][0] ) ; + if ( recvtype == MPI_INT ) { + delete [] (int *) (*_DataMessages)[target][0] ; + } + else { + delete [] (double *) (*_DataMessages)[target][0] ; + } + (*_DataMessages)[target][0] = (*_DataMessages)[target][1] ; +// (*_DataMessages)[target][1] = malloc( recvsize ) ; + if ( recvtype == MPI_INT ) { + (*_DataMessages)[target][1] = new int[recvcount] ; + } + else { + (*_DataMessages)[target][1] = new double[recvcount] ; + } + sts = Recv( (*_DataMessages)[target][1] , recvcount , recvtype , target , + RecvDataRequestId ) ; + } + + if ( _t > (*_TimeMessages)[target][0].time && + _t <= (*_TimeMessages)[target][1].time ) { +//Ok + } + else { + _OutOfTime = true ; + } + } + + //cout << "CheckTime" << _MyRank << " TimeMessage target " << target << " Time " + // << (*_TimeMessages)[target][1].time << " DataMessage" ; + //int i ; + //void * buff = (*_DataMessages)[target][1] ; + //for ( i = 0 ; i < recvcount ; i++ ) { + // cout << " " << ((int *) buff)[i] ; + //} + //cout << endl ; + return sts ; +} + +int MPI_AccessDEC::CheckSent(bool WithWait) { + int sts = MPI_SUCCESS ; + int flag = WithWait ; + int size = _MPIAccess->SendRequestIdsSize() ; + int * ArrayOfSendRequests = new int[ size ] ; + int nSendRequest = _MPIAccess->SendRequestIds( size , ArrayOfSendRequests ) ; + //cout << "CheckSent" << _MyRank << " nSendRequest " << nSendRequest << " :" << endl ; + int i ; + for ( i = 0 ; i < nSendRequest ; i++ ) { + if ( WithWait ) { + sts = _MPIAccess->Wait( ArrayOfSendRequests[i] ) ; + } + else { + sts = _MPIAccess->Test( ArrayOfSendRequests[i] , flag ) ; + } + if ( flag ) { + _MPIAccess->DeleteRequest( ArrayOfSendRequests[i] ) ; +// cout << "CheckSent" << _MyRank << " " << i << "./" << nSendRequest +// << " SendRequestId " << ArrayOfSendRequests[i] +// << " flag " << flag << " SendBuffStruct/SendBuffer " +// << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ] << "/" +// << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer +// << " Counter " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter +// << " TimeBuf " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->TimeBuf +// << endl ; + (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter -= 1 ; +// if ( (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->TimeBuf ) { +// cout << "CheckTimeSent" << _MyRank << " Request " ; +// } +// else { +// cout << "CheckDataSent" << _MyRank << " Request " ; +// } +// cout << ArrayOfSendRequests[i] +// << " _MapOfSendBuffers->SendBuffer " +// << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer +// << " Counter " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter +// << endl ; + if ( (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter == 0 ) { +// cout << "CheckSent" << _MyRank << " SendRequestId " << ArrayOfSendRequests[i] +// << " Counter " << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->Counter +// << " flag " << flag << " SendBuffer " +// << (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer +// << " deleted. Erase in _MapOfSendBuffers :" << endl ; + if ( (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->DataType == + _MPIAccess->TimeType() ) { + delete (TimeMessage * ) (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer ; + } + else { +// free( (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer ) ; + if ( (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->DataType == MPI_INT ) { + delete [] (int *) (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer ; + } + else { + delete [] (double *) (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ]->SendBuffer ; + } + } + delete (*_MapOfSendBuffers)[ ArrayOfSendRequests[i] ] ; + } +// cout << "CheckSent" << _MyRank << " Erase in _MapOfSendBuffers SendRequestId " +// << ArrayOfSendRequests[i] << endl ; + (*_MapOfSendBuffers).erase( ArrayOfSendRequests[i] ) ; + } + } + delete [] ArrayOfSendRequests ; + return sts ; +} + +int MPI_AccessDEC::CheckFinalRecv() { + int target ; + for ( target = 0 ; target < _GroupSize ; target++ ) { + if ( (*_DataMessages)[target][0] != NULL ) { + if ( _DataMessagesType == MPI_INT ) { + delete [] (int *) (*_DataMessages)[target][0] ; + } + else { + delete [] (double *) (*_DataMessages)[target][0] ; + } + } + if ( (*_DataMessages)[target][1] != NULL ) { + if ( _DataMessagesType == MPI_INT ) { + delete [] (int *) (*_DataMessages)[target][1] ; + } + else { + delete [] (double *) (*_DataMessages)[target][1] ; + } + } + } + return _MPIAccess->CancelAll() ; +} + +ostream & operator<< (ostream & f ,const TimeInterpolationMethod & interpolationmethod ) { + switch (interpolationmethod) { + case WithoutTimeInterp : + f << " WithoutTimeInterpolation "; + break; + case LinearTimeInterp : + f << " LinearTimeInterpolation "; + break; + default : + f << " UnknownTimeInterpolation "; + break; + } + + return f; +} + + +} + diff --git a/src/ParaMEDMEM/MPI_AccessDEC.hxx b/src/ParaMEDMEM/MPI_AccessDEC.hxx new file mode 100644 index 000000000..6d27f9dd0 --- /dev/null +++ b/src/ParaMEDMEM/MPI_AccessDEC.hxx @@ -0,0 +1,201 @@ +#ifndef MPI_ACESSDEC_HXX_ +#define MPI_ACESSDEC_HXX_ + +#include +#include + +#include "MPI_Access.hxx" +#include "DEC.hxx" +#include "LinearTimeInterpolator.hxx" + +namespace ParaMEDMEM { + + typedef enum{WithoutTimeInterp,LinearTimeInterp} TimeInterpolationMethod; + + class MPI_AccessDEC { + + public: + MPI_AccessDEC( const ProcessorGroup& local_group, const ProcessorGroup& distant_group, + bool Asynchronous = true ) ; + //MPI_AccessDEC( ProcessorGroup& local_group, ProcessorGroup& distant_group, + // TimeInterpolator * aTimeInterpolator , + // bool Asynchronous = true ) ; + virtual ~MPI_AccessDEC(); + MPI_Access * MPIAccess() ; + const MPI_Comm* GetComm() ; + + void Asynchronous( bool Asynchronous = true ) ; + void SetTimeInterpolator( TimeInterpolationMethod anInterp , double InterpPrecision=0 , + int nStepBefore=1, int nStepAfter=1 ) ; + + void SetTime( double t ) ; + void SetTime( double t , double dt ) ; + bool OutOfTime() ; + + int Send( void* sendbuf, int sendcount , MPI_Datatype sendtype , int target ) ; + int Recv( void* recvbuf, int recvcount , MPI_Datatype recvtype , int target ) ; + int Recv( void* recvbuf, int recvcount , MPI_Datatype recvtype , int target , + int RecvRequestId , bool Asynchronous=false ) ; + int SendRecv( void* sendbuf, int sendcount , MPI_Datatype sendtype , + void* recvbuf, int recvcount , MPI_Datatype recvtype , int target ) ; + + int AllToAll( void* sendbuf, int sendcount, MPI_Datatype sendtype , + void* recvbuf, int recvcount, MPI_Datatype recvtype ) ; + int AllToAllv( void* sendbuf, int* sendcounts, int* sdispls, MPI_Datatype sendtype , + void* recvbuf, int* recvcounts, int* rdispls, MPI_Datatype recvtype ) ; + + int AllToAllTime( void* sendbuf, int sendcount , MPI_Datatype sendtype , + void* recvbuf, int recvcount , MPI_Datatype recvtype ) ; + int AllToAllvTime( void* sendbuf, int* sendcounts, int* sdispls, + MPI_Datatype sendtype , + void* recvbuf, int* recvcounts, int* rdispls, + MPI_Datatype recvtype ) ; +// int CheckTime( int recvcount , int recvsize , MPI_Datatype recvtype , int target , +// bool &OutOfTime ) ; + int CheckTime( int recvcount , int recvsize , MPI_Datatype recvtype , int target ) ; + int CheckSent(bool WithWait=false) ; + int CheckFinalSent() { + return CheckSent( true ) ; } ; + int CheckFinalRecv() ; + + protected: + int Send( void* sendbuf, int sendcount , int sendoffset , MPI_Datatype sendtype , + int target ) ; + int Recv( void* recvbuf, int recvcount , int recvoffset , MPI_Datatype recvtype , + int target ) ; + + int Send( void* sendbuf, int sendcount , int sendoffset , MPI_Datatype sendtype , + int target, int &SendRequestId ) ; + int Recv( void* recvbuf, int recvcount , int recvoffset , MPI_Datatype recvtype , + int target, int &RecvRequestId ) ; + int SendRecv( void* sendbuf, int sendcount , int sendoffset , + MPI_Datatype sendtype , + void* recvbuf, int recvcount , int recvoffset , + MPI_Datatype recvtype , int target , + int &SendRequestId ,int &RecvRequestId ) ; + + private : + bool _Asynchronous ; + const ProcessorGroup * _local_group ; + const ProcessorGroup * _distant_group ; + MPIProcessorGroup * _MPI_union_group ; + + TimeInterpolator * _TimeInterpolator ; + int _nStepBefore ; + int _nStepAfter ; + + int _MyRank ; + int _GroupSize ; + MPI_Access * _MPIAccess ; + + double _t ; + double _dt ; + bool _OutOfTime ; + +// TimeMessages from each target _TimeMessages[target][Step] : TimeMessage + vector< vector< TimeMessage > > *_TimeMessages ; +// Corresponding DataMessages from each target _DataMessages[target][~TimeStep] + MPI_Datatype _DataMessagesType ; + vector< vector< void * > > *_DataMessages ; + + typedef struct { void * SendBuffer ; + int Counter ; + MPI_Datatype DataType ; } SendBuffStruct ; +// int RequestId -> SendBuffStruct : + map< int , SendBuffStruct * > *_MapOfSendBuffers ; + + }; + + inline MPI_Access * MPI_AccessDEC::MPIAccess() { + return _MPIAccess ; } ; + inline const MPI_Comm* MPI_AccessDEC::GetComm() { + return _MPI_union_group->getComm() ; } ; + + inline void MPI_AccessDEC::Asynchronous( bool Asynchronous ) { + _Asynchronous = Asynchronous ; } ; + + inline void MPI_AccessDEC::SetTime( double t ) { + _t = t ; _dt = -1 ; } ; + inline void MPI_AccessDEC::SetTime( double t , double dt ) { + _t = t ; _dt = dt ; } ; + inline bool MPI_AccessDEC::OutOfTime() { + return _OutOfTime ; } ; + + inline int MPI_AccessDEC::Send( void* sendbuf, int sendcount , MPI_Datatype sendtype , + int target ) { + int SendRequestId ; + int sts ; + if ( _Asynchronous ) { + sts = _MPIAccess->ISend( sendbuf , sendcount , sendtype , target , + SendRequestId ) ; + } + else { + sts = _MPIAccess->Send( sendbuf , sendcount , sendtype , target , + SendRequestId ) ; + if ( sts == MPI_SUCCESS ) { + free( sendbuf ) ; + } + } + return sts ; } ; + inline int MPI_AccessDEC::Recv( void* recvbuf, int recvcount , MPI_Datatype recvtype , + int target ) { + int RecvRequestId ; + int sts ; + if ( _Asynchronous ) { + sts = _MPIAccess->IRecv( recvbuf , recvcount , recvtype , target , + RecvRequestId ) ; + } + else { + sts = _MPIAccess->Recv( recvbuf , recvcount , recvtype , target , + RecvRequestId ) ; + } + return sts ; } ; + inline int MPI_AccessDEC::Recv( void* recvbuf, int recvcount , MPI_Datatype recvtype , + int target , int RecvRequestId , bool Asynchronous ) { + int sts ; + if ( Asynchronous ) { + sts = _MPIAccess->IRecv( recvbuf , recvcount , recvtype , target , + RecvRequestId ) ; + } + else { + sts = _MPIAccess->Recv( recvbuf , recvcount , recvtype , target , + RecvRequestId ) ; + } + return sts ; } ; + inline int MPI_AccessDEC::SendRecv( void* sendbuf, int sendcount , MPI_Datatype sendtype , + void* recvbuf, int recvcount , MPI_Datatype recvtype , + int target ) { + int SendRequestId ; + int RecvRequestId ; + int sts ; + if ( _Asynchronous ) { + sts = _MPIAccess->ISendRecv( sendbuf , sendcount , sendtype , target , + SendRequestId , + recvbuf , recvcount , recvtype , target , + RecvRequestId ) ; + } + else { + sts = _MPIAccess->SendRecv( sendbuf , sendcount , sendtype , target , + SendRequestId , + recvbuf , recvcount , recvtype , target , + RecvRequestId ) ; + } + return sts ; } ; + + inline int MPI_AccessDEC::Send( void* sendbuf, int sendcount , int sendoffset , + MPI_Datatype sendtype , int target ) { + int SendRequestId ; + return Send( sendbuf , sendcount , sendoffset , sendtype , target , + SendRequestId ) ; } ; + inline int MPI_AccessDEC::Recv( void* recvbuf, int recvcount , int recvoffset , + MPI_Datatype recvtype , int target ) { + int RecvRequestId ; + return Recv( recvbuf , recvcount , recvoffset , recvtype , target , + RecvRequestId ) ; } ; + + +ostream & operator<< (ostream &,const TimeInterpolationMethod &); + +} + +#endif diff --git a/src/ParaMEDMEM/Makefile.in b/src/ParaMEDMEM/Makefile.in index 6242a23c1..777fe17cd 100644 --- a/src/ParaMEDMEM/Makefile.in +++ b/src/ParaMEDMEM/Makefile.in @@ -66,6 +66,10 @@ ICoCoField.hxx \ ICoCoMEDField.hxx \ ICoCoTrioField.hxx \ BBTree.H\ +MPI_Access.hxx \ +MPI_AccessDEC.hxx \ +TimeInterpolator.hxx \ +LinearTimeInterpolator.hxx # Libraries targets @@ -91,6 +95,10 @@ UnstructuredParaSUPPORT.cxx\ ElementLocator.cxx\ ExplicitTopology.cxx\ ICoCoMEDField.cxx +MPI_Access.cxx \ +MPI_AccessDEC.cxx \ +TimeInterpolator.cxx \ +LinearTimeInterpolator.cxx # Executables targets BIN = diff --git a/src/ParaMEDMEM/MxN_Mapping.cxx b/src/ParaMEDMEM/MxN_Mapping.cxx index 729f1bd8c..538b46291 100644 --- a/src/ParaMEDMEM/MxN_Mapping.cxx +++ b/src/ParaMEDMEM/MxN_Mapping.cxx @@ -9,6 +9,7 @@ namespace ParaMEDMEM MxN_Mapping::MxN_Mapping(const ProcessorGroup& local_group, const ProcessorGroup& distant_group) : _union_group(local_group.fuse(distant_group)) { + _accessDEC = new MPI_AccessDEC(local_group,distant_group); _send_proc_offsets.resize(_union_group->size()+1,0); _recv_proc_offsets.resize(_union_group->size()+1,0); @@ -17,6 +18,7 @@ MxN_Mapping::MxN_Mapping(const ProcessorGroup& local_group, const ProcessorGroup MxN_Mapping::~MxN_Mapping() { delete _union_group; + delete _accessDEC ; } @@ -101,74 +103,73 @@ void MxN_Mapping::prepareSendRecv() delete[] recvdispls; } - -/*! Exchanging field data between two groups of processes - * - * \param field MEDMEM field containing the values to be sent - * - * The ids that were defined by addElementFromSource method - * are sent. - */ -void MxN_Mapping::sendRecv(MEDMEM::FIELD& field) -{ - CommInterface comm_interface=_union_group->getCommInterface(); - const MPIProcessorGroup* group = static_cast(_union_group); +// /*! Exchanging field data between two groups of processes +// * +// * \param field MEDMEM field containing the values to be sent +// * +// * The ids that were defined by addElementFromSource method +// * are sent. +// */ +// void MxN_Mapping::sendRecv(MEDMEM::FIELD& field) +// { +// CommInterface comm_interface=_union_group->getCommInterface(); +// const MPIProcessorGroup* group = static_cast(_union_group); - int nbcomp=field.getNumberOfComponents(); - double* sendbuf=0; - double* recvbuf=0; - if (_sending_ids.size() >0) - sendbuf = new double[_sending_ids.size()*nbcomp]; - if (_recv_ids.size()>0) - recvbuf = new double[_recv_ids.size()*nbcomp]; +// int nbcomp=field.getNumberOfComponents(); +// double* sendbuf=0; +// double* recvbuf=0; +// if (_sending_ids.size() >0) +// sendbuf = new double[_sending_ids.size()*nbcomp]; +// if (_recv_ids.size()>0) +// recvbuf = new double[_recv_ids.size()*nbcomp]; - int* sendcounts = new int[_union_group->size()]; - int* senddispls=new int[_union_group->size()]; - int* recvcounts=new int[_union_group->size()]; - int* recvdispls=new int[_union_group->size()]; +// int* sendcounts = new int[_union_group->size()]; +// int* senddispls=new int[_union_group->size()]; +// int* recvcounts=new int[_union_group->size()]; +// int* recvdispls=new int[_union_group->size()]; - for (int i=0; i< _union_group->size(); i++) - { - sendcounts[i]=nbcomp*(_send_proc_offsets[i+1]-_send_proc_offsets[i]); - senddispls[i]=nbcomp*(_send_proc_offsets[i]); - recvcounts[i]=nbcomp*(_recv_proc_offsets[i+1]-_recv_proc_offsets[i]); - recvdispls[i]=nbcomp*(_recv_proc_offsets[i]); - } - //building the buffer of the elements to be sent - vector offsets = _send_proc_offsets; - for (int i=0; i<_sending_ids.size();i++) - { - int iproc = _sending_ids[i].first; - for (int icomp=0; icompsize(); i++) +// { +// sendcounts[i]=nbcomp*(_send_proc_offsets[i+1]-_send_proc_offsets[i]); +// senddispls[i]=nbcomp*(_send_proc_offsets[i]); +// recvcounts[i]=nbcomp*(_recv_proc_offsets[i+1]-_recv_proc_offsets[i]); +// recvdispls[i]=nbcomp*(_recv_proc_offsets[i]); +// } +// //building the buffer of the elements to be sent +// vector offsets = _send_proc_offsets; +// for (int i=0; i<_sending_ids.size();i++) +// { +// int iproc = _sending_ids[i].first; +// for (int icomp=0; icompgetComm(); - comm_interface.allToAllV(sendbuf, sendcounts, senddispls, MPI_INT, - recvbuf, recvcounts, recvdispls, MPI_INT, - *comm); +// //communication phase +// const MPI_Comm* comm = group->getComm(); +// comm_interface.allToAllV(sendbuf, sendcounts, senddispls, MPI_INT, +// recvbuf, recvcounts, recvdispls, MPI_INT, +// *comm); - //setting the received values in the field - double* recvptr; - for (int i=0; i< _recv_proc_offsets[_union_group->size()]; i++) - { - for (int icomp=0; icompsize()]; i++) +// { +// for (int icomp=0; icomp& field) cons } //communication phase - const MPI_Comm* comm = group->getComm(); - comm_interface.allToAllV(sendbuf, sendcounts, senddispls, MPI_DOUBLE, - recvbuf, recvcounts, recvdispls, MPI_DOUBLE, - *comm); - + switch (_allToAllMethod) { + case Native: + const MPI_Comm* comm = group->getComm(); + comm_interface.allToAllV(sendbuf, sendcounts, senddispls, MPI_DOUBLE, + recvbuf, recvcounts, recvdispls, MPI_DOUBLE, + *comm); + break; + case PointToPoint: + _accessDEC->AllToAllv(sendbuf, sendcounts, senddispls, MPI_DOUBLE, + recvbuf, recvcounts, recvdispls, MPI_DOUBLE); + break; + } //setting zero values in the field // for (int i=0; i< field.getSupport()->getNumberOfElements(MED_EN::MED_ALL_ELEMENTS);i++) @@ -236,7 +244,8 @@ void MxN_Mapping::sendRecv(double* sendfield, MEDMEM::FIELD& field) cons } } - if (sendbuf!=0) delete[] sendbuf; +// if (sendbuf!=0) delete[] sendbuf; + if (sendbuf!=0 && _allToAllMethod == Native) delete[] sendbuf; if (recvbuf!=0) delete[] recvbuf; delete[] sendcounts; delete[] recvcounts; @@ -298,10 +307,18 @@ void MxN_Mapping::reverseSendRecv(double* recvfield, MEDMEM::FIELD& fiel // } //communication phase - const MPI_Comm* comm = group->getComm(); - comm_interface.allToAllV(sendbuf, sendcounts, senddispls, MPI_DOUBLE, - recvbuf, recvcounts, recvdispls, MPI_DOUBLE, - *comm); + switch (_allToAllMethod) { + case Native: + const MPI_Comm* comm = group->getComm(); + comm_interface.allToAllV(sendbuf, sendcounts, senddispls, MPI_DOUBLE, + recvbuf, recvcounts, recvdispls, MPI_DOUBLE, + *comm); + break; + case PointToPoint: + _accessDEC->AllToAllv(sendbuf, sendcounts, senddispls, MPI_DOUBLE, + recvbuf, recvcounts, recvdispls, MPI_DOUBLE); + break; + } //setting zero values in the field @@ -322,7 +339,8 @@ void MxN_Mapping::reverseSendRecv(double* recvfield, MEDMEM::FIELD& fiel } - if (sendbuf!=0) delete[] sendbuf; +// if (sendbuf!=0) delete[] sendbuf; + if (sendbuf!=0 && _allToAllMethod == Native) delete[] sendbuf; if (recvbuf!=0) delete[] recvbuf; delete[] sendcounts; delete[] recvcounts; @@ -332,4 +350,20 @@ void MxN_Mapping::reverseSendRecv(double* recvfield, MEDMEM::FIELD& fiel } +ostream & operator<< (ostream & f ,const AllToAllMethod & alltoallmethod ) { + switch (alltoallmethod) { + case Native : + f << " Native "; + break; + case PointToPoint : + f << " PointToPoint "; + break; + default : + f << " UnknownAllToAllMethod "; + break; + } + + return f; +} + } diff --git a/src/ParaMEDMEM/MxN_Mapping.hxx b/src/ParaMEDMEM/MxN_Mapping.hxx index 51136c0ad..edc039c40 100644 --- a/src/ParaMEDMEM/MxN_Mapping.hxx +++ b/src/ParaMEDMEM/MxN_Mapping.hxx @@ -4,10 +4,13 @@ #include #include "MEDMEM_Field.hxx" +#include "MPI_AccessDEC.hxx" namespace ParaMEDMEM { +typedef enum{Native,PointToPoint} AllToAllMethod; + class ProcessorGroup; class MxN_Mapping @@ -21,19 +24,23 @@ public: void sendRecv(MEDMEM::FIELD& field); void sendRecv(double* field, MEDMEM::FIELD& field) const ; void reverseSendRecv(double* field, MEDMEM::FIELD& field) const ; - + void setAllToAllMethod(const AllToAllMethod& method){_allToAllMethod = method ;}; + MPI_AccessDEC* getAccessDEC(){return _accessDEC;} private : // ProcessorGroup& _local_group; // ProcessorGroup& _distant_group; ProcessorGroup* _union_group; + MPI_AccessDEC * _accessDEC; + AllToAllMethod _allToAllMethod ; int _nb_comps; std::vector > _sending_ids; std::vector _recv_ids; std::vector _send_proc_offsets; std::vector _recv_proc_offsets; - }; +ostream & operator<< (ostream &,const AllToAllMethod &); + } #endif /*MXN_MAPPING_HXX_*/ diff --git a/src/ParaMEDMEM/Test/ParaMEDMEMTest.hxx b/src/ParaMEDMEM/Test/ParaMEDMEMTest.hxx index b7164b056..fe9c62b17 100644 --- a/src/ParaMEDMEM/Test/ParaMEDMEMTest.hxx +++ b/src/ParaMEDMEM/Test/ParaMEDMEMTest.hxx @@ -38,6 +38,17 @@ class ParaMEDMEMTest : public CppUnit::TestFixture CPPUNIT_TEST(testBlockTopology_constructor); CPPUNIT_TEST(testBlockTopology_serialize); CPPUNIT_TEST(testIntersectionDEC_2D); + CPPUNIT_TEST(testSynchronousEqualIntersectionWithoutInterpDEC_2D); + CPPUNIT_TEST(testSynchronousEqualIntersectionDEC_2D); + CPPUNIT_TEST(testSynchronousFasterSourceIntersectionDEC_2D); + CPPUNIT_TEST(testSynchronousSlowerSourceIntersectionDEC_2D); + CPPUNIT_TEST(testSynchronousSlowSourceIntersectionDEC_2D); + CPPUNIT_TEST(testSynchronousFastSourceIntersectionDEC_2D); + CPPUNIT_TEST(testAsynchronousEqualIntersectionDEC_2D); + CPPUNIT_TEST(testAsynchronousFasterSourceIntersectionDEC_2D); + CPPUNIT_TEST(testAsynchronousSlowerSourceIntersectionDEC_2D); + CPPUNIT_TEST(testAsynchronousSlowSourceIntersectionDEC_2D); + CPPUNIT_TEST(testAsynchronousFastSourceIntersectionDEC_2D); //can be added again after FVM correction for 2D // CPPUNIT_TEST(testNonCoincidentDEC_2D); CPPUNIT_TEST(testNonCoincidentDEC_3D); @@ -58,6 +69,12 @@ public: void testIntersectionDEC_2D(); void testNonCoincidentDEC_2D(); void testNonCoincidentDEC_3D(); + void testSynchronousEqualIntersectionWithoutInterpDEC_2D(); + void testSynchronousEqualIntersectionDEC_2D(); + void testSynchronousFasterSourceIntersectionDEC_2D(); + void testSynchronousSlowerSourceIntersectionDEC_2D(); + void testSynchronousSlowSourceIntersectionDEC_2D(); + void testSynchronousFastSourceIntersectionDEC_2D(); void testAsynchronousEqualIntersectionDEC_2D(); void testAsynchronousFasterSourceIntersectionDEC_2D(); void testAsynchronousSlowerSourceIntersectionDEC_2D(); @@ -72,7 +89,7 @@ private: const std::string& meshname2, int nbprocsource); void testAsynchronousIntersectionDEC_2D(double dtA, double tmaxA, - double dtB, double tmaxB); + double dtB, double tmaxB, bool Asynchronous, bool WithInterp ); }; diff --git a/src/ParaMEDMEM/Test/ParaMEDMEMTest_IntersectionDEC.cxx b/src/ParaMEDMEM/Test/ParaMEDMEMTest_IntersectionDEC.cxx index 7887cbd3a..68431c0e5 100644 --- a/src/ParaMEDMEM/Test/ParaMEDMEMTest_IntersectionDEC.cxx +++ b/src/ParaMEDMEM/Test/ParaMEDMEMTest_IntersectionDEC.cxx @@ -26,6 +26,7 @@ #include "MPIProcessorGroup.hxx" #include "Topology.hxx" #include "DEC.hxx" +#include "MxN_Mapping.hxx" #include "IntersectionDEC.hxx" #include "ParaMESH.hxx" #include "ParaFIELD.hxx" @@ -232,26 +233,54 @@ void ParaMEDMEMTest::testIntersectionDEC_2D() cout << "end of IntersectionDEC_2D test"<* field; +// MEDMEM::FIELD* field; ParaMEDMEM::ParaMESH* paramesh; ParaMEDMEM::ParaFIELD* parafield; + ParaMEDMEM::ParaSUPPORT* parasupport ; + double * value ; + ICoCo::Field* icocofield ; + string data_dir = getenv("MED_ROOT_DIR"); string tmp_dir = getenv("TMP"); if (tmp_dir == "") @@ -310,6 +343,7 @@ void ParaMEDMEMTest::testAsynchronousIntersectionDEC_2D(double dtA, double tmaxA ParaMEDMEMTest_TmpFilesRemover aRemover; MPI_Barrier(MPI_COMM_WORLD); + if (source_group->containsMyRank()) { string master = filename_xml1; @@ -324,18 +358,21 @@ void ParaMEDMEMTest::testAsynchronousIntersectionDEC_2D(double dtA, double tmaxA paramesh=new ParaMESH (*mesh,*source_group,"source mesh"); - ParaMEDMEM::ParaSUPPORT* parasupport=new UnstructuredParaSUPPORT( support,*source_group); +// ParaMEDMEM::ParaSUPPORT* parasupport=new UnstructuredParaSUPPORT( support,*source_group); + parasupport=new UnstructuredParaSUPPORT( support,*source_group); ParaMEDMEM::ComponentTopology comptopo; parafield = new ParaFIELD(parasupport, comptopo); int nb_local=support->getNumberOfElements(MED_EN::MED_ALL_ELEMENTS); - double * value= new double[nb_local]; +// double * value= new double[nb_local]; + value = new double[nb_local]; for(int ielem=0; ielemgetField()->setValue(value); - ICoCo::Field* icocofield=new ICoCo::MEDField(paramesh,parafield); +// ICoCo::Field* icocofield=new ICoCo::MEDField(paramesh,parafield); + icocofield=new ICoCo::MEDField(paramesh,parafield); dec.attachLocalField(icocofield); } @@ -353,17 +390,20 @@ void ParaMEDMEMTest::testAsynchronousIntersectionDEC_2D(double dtA, double tmaxA support=new MEDMEM::SUPPORT(mesh,"all elements",MED_EN::MED_CELL); paramesh=new ParaMESH (*mesh,*target_group,"target mesh"); - ParaMEDMEM::ParaSUPPORT* parasupport=new UnstructuredParaSUPPORT(support,*target_group); +// ParaMEDMEM::ParaSUPPORT* parasupport=new UnstructuredParaSUPPORT(support,*target_group); + parasupport=new UnstructuredParaSUPPORT(support,*target_group); ParaMEDMEM::ComponentTopology comptopo; parafield = new ParaFIELD(parasupport, comptopo); int nb_local=support->getNumberOfElements(MED_EN::MED_ALL_ELEMENTS); - double * value= new double[nb_local]; +// double * value= new double[nb_local]; + value = new double[nb_local]; for(int ielem=0; ielemgetField()->setValue(value); - ICoCo::Field* icocofield=new ICoCo::MEDField(paramesh,parafield); +// ICoCo::Field* icocofield=new ICoCo::MEDField(paramesh,parafield); + icocofield=new ICoCo::MEDField(paramesh,parafield); dec.attachLocalField(icocofield); } @@ -373,17 +413,28 @@ void ParaMEDMEMTest::testAsynchronousIntersectionDEC_2D(double dtA, double tmaxA if (source_group->containsMyRank()) { - dec.synchronize(); cout<<"DEC usage"< (parafield->getField()->getValue()); - int nb_local=parafield->getField()->getSupport()->getNumberOfElements(MED_EN::MED_ALL_ELEMENTS); - for (int i=0; i (parafield->getField()->getValue()); + int nb_local=parafield->getField()->getSupport()->getNumberOfElements(MED_EN::MED_ALL_ELEMENTS); + for (int i=0; icontainsMyRank()) { + cout<<"DEC usage"< times; for (double time=0; timegetVolumeIntegral(1) + << " time*10000 " << time*10000 << endl ; CPPUNIT_ASSERT_DOUBLES_EQUAL(parafield->getVolumeIntegral(1),time*10000,0.001); } } - delete source_group; + delete source_group; delete target_group; delete self_group; + delete mesh ; + delete support ; + delete paramesh ; + delete parafield ; + delete parasupport ; + delete [] value ; + delete icocofield ; + cout << "testAsynchronousIntersectionDEC_2D" << rank << " MPI_Barrier " << endl ; MPI_Barrier(MPI_COMM_WORLD); cout << "end of IntersectionDEC_2D test"< +#include + +#include "ProcessorGroup.hxx" + +namespace ParaMEDMEM { + + class TimeInterpolator { + + public: + TimeInterpolator( double InterpPrecision, int nStepBefore=1, int nStepAfter=1 ) ; + virtual ~TimeInterpolator(); + + void SetInterpParams( double InterpPrecision, int nStepBefore=1, int nStepAfter=1 ) { + _InterpPrecision = InterpPrecision ; + _nStepBefore = nStepBefore ; + _nStepAfter = nStepAfter ; } ; + void Steps( int &nStepBefore, int &nStepAfter ) { + nStepBefore = _nStepBefore ; + nStepAfter = _nStepAfter ; } ; + virtual void DoInterp( double time0, double time1, double time, int recvcount , + int nbuff0, int nbuff1, + int **recvbuff0, int **recvbuff1, int *result )= 0 ; + virtual void DoInterp( double time0, double time1, double time, int recvcount , + int nbuff0, int nbuff1, + double **recvbuff0, double **recvbuff1, double *result )= 0 ; + + protected : + double _InterpPrecision ; + int _nStepBefore ; + int _nStepAfter ; + }; +} + +#endif diff --git a/src/ParaMEDMEM/test_IntersectionDEC.cxx b/src/ParaMEDMEM/test_IntersectionDEC.cxx index 4d617dc44..d0c3e7d2f 100644 --- a/src/ParaMEDMEM/test_IntersectionDEC.cxx +++ b/src/ParaMEDMEM/test_IntersectionDEC.cxx @@ -34,7 +34,7 @@ int main(int argc, char** argv) if (argc !=2 || size < 2) { cout << "usage :"< test_NonCoincidentDEC "< test_IntersectionDEC "<=2)"<