X-Git-Url: http://git.salome-platform.org/gitweb/?a=blobdiff_plain;f=src%2FParaMEDMEM%2FOverlapMapping.cxx;h=6871f1f3514232d7b22633d943386b8b238d5e7e;hb=eb89ffa7833f70341c6daf6f5b9b848e10a57711;hp=abb7a1d1bf93cbb59a0801ee9572870d27dd1495;hpb=0a5983902df7b8197b97ac8b146226867ab37553;p=modules%2Fmed.git diff --git a/src/ParaMEDMEM/OverlapMapping.cxx b/src/ParaMEDMEM/OverlapMapping.cxx index abb7a1d1b..6871f1f35 100644 --- a/src/ParaMEDMEM/OverlapMapping.cxx +++ b/src/ParaMEDMEM/OverlapMapping.cxx @@ -36,46 +36,51 @@ OverlapMapping::OverlapMapping(const ProcessorGroup& group):_group(group) } /*! - * This method keeps tracks of source ids to know in step 6 of main algorithm, which tuple ids to send away. - * This method incarnates item#1 of step2 algorithm. + * Keeps the link between a given a proc holding source mesh data, and the corresponding cell IDs. */ void OverlapMapping::keepTracksOfSourceIds(int procId, DataArrayInt *ids) { ids->incrRef(); - _src_ids_st2.push_back(ids); - _src_proc_st2.push_back(procId); + _sent_src_ids_st2.push_back(ids); + _sent_src_proc_st2.push_back(procId); } /*! - * This method keeps tracks of target ids to know in step 6 of main algorithm. - * This method incarnates item#0 of step2 algorithm. - */ + * Same as keepTracksOfSourceIds() but for target mesh data. +*/ void OverlapMapping::keepTracksOfTargetIds(int procId, DataArrayInt *ids) { ids->incrRef(); - _trg_ids_st2.push_back(ids); - _trg_proc_st2.push_back(procId); + _sent_trg_ids_st2.push_back(ids); + _sent_trg_proc_st2.push_back(procId); } /*! - * This method stores from a matrix in format Target(rows)/Source(cols) for a source procId 'srcProcId' and for a target procId 'trgProcId'. - * All ids (source and target) are in format of local ids. + * This method stores in the local members the contribution coming from a matrix in format + * Target(rows)/Source(cols) for a source procId 'srcProcId' and for a target procId 'trgProcId'. + * All IDs received here (source and target) are in the format of local IDs. + * + * @param srcIds is null if the source mesh is on the local proc + * @param trgIds is null if the source mesh is on the local proc + * + * One of the 2 is necessarily null (the two can be null together) */ -void OverlapMapping::addContributionST(const std::vector< std::map >& matrixST, const DataArrayInt *srcIds, int srcProcId, const DataArrayInt *trgIds, int trgProcId) +void OverlapMapping::addContributionST(const std::vector< SparseDoubleVec >& matrixST, const DataArrayInt *srcIds, int srcProcId, const DataArrayInt *trgIds, int trgProcId) { _matrixes_st.push_back(matrixST); _source_proc_id_st.push_back(srcProcId); _target_proc_id_st.push_back(trgProcId); - if(srcIds) + if(srcIds) // source mesh part is remote {//item#1 of step2 algorithm in proc m. Only to know in advanced nb of recv ids [ (0,1) computed on proc1 and Matrix-Vector on proc1 ] _nb_of_src_ids_proc_st2.push_back(srcIds->getNumberOfTuples()); _src_ids_proc_st2.push_back(srcProcId); } - else + else // source mesh part is local {//item#0 of step2 algorithm in proc k std::set s; - for(std::vector< std::map >::const_iterator it1=matrixST.begin();it1!=matrixST.end();it1++) - for(std::map::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) + // For all source IDs (=col indices) in the sparse matrix: + for(std::vector< SparseDoubleVec >::const_iterator it1=matrixST.begin();it1!=matrixST.end();it1++) + for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) s.insert((*it2).first); _src_ids_zip_st2.resize(_src_ids_zip_st2.size()+1); _src_ids_zip_st2.back().insert(_src_ids_zip_st2.back().end(),s.begin(),s.end()); @@ -84,14 +89,17 @@ void OverlapMapping::addContributionST(const std::vector< std::map > } /*! - * 'procsInInteraction' gives the global view of interaction between procs. - * In 'procsInInteraction' for a proc with id i, is in interaction with procs listed in procsInInteraction[i]. + * This method is in charge to send matrices in AlltoAll mode. + * + * 'procsToSendField' gives the list of procs field data has to be sent to. + * See OverlapElementLocator::computeBoundingBoxesAndTodoList() * - * This method is in charge to send matrixes in AlltoAll mode. - * After the call of this method 'this' contains the matrixST for all source elements of the current proc + * After the call of this method, 'this' contains the matrixST for all source cells of the current proc */ -void OverlapMapping::prepare(const std::vector< std::vector >& procsInInteraction, int nbOfTrgElems) +void OverlapMapping::prepare(const std::vector< int >& procsToSendField, int nbOfTrgElems) { +// printMatrixesST(); + CommInterface commInterface=_group.getCommInterface(); const MPIProcessorGroup *group=static_cast(&_group); const MPI_Comm *comm=group->getComm(); @@ -101,12 +109,6 @@ void OverlapMapping::prepare(const std::vector< std::vector >& procsInInter INTERP_KERNEL::AutoPtr nbsend3=new int[grpSize]; std::fill(nbsend,nbsend+grpSize,0); int myProcId=_group.myRank(); - _proc_ids_to_recv_vector_st.clear(); - int curProc=0; - for(std::vector< std::vector >::const_iterator it1=procsInInteraction.begin();it1!=procsInInteraction.end();it1++,curProc++) - if(std::find((*it1).begin(),(*it1).end(),myProcId)!=(*it1).end()) - _proc_ids_to_recv_vector_st.push_back(curProc); - _proc_ids_to_send_vector_st=procsInInteraction[myProcId]; for(std::size_t i=0;i<_matrixes_st.size();i++) if(_source_proc_id_st[i]==myProcId) nbsend[_target_proc_id_st[i]]=_matrixes_st[i].size(); @@ -149,9 +151,46 @@ void OverlapMapping::prepare(const std::vector< std::vector >& procsInInter bigArrRecv2,bigArrDRecv2,nbrecv3,nbrecv4); //updating _src_ids_zip_st2 and _src_ids_zip_st2 with received matrix. updateZipSourceIdsForFuture(); - //finish to fill _the_matrix_st with already in place matrix in _matrixes_st + //finish to fill _the_matrix_st with already in place matrix in _matrixes_st (local computation) finishToFillFinalMatrixST(); - //printTheMatrix(); + // Prepare proc lists for future field data exchange (mutliply()): + fillProcToSendRcvForMultiply(procsToSendField); + // Make some space on local proc: + _matrixes_st.clear(); + +// std::stringstream scout; +// scout << "\n(" << myProcId << ") to send:"; +// for (std::vector< int >::const_iterator itdbg=_proc_ids_to_send_vector_st.begin(); itdbg!=_proc_ids_to_send_vector_st.end(); itdbg++) +// scout << "," << *itdbg; +// scout << "\n(" << myProcId << ") to recv:"; +// for (std::vector< int >::const_iterator itdbg=_proc_ids_to_recv_vector_st.begin(); itdbg!=_proc_ids_to_recv_vector_st.end(); itdbg++) +// scout << "," << *itdbg; +// std::cout << scout.str() << "\n"; +// +// printTheMatrix(); +} + +/** + * Fill the members _proc_ids_to_send_vector_st and _proc_ids_to_recv_vector_st + * indicating for each proc to which proc it should send its source field data + * and from which proc it should receive source field data. + * + * Should be called once finishToFillFinalMatrixST() has been executed, i.e. when the + * local matrices are completly ready. + */ +void OverlapMapping::fillProcToSendRcvForMultiply(const std::vector< int >& procsToSendField) +{ +// _proc_ids_to_send_vector_st.clear(); + _proc_ids_to_recv_vector_st.clear(); + // Receiving side is easy - just inspect non-void terms in the final matrix + int i=0; + std::vector< std::vector< SparseDoubleVec > >::const_iterator it; + std::vector< SparseDoubleVec >::const_iterator it2; + for(it=_the_matrix_st.begin(); it!=_the_matrix_st.end(); it++, i++) + _proc_ids_to_recv_vector_st.push_back(_the_matrix_st_source_proc_id[i]); + + // Sending side was computed in OverlapElementLocator::computeBoundingBoxesAndTodoList() + _proc_ids_to_send_vector_st = procsToSendField; } /*! @@ -169,11 +208,11 @@ void OverlapMapping::computeDenoGlobConstraint() for(std::size_t j=0;j& mToFill=_the_deno_st[i][j]; - const std::map& m=_the_matrix_st[i][j]; - for(std::map::const_iterator it=m.begin();it!=m.end();it++) + SparseDoubleVec& mToFill=_the_deno_st[i][j]; + const SparseDoubleVec& m=_the_matrix_st[i][j]; + for(SparseDoubleVec::const_iterator it=m.begin();it!=m.end();it++) sum+=(*it).second; - for(std::map::const_iterator it=m.begin();it!=m.end();it++) + for(SparseDoubleVec::const_iterator it=m.begin();it!=m.end();it++) mToFill[(*it).first]=sum; } } @@ -184,7 +223,6 @@ void OverlapMapping::computeDenoGlobConstraint() */ void OverlapMapping::computeDenoConservativeVolumic(int nbOfTuplesTrg) { - CommInterface commInterface=_group.getCommInterface(); int myProcId=_group.myRank(); // _the_deno_st.clear(); @@ -193,24 +231,24 @@ void OverlapMapping::computeDenoConservativeVolumic(int nbOfTuplesTrg) std::vector deno(nbOfTuplesTrg); for(std::size_t i=0;i >& mat=_the_matrix_st[i]; + const std::vector< SparseDoubleVec >& mat=_the_matrix_st[i]; int curSrcId=_the_matrix_st_source_proc_id[i]; - std::vector::iterator isItem1=std::find(_trg_proc_st2.begin(),_trg_proc_st2.end(),curSrcId); + std::vector::iterator isItem1=std::find(_sent_trg_proc_st2.begin(),_sent_trg_proc_st2.end(),curSrcId); int rowId=0; - if(isItem1==_trg_proc_st2.end() || curSrcId==myProcId)//item1 of step2 main algo. Simple, because rowId of mat are directly target ids. + if(isItem1==_sent_trg_proc_st2.end() || curSrcId==myProcId)//item1 of step2 main algo. Simple, because rowId of mat are directly target ids. { - for(std::vector< std::map >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++) - for(std::map::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) + for(std::vector< SparseDoubleVec >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++) + for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) deno[rowId]+=(*it2).second; } else {//item0 of step2 main algo. More complicated. std::vector::iterator fnd=isItem1;//std::find(_trg_proc_st2.begin(),_trg_proc_st2.end(),curSrcId); - int locId=std::distance(_trg_proc_st2.begin(),fnd); - const DataArrayInt *trgIds=_trg_ids_st2[locId]; + int locId=std::distance(_sent_trg_proc_st2.begin(),fnd); + const DataArrayInt *trgIds=_sent_trg_ids_st2[locId]; const int *trgIds2=trgIds->getConstPointer(); - for(std::vector< std::map >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++) - for(std::map::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) + for(std::vector< SparseDoubleVec >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++) + for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) deno[trgIds2[rowId]]+=(*it2).second; } } @@ -218,26 +256,26 @@ void OverlapMapping::computeDenoConservativeVolumic(int nbOfTuplesTrg) for(std::size_t i=0;i >& mat=_the_matrix_st[i]; + const std::vector< SparseDoubleVec >& mat=_the_matrix_st[i]; int curSrcId=_the_matrix_st_source_proc_id[i]; - std::vector::iterator isItem1=std::find(_trg_proc_st2.begin(),_trg_proc_st2.end(),curSrcId); - std::vector< std::map >& denoM=_the_deno_st[i]; + std::vector::iterator isItem1=std::find(_sent_trg_proc_st2.begin(),_sent_trg_proc_st2.end(),curSrcId); + std::vector< SparseDoubleVec >& denoM=_the_deno_st[i]; denoM.resize(mat.size()); - if(isItem1==_trg_proc_st2.end() || curSrcId==myProcId)//item1 of step2 main algo. Simple, because rowId of mat are directly target ids. + if(isItem1==_sent_trg_proc_st2.end() || curSrcId==myProcId)//item1 of step2 main algo. Simple, because rowId of mat are directly target ids. { int rowId=0; - for(std::vector< std::map >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++) - for(std::map::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) + for(std::vector< SparseDoubleVec >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++) + for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) denoM[rowId][(*it2).first]=deno[rowId]; } else { std::vector::iterator fnd=isItem1; - int locId=std::distance(_trg_proc_st2.begin(),fnd); - const DataArrayInt *trgIds=_trg_ids_st2[locId]; + int locId=std::distance(_sent_trg_proc_st2.begin(),fnd); + const DataArrayInt *trgIds=_sent_trg_ids_st2[locId]; const int *trgIds2=trgIds->getConstPointer(); - for(std::vector< std::map >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++) - for(std::map::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) + for(std::vector< SparseDoubleVec >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++) + for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) denoM[rowId][(*it2).first]=deno[trgIds2[rowId]]; } } @@ -275,8 +313,8 @@ void OverlapMapping::serializeMatrixStep0ST(const int *nbOfElemsSrc, int *&bigAr int start=offsets[_target_proc_id_st[i]]; int *work=bigArr+start; *work=0; - const std::vector< std::map >& mat=_matrixes_st[i]; - for(std::vector< std::map >::const_iterator it=mat.begin();it!=mat.end();it++,work++) + const std::vector< SparseDoubleVec >& mat=_matrixes_st[i]; + for(std::vector< SparseDoubleVec >::const_iterator it=mat.begin();it!=mat.end();it++,work++) work[1]=work[0]+(*it).size(); } } @@ -295,6 +333,7 @@ void OverlapMapping::serializeMatrixStep0ST(const int *nbOfElemsSrc, int *&bigAr /*! * This method performs step#1 and step#2/3. It returns the size of expected array to get allToAllV. + * It is where the locally computed matrices are serialized to be sent to adequate final proc. */ int OverlapMapping::serializeMatrixStep1ST(const int *nbOfElemsSrc, const int *recvStep0, const int *countStep0, const int *offsStep0, int *&bigArrI, double *&bigArrD, int *count, int *offsets, @@ -322,9 +361,9 @@ int OverlapMapping::serializeMatrixStep1ST(const int *nbOfElemsSrc, const int *r { if(_source_proc_id_st[i]==myProcId) { - const std::vector< std::map >& mat=_matrixes_st[i]; + const std::vector< SparseDoubleVec >& mat=_matrixes_st[i]; int lgthToSend=0; - for(std::vector< std::map >::const_iterator it=mat.begin();it!=mat.end();it++) + for(std::vector< SparseDoubleVec >::const_iterator it=mat.begin();it!=mat.end();it++) lgthToSend+=(*it).size(); count[_target_proc_id_st[i]]=lgthToSend; fullLgth+=lgthToSend; @@ -341,11 +380,11 @@ int OverlapMapping::serializeMatrixStep1ST(const int *nbOfElemsSrc, const int *r { if(_source_proc_id_st[i]==myProcId) { - const std::vector< std::map >& mat=_matrixes_st[i]; - for(std::vector< std::map >::const_iterator it1=mat.begin();it1!=mat.end();it1++) + const std::vector< SparseDoubleVec >& mat=_matrixes_st[i]; + for(std::vector< SparseDoubleVec >::const_iterator it1=mat.begin();it1!=mat.end();it1++) { int j=0; - for(std::map::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++,j++) + for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++,j++) { bigArrI[fullLgth+j]=(*it2).first; bigArrD[fullLgth+j]=(*it2).second; @@ -396,9 +435,10 @@ void OverlapMapping::unserializationST(int nbOfTrgElems, } /*! - * This method should be called when all remote matrix with sourceProcId==thisProcId have been retrieved and are in 'this->_the_matrix_st' and 'this->_the_matrix_st_target_proc_id' - * and 'this->_the_matrix_st_target_ids'. - * This method finish the job of filling 'this->_the_matrix_st' and 'this->_the_matrix_st_target_proc_id' by putting candidates in 'this->_matrixes_st' into them. + * This method should be called when all remote matrix with sourceProcId==thisProcId have been retrieved and are + * in 'this->_the_matrix_st' and 'this->_the_matrix_st_target_proc_id' and 'this->_the_matrix_st_target_ids'. + * This method finish the job of filling 'this->_the_matrix_st' and 'this->_the_matrix_st_target_proc_id' + * by putting candidates in 'this->_matrixes_st' into them (i.e. local computation result). */ void OverlapMapping::finishToFillFinalMatrixST() { @@ -417,64 +457,11 @@ void OverlapMapping::finishToFillFinalMatrixST() for(int i=0;i >& mat=_matrixes_st[i]; + const std::vector& mat=_matrixes_st[i]; _the_matrix_st[j]=mat; _the_matrix_st_source_proc_id.push_back(_source_proc_id_st[i]); j++; } - _matrixes_st.clear(); -} - -/*! - * This method performs the operation of target ids broadcasting. - */ -void OverlapMapping::prepareIdsToSendST() -{ - CommInterface commInterface=_group.getCommInterface(); - const MPIProcessorGroup *group=static_cast(&_group); - const MPI_Comm *comm=group->getComm(); - int grpSize=_group.size(); - _source_ids_to_send_st.clear(); - _source_ids_to_send_st.resize(grpSize); - INTERP_KERNEL::AutoPtr nbsend=new int[grpSize]; - std::fill(nbsend,nbsend+grpSize,0); - for(std::size_t i=0;i<_the_matrix_st_source_proc_id.size();i++) - nbsend[_the_matrix_st_source_proc_id[i]]=_the_matrix_st_source_ids[i].size(); - INTERP_KERNEL::AutoPtr nbrecv=new int[grpSize]; - commInterface.allToAll(nbsend,1,MPI_INT,nbrecv,1,MPI_INT,*comm); - // - INTERP_KERNEL::AutoPtr nbsend2=new int[grpSize]; - std::copy((int *)nbsend,((int *)nbsend)+grpSize,(int *)nbsend2); - INTERP_KERNEL::AutoPtr nbsend3=new int[grpSize]; - nbsend3[0]=0; - for(int i=1;i bigDataSend=new int[sendSz]; - for(std::size_t i=0;i<_the_matrix_st_source_proc_id.size();i++) - { - int offset=nbsend3[_the_matrix_st_source_proc_id[i]]; - std::copy(_the_matrix_st_source_ids[i].begin(),_the_matrix_st_source_ids[i].end(),((int *)nbsend3)+offset); - } - INTERP_KERNEL::AutoPtr nbrecv2=new int[grpSize]; - INTERP_KERNEL::AutoPtr nbrecv3=new int[grpSize]; - std::copy((int *)nbrecv,((int *)nbrecv)+grpSize,(int *)nbrecv2); - nbrecv3[0]=0; - for(int i=1;i bigDataRecv=new int[recvSz]; - // - commInterface.allToAllV(bigDataSend,nbsend2,nbsend3,MPI_INT, - bigDataRecv,nbrecv2,nbrecv3,MPI_INT, - *comm); - for(int i=0;i0) - { - _source_ids_to_send_st[i].insert(_source_ids_to_send_st[i].end(),((int *)bigDataRecv)+nbrecv3[i],((int *)bigDataRecv)+nbrecv3[i]+nbrecv2[i]); - } - } } /*! @@ -499,20 +486,29 @@ void OverlapMapping::multiply(const MEDCouplingFieldDouble *fieldInput, MEDCoupl nbsend2[0]=0; nbrecv2[0]=0; std::vector valsToSend; + + /* + * FIELD VALUE XCHGE + */ for(int i=0;i::const_iterator isItem1=std::find(_src_proc_st2.begin(),_src_proc_st2.end(),i); + std::vector::const_iterator isItem1=std::find(_sent_src_proc_st2.begin(),_sent_src_proc_st2.end(),i); MEDCouplingAutoRefCountObjectPtr vals; - if(isItem1!=_src_proc_st2.end())//item1 of step2 main algo + if(isItem1!=_sent_src_proc_st2.end()) // source data was sent to proc 'i' { - int id=std::distance(_src_proc_st2.begin(),isItem1); - vals=fieldInput->getArray()->selectByTupleId(_src_ids_st2[id]->getConstPointer(),_src_ids_st2[id]->getConstPointer()+_src_ids_st2[id]->getNumberOfTuples()); + // Prepare local field data to send to proc i + int id=std::distance(_sent_src_proc_st2.begin(),isItem1); + vals=fieldInput->getArray()->selectByTupleId(*_sent_src_ids_st2[id]); } - else + else // no source data was sent to proc 'i' {//item0 of step2 main algo - int id=std::distance(_src_ids_zip_proc_st2.begin(),std::find(_src_ids_zip_proc_st2.begin(),_src_ids_zip_proc_st2.end(),i)); + std::vector::const_iterator isItem22 = std::find(_src_ids_zip_proc_st2.begin(),_src_ids_zip_proc_st2.end(),i ); + int id=std::distance(_src_ids_zip_proc_st2.begin(),isItem22); + if (isItem22 == _src_ids_zip_proc_st2.end()) + std::cout << "(" << myProcId << ") has end iterator!!!!!\n"; vals=fieldInput->getArray()->selectByTupleId(&(_src_ids_zip_st2[id])[0],&(_src_ids_zip_st2[id])[0]+_src_ids_zip_st2[id].size()); } nbsend[i]=vals->getNbOfElems(); @@ -520,8 +516,8 @@ void OverlapMapping::multiply(const MEDCouplingFieldDouble *fieldInput, MEDCoupl } if(std::find(_proc_ids_to_recv_vector_st.begin(),_proc_ids_to_recv_vector_st.end(),i)!=_proc_ids_to_recv_vector_st.end()) { - std::vector::const_iterator isItem0=std::find(_trg_proc_st2.begin(),_trg_proc_st2.end(),i); - if(isItem0==_trg_proc_st2.end())//item1 of step2 main algo [ (0,1) computed on proc1 and Matrix-Vector on proc1 ] + std::vector::const_iterator isItem0=std::find(_sent_trg_proc_st2.begin(),_sent_trg_proc_st2.end(),i); + if(isItem0==_sent_trg_proc_st2.end())//item1 of step2 main algo [ (0,1) computed on proc1 and Matrix-Vector on proc1 ] { std::vector::const_iterator it1=std::find(_src_ids_proc_st2.begin(),_src_ids_proc_st2.end(),i); if(it1!=_src_ids_proc_st2.end()) @@ -529,12 +525,12 @@ void OverlapMapping::multiply(const MEDCouplingFieldDouble *fieldInput, MEDCoupl int id=std::distance(_src_ids_proc_st2.begin(),it1); nbrecv[i]=_nb_of_src_ids_proc_st2[id]*nbOfCompo; } - else if(i==myProcId) + else if(i==myProcId) // diagonal element (i.e. proc #i talking to same proc #i) { - nbrecv[i]=fieldInput->getNumberOfTuplesExpected()*nbOfCompo; + nbrecv[i] = nbsend[i]; } else - throw INTERP_KERNEL::Exception("Plouff ! send email to anthony.geay@cea.fr ! "); + throw INTERP_KERNEL::Exception("OverlapMapping::multiply(): Internal error at field values exchange!"); } else {//item0 of step2 main algo [ (2,1) computed on proc2 but Matrix-Vector on proc1 ] [(1,0) computed on proc1 but Matrix-Vector on proc0] @@ -549,8 +545,19 @@ void OverlapMapping::multiply(const MEDCouplingFieldDouble *fieldInput, MEDCoupl nbrecv2[i]=nbrecv2[i-1]+nbrecv[i-1]; } INTERP_KERNEL::AutoPtr bigArr=new double[nbrecv2[grpSize-1]+nbrecv[grpSize-1]]; + +// scout << "(" << myProcId << ") nbsend :" << nbsend[0] << "," << nbsend[1] << "," << nbsend[2] << "\n"; +// scout << "(" << myProcId << ") nbrecv :" << nbrecv[0] << "," << nbrecv[1] << "," << nbrecv[2] << "\n"; +// std::cout << scout.str() << "\n"; + /* + * *********************** ALL-TO-ALL + */ commInterface.allToAllV(&valsToSend[0],nbsend,nbsend2,MPI_DOUBLE, bigArr,nbrecv,nbrecv2,MPI_DOUBLE,*comm); + /* + * + * TARGET FIELD COMPUTATION (matrix-vec computation) + */ fieldOutput->getArray()->fillWithZero(); INTERP_KERNEL::AutoPtr tmp=new double[nbOfCompo]; for(int i=0;i >& mat=_the_matrix_st[id]; - const std::vector< std::map >& deno=_the_deno_st[id]; - std::vector::const_iterator isItem0=std::find(_trg_proc_st2.begin(),_trg_proc_st2.end(),i); - if(isItem0==_trg_proc_st2.end())//item1 of step2 main algo [ (0,1) computed on proc1 and Matrix-Vector on proc1 ] + const std::vector< SparseDoubleVec >& mat=_the_matrix_st[id]; + const std::vector< SparseDoubleVec >& deno=_the_deno_st[id]; + std::vector::const_iterator isItem0=std::find(_sent_trg_proc_st2.begin(),_sent_trg_proc_st2.end(),i); + if(isItem0==_sent_trg_proc_st2.end())//item1 of step2 main algo [ (0,1) computed on proc1 and Matrix-Vector on proc1 ] { int nbOfTrgTuples=mat.size(); for(int j=0;j& mat1=mat[j]; - const std::map& deno1=deno[j]; - std::map::const_iterator it4=deno1.begin(); - for(std::map::const_iterator it3=mat1.begin();it3!=mat1.end();it3++,it4++) + const SparseDoubleVec& mat1=mat[j]; + const SparseDoubleVec& deno1=deno[j]; + SparseDoubleVec::const_iterator it4=deno1.begin(); + for(SparseDoubleVec::const_iterator it3=mat1.begin();it3!=mat1.end();it3++,it4++) { - std::transform(bigArr+nbrecv2[i]+((*it3).first)*nbOfCompo,bigArr+nbrecv2[i]+((*it3).first+1)*(nbOfCompo),(double *)tmp,std::bind2nd(std::multiplies(),(*it3).second/(*it4).second)); + std::transform(bigArr+nbrecv2[i]+((*it3).first)*nbOfCompo, + bigArr+nbrecv2[i]+((*it3).first+1)*(nbOfCompo), + (double *)tmp, + std::bind2nd(std::multiplies(),(*it3).second/(*it4).second)); std::transform((double *)tmp,(double *)tmp+nbOfCompo,pt,pt,std::plus()); } } @@ -589,21 +599,24 @@ void OverlapMapping::multiply(const MEDCouplingFieldDouble *fieldInput, MEDCoupl int newId=0; for(std::vector::const_iterator it=zipIds.begin();it!=zipIds.end();it++,newId++) zipCor[*it]=newId; - int id2=std::distance(_trg_proc_st2.begin(),std::find(_trg_proc_st2.begin(),_trg_proc_st2.end(),i)); - const DataArrayInt *tgrIds=_trg_ids_st2[id2]; + int id2=std::distance(_sent_trg_proc_st2.begin(),std::find(_sent_trg_proc_st2.begin(),_sent_trg_proc_st2.end(),i)); + const DataArrayInt *tgrIds=_sent_trg_ids_st2[id2]; const int *tgrIds2=tgrIds->getConstPointer(); int nbOfTrgTuples=mat.size(); for(int j=0;j& mat1=mat[j]; - const std::map& deno1=deno[j]; - std::map::const_iterator it5=deno1.begin(); - for(std::map::const_iterator it3=mat1.begin();it3!=mat1.end();it3++,it5++) + const SparseDoubleVec& mat1=mat[j]; + const SparseDoubleVec& deno1=deno[j]; + SparseDoubleVec::const_iterator it5=deno1.begin(); + for(SparseDoubleVec::const_iterator it3=mat1.begin();it3!=mat1.end();it3++,it5++) { std::map::const_iterator it4=zipCor.find((*it3).first); if(it4==zipCor.end()) - throw INTERP_KERNEL::Exception("Hmmmmm send e mail to anthony.geay@cea.fr !"); - std::transform(bigArr+nbrecv2[i]+((*it4).second)*nbOfCompo,bigArr+nbrecv2[i]+((*it4).second+1)*(nbOfCompo),(double *)tmp,std::bind2nd(std::multiplies(),(*it3).second/(*it5).second)); + throw INTERP_KERNEL::Exception("OverlapMapping::multiply(): Internal error in final value computation!"); + std::transform(bigArr+nbrecv2[i]+((*it4).second)*nbOfCompo, + bigArr+nbrecv2[i]+((*it4).second+1)*(nbOfCompo), + (double *)tmp, + std::bind2nd(std::multiplies(),(*it3).second/(*it5).second)); std::transform((double *)tmp,(double *)tmp+nbOfCompo,pt+tgrIds2[j]*nbOfCompo,pt+tgrIds2[j]*nbOfCompo,std::plus()); } } @@ -621,11 +634,16 @@ void OverlapMapping::transposeMultiply(const MEDCouplingFieldDouble *fieldInput, } /*! - * This method should be called immediately after _the_matrix_st has been filled with remote computed matrix put in this proc for Matrix-Vector. - * This method computes for these matrix the minimal set of source ids corresponding to the source proc id. + * This method should be called immediately after _the_matrix_st has been filled with remote computed matrix + * put in this proc for Matrix-Vector. + * This method computes for these matrix the minimal set of source ids corresponding to the source proc id. + * */ void OverlapMapping::updateZipSourceIdsForFuture() { + /* When it is called, only the bits received from other processors (i.e. the remotely executed jobs) are in the + big matrix _the_matrix_st. */ + CommInterface commInterface=_group.getCommInterface(); int myProcId=_group.myRank(); int nbOfMatrixRecveived=_the_matrix_st_source_proc_id.size(); @@ -634,20 +652,22 @@ void OverlapMapping::updateZipSourceIdsForFuture() int curSrcProcId=_the_matrix_st_source_proc_id[i]; if(curSrcProcId!=myProcId) { - const std::vector< std::map >& mat=_the_matrix_st[i]; + const std::vector< SparseDoubleVec >& mat=_the_matrix_st[i]; _src_ids_zip_proc_st2.push_back(curSrcProcId); _src_ids_zip_st2.resize(_src_ids_zip_st2.size()+1); std::set s; - for(std::vector< std::map >::const_iterator it1=mat.begin();it1!=mat.end();it1++) - for(std::map::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) + for(std::vector< SparseDoubleVec >::const_iterator it1=mat.begin();it1!=mat.end();it1++) + for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) s.insert((*it2).first); _src_ids_zip_st2.back().insert(_src_ids_zip_st2.back().end(),s.begin(),s.end()); + +// std::stringstream scout; +// scout << "(" << myProcId << ") pushed " << _src_ids_zip_st2.back().size() << " values for tgt proc " << curSrcProcId; +// std::cout << scout.str() << "\n"; } } } -// #include - // void OverlapMapping::printTheMatrix() const // { // CommInterface commInterface=_group.getCommInterface(); @@ -655,19 +675,55 @@ void OverlapMapping::updateZipSourceIdsForFuture() // const MPI_Comm *comm=group->getComm(); // int grpSize=_group.size(); // int myProcId=_group.myRank(); -// std::cerr << "I am proc #" << myProcId << std::endl; +// std::stringstream oscerr; // int nbOfMat=_the_matrix_st.size(); -// std::cerr << "I do manage " << nbOfMat << "matrix : "<< std::endl; +// oscerr << "(" << myProcId << ") I hold " << nbOfMat << " matrix(ces) : "<< std::endl; // for(int i=0;i >& locMat=_the_matrix_st[i]; -// for(std::vector< std::map >::const_iterator it1=locMat.begin();it1!=locMat.end();it1++) +// oscerr << " - Matrix #" << i << " coming from source proc #" << _the_matrix_st_source_proc_id[i] << ": "; +// const std::vector< SparseDoubleVec >& locMat=_the_matrix_st[i]; +// int j = 0; +// for(std::vector< SparseDoubleVec >::const_iterator it1=locMat.begin();it1!=locMat.end();it1++, j++) // { -// for(std::map::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) -// std::cerr << "(" << (*it2).first << "," << (*it2).second << "), "; -// std::cerr << std::endl; +// oscerr << " Target Cell #" << j; +// for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) +// oscerr << " (" << (*it2).first << "," << (*it2).second << "), "; +// oscerr << std::endl; // } // } -// std::cerr << "*********" << std::endl; +// oscerr << "*********" << std::endl; +// +// // Hope this will be flushed in one go: +// std::cerr << oscerr.str() << std::endl; +//// if(myProcId != 0) +//// MPI_Barrier(MPI_COMM_WORLD); // } +// +// void OverlapMapping::printMatrixesST() const +// { +// CommInterface commInterface=_group.getCommInterface(); +// const MPIProcessorGroup *group=static_cast(&_group); +// const MPI_Comm *comm=group->getComm(); +// int grpSize=_group.size(); +// int myProcId=_group.myRank(); +// std::stringstream oscerr; +// int nbOfMat=_matrixes_st.size(); +// oscerr << "(" << myProcId << ") I hold " << nbOfMat << " LOCAL matrix(ces) : "<< std::endl; +// for(int i=0;i& locMat=_matrixes_st[i]; +// int j = 0; +// for(std::vector< SparseDoubleVec >::const_iterator it1=locMat.begin();it1!=locMat.end();it1++, j++) +// { +// oscerr << " Target Cell #" << j; +// for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++) +// oscerr << " (" << (*it2).first << "," << (*it2).second << "), "; +// oscerr << std::endl; +// } +// } +// oscerr << "*********" << std::endl; +// +// // Hope this will be flushed in one go: +// std::cerr << oscerr.str() << std::endl; +// }