}
/*!
- * This method keeps tracks of source ids to know in step 6 of main algorithm, which tuple ids to send away.
- * This method incarnates item#1 of step2 algorithm.
+ * Keeps the link between a given a proc holding source mesh data, and the corresponding cell IDs.
*/
void OverlapMapping::keepTracksOfSourceIds(int procId, DataArrayInt *ids)
{
ids->incrRef();
- _src_ids_st2.push_back(ids);
- _src_proc_st2.push_back(procId);
+ _sent_src_ids_st2.push_back(ids);
+ _sent_src_proc_st2.push_back(procId);
}
/*!
- * This method keeps tracks of target ids to know in step 6 of main algorithm.
- * This method incarnates item#0 of step2 algorithm.
- */
+ * Same as keepTracksOfSourceIds() but for target mesh data.
+*/
void OverlapMapping::keepTracksOfTargetIds(int procId, DataArrayInt *ids)
{
ids->incrRef();
- _trg_ids_st2.push_back(ids);
- _trg_proc_st2.push_back(procId);
+ _sent_trg_ids_st2.push_back(ids);
+ _sent_trg_proc_st2.push_back(procId);
}
/*!
- * This method stores from a matrix in format Target(rows)/Source(cols) for a source procId 'srcProcId' and for a target procId 'trgProcId'.
- * All ids (source and target) are in format of local ids.
+ * This method stores in the local members the contribution coming from a matrix in format
+ * Target(rows)/Source(cols) for a source procId 'srcProcId' and for a target procId 'trgProcId'.
+ * All IDs received here (source and target) are in the format of local IDs.
+ *
+ * @param srcIds is null if the source mesh is on the local proc
+ * @param trgIds is null if the source mesh is on the local proc
+ *
+ * One of the 2 is necessarily null (the two can be null together)
*/
-void OverlapMapping::addContributionST(const std::vector< std::map<int,double> >& matrixST, const DataArrayInt *srcIds, int srcProcId, const DataArrayInt *trgIds, int trgProcId)
+void OverlapMapping::addContributionST(const std::vector< SparseDoubleVec >& matrixST, const DataArrayInt *srcIds, int srcProcId, const DataArrayInt *trgIds, int trgProcId)
{
_matrixes_st.push_back(matrixST);
_source_proc_id_st.push_back(srcProcId);
_target_proc_id_st.push_back(trgProcId);
- if(srcIds)
+ if(srcIds) // source mesh part is remote
{//item#1 of step2 algorithm in proc m. Only to know in advanced nb of recv ids [ (0,1) computed on proc1 and Matrix-Vector on proc1 ]
_nb_of_src_ids_proc_st2.push_back(srcIds->getNumberOfTuples());
_src_ids_proc_st2.push_back(srcProcId);
}
- else
+ else // source mesh part is local
{//item#0 of step2 algorithm in proc k
std::set<int> s;
- for(std::vector< std::map<int,double> >::const_iterator it1=matrixST.begin();it1!=matrixST.end();it1++)
- for(std::map<int,double>::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
+ // For all source IDs (=col indices) in the sparse matrix:
+ for(std::vector< SparseDoubleVec >::const_iterator it1=matrixST.begin();it1!=matrixST.end();it1++)
+ for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
s.insert((*it2).first);
_src_ids_zip_st2.resize(_src_ids_zip_st2.size()+1);
_src_ids_zip_st2.back().insert(_src_ids_zip_st2.back().end(),s.begin(),s.end());
}
/*!
- * 'procsInInteraction' gives the global view of interaction between procs.
- * In 'procsInInteraction' for a proc with id i, is in interaction with procs listed in procsInInteraction[i].
+ * This method is in charge to send matrices in AlltoAll mode.
+ *
+ * 'procsToSendField' gives the list of procs field data has to be sent to.
+ * See OverlapElementLocator::computeBoundingBoxesAndTodoList()
*
- * This method is in charge to send matrixes in AlltoAll mode.
- * After the call of this method 'this' contains the matrixST for all source elements of the current proc
+ * After the call of this method, 'this' contains the matrixST for all source cells of the current proc
*/
-void OverlapMapping::prepare(const std::vector< std::vector<int> >& procsInInteraction, int nbOfTrgElems)
+void OverlapMapping::prepare(const std::vector< int >& procsToSendField, int nbOfTrgElems)
{
+// printMatrixesST();
+
CommInterface commInterface=_group.getCommInterface();
const MPIProcessorGroup *group=static_cast<const MPIProcessorGroup*>(&_group);
const MPI_Comm *comm=group->getComm();
INTERP_KERNEL::AutoPtr<int> nbsend3=new int[grpSize];
std::fill<int *>(nbsend,nbsend+grpSize,0);
int myProcId=_group.myRank();
- _proc_ids_to_recv_vector_st.clear();
- int curProc=0;
- for(std::vector< std::vector<int> >::const_iterator it1=procsInInteraction.begin();it1!=procsInInteraction.end();it1++,curProc++)
- if(std::find((*it1).begin(),(*it1).end(),myProcId)!=(*it1).end())
- _proc_ids_to_recv_vector_st.push_back(curProc);
- _proc_ids_to_send_vector_st=procsInInteraction[myProcId];
for(std::size_t i=0;i<_matrixes_st.size();i++)
if(_source_proc_id_st[i]==myProcId)
nbsend[_target_proc_id_st[i]]=_matrixes_st[i].size();
bigArrRecv2,bigArrDRecv2,nbrecv3,nbrecv4);
//updating _src_ids_zip_st2 and _src_ids_zip_st2 with received matrix.
updateZipSourceIdsForFuture();
- //finish to fill _the_matrix_st with already in place matrix in _matrixes_st
+ //finish to fill _the_matrix_st with already in place matrix in _matrixes_st (local computation)
finishToFillFinalMatrixST();
- //printTheMatrix();
+ // Prepare proc lists for future field data exchange (mutliply()):
+ fillProcToSendRcvForMultiply(procsToSendField);
+ // Make some space on local proc:
+ _matrixes_st.clear();
+
+// std::stringstream scout;
+// scout << "\n(" << myProcId << ") to send:";
+// for (std::vector< int >::const_iterator itdbg=_proc_ids_to_send_vector_st.begin(); itdbg!=_proc_ids_to_send_vector_st.end(); itdbg++)
+// scout << "," << *itdbg;
+// scout << "\n(" << myProcId << ") to recv:";
+// for (std::vector< int >::const_iterator itdbg=_proc_ids_to_recv_vector_st.begin(); itdbg!=_proc_ids_to_recv_vector_st.end(); itdbg++)
+// scout << "," << *itdbg;
+// std::cout << scout.str() << "\n";
+//
+// printTheMatrix();
+}
+
+/**
+ * Fill the members _proc_ids_to_send_vector_st and _proc_ids_to_recv_vector_st
+ * indicating for each proc to which proc it should send its source field data
+ * and from which proc it should receive source field data.
+ *
+ * Should be called once finishToFillFinalMatrixST() has been executed, i.e. when the
+ * local matrices are completly ready.
+ */
+void OverlapMapping::fillProcToSendRcvForMultiply(const std::vector< int >& procsToSendField)
+{
+// _proc_ids_to_send_vector_st.clear();
+ _proc_ids_to_recv_vector_st.clear();
+ // Receiving side is easy - just inspect non-void terms in the final matrix
+ int i=0;
+ std::vector< std::vector< SparseDoubleVec > >::const_iterator it;
+ std::vector< SparseDoubleVec >::const_iterator it2;
+ for(it=_the_matrix_st.begin(); it!=_the_matrix_st.end(); it++, i++)
+ _proc_ids_to_recv_vector_st.push_back(_the_matrix_st_source_proc_id[i]);
+
+ // Sending side was computed in OverlapElementLocator::computeBoundingBoxesAndTodoList()
+ _proc_ids_to_send_vector_st = procsToSendField;
}
/*!
for(std::size_t j=0;j<sz2;j++)
{
double sum=0;
- std::map<int,double>& mToFill=_the_deno_st[i][j];
- const std::map<int,double>& m=_the_matrix_st[i][j];
- for(std::map<int,double>::const_iterator it=m.begin();it!=m.end();it++)
+ SparseDoubleVec& mToFill=_the_deno_st[i][j];
+ const SparseDoubleVec& m=_the_matrix_st[i][j];
+ for(SparseDoubleVec::const_iterator it=m.begin();it!=m.end();it++)
sum+=(*it).second;
- for(std::map<int,double>::const_iterator it=m.begin();it!=m.end();it++)
+ for(SparseDoubleVec::const_iterator it=m.begin();it!=m.end();it++)
mToFill[(*it).first]=sum;
}
}
*/
void OverlapMapping::computeDenoConservativeVolumic(int nbOfTuplesTrg)
{
- CommInterface commInterface=_group.getCommInterface();
int myProcId=_group.myRank();
//
_the_deno_st.clear();
std::vector<double> deno(nbOfTuplesTrg);
for(std::size_t i=0;i<sz1;i++)
{
- const std::vector< std::map<int,double> >& mat=_the_matrix_st[i];
+ const std::vector< SparseDoubleVec >& mat=_the_matrix_st[i];
int curSrcId=_the_matrix_st_source_proc_id[i];
- std::vector<int>::iterator isItem1=std::find(_trg_proc_st2.begin(),_trg_proc_st2.end(),curSrcId);
+ std::vector<int>::iterator isItem1=std::find(_sent_trg_proc_st2.begin(),_sent_trg_proc_st2.end(),curSrcId);
int rowId=0;
- if(isItem1==_trg_proc_st2.end() || curSrcId==myProcId)//item1 of step2 main algo. Simple, because rowId of mat are directly target ids.
+ if(isItem1==_sent_trg_proc_st2.end() || curSrcId==myProcId)//item1 of step2 main algo. Simple, because rowId of mat are directly target ids.
{
- for(std::vector< std::map<int,double> >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++)
- for(std::map<int,double>::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
+ for(std::vector< SparseDoubleVec >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++)
+ for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
deno[rowId]+=(*it2).second;
}
else
{//item0 of step2 main algo. More complicated.
std::vector<int>::iterator fnd=isItem1;//std::find(_trg_proc_st2.begin(),_trg_proc_st2.end(),curSrcId);
- int locId=std::distance(_trg_proc_st2.begin(),fnd);
- const DataArrayInt *trgIds=_trg_ids_st2[locId];
+ int locId=std::distance(_sent_trg_proc_st2.begin(),fnd);
+ const DataArrayInt *trgIds=_sent_trg_ids_st2[locId];
const int *trgIds2=trgIds->getConstPointer();
- for(std::vector< std::map<int,double> >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++)
- for(std::map<int,double>::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
+ for(std::vector< SparseDoubleVec >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++)
+ for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
deno[trgIds2[rowId]]+=(*it2).second;
}
}
for(std::size_t i=0;i<sz1;i++)
{
int rowId=0;
- const std::vector< std::map<int,double> >& mat=_the_matrix_st[i];
+ const std::vector< SparseDoubleVec >& mat=_the_matrix_st[i];
int curSrcId=_the_matrix_st_source_proc_id[i];
- std::vector<int>::iterator isItem1=std::find(_trg_proc_st2.begin(),_trg_proc_st2.end(),curSrcId);
- std::vector< std::map<int,double> >& denoM=_the_deno_st[i];
+ std::vector<int>::iterator isItem1=std::find(_sent_trg_proc_st2.begin(),_sent_trg_proc_st2.end(),curSrcId);
+ std::vector< SparseDoubleVec >& denoM=_the_deno_st[i];
denoM.resize(mat.size());
- if(isItem1==_trg_proc_st2.end() || curSrcId==myProcId)//item1 of step2 main algo. Simple, because rowId of mat are directly target ids.
+ if(isItem1==_sent_trg_proc_st2.end() || curSrcId==myProcId)//item1 of step2 main algo. Simple, because rowId of mat are directly target ids.
{
int rowId=0;
- for(std::vector< std::map<int,double> >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++)
- for(std::map<int,double>::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
+ for(std::vector< SparseDoubleVec >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++)
+ for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
denoM[rowId][(*it2).first]=deno[rowId];
}
else
{
std::vector<int>::iterator fnd=isItem1;
- int locId=std::distance(_trg_proc_st2.begin(),fnd);
- const DataArrayInt *trgIds=_trg_ids_st2[locId];
+ int locId=std::distance(_sent_trg_proc_st2.begin(),fnd);
+ const DataArrayInt *trgIds=_sent_trg_ids_st2[locId];
const int *trgIds2=trgIds->getConstPointer();
- for(std::vector< std::map<int,double> >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++)
- for(std::map<int,double>::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
+ for(std::vector< SparseDoubleVec >::const_iterator it1=mat.begin();it1!=mat.end();it1++,rowId++)
+ for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
denoM[rowId][(*it2).first]=deno[trgIds2[rowId]];
}
}
int start=offsets[_target_proc_id_st[i]];
int *work=bigArr+start;
*work=0;
- const std::vector< std::map<int,double> >& mat=_matrixes_st[i];
- for(std::vector< std::map<int,double> >::const_iterator it=mat.begin();it!=mat.end();it++,work++)
+ const std::vector< SparseDoubleVec >& mat=_matrixes_st[i];
+ for(std::vector< SparseDoubleVec >::const_iterator it=mat.begin();it!=mat.end();it++,work++)
work[1]=work[0]+(*it).size();
}
}
/*!
* This method performs step#1 and step#2/3. It returns the size of expected array to get allToAllV.
+ * It is where the locally computed matrices are serialized to be sent to adequate final proc.
*/
int OverlapMapping::serializeMatrixStep1ST(const int *nbOfElemsSrc, const int *recvStep0, const int *countStep0, const int *offsStep0,
int *&bigArrI, double *&bigArrD, int *count, int *offsets,
{
if(_source_proc_id_st[i]==myProcId)
{
- const std::vector< std::map<int,double> >& mat=_matrixes_st[i];
+ const std::vector< SparseDoubleVec >& mat=_matrixes_st[i];
int lgthToSend=0;
- for(std::vector< std::map<int,double> >::const_iterator it=mat.begin();it!=mat.end();it++)
+ for(std::vector< SparseDoubleVec >::const_iterator it=mat.begin();it!=mat.end();it++)
lgthToSend+=(*it).size();
count[_target_proc_id_st[i]]=lgthToSend;
fullLgth+=lgthToSend;
{
if(_source_proc_id_st[i]==myProcId)
{
- const std::vector< std::map<int,double> >& mat=_matrixes_st[i];
- for(std::vector< std::map<int,double> >::const_iterator it1=mat.begin();it1!=mat.end();it1++)
+ const std::vector< SparseDoubleVec >& mat=_matrixes_st[i];
+ for(std::vector< SparseDoubleVec >::const_iterator it1=mat.begin();it1!=mat.end();it1++)
{
int j=0;
- for(std::map<int,double>::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++,j++)
+ for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++,j++)
{
bigArrI[fullLgth+j]=(*it2).first;
bigArrD[fullLgth+j]=(*it2).second;
}
/*!
- * This method should be called when all remote matrix with sourceProcId==thisProcId have been retrieved and are in 'this->_the_matrix_st' and 'this->_the_matrix_st_target_proc_id'
- * and 'this->_the_matrix_st_target_ids'.
- * This method finish the job of filling 'this->_the_matrix_st' and 'this->_the_matrix_st_target_proc_id' by putting candidates in 'this->_matrixes_st' into them.
+ * This method should be called when all remote matrix with sourceProcId==thisProcId have been retrieved and are
+ * in 'this->_the_matrix_st' and 'this->_the_matrix_st_target_proc_id' and 'this->_the_matrix_st_target_ids'.
+ * This method finish the job of filling 'this->_the_matrix_st' and 'this->_the_matrix_st_target_proc_id'
+ * by putting candidates in 'this->_matrixes_st' into them (i.e. local computation result).
*/
void OverlapMapping::finishToFillFinalMatrixST()
{
for(int i=0;i<sz;i++)
if(_source_proc_id_st[i]!=myProcId)
{
- const std::vector<std::map<int,double> >& mat=_matrixes_st[i];
+ const std::vector<SparseDoubleVec >& mat=_matrixes_st[i];
_the_matrix_st[j]=mat;
_the_matrix_st_source_proc_id.push_back(_source_proc_id_st[i]);
j++;
}
- _matrixes_st.clear();
-}
-
-/*!
- * This method performs the operation of target ids broadcasting.
- */
-void OverlapMapping::prepareIdsToSendST()
-{
- CommInterface commInterface=_group.getCommInterface();
- const MPIProcessorGroup *group=static_cast<const MPIProcessorGroup*>(&_group);
- const MPI_Comm *comm=group->getComm();
- int grpSize=_group.size();
- _source_ids_to_send_st.clear();
- _source_ids_to_send_st.resize(grpSize);
- INTERP_KERNEL::AutoPtr<int> nbsend=new int[grpSize];
- std::fill<int *>(nbsend,nbsend+grpSize,0);
- for(std::size_t i=0;i<_the_matrix_st_source_proc_id.size();i++)
- nbsend[_the_matrix_st_source_proc_id[i]]=_the_matrix_st_source_ids[i].size();
- INTERP_KERNEL::AutoPtr<int> nbrecv=new int[grpSize];
- commInterface.allToAll(nbsend,1,MPI_INT,nbrecv,1,MPI_INT,*comm);
- //
- INTERP_KERNEL::AutoPtr<int> nbsend2=new int[grpSize];
- std::copy((int *)nbsend,((int *)nbsend)+grpSize,(int *)nbsend2);
- INTERP_KERNEL::AutoPtr<int> nbsend3=new int[grpSize];
- nbsend3[0]=0;
- for(int i=1;i<grpSize;i++)
- nbsend3[i]=nbsend3[i-1]+nbsend2[i-1];
- int sendSz=nbsend3[grpSize-1]+nbsend2[grpSize-1];
- INTERP_KERNEL::AutoPtr<int> bigDataSend=new int[sendSz];
- for(std::size_t i=0;i<_the_matrix_st_source_proc_id.size();i++)
- {
- int offset=nbsend3[_the_matrix_st_source_proc_id[i]];
- std::copy(_the_matrix_st_source_ids[i].begin(),_the_matrix_st_source_ids[i].end(),((int *)nbsend3)+offset);
- }
- INTERP_KERNEL::AutoPtr<int> nbrecv2=new int[grpSize];
- INTERP_KERNEL::AutoPtr<int> nbrecv3=new int[grpSize];
- std::copy((int *)nbrecv,((int *)nbrecv)+grpSize,(int *)nbrecv2);
- nbrecv3[0]=0;
- for(int i=1;i<grpSize;i++)
- nbrecv3[i]=nbrecv3[i-1]+nbrecv2[i-1];
- int recvSz=nbrecv3[grpSize-1]+nbrecv2[grpSize-1];
- INTERP_KERNEL::AutoPtr<int> bigDataRecv=new int[recvSz];
- //
- commInterface.allToAllV(bigDataSend,nbsend2,nbsend3,MPI_INT,
- bigDataRecv,nbrecv2,nbrecv3,MPI_INT,
- *comm);
- for(int i=0;i<grpSize;i++)
- {
- if(nbrecv2[i]>0)
- {
- _source_ids_to_send_st[i].insert(_source_ids_to_send_st[i].end(),((int *)bigDataRecv)+nbrecv3[i],((int *)bigDataRecv)+nbrecv3[i]+nbrecv2[i]);
- }
- }
}
/*!
nbsend2[0]=0;
nbrecv2[0]=0;
std::vector<double> valsToSend;
+
+ /*
+ * FIELD VALUE XCHGE
+ */
for(int i=0;i<grpSize;i++)
{
+ // Prepare field data to be sent/received through a MPI_AllToAllV
if(std::find(_proc_ids_to_send_vector_st.begin(),_proc_ids_to_send_vector_st.end(),i)!=_proc_ids_to_send_vector_st.end())
{
- std::vector<int>::const_iterator isItem1=std::find(_src_proc_st2.begin(),_src_proc_st2.end(),i);
+ std::vector<int>::const_iterator isItem1=std::find(_sent_src_proc_st2.begin(),_sent_src_proc_st2.end(),i);
MEDCouplingAutoRefCountObjectPtr<DataArrayDouble> vals;
- if(isItem1!=_src_proc_st2.end())//item1 of step2 main algo
+ if(isItem1!=_sent_src_proc_st2.end()) // source data was sent to proc 'i'
{
- int id=std::distance(_src_proc_st2.begin(),isItem1);
- vals=fieldInput->getArray()->selectByTupleId(_src_ids_st2[id]->getConstPointer(),_src_ids_st2[id]->getConstPointer()+_src_ids_st2[id]->getNumberOfTuples());
+ // Prepare local field data to send to proc i
+ int id=std::distance(_sent_src_proc_st2.begin(),isItem1);
+ vals=fieldInput->getArray()->selectByTupleId(*_sent_src_ids_st2[id]);
}
- else
+ else // no source data was sent to proc 'i'
{//item0 of step2 main algo
- int id=std::distance(_src_ids_zip_proc_st2.begin(),std::find(_src_ids_zip_proc_st2.begin(),_src_ids_zip_proc_st2.end(),i));
+ std::vector<int>::const_iterator isItem22 = std::find(_src_ids_zip_proc_st2.begin(),_src_ids_zip_proc_st2.end(),i );
+ int id=std::distance(_src_ids_zip_proc_st2.begin(),isItem22);
+ if (isItem22 == _src_ids_zip_proc_st2.end())
+ std::cout << "(" << myProcId << ") has end iterator!!!!!\n";
vals=fieldInput->getArray()->selectByTupleId(&(_src_ids_zip_st2[id])[0],&(_src_ids_zip_st2[id])[0]+_src_ids_zip_st2[id].size());
}
nbsend[i]=vals->getNbOfElems();
}
if(std::find(_proc_ids_to_recv_vector_st.begin(),_proc_ids_to_recv_vector_st.end(),i)!=_proc_ids_to_recv_vector_st.end())
{
- std::vector<int>::const_iterator isItem0=std::find(_trg_proc_st2.begin(),_trg_proc_st2.end(),i);
- if(isItem0==_trg_proc_st2.end())//item1 of step2 main algo [ (0,1) computed on proc1 and Matrix-Vector on proc1 ]
+ std::vector<int>::const_iterator isItem0=std::find(_sent_trg_proc_st2.begin(),_sent_trg_proc_st2.end(),i);
+ if(isItem0==_sent_trg_proc_st2.end())//item1 of step2 main algo [ (0,1) computed on proc1 and Matrix-Vector on proc1 ]
{
std::vector<int>::const_iterator it1=std::find(_src_ids_proc_st2.begin(),_src_ids_proc_st2.end(),i);
if(it1!=_src_ids_proc_st2.end())
int id=std::distance(_src_ids_proc_st2.begin(),it1);
nbrecv[i]=_nb_of_src_ids_proc_st2[id]*nbOfCompo;
}
- else if(i==myProcId)
+ else if(i==myProcId) // diagonal element (i.e. proc #i talking to same proc #i)
{
- nbrecv[i]=fieldInput->getNumberOfTuplesExpected()*nbOfCompo;
+ nbrecv[i] = nbsend[i];
}
else
- throw INTERP_KERNEL::Exception("Plouff ! send email to anthony.geay@cea.fr ! ");
+ throw INTERP_KERNEL::Exception("OverlapMapping::multiply(): Internal error at field values exchange!");
}
else
{//item0 of step2 main algo [ (2,1) computed on proc2 but Matrix-Vector on proc1 ] [(1,0) computed on proc1 but Matrix-Vector on proc0]
nbrecv2[i]=nbrecv2[i-1]+nbrecv[i-1];
}
INTERP_KERNEL::AutoPtr<double> bigArr=new double[nbrecv2[grpSize-1]+nbrecv[grpSize-1]];
+
+// scout << "(" << myProcId << ") nbsend :" << nbsend[0] << "," << nbsend[1] << "," << nbsend[2] << "\n";
+// scout << "(" << myProcId << ") nbrecv :" << nbrecv[0] << "," << nbrecv[1] << "," << nbrecv[2] << "\n";
+// std::cout << scout.str() << "\n";
+ /*
+ * *********************** ALL-TO-ALL
+ */
commInterface.allToAllV(&valsToSend[0],nbsend,nbsend2,MPI_DOUBLE,
bigArr,nbrecv,nbrecv2,MPI_DOUBLE,*comm);
+ /*
+ *
+ * TARGET FIELD COMPUTATION (matrix-vec computation)
+ */
fieldOutput->getArray()->fillWithZero();
INTERP_KERNEL::AutoPtr<double> tmp=new double[nbOfCompo];
for(int i=0;i<grpSize;i++)
if(it==_the_matrix_st_source_proc_id.end())
throw INTERP_KERNEL::Exception("Big problem !");
int id=std::distance(_the_matrix_st_source_proc_id.begin(),it);
- const std::vector< std::map<int,double> >& mat=_the_matrix_st[id];
- const std::vector< std::map<int,double> >& deno=_the_deno_st[id];
- std::vector<int>::const_iterator isItem0=std::find(_trg_proc_st2.begin(),_trg_proc_st2.end(),i);
- if(isItem0==_trg_proc_st2.end())//item1 of step2 main algo [ (0,1) computed on proc1 and Matrix-Vector on proc1 ]
+ const std::vector< SparseDoubleVec >& mat=_the_matrix_st[id];
+ const std::vector< SparseDoubleVec >& deno=_the_deno_st[id];
+ std::vector<int>::const_iterator isItem0=std::find(_sent_trg_proc_st2.begin(),_sent_trg_proc_st2.end(),i);
+ if(isItem0==_sent_trg_proc_st2.end())//item1 of step2 main algo [ (0,1) computed on proc1 and Matrix-Vector on proc1 ]
{
int nbOfTrgTuples=mat.size();
for(int j=0;j<nbOfTrgTuples;j++,pt+=nbOfCompo)
{
- const std::map<int,double>& mat1=mat[j];
- const std::map<int,double>& deno1=deno[j];
- std::map<int,double>::const_iterator it4=deno1.begin();
- for(std::map<int,double>::const_iterator it3=mat1.begin();it3!=mat1.end();it3++,it4++)
+ const SparseDoubleVec& mat1=mat[j];
+ const SparseDoubleVec& deno1=deno[j];
+ SparseDoubleVec::const_iterator it4=deno1.begin();
+ for(SparseDoubleVec::const_iterator it3=mat1.begin();it3!=mat1.end();it3++,it4++)
{
- std::transform(bigArr+nbrecv2[i]+((*it3).first)*nbOfCompo,bigArr+nbrecv2[i]+((*it3).first+1)*(nbOfCompo),(double *)tmp,std::bind2nd(std::multiplies<double>(),(*it3).second/(*it4).second));
+ std::transform(bigArr+nbrecv2[i]+((*it3).first)*nbOfCompo,
+ bigArr+nbrecv2[i]+((*it3).first+1)*(nbOfCompo),
+ (double *)tmp,
+ std::bind2nd(std::multiplies<double>(),(*it3).second/(*it4).second));
std::transform((double *)tmp,(double *)tmp+nbOfCompo,pt,pt,std::plus<double>());
}
}
int newId=0;
for(std::vector<int>::const_iterator it=zipIds.begin();it!=zipIds.end();it++,newId++)
zipCor[*it]=newId;
- int id2=std::distance(_trg_proc_st2.begin(),std::find(_trg_proc_st2.begin(),_trg_proc_st2.end(),i));
- const DataArrayInt *tgrIds=_trg_ids_st2[id2];
+ int id2=std::distance(_sent_trg_proc_st2.begin(),std::find(_sent_trg_proc_st2.begin(),_sent_trg_proc_st2.end(),i));
+ const DataArrayInt *tgrIds=_sent_trg_ids_st2[id2];
const int *tgrIds2=tgrIds->getConstPointer();
int nbOfTrgTuples=mat.size();
for(int j=0;j<nbOfTrgTuples;j++)
{
- const std::map<int,double>& mat1=mat[j];
- const std::map<int,double>& deno1=deno[j];
- std::map<int,double>::const_iterator it5=deno1.begin();
- for(std::map<int,double>::const_iterator it3=mat1.begin();it3!=mat1.end();it3++,it5++)
+ const SparseDoubleVec& mat1=mat[j];
+ const SparseDoubleVec& deno1=deno[j];
+ SparseDoubleVec::const_iterator it5=deno1.begin();
+ for(SparseDoubleVec::const_iterator it3=mat1.begin();it3!=mat1.end();it3++,it5++)
{
std::map<int,int>::const_iterator it4=zipCor.find((*it3).first);
if(it4==zipCor.end())
- throw INTERP_KERNEL::Exception("Hmmmmm send e mail to anthony.geay@cea.fr !");
- std::transform(bigArr+nbrecv2[i]+((*it4).second)*nbOfCompo,bigArr+nbrecv2[i]+((*it4).second+1)*(nbOfCompo),(double *)tmp,std::bind2nd(std::multiplies<double>(),(*it3).second/(*it5).second));
+ throw INTERP_KERNEL::Exception("OverlapMapping::multiply(): Internal error in final value computation!");
+ std::transform(bigArr+nbrecv2[i]+((*it4).second)*nbOfCompo,
+ bigArr+nbrecv2[i]+((*it4).second+1)*(nbOfCompo),
+ (double *)tmp,
+ std::bind2nd(std::multiplies<double>(),(*it3).second/(*it5).second));
std::transform((double *)tmp,(double *)tmp+nbOfCompo,pt+tgrIds2[j]*nbOfCompo,pt+tgrIds2[j]*nbOfCompo,std::plus<double>());
}
}
}
/*!
- * This method should be called immediately after _the_matrix_st has been filled with remote computed matrix put in this proc for Matrix-Vector.
- * This method computes for these matrix the minimal set of source ids corresponding to the source proc id.
+ * This method should be called immediately after _the_matrix_st has been filled with remote computed matrix
+ * put in this proc for Matrix-Vector.
+ * This method computes for these matrix the minimal set of source ids corresponding to the source proc id.
+ *
*/
void OverlapMapping::updateZipSourceIdsForFuture()
{
+ /* When it is called, only the bits received from other processors (i.e. the remotely executed jobs) are in the
+ big matrix _the_matrix_st. */
+
CommInterface commInterface=_group.getCommInterface();
int myProcId=_group.myRank();
int nbOfMatrixRecveived=_the_matrix_st_source_proc_id.size();
int curSrcProcId=_the_matrix_st_source_proc_id[i];
if(curSrcProcId!=myProcId)
{
- const std::vector< std::map<int,double> >& mat=_the_matrix_st[i];
+ const std::vector< SparseDoubleVec >& mat=_the_matrix_st[i];
_src_ids_zip_proc_st2.push_back(curSrcProcId);
_src_ids_zip_st2.resize(_src_ids_zip_st2.size()+1);
std::set<int> s;
- for(std::vector< std::map<int,double> >::const_iterator it1=mat.begin();it1!=mat.end();it1++)
- for(std::map<int,double>::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
+ for(std::vector< SparseDoubleVec >::const_iterator it1=mat.begin();it1!=mat.end();it1++)
+ for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
s.insert((*it2).first);
_src_ids_zip_st2.back().insert(_src_ids_zip_st2.back().end(),s.begin(),s.end());
+
+// std::stringstream scout;
+// scout << "(" << myProcId << ") pushed " << _src_ids_zip_st2.back().size() << " values for tgt proc " << curSrcProcId;
+// std::cout << scout.str() << "\n";
}
}
}
-// #include <iostream>
-
// void OverlapMapping::printTheMatrix() const
// {
// CommInterface commInterface=_group.getCommInterface();
// const MPI_Comm *comm=group->getComm();
// int grpSize=_group.size();
// int myProcId=_group.myRank();
-// std::cerr << "I am proc #" << myProcId << std::endl;
+// std::stringstream oscerr;
// int nbOfMat=_the_matrix_st.size();
-// std::cerr << "I do manage " << nbOfMat << "matrix : "<< std::endl;
+// oscerr << "(" << myProcId << ") I hold " << nbOfMat << " matrix(ces) : "<< std::endl;
// for(int i=0;i<nbOfMat;i++)
// {
-// std::cerr << " - Matrix #" << i << " on source proc #" << _the_matrix_st_source_proc_id[i];
-// const std::vector< std::map<int,double> >& locMat=_the_matrix_st[i];
-// for(std::vector< std::map<int,double> >::const_iterator it1=locMat.begin();it1!=locMat.end();it1++)
+// oscerr << " - Matrix #" << i << " coming from source proc #" << _the_matrix_st_source_proc_id[i] << ": ";
+// const std::vector< SparseDoubleVec >& locMat=_the_matrix_st[i];
+// int j = 0;
+// for(std::vector< SparseDoubleVec >::const_iterator it1=locMat.begin();it1!=locMat.end();it1++, j++)
// {
-// for(std::map<int,double>::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
-// std::cerr << "(" << (*it2).first << "," << (*it2).second << "), ";
-// std::cerr << std::endl;
+// oscerr << " Target Cell #" << j;
+// for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
+// oscerr << " (" << (*it2).first << "," << (*it2).second << "), ";
+// oscerr << std::endl;
// }
// }
-// std::cerr << "*********" << std::endl;
+// oscerr << "*********" << std::endl;
+//
+// // Hope this will be flushed in one go:
+// std::cerr << oscerr.str() << std::endl;
+//// if(myProcId != 0)
+//// MPI_Barrier(MPI_COMM_WORLD);
// }
+//
+// void OverlapMapping::printMatrixesST() const
+// {
+// CommInterface commInterface=_group.getCommInterface();
+// const MPIProcessorGroup *group=static_cast<const MPIProcessorGroup*>(&_group);
+// const MPI_Comm *comm=group->getComm();
+// int grpSize=_group.size();
+// int myProcId=_group.myRank();
+// std::stringstream oscerr;
+// int nbOfMat=_matrixes_st.size();
+// oscerr << "(" << myProcId << ") I hold " << nbOfMat << " LOCAL matrix(ces) : "<< std::endl;
+// for(int i=0;i<nbOfMat;i++)
+// {
+// oscerr << " - Matrix #" << i << ": (source proc #" << _source_proc_id_st[i] << " / tgt proc#" << _target_proc_id_st[i] << "):";
+// const std::vector< SparseDoubleVec >& locMat=_matrixes_st[i];
+// int j = 0;
+// for(std::vector< SparseDoubleVec >::const_iterator it1=locMat.begin();it1!=locMat.end();it1++, j++)
+// {
+// oscerr << " Target Cell #" << j;
+// for(SparseDoubleVec::const_iterator it2=(*it1).begin();it2!=(*it1).end();it2++)
+// oscerr << " (" << (*it2).first << "," << (*it2).second << "), ";
+// oscerr << std::endl;
+// }
+// }
+// oscerr << "*********" << std::endl;
+//
+// // Hope this will be flushed in one go:
+// std::cerr << oscerr.str() << std::endl;
+// }
