From: Anthony GEAY Date: Tue, 10 Nov 2020 15:21:28 +0000 (+0100) Subject: Optimize memory peak during coordinates loading implied by MEDFileUMesh.LoadPart. X-Git-Tag: V9_6_asterxx_0~5 X-Git-Url: http://git.salome-platform.org/gitweb/?a=commitdiff_plain;h=4f627963e2f8e90b80c9dc6a06cd97079a90b025;p=tools%2Fmedcoupling.git Optimize memory peak during coordinates loading implied by MEDFileUMesh.LoadPart. --- diff --git a/src/MEDCoupling/MEDCouplingPartDefinition.hxx b/src/MEDCoupling/MEDCouplingPartDefinition.hxx index 0a8e78c3b..39ccd539e 100644 --- a/src/MEDCoupling/MEDCouplingPartDefinition.hxx +++ b/src/MEDCoupling/MEDCouplingPartDefinition.hxx @@ -18,8 +18,7 @@ // // Author : Anthony Geay (EDF R&D) -#ifndef __PARAMEDMEM_MEDCOUPLINGPARTDEFINITION_HXX__ -#define __PARAMEDMEM_MEDCOUPLINGPARTDEFINITION_HXX__ +#pragma once #include "MEDCoupling.hxx" #include "MEDCouplingMemArray.hxx" @@ -110,5 +109,3 @@ namespace MEDCoupling mcIdType _step; }; } - -#endif diff --git a/src/MEDLoader/MEDFileMeshLL.cxx b/src/MEDLoader/MEDFileMeshLL.cxx index 5809f3595..3e476b2b9 100644 --- a/src/MEDLoader/MEDFileMeshLL.cxx +++ b/src/MEDLoader/MEDFileMeshLL.cxx @@ -560,13 +560,26 @@ void MEDFileUMeshL2::loadPart(med_idt fid, const MeshOrStructMeshCls *mId, const for(std::vector< std::vector< MCAuto > >::const_iterator it0=_per_type_mesh.begin();it0!=_per_type_mesh.end();it0++) for(std::vector< MCAuto >::const_iterator it1=(*it0).begin();it1!=(*it0).end();it1++) (*it1)->getMesh()->computeNodeIdsAlg(fetchedNodeIds); - mcIdType nMin(ToIdType(std::distance(fetchedNodeIds.begin(),std::find(fetchedNodeIds.begin(),fetchedNodeIds.end(),true)))); - mcIdType nMax(ToIdType(std::distance(fetchedNodeIds.rbegin(),std::find(fetchedNodeIds.rbegin(),fetchedNodeIds.rend(),true)))); - nMax=nCoords-nMax; - for(std::vector< std::vector< MCAuto > >::const_iterator it0=_per_type_mesh.begin();it0!=_per_type_mesh.end();it0++) - for(std::vector< MCAuto >::const_iterator it1=(*it0).begin();it1!=(*it0).end();it1++) - (*it1)->getMesh()->renumberNodesWithOffsetInConn(-nMin); - loadPartCoords(fid,infosOnComp,mName,dt,it,nMin,nMax); + if(!mrs || mrs->getNumberOfCoordsLoadSessions()==1) + { + mcIdType nMin(ToIdType(std::distance(fetchedNodeIds.begin(),std::find(fetchedNodeIds.begin(),fetchedNodeIds.end(),true)))); + mcIdType nMax(ToIdType(std::distance(fetchedNodeIds.rbegin(),std::find(fetchedNodeIds.rbegin(),fetchedNodeIds.rend(),true)))); + nMax=nCoords-nMax; + for(std::vector< std::vector< MCAuto > >::const_iterator it0=_per_type_mesh.begin();it0!=_per_type_mesh.end();it0++) + for(std::vector< MCAuto >::const_iterator it1=(*it0).begin();it1!=(*it0).end();it1++) + (*it1)->getMesh()->renumberNodesWithOffsetInConn(-nMin); + this->loadPartCoords(fid,infosOnComp,mName,dt,it,nMin,nMax); + } + else + { + mcIdType nbOfCooLS(mrs->getNumberOfCoordsLoadSessions()); + MCAuto fni(DataArrayIdType::BuildListOfSwitchedOn(fetchedNodeIds)); + MCAuto< MapKeyVal > o2n(fni->invertArrayN2O2O2NOptimized()); + for(std::vector< std::vector< MCAuto > >::const_iterator it0=_per_type_mesh.begin();it0!=_per_type_mesh.end();it0++) + for(std::vector< MCAuto >::const_iterator it1=(*it0).begin();it1!=(*it0).end();it1++) + (*it1)->getMesh()->renumberNodesInConn(o2n->data()); + this->loadPartCoordsSlice(fid,infosOnComp,mName,dt,it,fni,nbOfCooLS); + } } void MEDFileUMeshL2::loadConnectivity(med_idt fid, int mdim, const std::string& mName, int dt, int it, MEDFileMeshReadSelector *mrs) @@ -698,11 +711,76 @@ MCAuto& _coords, MCAuto& _part_coords, MCAutosetInfoOnComponents(infosOnComp); } +/*! + * For performance reasons LoadPartCoordsArray method calls LoadPartCoords + */ +void MEDFileUMeshL2::LoadPartCoordsArray(med_idt fid, const std::vector& infosOnComp, const std::string& mName, int dt, int it, const DataArrayIdType *nodeIds, +MCAuto& _coords, MCAuto& _fam_coords, MCAuto& _num_coords, MCAuto& _name_coords) +{ + MCAuto useless; + nodeIds->checkAllocated(); + nodeIds->checkNbOfComps(1,"loadPartCoordsSlice : Only one component expected !"); + mcIdType nMin(0),nMax(0); + if(!nodeIds->empty()) + { nMin = nodeIds->front(); nMax = nodeIds->back()+1; } + LoadPartCoords(fid,infosOnComp,mName,dt,it,nMin,nMax,_coords,useless,_fam_coords,_num_coords,_name_coords); + if(nodeIds->empty()) + return ; + MCAuto nodeIds2(nodeIds->deepCopy()); + nodeIds2->applyLin(1,-nMin); + _coords = _coords->selectByTupleIdSafe(nodeIds2->begin(),nodeIds2->end()); + if(_fam_coords.isNotNull()) + _fam_coords = _fam_coords->selectByTupleIdSafe(nodeIds2->begin(),nodeIds2->end()); + if(_num_coords.isNotNull()) + _num_coords = _num_coords->selectByTupleIdSafe(nodeIds2->begin(),nodeIds2->end()); + if(_name_coords.isNotNull()) + { + MCAuto tmp(_name_coords->selectByTupleIdSafe(nodeIds2->begin(),nodeIds2->end())); + _name_coords = DynamicCastSafe( tmp ); + } +} + void MEDFileUMeshL2::loadPartCoords(med_idt fid, const std::vector& infosOnComp, const std::string& mName, int dt, int it, mcIdType nMin, mcIdType nMax) { LoadPartCoords(fid,infosOnComp,mName,dt,it,nMin,nMax,_coords,_part_coords,_fam_coords,_num_coords,_name_coords); } +void MEDFileUMeshL2::loadPartCoordsSlice(med_idt fid, const std::vector& infosOnComp, const std::string& mName, int dt, int it, const DataArrayIdType *nodeIds, mcIdType nbOfCoordLS) +{ + nodeIds->checkAllocated(); + nodeIds->checkNbOfComps(1,"loadPartCoordsSlice : Only one component expected !"); + if(nodeIds->empty()) + return ; + if( nbOfCoordLS<1 ) + throw INTERP_KERNEL::Exception("MEDFileUMeshL2::loadPartCoordsSlice : nb of coords load session must be >=1 !"); + mcIdType nMin(nodeIds->front()),nMax(nodeIds->back()+1); + std::vector< MCAuto > coords(nbOfCoordLS); + std::vector< MCAuto > famCoords(nbOfCoordLS); + std::vector< MCAuto > numCoords(nbOfCoordLS); + std::vector< MCAuto > nameCoords(nbOfCoordLS); + for(mcIdType ipart = 0 ; ipart < nbOfCoordLS ; ++ipart) + { + mcIdType partStart,partStop; + DataArray::GetSlice(nMin,nMax,1,ipart,nbOfCoordLS,partStart,partStop); + MCAuto idsNodeIdsToKeep(nodeIds->findIdsInRange(partStart,partStop)); + MCAuto nodeIdsToKeep( nodeIds->selectByTupleIdSafe(idsNodeIdsToKeep->begin(),idsNodeIdsToKeep->end()) ); + LoadPartCoordsArray(fid,infosOnComp,mName,dt,it,nodeIdsToKeep,coords[ipart],famCoords[ipart],numCoords[ipart],nameCoords[ipart]); + } + _coords = DataArrayDouble::Aggregate(ToConstVect(coords)); + if(famCoords[0].isNotNull()) + _fam_coords = DataArrayIdType::Aggregate(ToConstVect(famCoords)); + if(numCoords[0].isNotNull()) + _num_coords = DataArrayIdType::Aggregate(ToConstVect(numCoords)); + if(nameCoords[0].isNotNull()) + { + std::vector< MCAuto > nameCoords2(nameCoords.begin(),nameCoords.end()); + std::for_each(nameCoords2.begin(),nameCoords2.end(),[](MCAuto& elt){ elt->incrRef(); }); + MCAuto tmp( DataArrayChar::Aggregate(ToConstVect(nameCoords2)) ); + _name_coords = DynamicCastSafe( tmp ); + } + _part_coords = DataArrayPartDefinition::New( const_cast(nodeIds) ); +} + void MEDFileUMeshL2::sortTypes() { std::set mdims; diff --git a/src/MEDLoader/MEDFileMeshLL.hxx b/src/MEDLoader/MEDFileMeshLL.hxx index 7552e158f..5c9a667a2 100644 --- a/src/MEDLoader/MEDFileMeshLL.hxx +++ b/src/MEDLoader/MEDFileMeshLL.hxx @@ -125,6 +125,7 @@ namespace MEDCoupling void loadPartOfConnectivity(med_idt fid, int mdim, const std::string& mName, const std::vector& types, const std::vector& slicPerTyp, int dt, int it, MEDFileMeshReadSelector *mrs); void loadCoords(med_idt fid, const std::vector& infosOnComp, const std::string& mName, int dt, int it); void loadPartCoords(med_idt fid, const std::vector& infosOnComp, const std::string& mName, int dt, int it, mcIdType nMin, mcIdType nMax); + void loadPartCoordsSlice(med_idt fid, const std::vector& infosOnComp, const std::string& mName, int dt, int it, const DataArrayIdType *nodeIds, mcIdType nbOfCoordLS); int getNumberOfLevels() const { return (int)_per_type_mesh.size(); } bool emptyLev(int levId) const { return _per_type_mesh[levId].empty(); } const std::vector< MCAuto >& getLev(int levId) const { return _per_type_mesh[levId]; } @@ -139,6 +140,8 @@ namespace MEDCoupling static void WriteCoords(med_idt fid, const std::string& mname, int dt, int it, double time, const DataArrayDouble *coords, const DataArrayIdType *famCoords, const DataArrayIdType *numCoords, const DataArrayAsciiChar *nameCoords, const DataArrayIdType *globalNumCoords); static void LoadPartCoords(med_idt fid, const std::vector& infosOnComp, const std::string& mName, int dt, int it, mcIdType nMin, mcIdType nMax, MCAuto& _coords, MCAuto& _part_coords, MCAuto& _fam_coords, MCAuto& _num_coords, MCAuto& _name_coords); + static void LoadPartCoordsArray(med_idt fid, const std::vector& infosOnComp, const std::string& mName, int dt, int it, const DataArrayIdType *nodeIds, +MCAuto& _coords, MCAuto& _fam_coords, MCAuto& _num_coords, MCAuto& _name_coords); private: void sortTypes(); private: diff --git a/src/MEDLoader/MEDFileMeshReadSelector.cxx b/src/MEDLoader/MEDFileMeshReadSelector.cxx index eb717c10e..241086d43 100644 --- a/src/MEDLoader/MEDFileMeshReadSelector.cxx +++ b/src/MEDLoader/MEDFileMeshReadSelector.cxx @@ -16,13 +16,17 @@ // // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com // -// Author : Anthony Geay (CEA/DEN) +// Author : Anthony Geay (EDF R&D) #include "MEDFileMeshReadSelector.hxx" +#include "InterpKernelException.hxx" + +#include + using namespace MEDCoupling; -MEDFileMeshReadSelector::MEDFileMeshReadSelector():_code(0xFFFFFFFF) +MEDFileMeshReadSelector::MEDFileMeshReadSelector():_nb_coords_load_sessions(1),_code(0xFFFFFFFF) { } @@ -40,6 +44,13 @@ void MEDFileMeshReadSelector::setCode(unsigned int newCode) _code=newCode; } +void MEDFileMeshReadSelector::setNumberOfCoordsLoadSessions(mcIdType newNbOfCoordsLoadSessions) +{ + if(newNbOfCoordsLoadSessions < 1) + throw INTERP_KERNEL::Exception("MEDFileMeshReadSelector::setNumberOfCoordsLoadSessions : input must be >= 1 !"); + _nb_coords_load_sessions = newNbOfCoordsLoadSessions; +} + bool MEDFileMeshReadSelector::isCellFamilyFieldReading() const { return _code & 0x00000001; @@ -141,6 +152,7 @@ void MEDFileMeshReadSelector::setGlobalNodeNumFieldReading(bool b) void MEDFileMeshReadSelector::reprAll(std::ostream& str) const { str << "MEDFileMeshReadSelector (code=" << _code << ") : \n"; + str << "Number of coords load part sessions : " << this->_nb_coords_load_sessions << std::endl; str << "Read family field on cells : " << ReprStatus(isCellFamilyFieldReading()) << std::endl; str << "Read family field on nodes : " << ReprStatus(isNodeFamilyFieldReading()) << std::endl; str << "Read name field on cells : " << ReprStatus(isCellNameFieldReading()) << std::endl; diff --git a/src/MEDLoader/MEDFileMeshReadSelector.hxx b/src/MEDLoader/MEDFileMeshReadSelector.hxx index 514ebea62..8db3a3c3b 100644 --- a/src/MEDLoader/MEDFileMeshReadSelector.hxx +++ b/src/MEDLoader/MEDFileMeshReadSelector.hxx @@ -16,14 +16,14 @@ // // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com // -// Author : Anthony Geay (CEA/DEN) +// Author : Anthony Geay (EDF R&D) -#ifndef __MEDFILEMESHREADSELECTOR_HXX__ -#define __MEDFILEMESHREADSELECTOR_HXX__ +#pragma once #include "MEDLoaderDefines.hxx" -#include +#include "MCIdType.hxx" + #include namespace MEDCoupling @@ -35,6 +35,8 @@ namespace MEDCoupling MEDFileMeshReadSelector(unsigned int code); unsigned int getCode() const; void setCode(unsigned int newCode); + mcIdType getNumberOfCoordsLoadSessions() const { return _nb_coords_load_sessions; } + void setNumberOfCoordsLoadSessions(mcIdType newNbOfCoordsLoadSessions); bool isCellFamilyFieldReading() const; bool isNodeFamilyFieldReading() const; bool isCellNameFieldReading() const; @@ -53,6 +55,7 @@ namespace MEDCoupling private: static std::string ReprStatus(bool v); private: + mcIdType _nb_coords_load_sessions; //bit #0 cell family field //bit #1 node family field //bit #2 cell name field @@ -63,4 +66,3 @@ namespace MEDCoupling }; } -#endif diff --git a/src/MEDLoader/Swig/MEDLoaderCommon.i b/src/MEDLoader/Swig/MEDLoaderCommon.i index 955e5f565..b1176fe83 100644 --- a/src/MEDLoader/Swig/MEDLoaderCommon.i +++ b/src/MEDLoader/Swig/MEDLoaderCommon.i @@ -672,6 +672,8 @@ namespace MEDCoupling public: MEDFileMeshReadSelector(); MEDFileMeshReadSelector(unsigned int code); + mcIdType getNumberOfCoordsLoadSessions(); + void setNumberOfCoordsLoadSessions(mcIdType newNbOfCoordsLoadSessions); unsigned int getCode() const; void setCode(unsigned int newCode); bool isCellFamilyFieldReading() const; diff --git a/src/MEDLoader/Swig/MEDLoaderTest3.py b/src/MEDLoader/Swig/MEDLoaderTest3.py index de1a827a6..cc6f3ad12 100644 --- a/src/MEDLoader/Swig/MEDLoaderTest3.py +++ b/src/MEDLoader/Swig/MEDLoaderTest3.py @@ -4159,6 +4159,28 @@ class MEDLoaderTest3(unittest.TestCase): arr=DataArrayDouble([(204,304),(205,305),(206,306),(207,307),(210,310),(211,311),(212,312),(213,313)]) arr.setInfoOnComponents(compos) self.assertTrue(fs[1][0].getUndergroundDataArray().isEqual(arr,1e-12)) + m_ref = mm[0].deepCopy() + # now read it in 2 load sessions to avoid memory peak. zipCoords is no more requested here. + ms=MEDFileMeshes() + mrs = MEDFileMeshReadSelector() + mrs.setNumberOfCoordsLoadSessions(2) + mm=MEDFileUMesh.LoadPartOf(fileName,meshName,[NORM_QUAD4],[4,6,1],-1,-1,mrs) + ms.pushMesh(mm) + spd=mm.getPartDefAtLevel(0,NORM_QUAD4) + self.assertEqual(spd.getSlice(),slice(4,6,1)) + spd=mm.getPartDefAtLevel(1) + self.assertTrue(spd.getNumberOfElems()==8 and spd.getNumberOfElems()==mm.getNumberOfNodes()) + self.assertTrue(spd.toDAI().isEqual(DataArrayInt([4,5,6,7,10,11,12,13]))) + fs=MEDFileFields.LoadPartOf(fileName,False,ms) + fs[0][0].loadArrays() + arr=DataArrayDouble([(4,104),(5,105)]) + arr.setInfoOnComponents(compos) + self.assertTrue(fs[0][0].getUndergroundDataArray().isEqual(arr,1e-12)) + fs[1][0].loadArrays() + arr=DataArrayDouble([(204,304),(205,305),(206,306),(207,307),(210,310),(211,311),(212,312),(213,313)]) + arr.setInfoOnComponents(compos) + self.assertTrue(fs[1][0].getUndergroundDataArray().isEqual(arr,1e-12)) + self.assertTrue( mm[0].deepCopy().isEqual(m_ref,1e-12) ) pass @WriteInTmpDir