Optimize memory peak during coordinates loading implied by MEDFileUMesh.LoadPart.

diff --git a/src/MEDCoupling/MEDCouplingPartDefinition.hxx b/src/MEDCoupling/MEDCouplingPartDefinition.hxx
index 0a8e78c3b2e747364ce0038165ae235b712a4213..39ccd539e9d8bfa84c33e88a0fe686bc26a1a023 100644 (file)
--- a/src/MEDCoupling/MEDCouplingPartDefinition.hxx
+++ b/src/MEDCoupling/MEDCouplingPartDefinition.hxx
@@ -18,8 +18,7 @@
 //
 // Author : Anthony Geay (EDF R&D)
 
-#ifndef __PARAMEDMEM_MEDCOUPLINGPARTDEFINITION_HXX__
-#define __PARAMEDMEM_MEDCOUPLINGPARTDEFINITION_HXX__
+#pragma once
 
 #include "MEDCoupling.hxx"
 #include "MEDCouplingMemArray.hxx"
@@ -110,5 +109,3 @@ namespace MEDCoupling
     mcIdType _step;
   };
 }
-
-#endif

diff --git a/src/MEDLoader/MEDFileMeshLL.cxx b/src/MEDLoader/MEDFileMeshLL.cxx
index 5809f35957bd23105ace385fdcecc0ed59fdb42a..3e476b2b969e0bb439fa5b2d6ca59e720a4010e7 100644 (file)
--- a/src/MEDLoader/MEDFileMeshLL.cxx
+++ b/src/MEDLoader/MEDFileMeshLL.cxx
@@ -560,13 +560,26 @@ void MEDFileUMeshL2::loadPart(med_idt fid, const MeshOrStructMeshCls *mId, const
   for(std::vector< std::vector< MCAuto<MEDFileUMeshPerType> > >::const_iterator it0=_per_type_mesh.begin();it0!=_per_type_mesh.end();it0++)
     for(std::vector< MCAuto<MEDFileUMeshPerType> >::const_iterator it1=(*it0).begin();it1!=(*it0).end();it1++)
       (*it1)->getMesh()->computeNodeIdsAlg(fetchedNodeIds);
-  mcIdType nMin(ToIdType(std::distance(fetchedNodeIds.begin(),std::find(fetchedNodeIds.begin(),fetchedNodeIds.end(),true))));
-  mcIdType nMax(ToIdType(std::distance(fetchedNodeIds.rbegin(),std::find(fetchedNodeIds.rbegin(),fetchedNodeIds.rend(),true))));
-  nMax=nCoords-nMax;
-  for(std::vector< std::vector< MCAuto<MEDFileUMeshPerType> > >::const_iterator it0=_per_type_mesh.begin();it0!=_per_type_mesh.end();it0++)
-    for(std::vector< MCAuto<MEDFileUMeshPerType> >::const_iterator it1=(*it0).begin();it1!=(*it0).end();it1++)
-      (*it1)->getMesh()->renumberNodesWithOffsetInConn(-nMin);
-  loadPartCoords(fid,infosOnComp,mName,dt,it,nMin,nMax);
+  if(!mrs || mrs->getNumberOfCoordsLoadSessions()==1)
+  {
+    mcIdType nMin(ToIdType(std::distance(fetchedNodeIds.begin(),std::find(fetchedNodeIds.begin(),fetchedNodeIds.end(),true))));
+    mcIdType nMax(ToIdType(std::distance(fetchedNodeIds.rbegin(),std::find(fetchedNodeIds.rbegin(),fetchedNodeIds.rend(),true))));
+    nMax=nCoords-nMax;
+    for(std::vector< std::vector< MCAuto<MEDFileUMeshPerType> > >::const_iterator it0=_per_type_mesh.begin();it0!=_per_type_mesh.end();it0++)
+      for(std::vector< MCAuto<MEDFileUMeshPerType> >::const_iterator it1=(*it0).begin();it1!=(*it0).end();it1++)
+        (*it1)->getMesh()->renumberNodesWithOffsetInConn(-nMin);
+    this->loadPartCoords(fid,infosOnComp,mName,dt,it,nMin,nMax);
+  }
+  else
+  {
+    mcIdType nbOfCooLS(mrs->getNumberOfCoordsLoadSessions());
+    MCAuto<DataArrayIdType> fni(DataArrayIdType::BuildListOfSwitchedOn(fetchedNodeIds));
+    MCAuto< MapKeyVal<mcIdType, mcIdType> > o2n(fni->invertArrayN2O2O2NOptimized());
+    for(std::vector< std::vector< MCAuto<MEDFileUMeshPerType> > >::const_iterator it0=_per_type_mesh.begin();it0!=_per_type_mesh.end();it0++)
+      for(std::vector< MCAuto<MEDFileUMeshPerType> >::const_iterator it1=(*it0).begin();it1!=(*it0).end();it1++)
+        (*it1)->getMesh()->renumberNodesInConn(o2n->data());
+    this->loadPartCoordsSlice(fid,infosOnComp,mName,dt,it,fni,nbOfCooLS);
+  }
 }
 
 void MEDFileUMeshL2::loadConnectivity(med_idt fid, int mdim, const std::string& mName, int dt, int it, MEDFileMeshReadSelector *mrs)
@@ -698,11 +711,76 @@ MCAuto<DataArrayDouble>& _coords, MCAuto<PartDefinition>& _part_coords, MCAuto<D
   _coords->setInfoOnComponents(infosOnComp);
 }
 
+/*!
+ * For performance reasons LoadPartCoordsArray method calls LoadPartCoords
+ */
+void MEDFileUMeshL2::LoadPartCoordsArray(med_idt fid, const std::vector<std::string>& infosOnComp, const std::string& mName, int dt, int it, const DataArrayIdType *nodeIds,
+MCAuto<DataArrayDouble>& _coords, MCAuto<DataArrayIdType>& _fam_coords, MCAuto<DataArrayIdType>& _num_coords, MCAuto<DataArrayAsciiChar>& _name_coords)
+{
+  MCAuto<PartDefinition> useless;
+  nodeIds->checkAllocated();
+  nodeIds->checkNbOfComps(1,"loadPartCoordsSlice : Only one component expected !");
+  mcIdType nMin(0),nMax(0);
+  if(!nodeIds->empty())
+  { nMin = nodeIds->front(); nMax = nodeIds->back()+1; }
+  LoadPartCoords(fid,infosOnComp,mName,dt,it,nMin,nMax,_coords,useless,_fam_coords,_num_coords,_name_coords);
+  if(nodeIds->empty())
+    return ;
+  MCAuto<DataArrayIdType> nodeIds2(nodeIds->deepCopy());
+  nodeIds2->applyLin(1,-nMin);
+  _coords = _coords->selectByTupleIdSafe(nodeIds2->begin(),nodeIds2->end());
+  if(_fam_coords.isNotNull())
+    _fam_coords = _fam_coords->selectByTupleIdSafe(nodeIds2->begin(),nodeIds2->end());
+  if(_num_coords.isNotNull())
+    _num_coords = _num_coords->selectByTupleIdSafe(nodeIds2->begin(),nodeIds2->end());
+  if(_name_coords.isNotNull())
+    {
+      MCAuto<DataArrayChar> tmp(_name_coords->selectByTupleIdSafe(nodeIds2->begin(),nodeIds2->end()));
+      _name_coords = DynamicCastSafe<DataArrayChar,DataArrayAsciiChar>( tmp );
+    }
+}
+
 void MEDFileUMeshL2::loadPartCoords(med_idt fid, const std::vector<std::string>& infosOnComp, const std::string& mName, int dt, int it, mcIdType nMin, mcIdType nMax)
 {
   LoadPartCoords(fid,infosOnComp,mName,dt,it,nMin,nMax,_coords,_part_coords,_fam_coords,_num_coords,_name_coords);
 }
 
+void MEDFileUMeshL2::loadPartCoordsSlice(med_idt fid, const std::vector<std::string>& infosOnComp, const std::string& mName, int dt, int it, const DataArrayIdType *nodeIds, mcIdType nbOfCoordLS)
+{
+  nodeIds->checkAllocated();
+  nodeIds->checkNbOfComps(1,"loadPartCoordsSlice : Only one component expected !");
+  if(nodeIds->empty())
+    return ;
+  if( nbOfCoordLS<1 )
+    throw INTERP_KERNEL::Exception("MEDFileUMeshL2::loadPartCoordsSlice : nb of coords load session must be >=1 !");
+  mcIdType nMin(nodeIds->front()),nMax(nodeIds->back()+1);
+  std::vector< MCAuto<DataArrayDouble> > coords(nbOfCoordLS);
+  std::vector< MCAuto<DataArrayIdType> > famCoords(nbOfCoordLS);
+  std::vector< MCAuto<DataArrayIdType> > numCoords(nbOfCoordLS);
+  std::vector< MCAuto<DataArrayAsciiChar> > nameCoords(nbOfCoordLS);
+  for(mcIdType ipart = 0 ; ipart < nbOfCoordLS ; ++ipart)
+    {
+      mcIdType partStart,partStop;
+      DataArray::GetSlice(nMin,nMax,1,ipart,nbOfCoordLS,partStart,partStop);
+      MCAuto<DataArrayIdType> idsNodeIdsToKeep(nodeIds->findIdsInRange(partStart,partStop));
+      MCAuto<DataArrayIdType> nodeIdsToKeep( nodeIds->selectByTupleIdSafe(idsNodeIdsToKeep->begin(),idsNodeIdsToKeep->end()) );
+      LoadPartCoordsArray(fid,infosOnComp,mName,dt,it,nodeIdsToKeep,coords[ipart],famCoords[ipart],numCoords[ipart],nameCoords[ipart]);
+    }
+  _coords = DataArrayDouble::Aggregate(ToConstVect<DataArrayDouble>(coords));
+  if(famCoords[0].isNotNull())
+    _fam_coords = DataArrayIdType::Aggregate(ToConstVect<DataArrayIdType>(famCoords));
+  if(numCoords[0].isNotNull())
+    _num_coords = DataArrayIdType::Aggregate(ToConstVect<DataArrayIdType>(numCoords));
+  if(nameCoords[0].isNotNull())
+  {
+    std::vector< MCAuto<DataArrayChar> > nameCoords2(nameCoords.begin(),nameCoords.end());
+    std::for_each(nameCoords2.begin(),nameCoords2.end(),[](MCAuto<DataArrayChar>& elt){ elt->incrRef(); });
+    MCAuto<DataArrayChar> tmp( DataArrayChar::Aggregate(ToConstVect<DataArrayChar>(nameCoords2)) );
+    _name_coords = DynamicCastSafe<DataArrayChar,DataArrayAsciiChar>( tmp );
+  }
+  _part_coords = DataArrayPartDefinition::New( const_cast<DataArrayIdType *>(nodeIds) );
+}
+
 void MEDFileUMeshL2::sortTypes()
 {
   std::set<int> mdims;

diff --git a/src/MEDLoader/MEDFileMeshLL.hxx b/src/MEDLoader/MEDFileMeshLL.hxx
index 7552e158fc7fa763a7f20cf109a2198cce929794..5c9a667a25d3929e0a2eae68abcd3f2392262e82 100644 (file)
--- a/src/MEDLoader/MEDFileMeshLL.hxx
+++ b/src/MEDLoader/MEDFileMeshLL.hxx
@@ -125,6 +125,7 @@ namespace MEDCoupling
     void loadPartOfConnectivity(med_idt fid, int mdim, const std::string& mName, const std::vector<INTERP_KERNEL::NormalizedCellType>& types, const std::vector<mcIdType>& slicPerTyp, int dt, int it, MEDFileMeshReadSelector *mrs);
     void loadCoords(med_idt fid, const std::vector<std::string>& infosOnComp, const std::string& mName, int dt, int it);
     void loadPartCoords(med_idt fid, const std::vector<std::string>& infosOnComp, const std::string& mName, int dt, int it, mcIdType nMin, mcIdType nMax);
+    void loadPartCoordsSlice(med_idt fid, const std::vector<std::string>& infosOnComp, const std::string& mName, int dt, int it, const DataArrayIdType *nodeIds, mcIdType nbOfCoordLS);
     int getNumberOfLevels() const { return (int)_per_type_mesh.size(); }
     bool emptyLev(int levId) const { return _per_type_mesh[levId].empty(); }
     const std::vector< MCAuto<MEDFileUMeshPerType> >& getLev(int levId) const { return _per_type_mesh[levId]; }
@@ -139,6 +140,8 @@ namespace MEDCoupling
     static void WriteCoords(med_idt fid, const std::string& mname, int dt, int it, double time, const DataArrayDouble *coords, const DataArrayIdType *famCoords, const DataArrayIdType *numCoords, const DataArrayAsciiChar *nameCoords, const DataArrayIdType *globalNumCoords);
     static void LoadPartCoords(med_idt fid, const std::vector<std::string>& infosOnComp, const std::string& mName, int dt, int it, mcIdType nMin, mcIdType nMax,
 MCAuto<DataArrayDouble>& _coords, MCAuto<PartDefinition>& _part_coords, MCAuto<DataArrayIdType>& _fam_coords, MCAuto<DataArrayIdType>& _num_coords, MCAuto<DataArrayAsciiChar>& _name_coords);
+    static void LoadPartCoordsArray(med_idt fid, const std::vector<std::string>& infosOnComp, const std::string& mName, int dt, int it, const DataArrayIdType *nodeIds,
+MCAuto<DataArrayDouble>& _coords, MCAuto<DataArrayIdType>& _fam_coords, MCAuto<DataArrayIdType>& _num_coords, MCAuto<DataArrayAsciiChar>& _name_coords);
   private:
     void sortTypes();
   private:

diff --git a/src/MEDLoader/MEDFileMeshReadSelector.cxx b/src/MEDLoader/MEDFileMeshReadSelector.cxx
index eb717c10e54c67404a4c7c939abbdba9d14e2a9f..241086d43054d32c0ef661739f839e4da48d1b81 100644 (file)
--- a/src/MEDLoader/MEDFileMeshReadSelector.cxx
+++ b/src/MEDLoader/MEDFileMeshReadSelector.cxx
@@ -16,13 +16,17 @@
 //
 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
 //
-// Author : Anthony Geay (CEA/DEN)
+// Author : Anthony Geay (EDF R&D)
 
 #include "MEDFileMeshReadSelector.hxx"
 
+#include "InterpKernelException.hxx"
+
+#include <sstream>
+
 using namespace MEDCoupling;
 
-MEDFileMeshReadSelector::MEDFileMeshReadSelector():_code(0xFFFFFFFF)
+MEDFileMeshReadSelector::MEDFileMeshReadSelector():_nb_coords_load_sessions(1),_code(0xFFFFFFFF)
 {
 }
 
@@ -40,6 +44,13 @@ void MEDFileMeshReadSelector::setCode(unsigned int newCode)
   _code=newCode;
 }
 
+void MEDFileMeshReadSelector::setNumberOfCoordsLoadSessions(mcIdType newNbOfCoordsLoadSessions)
+{
+  if(newNbOfCoordsLoadSessions < 1)
+    throw INTERP_KERNEL::Exception("MEDFileMeshReadSelector::setNumberOfCoordsLoadSessions : input must be >= 1 !");
+  _nb_coords_load_sessions = newNbOfCoordsLoadSessions;
+}
+
 bool MEDFileMeshReadSelector::isCellFamilyFieldReading() const
 {
   return _code & 0x00000001;
@@ -141,6 +152,7 @@ void MEDFileMeshReadSelector::setGlobalNodeNumFieldReading(bool b)
 void MEDFileMeshReadSelector::reprAll(std::ostream& str) const
 {
   str << "MEDFileMeshReadSelector (code=" << _code << ") : \n";
+  str << "Number of coords load part sessions : " << this->_nb_coords_load_sessions << std::endl;
   str << "Read family field on cells : " << ReprStatus(isCellFamilyFieldReading()) << std::endl;
   str << "Read family field on nodes : " << ReprStatus(isNodeFamilyFieldReading()) << std::endl;
   str << "Read name field on cells : " << ReprStatus(isCellNameFieldReading()) << std::endl;

diff --git a/src/MEDLoader/MEDFileMeshReadSelector.hxx b/src/MEDLoader/MEDFileMeshReadSelector.hxx
index 514ebea6285bdf81f9dd70573e3a1a1338c0a7db..8db3a3c3ba0e3672527c7f00b17114a75ca38d71 100644 (file)
--- a/src/MEDLoader/MEDFileMeshReadSelector.hxx
+++ b/src/MEDLoader/MEDFileMeshReadSelector.hxx
@@ -16,14 +16,14 @@
 //
 // See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
 //
-// Author : Anthony Geay (CEA/DEN)
+// Author : Anthony Geay (EDF R&D)
 
-#ifndef __MEDFILEMESHREADSELECTOR_HXX__
-#define __MEDFILEMESHREADSELECTOR_HXX__
+#pragma once
 
 #include "MEDLoaderDefines.hxx"
 
-#include <sstream>
+#include "MCIdType.hxx"
+
 #include <string>
 
 namespace MEDCoupling
@@ -35,6 +35,8 @@ namespace MEDCoupling
     MEDFileMeshReadSelector(unsigned int code);
     unsigned int getCode() const;
     void setCode(unsigned int newCode);
+    mcIdType getNumberOfCoordsLoadSessions() const { return _nb_coords_load_sessions; }
+    void setNumberOfCoordsLoadSessions(mcIdType newNbOfCoordsLoadSessions);
     bool isCellFamilyFieldReading() const;
     bool isNodeFamilyFieldReading() const;
     bool isCellNameFieldReading() const;
@@ -53,6 +55,7 @@ namespace MEDCoupling
   private:
     static std::string ReprStatus(bool v);
   private:
+    mcIdType _nb_coords_load_sessions;
     //bit #0 cell family field
     //bit #1 node family field
     //bit #2 cell name field
@@ -63,4 +66,3 @@ namespace MEDCoupling
   };
 }
 
-#endif

diff --git a/src/MEDLoader/Swig/MEDLoaderCommon.i b/src/MEDLoader/Swig/MEDLoaderCommon.i
index 955e5f565440b9268e597d1f9e095a297a3396d3..b1176fe839668922a3566577aad67271553fe1fe 100644 (file)
--- a/src/MEDLoader/Swig/MEDLoaderCommon.i
+++ b/src/MEDLoader/Swig/MEDLoaderCommon.i
@@ -672,6 +672,8 @@ namespace MEDCoupling
   public:
     MEDFileMeshReadSelector();
     MEDFileMeshReadSelector(unsigned int code);
+    mcIdType getNumberOfCoordsLoadSessions();
+    void setNumberOfCoordsLoadSessions(mcIdType newNbOfCoordsLoadSessions);
     unsigned int getCode() const;
     void setCode(unsigned int newCode);
     bool isCellFamilyFieldReading() const;

diff --git a/src/MEDLoader/Swig/MEDLoaderTest3.py b/src/MEDLoader/Swig/MEDLoaderTest3.py
index de1a827a62f636e9bf5b41e972c5e2f23c0b8888..cc6f3ad128ff6af16058fffe02f6526a82981ec1 100644 (file)
--- a/src/MEDLoader/Swig/MEDLoaderTest3.py
+++ b/src/MEDLoader/Swig/MEDLoaderTest3.py
@@ -4159,6 +4159,28 @@ class MEDLoaderTest3(unittest.TestCase):
         arr=DataArrayDouble([(204,304),(205,305),(206,306),(207,307),(210,310),(211,311),(212,312),(213,313)])
         arr.setInfoOnComponents(compos)
         self.assertTrue(fs[1][0].getUndergroundDataArray().isEqual(arr,1e-12))
+        m_ref = mm[0].deepCopy()
+        # now read it in 2 load sessions to avoid memory peak. zipCoords is no more requested here.
+        ms=MEDFileMeshes()
+        mrs = MEDFileMeshReadSelector()
+        mrs.setNumberOfCoordsLoadSessions(2)
+        mm=MEDFileUMesh.LoadPartOf(fileName,meshName,[NORM_QUAD4],[4,6,1],-1,-1,mrs)
+        ms.pushMesh(mm)
+        spd=mm.getPartDefAtLevel(0,NORM_QUAD4)
+        self.assertEqual(spd.getSlice(),slice(4,6,1))
+        spd=mm.getPartDefAtLevel(1)
+        self.assertTrue(spd.getNumberOfElems()==8 and spd.getNumberOfElems()==mm.getNumberOfNodes())
+        self.assertTrue(spd.toDAI().isEqual(DataArrayInt([4,5,6,7,10,11,12,13])))
+        fs=MEDFileFields.LoadPartOf(fileName,False,ms)
+        fs[0][0].loadArrays()
+        arr=DataArrayDouble([(4,104),(5,105)])
+        arr.setInfoOnComponents(compos)
+        self.assertTrue(fs[0][0].getUndergroundDataArray().isEqual(arr,1e-12))
+        fs[1][0].loadArrays()
+        arr=DataArrayDouble([(204,304),(205,305),(206,306),(207,307),(210,310),(211,311),(212,312),(213,313)])
+        arr.setInfoOnComponents(compos)
+        self.assertTrue(fs[1][0].getUndergroundDataArray().isEqual(arr,1e-12))
+        self.assertTrue( mm[0].deepCopy().isEqual(m_ref,1e-12) )
         pass
 
     @WriteInTmpDir