From 30c835f259df6124be8f2134ce3268a517c132e4 Mon Sep 17 00:00:00 2001 From: geay Date: Tue, 25 Feb 2014 10:37:54 +0100 Subject: [PATCH] Addition unloadArraysWithoutDataLoss methods in MEDFileFields, MEDFileAnyTypeFieldMultiTS, MEDFileAnyTypeField1TS to give opportunity of new MEDReader to have different cache strategy --- src/MEDLoader/MEDFileField.cxx | 71 +++++++++++++++++++++------- src/MEDLoader/MEDFileField.hxx | 5 +- src/MEDLoader/Swig/MEDLoaderCommon.i | 3 ++ src/MEDLoader/Swig/MEDLoaderTest3.py | 63 ++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 19 deletions(-) diff --git a/src/MEDLoader/MEDFileField.cxx b/src/MEDLoader/MEDFileField.cxx index e69aaf127..04ad5b5b7 100644 --- a/src/MEDLoader/MEDFileField.cxx +++ b/src/MEDLoader/MEDFileField.cxx @@ -3398,11 +3398,6 @@ std::string MEDFileFieldGlobsReal::getFileName() const return contentNotNull()->getFileName(); } -std::string MEDFileFieldGlobsReal::getFileName2() const -{ - return contentNotNull()->getFileName2(); -} - /*! * Returns a localization object by its name. * \param [in] locName - the name of the localization of interest. @@ -5598,15 +5593,29 @@ void MEDFileAnyTypeField1TS::loadArraysIfNecessary() /*! * This method releases potentially big data arrays and so returns to the same heap memory than status loaded with 'loadAll' parameter set to false. - * This method does not release arrays set outside the context of a MED file. + * \b WARNING, this method does release arrays even if \a this does not come from a load of a MED file. + * So this method can lead to a loss of data. If you want to unload arrays safely call MEDFileAnyTypeField1TS::unloadArraysWithoutDataLoss instead. * - * \sa MEDFileAnyTypeField1TS::loadArrays, MEDFileAnyTypeField1TS::loadArraysIfNecessary + * \sa MEDFileAnyTypeField1TS::loadArrays, MEDFileAnyTypeField1TS::loadArraysIfNecessary, MEDFileAnyTypeField1TS::unloadArraysWithoutDataLoss */ void MEDFileAnyTypeField1TS::unloadArrays() { contentNotNullBase()->unloadArrays(); } +/*! + * This method potentially releases big data arrays if \a this is coming from a file. If \a this has been built from scratch this method will have no effect. + * This method is the symetrical method of MEDFileAnyTypeField1TS::loadArraysIfNecessary. + * This method is useful to reduce \b safely amount of heap memory necessary for \a this by using MED file as database. + * + * \sa MEDFileAnyTypeField1TS::loadArraysIfNecessary + */ +void MEDFileAnyTypeField1TS::unloadArraysWithoutDataLoss() +{ + if(!getFileName().empty()) + contentNotNullBase()->unloadArrays(); +} + void MEDFileAnyTypeField1TS::writeLL(med_idt fid) const { int nbComp=getNumberOfComponents(); @@ -6117,7 +6126,7 @@ MEDFileField1TS::MEDFileField1TS() */ MEDCouplingFieldDouble *MEDFileField1TS::getFieldAtLevel(TypeOfField type, int meshDimRelToMax, int renumPol) const { - if(getFileName2().empty()) + if(getFileName().empty()) throw INTERP_KERNEL::Exception("MEDFileField1TS::getFieldAtLevel : Request for a method that can be used for instances coming from file loading ! Use getFieldOnMeshAtLevel method instead !"); MEDCouplingAutoRefCountObjectPtr arrOut; MEDCouplingAutoRefCountObjectPtr ret=contentNotNull()->getFieldAtLevel(type,meshDimRelToMax,std::string(),renumPol,this,arrOut,*contentNotNull()); @@ -6149,7 +6158,7 @@ MEDCouplingFieldDouble *MEDFileField1TS::getFieldAtLevel(TypeOfField type, int m */ MEDCouplingFieldDouble *MEDFileField1TS::getFieldAtTopLevel(TypeOfField type, int renumPol) const { - if(getFileName2().empty()) + if(getFileName().empty()) throw INTERP_KERNEL::Exception("MEDFileField1TS::getFieldAtTopLevel : Request for a method that can be used for instances coming from file loading ! Use getFieldOnMeshAtTopLevel method instead !"); MEDCouplingAutoRefCountObjectPtr arrOut; MEDCouplingAutoRefCountObjectPtr ret=contentNotNull()->getFieldAtTopLevel(type,std::string(),renumPol,this,arrOut,*contentNotNull()); @@ -6242,7 +6251,7 @@ MEDCouplingFieldDouble *MEDFileField1TS::getFieldOnMeshAtLevel(TypeOfField type, */ MEDCouplingFieldDouble *MEDFileField1TS::getFieldAtLevelOld(TypeOfField type, const std::string& mname, int meshDimRelToMax, int renumPol) const { - if(getFileName2().empty()) + if(getFileName().empty()) throw INTERP_KERNEL::Exception("MEDFileField1TS::getFieldAtLevelOld : Request for a method that can be used for instances coming from file loading ! Use getFieldOnMeshAtLevel method instead !"); MEDCouplingAutoRefCountObjectPtr arrOut; MEDCouplingAutoRefCountObjectPtr ret=contentNotNull()->getFieldAtLevel(type,meshDimRelToMax,mname,renumPol,this,arrOut,*contentNotNull()); @@ -6509,7 +6518,7 @@ const MEDFileIntField1TSWithoutSDA *MEDFileIntField1TS::contentNotNull() const MEDCouplingFieldDouble *MEDFileIntField1TS::getFieldAtLevel(TypeOfField type, int meshDimRelToMax, DataArrayInt* &arrOut, int renumPol) const { - if(getFileName2().empty()) + if(getFileName().empty()) throw INTERP_KERNEL::Exception("MEDFileIntField1TS::getFieldAtLevel : Request for a method that can be used for instances coming from file loading ! Use getFieldOnMeshAtLevel method instead !"); MEDCouplingAutoRefCountObjectPtr arrOut2; MEDCouplingAutoRefCountObjectPtr ret=contentNotNull()->getFieldAtLevel(type,meshDimRelToMax,std::string(),renumPol,this,arrOut2,*contentNotNull()); @@ -6556,7 +6565,7 @@ DataArrayInt *MEDFileIntField1TS::ReturnSafelyDataArrayInt(MEDCouplingAutoRefCou */ MEDCouplingFieldDouble *MEDFileIntField1TS::getFieldAtTopLevel(TypeOfField type, DataArrayInt* &arrOut, int renumPol) const { - if(getFileName2().empty()) + if(getFileName().empty()) throw INTERP_KERNEL::Exception("MEDFileField1TS::getFieldAtTopLevel : Request for a method that can be used for instances coming from file loading ! Use getFieldOnMeshAtTopLevel method instead !"); MEDCouplingAutoRefCountObjectPtr arr; MEDCouplingAutoRefCountObjectPtr ret=contentNotNull()->getFieldAtTopLevel(type,std::string(),renumPol,this,arr,*contentNotNull()); @@ -6652,7 +6661,7 @@ MEDCouplingFieldDouble *MEDFileIntField1TS::getFieldOnMeshAtLevel(TypeOfField ty */ MEDCouplingFieldDouble *MEDFileIntField1TS::getFieldAtLevelOld(TypeOfField type, const std::string& mname, int meshDimRelToMax, DataArrayInt* &arrOut, int renumPol) const { - if(getFileName2().empty()) + if(getFileName().empty()) throw INTERP_KERNEL::Exception("MEDFileField1TS::getFieldAtLevelOld : Request for a method that can be used for instances coming from file loading ! Use getFieldOnMeshAtLevel method instead !"); MEDCouplingAutoRefCountObjectPtr arr; MEDCouplingAutoRefCountObjectPtr ret=contentNotNull()->getFieldAtLevel(type,meshDimRelToMax,mname,renumPol,this,arr,*contentNotNull()); @@ -8181,15 +8190,29 @@ void MEDFileAnyTypeFieldMultiTS::loadArraysIfNecessary() /*! * This method releases potentially big data arrays and so returns to the same heap memory than status loaded with 'loadAll' parameter set to false. - * This method does not release arrays set outside the context of a MED file. + * \b WARNING, this method does release arrays even if \a this does not come from a load of a MED file. + * So this method can lead to a loss of data. If you want to unload arrays safely call MEDFileAnyTypeFieldMultiTS::unloadArraysWithoutDataLoss instead. * - * \sa MEDFileAnyTypeFieldMultiTS::loadArrays, MEDFileAnyTypeFieldMultiTS::loadArraysIfNecessary + * \sa MEDFileAnyTypeFieldMultiTS::loadArrays, MEDFileAnyTypeFieldMultiTS::loadArraysIfNecessary, MEDFileAnyTypeFieldMultiTS::unloadArraysWithoutDataLoss */ void MEDFileAnyTypeFieldMultiTS::unloadArrays() { contentNotNullBase()->unloadArrays(); } +/*! + * This method potentially releases big data arrays if \a this is coming from a file. If \a this has been built from scratch this method will have no effect. + * This method is the symetrical method of MEDFileAnyTypeFieldMultiTS::loadArraysIfNecessary. + * This method is useful to reduce \b safely amount of heap memory necessary for \a this by using MED file as database. + * + * \sa MEDFileAnyTypeFieldMultiTS::loadArraysIfNecessary + */ +void MEDFileAnyTypeFieldMultiTS::unloadArraysWithoutDataLoss() +{ + if(!getFileName().empty()) + contentNotNullBase()->unloadArrays(); +} + std::string MEDFileAnyTypeFieldMultiTS::simpleRepr() const { std::ostringstream oss; @@ -9639,9 +9662,10 @@ void MEDFileFields::loadArraysIfNecessary() /*! * This method releases potentially big data arrays and so returns to the same heap memory than status loaded with 'loadAll' parameter set to false. - * This method does not release arrays set outside the context of a MED file. + * \b WARNING, this method does release arrays even if \a this does not come from a load of a MED file. + * So this method can lead to a loss of data. If you want to unload arrays safely call MEDFileFields::unloadArraysWithoutDataLoss instead. * - * \sa MEDFileFields::loadArrays, MEDFileFields::loadArraysIfNecessary + * \sa MEDFileFields::loadArrays, MEDFileFields::loadArraysIfNecessary, MEDFileFields::unloadArraysWithoutDataLoss */ void MEDFileFields::unloadArrays() { @@ -9653,6 +9677,19 @@ void MEDFileFields::unloadArrays() } } +/*! + * This method potentially releases big data arrays if \a this is coming from a file. If \a this has been built from scratch this method will have no effect. + * This method is the symetrical method of MEDFileFields::loadArraysIfNecessary. + * This method is useful to reduce \b safely amount of heap memory necessary for \a this by using MED file as database. + * + * \sa MEDFileFields::loadArraysIfNecessary + */ +void MEDFileFields::unloadArraysWithoutDataLoss() +{ + if(!getFileName().empty()) + unloadArrays(); +} + std::vector MEDFileFields::getPflsReallyUsed() const { std::vector ret; diff --git a/src/MEDLoader/MEDFileField.hxx b/src/MEDLoader/MEDFileField.hxx index 06f929f3b..c63f95cf8 100644 --- a/src/MEDLoader/MEDFileField.hxx +++ b/src/MEDLoader/MEDFileField.hxx @@ -334,7 +334,6 @@ namespace ParaMEDMEM int getNbOfGaussPtPerCell(int locId) const; int getLocalizationId(const std::string& loc) const; std::string getFileName() const { return _file_name; } - std::string getFileName2() const { return _file_name; } const MEDFileFieldLoc& getLocalizationFromId(int locId) const; const MEDFileFieldLoc& getLocalization(const std::string& locName) const; const DataArrayInt *getProfileFromId(int pflId) const; @@ -412,7 +411,6 @@ namespace ParaMEDMEM MEDLOADER_EXPORT int getNbOfGaussPtPerCell(int locId) const; MEDLOADER_EXPORT int getLocalizationId(const std::string& loc) const; MEDLOADER_EXPORT std::string getFileName() const; - MEDLOADER_EXPORT std::string getFileName2() const; MEDLOADER_EXPORT const MEDFileFieldLoc& getLocalizationFromId(int locId) const; MEDLOADER_EXPORT const MEDFileFieldLoc& getLocalization(const std::string& locName) const; MEDLOADER_EXPORT MEDFileFieldLoc& getLocalizationFromId(int locId); @@ -665,6 +663,7 @@ namespace ParaMEDMEM MEDLOADER_EXPORT void loadArrays(); MEDLOADER_EXPORT void loadArraysIfNecessary(); MEDLOADER_EXPORT void unloadArrays(); + MEDLOADER_EXPORT void unloadArraysWithoutDataLoss(); MEDLOADER_EXPORT std::vector< MEDCouplingAutoRefCountObjectPtr< MEDFileAnyTypeField1TS > > splitComponents() const; MEDLOADER_EXPORT std::vector< MEDCouplingAutoRefCountObjectPtr< MEDFileAnyTypeField1TS > > splitDiscretizations() const; MEDLOADER_EXPORT MEDFileAnyTypeField1TS *deepCpy() const; @@ -913,6 +912,7 @@ namespace ParaMEDMEM MEDLOADER_EXPORT void loadArrays(); MEDLOADER_EXPORT void loadArraysIfNecessary(); MEDLOADER_EXPORT void unloadArrays(); + MEDLOADER_EXPORT void unloadArraysWithoutDataLoss(); MEDLOADER_EXPORT void write(const std::string& fileName, int mode) const; MEDLOADER_EXPORT void writeLL(med_idt fid) const; MEDLOADER_EXPORT std::size_t getHeapMemorySizeWithoutChildren() const; @@ -1085,6 +1085,7 @@ namespace ParaMEDMEM MEDLOADER_EXPORT void loadArrays(); MEDLOADER_EXPORT void loadArraysIfNecessary(); MEDLOADER_EXPORT void unloadArrays(); + MEDLOADER_EXPORT void unloadArraysWithoutDataLoss(); MEDLOADER_EXPORT int getNumberOfFields() const; MEDLOADER_EXPORT std::vector< std::pair > getCommonIterations(bool& areThereSomeForgottenTS) const; MEDLOADER_EXPORT std::vector getFieldsNames() const; diff --git a/src/MEDLoader/Swig/MEDLoaderCommon.i b/src/MEDLoader/Swig/MEDLoaderCommon.i index 7145ba888..63acae414 100644 --- a/src/MEDLoader/Swig/MEDLoaderCommon.i +++ b/src/MEDLoader/Swig/MEDLoaderCommon.i @@ -1226,6 +1226,7 @@ namespace ParaMEDMEM void loadArrays() throw(INTERP_KERNEL::Exception); void loadArraysIfNecessary() throw(INTERP_KERNEL::Exception); void unloadArrays() throw(INTERP_KERNEL::Exception); + void unloadArraysWithoutDataLoss() throw(INTERP_KERNEL::Exception); int getDimension() const throw(INTERP_KERNEL::Exception); int getIteration() const throw(INTERP_KERNEL::Exception); int getOrder() const throw(INTERP_KERNEL::Exception); @@ -1642,6 +1643,7 @@ namespace ParaMEDMEM void loadArrays() throw(INTERP_KERNEL::Exception); void loadArraysIfNecessary() throw(INTERP_KERNEL::Exception); void unloadArrays() throw(INTERP_KERNEL::Exception); + void unloadArraysWithoutDataLoss() throw(INTERP_KERNEL::Exception); // virtual MEDFileAnyTypeField1TS *getTimeStepAtPos(int pos) const throw(INTERP_KERNEL::Exception); MEDFileAnyTypeField1TS *getTimeStep(int iteration, int order) const throw(INTERP_KERNEL::Exception); @@ -2226,6 +2228,7 @@ namespace ParaMEDMEM void loadArrays() throw(INTERP_KERNEL::Exception); void loadArraysIfNecessary() throw(INTERP_KERNEL::Exception); void unloadArrays() throw(INTERP_KERNEL::Exception); + void unloadArraysWithoutDataLoss() throw(INTERP_KERNEL::Exception); void write(const std::string& fileName, int mode) const throw(INTERP_KERNEL::Exception); int getNumberOfFields() const; std::vector getFieldsNames() const throw(INTERP_KERNEL::Exception); diff --git a/src/MEDLoader/Swig/MEDLoaderTest3.py b/src/MEDLoader/Swig/MEDLoaderTest3.py index 6aac69079..e148051ce 100644 --- a/src/MEDLoader/Swig/MEDLoaderTest3.py +++ b/src/MEDLoader/Swig/MEDLoaderTest3.py @@ -3583,6 +3583,69 @@ class MEDLoaderTest(unittest.TestCase): self.assertEqual(mm.getMeshAtLevel(0).getName(),"abc") pass + def testMEDFileFieldsUnloadArraysWithoutDataLoss1(self): + fileName="Pyfile80.med" + m=MEDCouplingCMesh() ; m.setName("cmesh") + arr=DataArrayDouble(6) ; arr.iota() + m.setCoords(arr,arr) + nbCells=m.getNumberOfCells() + self.assertEqual(25,nbCells) + f=MEDCouplingFieldDouble(ON_CELLS) + f.setName("FieldOnCell") ; f.setMesh(m) + arr=DataArrayDouble(nbCells) ; arr.iota() + mm=MEDFileCMesh() + mm.setMesh(m) + # + fmts=MEDFileFieldMultiTS() + # + for i in xrange(nbCells): + t=(float(i)+0.1,i+1,-i-2) + f.setTime(*t) + arr2=DataArrayDouble(nbCells) + perm=DataArrayInt(nbCells) ; perm.iota(i) ; perm%=nbCells + arr2[perm]=arr + f.setArray(arr2) + f1ts=MEDFileField1TS() + f1ts.setFieldNoProfileSBT(f) + fmts.pushBackTimeStep(f1ts) + pass + fmts.unloadArraysWithoutDataLoss() + self.assertTrue(fmts[0].getUndergroundDataArray().isEqual(DataArrayDouble([0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.,11.,12.,13.,14.,15.,16.,17.,18.,19.,20.,21.,22.,23.,24.]),1e-12)) + fs=MEDFileFields() ; fs.pushField(fmts) + self.assertTrue(fs[0][0].getUndergroundDataArray().isEqual(DataArrayDouble([0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.,11.,12.,13.,14.,15.,16.,17.,18.,19.,20.,21.,22.,23.,24.]),1e-12)) + fs.unloadArraysWithoutDataLoss() + self.assertTrue(fs[0][0].getUndergroundDataArray().isEqual(DataArrayDouble([0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.,11.,12.,13.,14.,15.,16.,17.,18.,19.,20.,21.,22.,23.,24.]),1e-12)) + f1ts=fs[0][0] + self.assertTrue(f1ts.getUndergroundDataArray().isEqual(DataArrayDouble([0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.,11.,12.,13.,14.,15.,16.,17.,18.,19.,20.,21.,22.,23.,24.]),1e-12)) + f1ts.unloadArraysWithoutDataLoss() + self.assertTrue(f1ts.getUndergroundDataArray().isEqual(DataArrayDouble([0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.,11.,12.,13.,14.,15.,16.,17.,18.,19.,20.,21.,22.,23.,24.]),1e-12)) + mm.write(fileName,2) + fs.write(fileName,0) + del m,fmts,mm,f,f1ts + ## + mm=MEDFileMesh.New(fileName) + fmts=MEDFileFieldMultiTS(fileName) + self.assertTrue(fmts[0].getUndergroundDataArray().isEqual(DataArrayDouble([0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.,11.,12.,13.,14.,15.,16.,17.,18.,19.,20.,21.,22.,23.,24.]),1e-12)) + fmts.unloadArraysWithoutDataLoss() + self.assertTrue(not fmts[0].getUndergroundDataArray().isAllocated()) + fmts.loadArraysIfNecessary() + self.assertTrue(fmts[0].getUndergroundDataArray().isEqual(DataArrayDouble([0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.,11.,12.,13.,14.,15.,16.,17.,18.,19.,20.,21.,22.,23.,24.]),1e-12)) + del mm,fmts + fs=MEDFileFields(fileName) + self.assertTrue(fs[0][0].getUndergroundDataArray().isEqual(DataArrayDouble([0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.,11.,12.,13.,14.,15.,16.,17.,18.,19.,20.,21.,22.,23.,24.]),1e-12)) + fs.unloadArraysWithoutDataLoss() + self.assertTrue(not fs[0][0].getUndergroundDataArray().isAllocated()) + fs.loadArraysIfNecessary() + self.assertTrue(fs[0][0].getUndergroundDataArray().isEqual(DataArrayDouble([0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.,11.,12.,13.,14.,15.,16.,17.,18.,19.,20.,21.,22.,23.,24.]),1e-12)) + del fs + f1ts=MEDFileField1TS(fileName) + self.assertTrue(f1ts.getUndergroundDataArray().isEqual(DataArrayDouble([0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.,11.,12.,13.,14.,15.,16.,17.,18.,19.,20.,21.,22.,23.,24.]),1e-12)) + f1ts.unloadArraysWithoutDataLoss() + self.assertTrue(not f1ts.getUndergroundDataArray().isAllocated()) + f1ts.loadArraysIfNecessary() + self.assertTrue(f1ts.getUndergroundDataArray().isEqual(DataArrayDouble([0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.,11.,12.,13.,14.,15.,16.,17.,18.,19.,20.,21.,22.,23.,24.]),1e-12)) + pass + pass unittest.main() -- 2.39.2