From bc71be0fd888afadf940f305e6686d94170cccc1 Mon Sep 17 00:00:00 2001 From: sln Date: Thu, 29 Mar 2007 10:52:48 +0000 Subject: [PATCH] Problem of storing big files fixed (CTH14436, CTH15176) --- src/LightApp/LightApp_Driver.cxx | 671 ++++++++++++++++++++-------- src/LightApp/LightApp_Driver.h | 54 ++- src/LightApp/LightApp_HDFDriver.cxx | 123 +++-- 3 files changed, 629 insertions(+), 219 deletions(-) diff --git a/src/LightApp/LightApp_Driver.cxx b/src/LightApp/LightApp_Driver.cxx index e8e161dc5..6f1853a18 100644 --- a/src/LightApp/LightApp_Driver.cxx +++ b/src/LightApp/LightApp_Driver.cxx @@ -38,13 +38,24 @@ /*! Constructor.*/ LightApp_Driver::LightApp_Driver() -: myIsTemp( false ) +: myIsTemp( false ), + myCurrPos( 0 ), + myCurrBuff( 0 ), + myCurrFileIndex( 0 ), + myCurrIFile( 0 ), + mySizeToBeWritten( 0 ), + myFileSizes( 0 ), + myFileNameSizes( 0 ), + myCurrOFile( 0 ), + myNbFilles( 0 ) { } /*! Destructor.*/ LightApp_Driver::~LightApp_Driver() { + delete myFileSizes; + delete myFileNameSizes; } using namespace std; @@ -73,7 +84,7 @@ bool LightApp_Driver::SaveDatasInFile( const char* theFileName, bool isMultiFile aFileBufferSize += 4; //Add 4 bytes: a length of the module name aFileBufferSize += strlen(aModuleName[i])+1; std::string aName(aModuleName[i]); - PutFilesToStream(aName, aBuffer[i], aBufferSize[i], isMultiFile); +// PutFilesToStream(aName, aBuffer[i], aBufferSize[i], isMultiFile); aFileBufferSize += 8; //Add 8 bytes: a length of the buffer aFileBufferSize += aBufferSize[i]; i++; @@ -179,7 +190,8 @@ bool LightApp_Driver::ReadDatasFromFile( const char* theFileName, bool isMultiFi aCurrentPos += aBufferSize; // Put buffer to aListOfFiles and set to myMap - ListOfFiles aListOfFiles = PutStreamToFiles(aBuffer, aBufferSize, isMultiFile); +// ListOfFiles aListOfFiles = PutStreamToFiles(aBuffer, aBufferSize, isMultiFile); + ListOfFiles aListOfFiles; SetListOfFiles(aModuleName, aListOfFiles); delete[] aModuleName; @@ -225,180 +237,6 @@ void LightApp_Driver::SetListOfFiles( const char* theModuleName, const ListOfFil myMap[aName] = theListOfFiles; } -/*! - Converts files which was created from module into a byte sequence unsigned char -*/ -void LightApp_Driver::PutFilesToStream( const std::string& theModuleName, unsigned char*& theBuffer, - long& theBufferSize, bool theNamesOnly ) -{ - ListOfFiles aFiles = myMap[theModuleName]; - // aFiles must contain temporary directory name in its first item - // and names of files (relatively the temporary directory) in the others - - int i, aLength = aFiles.size() - 1; - if(aLength <= 0) { - theBufferSize = 0; - theBuffer = new unsigned char[theBufferSize]; - return; - } - //Get a temporary directory for saved a file - TCollection_AsciiString aTmpDir(const_cast(aFiles[0].c_str())); - - long aBufferSize = 0; - long aCurrentPos; - int aNbFiles = 0; - int* aFileNameSize= new int[aLength]; - long* aFileSize= new long[aLength]; - - //Determine the required size of the buffer - TCollection_AsciiString aFileName; - for (i = 0; i < aLength; i++) { - char* aFName = const_cast(aFiles[i+1].c_str()); - aFileName = aFName; - //Check if the file exists - if (!theNamesOnly) { // mpv 15.01.2003: if only file names must be stroed, then size of files is zero - TCollection_AsciiString aFullPath = aTmpDir + aFileName; - OSD_Path anOSDPath(aFullPath); - OSD_File anOSDFile(anOSDPath); - if(!anOSDFile.Exists()) continue; -#ifdef WNT - ifstream aFile(aFullPath.ToCString(), ios::binary); -#else - ifstream aFile(aFullPath.ToCString()); -#endif - aFile.seekg(0, ios::end); - aFileSize[i] = aFile.tellg(); - aBufferSize += aFileSize[i]; //Add a space to store the file - } - aFileNameSize[i] = strlen(aFName) + 1; - aBufferSize += aFileNameSize[i]; //Add a space to store the file name - aBufferSize += (theNamesOnly)?4:12; //Add 4 bytes: a length of the file name, - // 8 bytes: length of the file itself - aNbFiles++; - } - - aBufferSize += 4; //4 bytes for a number of the files that will be written to the stream; - theBuffer = new unsigned char[aBufferSize]; - if(theBuffer == NULL) { - theBufferSize = 0; - theBuffer = 0; - return; - } - //Initialize 4 bytes of the buffer by 0 - memset(theBuffer, 0, 4); - //Copy the number of files that will be written to the stream - memcpy(theBuffer, &aNbFiles, ((sizeof(int) > 4) ? 4 : sizeof(int))); - - aCurrentPos = 4; - - for(i = 0; i < aLength; i++) { - ifstream *aFile; - if (!theNamesOnly) { // mpv 15.01.2003: we don't open any file if theNamesOnly = true - TCollection_AsciiString aName(const_cast(aFiles[i+1].c_str())); - TCollection_AsciiString aFullPath = aTmpDir + aName; - OSD_Path anOSDPath(aFullPath); - OSD_File anOSDFile(anOSDPath); - if(!anOSDFile.Exists()) continue; -#ifdef WNT - aFile = new ifstream(aFullPath.ToCString(), ios::binary); -#else - aFile = new ifstream(aFullPath.ToCString()); -#endif - } - //Initialize 4 bytes of the buffer by 0 - memset((theBuffer + aCurrentPos), 0, 4); - //Copy the length of the file name to the buffer - memcpy((theBuffer + aCurrentPos), (aFileNameSize + i), ((sizeof(int) > 4) ? 4 : sizeof(int))); - aCurrentPos += 4; - - //Copy the file name to the buffer - char* aFName = const_cast(aFiles[i+1].c_str()); - memcpy((theBuffer + aCurrentPos), aFName, aFileNameSize[i]); - aCurrentPos += aFileNameSize[i]; - - if (!theNamesOnly) { // mpv 15.01.2003: we don't copy file content to the buffer if !theNamesOnly - //Initialize 8 bytes of the buffer by 0 - memset((theBuffer + aCurrentPos), 0, 8); - //Copy the length of the file to the buffer - memcpy((theBuffer + aCurrentPos), (aFileSize + i), ((sizeof(long) > 8) ? 8 : sizeof(long))); - aCurrentPos += 8; - - aFile->seekg(0, ios::beg); - aFile->read((char *)(theBuffer + aCurrentPos), aFileSize[i]); - aFile->close(); - delete(aFile); - aCurrentPos += aFileSize[i]; - } - } - delete[] aFileNameSize; - delete[] aFileSize; - - theBufferSize = aBufferSize; -} - -/*! - Converts a byte sequence to files and return list of them -*/ -LightApp_Driver::ListOfFiles LightApp_Driver::PutStreamToFiles( const unsigned char* theBuffer, - const long theBufferSize, bool theNamesOnly ) -{ - if(theBufferSize == 0 || theBuffer == 0) - return ListOfFiles(); - - // Create a temporary directory for the component's data files - std::string aDir = GetTmpDir(); - - // Remember that the files are in a temporary location that should be deleted - // when a study is closed - SetIsTemporary( true ); - - //Get a temporary directory for saving a file - TCollection_AsciiString aTmpDir(const_cast(aDir.c_str())); - - long aFileSize, aCurrentPos = 4; - int i, aFileNameSize, aNbFiles = 0; - - //Copy the number of files in the stream - memcpy(&aNbFiles, theBuffer, sizeof(int)); - - const int n = aNbFiles + 1; - ListOfFiles aFiles(n); - aFiles[0] = aDir; - - for(i = 0; i < aNbFiles; i++) { - //Put a length of the file name to aFileNameSize - memcpy(&aFileNameSize, (theBuffer + aCurrentPos), ((sizeof(int) > 4) ? 4 : sizeof(int))); - aCurrentPos += 4; - - char *aFileName = new char[aFileNameSize]; - //Put a file name to aFileName - memcpy(aFileName, (theBuffer + aCurrentPos), aFileNameSize); - aCurrentPos += aFileNameSize; - - //Put a length of the file to aFileSize - if (!theNamesOnly) { - memcpy(&aFileSize, (theBuffer + aCurrentPos), ((sizeof(long) > 8) ? 8 : sizeof(long))); - aCurrentPos += 8; - - TCollection_AsciiString aFullPath = aTmpDir + aFileName; - -#ifdef WNT - ofstream aFile(aFullPath.ToCString(), ios::out | ios::binary); -#else - ofstream aFile(aFullPath.ToCString()); -#endif - - aFile.write((char *)(theBuffer+aCurrentPos), aFileSize); - aFile.close(); - aCurrentPos += aFileSize; - } - std::string aStrFileName(aFileName); - aFiles[i+1] = aStrFileName; - delete[] aFileName; - } - return aFiles; -} - /*! Remove files. First item in is a directory with slash at the end. Other items are names of files. If is true, @@ -553,3 +391,482 @@ std::string LightApp_Driver::GetDirFromPath( const std::string& thePath ) { return aDirString.ToCString(); } +//============================================================================= +// Function : PutFilesToFirstStream +// Purpose : +//============================================================================= +bool LightApp_Driver::PutFilesToFirstStream( const std::string& theModuleName, unsigned char*& theBuffer, + long& theBufferSize, bool theNamesOnly ) +{ + myCurrPos = 0; + myCurrBuff = 0; + myCurrFileIndex = 0; + myCurrIFile = 0; + mySizeToBeWritten = 0; + if ( myFileSizes ) + { + delete[] myFileSizes; + myFileSizes = 0; + } + if ( myFileNameSizes ) + { + delete[] myFileNameSizes; + myFileNameSizes = 0; + } + + ListOfFiles aFiles = myMap[theModuleName]; + // aFiles must contain temporary directory name in its first item + // and names of files (relatively the temporary directory) in the others + + int i, aLength = aFiles.size() - 1; + if(aLength <= 0) { + theBufferSize = 0; + theBuffer = new unsigned char[theBufferSize]; + return true; + } + //Get a temporary directory for saved a file + TCollection_AsciiString aTmpDir(const_cast(aFiles[0].c_str())); + + int aNbFiles = 0; + myFileNameSizes = new size_t[aLength]; + myFileSizes = new size_t[aLength]; + + //Determine the required size of the buffer + TCollection_AsciiString aFileName; + for (i = 0; i < aLength; i++) + { + char* aFName = const_cast(aFiles[i+1].c_str()); + aFileName = aFName; + //Check if the file exists + if (!theNamesOnly) + { + TCollection_AsciiString aFullPath = aTmpDir + aFileName; + OSD_Path anOSDPath(aFullPath); + OSD_File anOSDFile(anOSDPath); + if(!anOSDFile.Exists()) + continue; +#ifdef WNT + ifstream aFile(aFullPath.ToCString(), ios::binary); +#else + ifstream aFile(aFullPath.ToCString()); +#endif + aFile.seekg(0, ios::end); + myFileSizes[i] = aFile.tellg(); + mySizeToBeWritten += myFileSizes[i]; //Add a space to store the file + } + myFileNameSizes[i] = strlen(aFName) + 1; + mySizeToBeWritten += myFileNameSizes[i]; //Add a space to store the file name + mySizeToBeWritten += (theNamesOnly)?4:12; //Add 4 bytes: a length of the file name, + // 8 bytes: length of the file itself + aNbFiles++; + } + + mySizeToBeWritten += 4; //4 bytes for a number of the files that will be written to the stream; + + size_t aCurrSize = mySizeToBeWritten <= GetMaxBuffSize() ? mySizeToBeWritten : GetMaxBuffSize(); + theBuffer = new unsigned char[ aCurrSize ]; + myCurrBuff = theBuffer; + + //Initialize 4 bytes of the buffer by 0 + memset( theBuffer, 0, 4); + //Copy the number of files that will be written to the stream + memcpy( theBuffer, &aNbFiles, ((sizeof(int) > 4) ? 4 : sizeof(int))); + + myCurrPos = 4; + int aCurrnetBuff = 0; + + for ( myCurrFileIndex = 0; myCurrFileIndex < aLength; myCurrFileIndex++ ) + { + if (!theNamesOnly) + { // mpv 15.01.2003: we don't open any file if theNamesOnly = true + TCollection_AsciiString aName(const_cast(aFiles[myCurrFileIndex+1].c_str())); + TCollection_AsciiString aFullPath = aTmpDir + aName; + OSD_Path anOSDPath(aFullPath); + OSD_File anOSDFile(anOSDPath); + if(!anOSDFile.Exists()) continue; +#ifdef WNT + myCurrIFile = new ifstream(aFullPath.ToCString(), ios::binary); +#else + myCurrIFile = new ifstream(aFullPath.ToCString()); +#endif + } + + //Initialize 4 bytes of the buffer by 0 + memset( (theBuffer + myCurrPos), 0, 4); + + //Copy the length of the file name to the buffer + memcpy((theBuffer + myCurrPos), (myFileNameSizes + myCurrFileIndex), ((sizeof(int) > 4) ? 4 : sizeof(int))); + myCurrPos += 4; + + //Copy the file name to the buffer + char* aFName = const_cast(aFiles[myCurrFileIndex+1].c_str()); + memcpy( myCurrBuff + myCurrPos, aFName, myFileNameSizes[ myCurrFileIndex ] ); + myCurrPos += myFileNameSizes[myCurrFileIndex]; + + if (!theNamesOnly) // mpv 15.01.2003: we don't copy file content to the buffer if !theNamesOnly + { + //Initialize 8 bytes of the buffer by 0 + memset( myCurrBuff + myCurrPos, 0, 8 ); + + //Copy the length of the file to the buffer + memcpy( myCurrBuff + myCurrPos, + (unsigned char*)(myFileSizes + myCurrFileIndex), ((sizeof(long) > 8) ? 8 : sizeof(long))); + myCurrPos += 8; + + // old code for small files + if ( aCurrSize < GetMaxBuffSize() ) + { + myCurrIFile->seekg(0, ios::beg); + myCurrIFile->read((char *)(theBuffer + myCurrPos), myFileSizes[myCurrFileIndex]); + myCurrIFile->close(); + delete(myCurrIFile); + myCurrIFile = 0; + myCurrPos += myFileSizes[myCurrFileIndex]; + mySizeToBeWritten = 0; + } + else // new code for big files + { + myCurrIFile->seekg(0, ios::beg); + int aPos = myCurrIFile->tellg(); + size_t aSizeToRead = GetMaxBuffSize() - myCurrPos; + if ( aSizeToRead > myFileSizes[myCurrFileIndex] - myCurrIFile->tellg() ) + aSizeToRead = myFileSizes[myCurrFileIndex] - myCurrIFile->tellg(); + myCurrIFile->read((char *)(theBuffer + myCurrPos), aSizeToRead ); + mySizeToBeWritten -= aSizeToRead - myCurrPos; + myCurrPos += aSizeToRead; + if ( myCurrPos == GetMaxBuffSize() ) + { + myCurrPos = 0; + break; + } + else // file was saved complitelly + { + myCurrIFile->close(); + delete(myCurrIFile); + myCurrIFile = 0; + } + if ( myCurrIFile ) + aPos = myCurrIFile->tellg(); + } + } + } + + if ( aCurrSize < GetMaxBuffSize() ) + theBufferSize = myCurrPos - 1; + else + theBufferSize = GetMaxBuffSize(); + + return aCurrSize < GetMaxBuffSize(); +} + +//============================================================================= +// Function : HasUnsavedData +// Purpose : +//============================================================================= +bool LightApp_Driver::HasUnsavedData() const +{ + return myCurrIFile && mySizeToBeWritten; +} + +//============================================================================= +// Function : PutFilesToNextStream +// Purpose : +//============================================================================= +bool LightApp_Driver::PutFilesToNextStream( const std::string& theModuleName, unsigned char*& theBuffer, + long& theBufferSize, bool theNamesOnly ) +{ + if ( theNamesOnly || !HasUnsavedData() ) + return true; + + ListOfFiles aFiles = myMap[ theModuleName ]; + int aLength = aFiles.size() - 1; + if ( aLength <= 0 ) + return true; + + TCollection_AsciiString aTmpDir(const_cast(aFiles[0].c_str())); + + size_t aCurrSize = mySizeToBeWritten <= GetMaxBuffSize() ? mySizeToBeWritten : GetMaxBuffSize(); + + for ( ; myCurrFileIndex < aLength; myCurrFileIndex++ ) + { + TCollection_AsciiString aName(const_cast(aFiles[myCurrFileIndex+1].c_str())); + TCollection_AsciiString aFullPath = aTmpDir + aName; + OSD_Path anOSDPath(aFullPath); + OSD_File anOSDFile(anOSDPath); + if(!anOSDFile.Exists()) + continue; + + if ( !myCurrIFile ) + { +#ifdef WNT + myCurrIFile = new ifstream( aFullPath.ToCString(), ios::binary ); +#else + myCurrIFile = new ifstream( aFullPath.ToCString() ); +#endif + + myCurrIFile->seekg(0, ios::beg); + + //////////// + //Initialize 4 bytes of the buffer by 0 + memset( (myCurrBuff + myCurrPos), 0, 4); + //Copy the length of the file name to the buffer + memcpy((myCurrBuff + myCurrPos), ( myFileNameSizes + myCurrFileIndex ), ((sizeof(int) > 4) ? 4 : sizeof(int))); + myCurrPos += 4; + + //Copy the file name to the buffer + char* aFName = const_cast(aFiles[myCurrFileIndex+1].c_str()); + memcpy( myCurrBuff + myCurrPos, aFName, myFileNameSizes[ myCurrFileIndex ] ); + myCurrPos += myFileNameSizes[myCurrFileIndex]; + ///////////// + + //Initialize 8 bytes of the buffer by 0 + memset( myCurrBuff + myCurrPos, 0, 8 ); + + //Copy the length of the file to the buffer + memcpy( myCurrBuff + myCurrPos, + (unsigned char*)(myFileSizes + myCurrFileIndex), ((sizeof(long) > 8) ? 8 : sizeof(long))); + myCurrPos += 8; + } + + // old code for small files + if ( aCurrSize < GetMaxBuffSize() ) + { + //myCurrIFile->seekg(0, ios::beg); + size_t aSizeToRead = myFileSizes[myCurrFileIndex] - myCurrIFile->tellg(); + myCurrIFile->read((char *)(theBuffer + myCurrPos), aSizeToRead ); + myCurrIFile->close(); + delete(myCurrIFile); + myCurrIFile = 0; + myCurrPos += aSizeToRead; + mySizeToBeWritten = 0; + } + else // new code for big files + { + int aPos = myCurrIFile->tellg(); + size_t aFileSize = myFileSizes[myCurrFileIndex]; + size_t aSizeToRead = GetMaxBuffSize() - myCurrPos; + if ( aSizeToRead > myFileSizes[myCurrFileIndex] - myCurrIFile->tellg() ) + aSizeToRead = myFileSizes[myCurrFileIndex] - myCurrIFile->tellg(); + myCurrIFile->read((char *)(theBuffer + myCurrPos), aSizeToRead ); + mySizeToBeWritten -= aSizeToRead - myCurrPos; + myCurrPos += aSizeToRead; + if ( myCurrPos == GetMaxBuffSize() ) + { + myCurrPos = 0; + break; + } + else // file was saved complitelly + { + myCurrIFile->close(); + delete(myCurrIFile); + myCurrIFile = 0; + } + if ( myCurrIFile ) + aPos = myCurrIFile->tellg(); + } + } + + if ( aCurrSize < GetMaxBuffSize() ) + theBufferSize = myCurrPos - 1; + else + theBufferSize = GetMaxBuffSize(); + + return aCurrSize < GetMaxBuffSize(); +} + +//============================================================================= +// Function : PutFirstStreamToFiles +// Purpose : +//============================================================================= +void LightApp_Driver::PutFirstStreamToFiles( ListOfFiles& theListOfFiles, + const unsigned char* theBuffer, + const long theBufferSize, + bool theNamesOnly ) +{ + theListOfFiles.resize( 0 ); + myCurrPos = 0; + myCurrOFile = 0; + mySizeToBeWritten = 0; + + if ( theBufferSize == 0 || theBuffer == 0 ) + return; + + delete[] myFileSizes; + delete[] myFileNameSizes; + + // Create a temporary directory for the component's data files + std::string aDir = GetTmpDir(); + + // Remember that the files are in a temporary location that should be deleted + // when a study is closed + SetIsTemporary( true ); + + //Get a temporary directory for saving a file + TCollection_AsciiString aTmpDir(const_cast(aDir.c_str())); + + myCurrPos = 4; + + //Copy the number of files in the stream + memcpy(&myNbFilles, theBuffer, sizeof(int)); + + const int n = myNbFilles + 1; + theListOfFiles.resize( n ); + myFileSizes = new size_t[ n ]; + myFileNameSizes = new size_t[ n ]; + theListOfFiles[ 0 ] = aDir; + myFileSizes[ 0 ] = 0; + myFileNameSizes[ 0 ] = 0; + + for( myCurrFileIndex = 1; myCurrFileIndex <= myNbFilles; myCurrFileIndex++) + { + //Put a length of the file name to myFileNameSizes[ myCurrFileIndex ] + memcpy(&myFileNameSizes[ myCurrFileIndex ], (theBuffer + myCurrPos), ((sizeof(int) > 4) ? 4 : sizeof(int))); + myCurrPos += 4; + + char *aFileName = new char[ myFileNameSizes[ myCurrFileIndex ] ]; + //Put a file name to aFileName + memcpy(aFileName, (theBuffer + myCurrPos), myFileNameSizes[ myCurrFileIndex ]); + myCurrPos += myFileNameSizes[ myCurrFileIndex ]; + + //Put a length of the file to myFileSizes + if (!theNamesOnly) + { + memcpy(&myFileSizes[ myCurrFileIndex ], (theBuffer + myCurrPos), ((sizeof(long) > 8) ? 8 : sizeof(long))); + myCurrPos += 8; + + TCollection_AsciiString aFullPath = aTmpDir + aFileName; + +#ifdef WNT + myCurrOFile = new ofstream(aFullPath.ToCString(), ios::out | ios::binary); +#else + myCurrOFile = new ofstream(aFullPath.ToCString()); +#endif + + std::string aStrFileName( aFileName ); + theListOfFiles[ myCurrFileIndex ] = aStrFileName; + delete[] aFileName; + + if ( myFileSizes[ myCurrFileIndex ] <= theBufferSize - myCurrPos ) + { + // old code + myCurrOFile->write((char *)(theBuffer+myCurrPos), myFileSizes[ myCurrFileIndex ] ); + myCurrOFile->close(); + delete myCurrOFile; + myCurrOFile = 0; + myCurrPos += myFileSizes[ myCurrFileIndex ]; + myFileSizes[ myCurrFileIndex ] = 0; + } + else + { + // old code for big files + size_t aSize = theBufferSize - myCurrPos; + myCurrOFile->write((char *)( theBuffer + myCurrPos ), aSize ); + myFileSizes[ myCurrFileIndex ] -= aSize; + myCurrPos = 0; + break; + } + } + } +} + +//============================================================================= +// Function : PutNextStreamToFiles +// Purpose : +//============================================================================= +void LightApp_Driver::PutNextStreamToFiles( ListOfFiles& theListOfFiles, + const unsigned char* theBuffer, + const long theBufferSize, + bool theNamesOnly ) +{ + if ( theNamesOnly ) + return; + + std::string aDir = GetTmpDir(); + + //Get a temporary directory for saving a file + TCollection_AsciiString aTmpDir(const_cast(aDir.c_str())); + + if ( !myNbFilles ) + return; + + for( ; myCurrFileIndex <= myNbFilles; myCurrFileIndex++) + { + if ( !myCurrOFile ) + { + //Put a length of the file name to myFileNameSizes + //int myFileNameSizes; + memcpy(&myFileNameSizes[ myCurrFileIndex ], (theBuffer + myCurrPos), ((sizeof(int) > 4) ? 4 : sizeof(int))); + myCurrPos += 4; + + char *aFileName = new char[ myFileNameSizes[ myCurrFileIndex ] ]; + //Put a file name to aFileName + memcpy(aFileName, (theBuffer + myCurrPos), myFileNameSizes[ myCurrFileIndex ]); + myCurrPos += myFileNameSizes[ myCurrFileIndex ]; + + std::string aStrFileName( aFileName ); + theListOfFiles[ myCurrFileIndex ] = aStrFileName; + + //Put a length of the file to myFileSizes + if ( theNamesOnly ) + { + delete[] aFileName; + continue; + } + + memcpy(&myFileSizes[ myCurrFileIndex ], (theBuffer + myCurrPos), ((sizeof(long) > 8) ? 8 : sizeof(long))); + myCurrPos += 8; + + TCollection_AsciiString aFullPath = aTmpDir + aFileName; + +#ifdef WNT + myCurrOFile = new ofstream(aFullPath.ToCString(), ios::out | ios::binary); +#else + myCurrOFile = new ofstream(aFullPath.ToCString()); +#endif + + delete[] aFileName; + + } //if ( !myCurrOFile ) + + if ( myFileSizes[ myCurrFileIndex ] <= theBufferSize - myCurrPos ) + { + // old code + myCurrOFile->write((char *)(theBuffer+myCurrPos), myFileSizes[ myCurrFileIndex ] ); + myCurrOFile->close(); + delete myCurrOFile; + myCurrOFile = 0; + myCurrPos += myFileSizes[ myCurrFileIndex ]; + myFileSizes[ myCurrFileIndex ] = 0; + } + else + { + // old code for big files + size_t aFileSize = myFileSizes[ myCurrFileIndex ]; + size_t aSize = theBufferSize - myCurrPos; + myCurrOFile->write((char *)( theBuffer + myCurrPos ), aSize ); + myFileSizes[ myCurrFileIndex ] -= aSize; + myCurrPos = 0; + break; + } + } // for +} + + + + + + + + + + + + + + + + + + + + diff --git a/src/LightApp/LightApp_Driver.h b/src/LightApp/LightApp_Driver.h index e130b86da..65839ff26 100644 --- a/src/LightApp/LightApp_Driver.h +++ b/src/LightApp/LightApp_Driver.h @@ -25,6 +25,8 @@ #include "vector" #include "map" +#include + #ifdef WIN32 #pragma warning( disable:4251 ) #endif @@ -52,10 +54,26 @@ public: virtual void ClearDriverContents(); protected: - void PutFilesToStream(const std::string& theModuleName, unsigned char*& theBuffer, + /*void PutFilesToStream(const std::string& theModuleName, unsigned char*& theBuffer, long& theBufferSize, bool theNamesOnly = false); ListOfFiles PutStreamToFiles(const unsigned char* theBuffer, - const long theBufferSize, bool theNamesOnly = false); + const long theBufferSize, bool theNamesOnly = false);*/ + + // New save + + bool PutFilesToFirstStream( const std::string& theModuleName, unsigned char*& theBuffer, + long& theBufferSize, bool theNamesOnly = false ); + bool PutFilesToNextStream( const std::string& theModuleName, unsigned char*& theBuffer, + long& theBufferSize, bool theNamesOnly = false ); + bool HasUnsavedData() const; + + // New read + void PutFirstStreamToFiles( ListOfFiles& theListOfFiles, const unsigned char* theBuffer, + const long theBufferSize, bool theNamesOnly = false); + void PutNextStreamToFiles( ListOfFiles& theListOfFiles, const unsigned char* theBuffer, + const long theBufferSize, bool theNamesOnly = false); + + inline static size_t GetMaxBuffSize(); std::string GetTmpDir(); std::string GetDirFromPath(const std::string& thePath); @@ -70,6 +88,38 @@ protected: private: bool myIsTemp; + + // reading/writing + int myCurrFileIndex; + + // writing + size_t myCurrPos; + unsigned char* myCurrBuff; + ifstream* myCurrIFile; + size_t mySizeToBeWritten; + size_t* myFileSizes; + size_t* myFileNameSizes; + + // reading + ofstream* myCurrOFile; + long myNbFilles; }; +inline size_t LightApp_Driver::GetMaxBuffSize() +{ + static size_t aValue = 0; + if ( !aValue ) + { + char* aVar = getenv( "HDF_BUFFER_SIZE" ); + if ( aVar ) + aValue = (size_t)atol( aVar ); + else + aValue = 50000000; + } + + return aValue; +} + #endif + + diff --git a/src/LightApp/LightApp_HDFDriver.cxx b/src/LightApp/LightApp_HDFDriver.cxx index 292cdd88b..c5c9fb2a1 100644 --- a/src/LightApp/LightApp_HDFDriver.cxx +++ b/src/LightApp/LightApp_HDFDriver.cxx @@ -69,14 +69,7 @@ bool LightApp_HDFDriver::SaveDatasInFile( const char* theFileName, bool isMultiF for (it = myMap.begin(); it != myMap.end(); ++it, ++tag) { std::string aName (it->first); char* aModuleName = const_cast(aName.c_str()); - unsigned char* aBuffer; - long aBufferSize; - PutFilesToStream(aName, aBuffer, aBufferSize, isMultiFile); - - //Handle(SALOMEDSImpl_SComponent) sco = itcomponent.Value(); - //TCollection_AsciiString scoid = sco->GetID(); - //hdf_sco_group = new HDFgroup(scoid.ToCString(), hdf_group_datacomponent); - + TCollection_AsciiString entry ("0:1:"); entry += TCollection_AsciiString(tag); mapNameEntry[aModuleName] = entry.ToCString(); @@ -85,14 +78,38 @@ bool LightApp_HDFDriver::SaveDatasInFile( const char* theFileName, bool isMultiF hdf_sco_group = new HDFgroup (entry.ToCString(), hdf_group_datacomponent); hdf_sco_group->CreateOnDisk(); - aHDFSize[0] = aBufferSize; + unsigned char* aBuffer; + long aBufferSize; + + /*while( LightApp_Driver::HasUnsavedData() ) + PutFilesToNextStream( aName, aBuffer, aBufferSize, isMultiFile );*/ - hdf_dataset = new HDFdataset ("FILE_STREAM", hdf_sco_group, HDF_STRING, aHDFSize, 1); + // first data set + TCollection_AsciiString aBaseName( (Standard_CString)"FILE_STREAM" ); + PutFilesToFirstStream( aName, aBuffer, aBufferSize, isMultiFile ); + aHDFSize[0] = aBufferSize; + hdf_dataset = new HDFdataset( aBaseName.ToCString(), hdf_sco_group, HDF_STRING, aHDFSize, 1); hdf_dataset->CreateOnDisk(); hdf_dataset->WriteOnDisk(aBuffer); //Save the stream in the HDF file hdf_dataset->CloseOnDisk(); hdf_dataset = 0; //will be deleted by hdf_sco_group destructor + // other data sets + int anIndex = 0; + while( LightApp_Driver::HasUnsavedData() ) + { + anIndex++; + TCollection_AsciiString aDataSetName = + aBaseName + TCollection_AsciiString( anIndex ); + PutFilesToNextStream( aName, aBuffer, aBufferSize, isMultiFile ); + aHDFSize[0] = aBufferSize; + hdf_dataset = new HDFdataset( aDataSetName.ToCString(), hdf_sco_group, HDF_STRING, aHDFSize, 1); + hdf_dataset->CreateOnDisk(); + hdf_dataset->WriteOnDisk(aBuffer); //Save the stream in the HDF file + hdf_dataset->CloseOnDisk(); + hdf_dataset = 0; //will be deleted by hdf_sco_group destructor + } + // store multifile state aHDFSize[0] = 2; hdf_dataset = new HDFdataset("MULTIFILE_STATE", hdf_sco_group, HDF_STRING, aHDFSize, 1); @@ -205,8 +222,10 @@ bool LightApp_HDFDriver::ReadDatasFromFile( const char* theFileName, bool isMult return false; } - try { - if (!hdf_file->ExistInternalObject("STUDY_STRUCTURE")) { + try + { + if (!hdf_file->ExistInternalObject("STUDY_STRUCTURE")) + { //_errorCode = "Study is empty"; isError = true; } else { @@ -253,41 +272,72 @@ bool LightApp_HDFDriver::ReadDatasFromFile( const char* theFileName, bool isMult hdf_group_study_structure->CloseOnDisk(); } - if (!hdf_file->ExistInternalObject("DATACOMPONENT")) { + if (!hdf_file->ExistInternalObject("DATACOMPONENT")) + { //_errorCode = "No components stored"; isError = true; - } else { + } + else + { hdf_group_datacomponent = new HDFgroup ("DATACOMPONENT", hdf_file); hdf_group_datacomponent->OpenOnDisk(); char name[HDF_NAME_MAX_LEN + 1]; Standard_Integer nbsons = hdf_group_datacomponent->nInternalObjects(); - for (Standard_Integer i = 0; i < nbsons; i++) { + for (Standard_Integer i = 0; i < nbsons; i++) + { hdf_group_datacomponent->InternalObjectIndentify(i, name); if (strncmp(name, "INTERNAL_COMPLEX", 16) == 0) continue; hdf_object_type type = hdf_group_datacomponent->InternalObjectType(name); - if (type == HDF_GROUP) { + if (type == HDF_GROUP) + { hdf_sco_group = new HDFgroup (name, hdf_group_datacomponent); hdf_sco_group->OpenOnDisk(); // Read component data - unsigned char* aStreamFile = NULL; - int aStreamSize = 0; - if (hdf_sco_group->ExistInternalObject("FILE_STREAM")) { - HDFdataset *hdf_dataset = new HDFdataset("FILE_STREAM", hdf_sco_group); - hdf_dataset->OpenOnDisk(); - aStreamSize = hdf_dataset->GetSize(); - aStreamFile = new unsigned char[aStreamSize]; - if (aStreamFile == NULL) { - isError = true; - } else { - hdf_dataset->ReadFromDisk(aStreamFile); + // read first file stream + TCollection_AsciiString aBaseName( (Standard_CString)"FILE_STREAM" ); + ListOfFiles aListOfFiles; + for ( int ds = 0; true; ds++ ) + { + unsigned char* aStreamFile = NULL; + int aStreamSize = 0; + + TCollection_AsciiString aDataSetName = aBaseName; + if ( ds > 0 ) + aDataSetName = aBaseName + TCollection_AsciiString( ds ); + + if ( hdf_sco_group->ExistInternalObject( aDataSetName.ToCString() ) ) + { + HDFdataset *hdf_dataset = new HDFdataset( aDataSetName.ToCString(), hdf_sco_group ); + hdf_dataset->OpenOnDisk(); + aStreamSize = hdf_dataset->GetSize(); + aStreamFile = new unsigned char[ aStreamSize ]; + if (aStreamFile == NULL) + isError = true; + else + hdf_dataset->ReadFromDisk( aStreamFile ); + + hdf_dataset->CloseOnDisk(); + hdf_dataset = 0; + } + else + break; + + if ( aStreamFile != NULL ) + { + if ( ds == 0 ) + PutFirstStreamToFiles( aListOfFiles, aStreamFile, aStreamSize, isMultiFile ); + else + PutNextStreamToFiles( aListOfFiles, aStreamFile, aStreamSize, isMultiFile ); + delete [] aStreamFile; } - hdf_dataset->CloseOnDisk(); - hdf_dataset = 0; - } + } // for ( int i = 0; true; i++ ) + + char* aCompDataType = (char*)(mapEntryName[name].c_str()); + SetListOfFiles( aCompDataType, aListOfFiles ); HDFdataset *multifile_hdf_dataset = new HDFdataset("MULTIFILE_STATE", hdf_sco_group); multifile_hdf_dataset->OpenOnDisk(); @@ -303,22 +353,15 @@ bool LightApp_HDFDriver::ReadDatasFromFile( const char* theFileName, bool isMult isASCII = (ASCIIfileState[0] == 'A') ? true : false; - if (aStreamFile != NULL) { - // Put buffer to aListOfFiles and set to myMap - ListOfFiles aListOfFiles = PutStreamToFiles(aStreamFile, aStreamSize, isMultiFile); - char* aCompDataType = (char*)(mapEntryName[name].c_str()); - SetListOfFiles(aCompDataType, aListOfFiles); - - delete [] aStreamFile; - } - hdf_sco_group->CloseOnDisk(); } } hdf_group_datacomponent->CloseOnDisk(); } - } catch (HDFexception) { + } + catch (HDFexception) + { isError = true; //Handle(TColStd_HSequenceOfAsciiString) aFilesToRemove = new TColStd_HSequenceOfAsciiString; -- 2.39.2