Ryzom
/
ryzom-core
1
1
Fork 0
Code Issues 38 Pull Requests Projects 1 Releases 1 Activity

Changed: Centralize BNP unpacking and packing methods, to be reused in other tools/programs

Browse Source 
--HG--
branch : develop
hg/feature/material-editor
kervala 9 years ago
parent 93e1289a77
commit 19aa093ea8
2 changed files with 383 additions and 100 deletions
Show Stats Download Patch File Download Diff File

94
code/nel/include/nel/misc/big_file.h
Unescape Escape View File

@ -20,7 +20,7 @@
#include "types_nl.h"
#include "tds.h"
#include "singleton.h"
#include "callback.h"
namespace NLMISC {
@ -86,6 +86,66 @@ public:
// Used for CPath only for the moment !
char *getFileNamePtr(const std::string &sFileName, const std::string &sBigFileName);
typedef CCallback<bool /* continue */, const std::string &/* filename */, uint32 /* currentSize */, uint32 /* totalSize */> TUnpackProgressCallback;
// Unpack all files in sBigFileName to sDestDir and send progress notifications to optional callback
static bool unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback = NULL);
// A BNPFile header (filename is a char* pointing on FileNames and is always lowercase)
struct BNPFile
{
BNPFile() : Name(NULL), Size(0), Pos(0) { }
char* Name;
uint32 Size;
uint32 Pos;
};
// A SBNPFile header (filename is a std::string and keeps the original case)
struct SBNPFile
{
SBNPFile() : Size(0), Pos(0) { }
std::string Name;
uint32 Size;
uint32 Pos;
};
// A BNP structure
struct BNP
{
BNP() : FileNames(NULL), ThreadFileId(0), CacheFileOnOpen(false), AlwaysOpened(false), InternalUse(false), OffsetFromBeginning(0) { }
// FileName of the BNP. important to open it in getFile() (for other threads or if not always opened).
std::string BigFileName;
// map of files in the BNP.
char *FileNames;
std::vector<BNPFile> Files;
std::vector<SBNPFile> SFiles;
// Since many seek may be done on a FILE*, each thread should have its own FILE opened.
uint32 ThreadFileId;
bool CacheFileOnOpen;
bool AlwaysOpened;
bool InternalUse;
// Offset written in BNP header
uint32 OffsetFromBeginning;
// Read BNP header from FILE* and init member variables
bool readHeader(FILE* file);
// Read BNP header from BigFileName and init member variables
bool readHeader();
// Append BNP header to the big file BigFileName (to use after appendFile calls)
bool appendHeader();
// Append a file to BigFileName
bool appendFile(const std::string &filename);
// Unpack BigFileName to sDestDir and send progress notifications to optional callback
bool unpack(const std::string &sDestDir, TUnpackProgressCallback *callback = NULL);
};
// ***************
private:
class CThreadFileArray;
@ -118,38 +178,6 @@ private:
uint32 _CurrentId;
};
// A BNPFile header
struct BNPFile
{
BNPFile() : Name(NULL), Size(0), Pos(0) { }
char *Name;
uint32 Size;
uint32 Pos;
};
struct CBNPFileComp
{
bool operator()(const BNPFile &f, const BNPFile &s )
{
return strcmp( f.Name, s.Name ) < 0;
}
};
// A BNP structure
struct BNP
{
BNP() : FileNames(NULL) { }
// FileName of the BNP. important to open it in getFile() (for other threads or if not always opened).
std::string BigFileName;
// map of files in the BNP.
char *FileNames;
std::vector<BNPFile> Files;
// Since many seek may be done on a FILE*, each thread should have its own FILE opened.
uint32 ThreadFileId;
bool CacheFileOnOpen;
bool AlwaysOpened;
};
private:
// CBigFile(); // Singleton mode -> access it with the getInstance function

389
code/nel/src/misc/big_file.cpp
Unescape Escape View File

@ -16,6 +16,7 @@
#include "stdmisc.h"
#include "nel/misc/file.h"
#include "nel/misc/big_file.h"
#include "nel/misc/path.h"
@ -138,43 +139,104 @@ bool CBigFile::add (const std::string &sBigFileName, uint32 nOptions)
handle.File = fopen (sBigFileName.c_str(), "rb");
if (handle.File == NULL)
return false;
uint32 nFileSize=CFile::getFileSize (handle.File);
//nlfseek64 (handle.File, 0, SEEK_END);
//uint32 nFileSize = ftell (handle.File);
// Result
if (nlfseek64 (handle.File, nFileSize-4, SEEK_SET) != 0)
// Used internally by CBigFile, use optimizations and lower case of filenames
bnp.InternalUse = true;
// read BNP header
if (!bnp.readHeader(handle.File))
{
fclose (handle.File);
handle.File = NULL;
return false;
}
if (nOptions&BF_CACHE_FILE_ON_OPEN)
bnp.CacheFileOnOpen = true;
else
bnp.CacheFileOnOpen = false;
uint32 nOffsetFromBeginning;
if (fread (&nOffsetFromBeginning, sizeof(uint32), 1, handle.File) != 1)
if (!(nOptions&BF_ALWAYS_OPENED))
{
fclose (handle.File);
handle.File = NULL;
bnp.AlwaysOpened = false;
}
else
{
bnp.AlwaysOpened = true;
}
//nldebug("BigFile : added bnp '%s' to the collection", bigfilenamealone.c_str());
return true;
}
// ***************************************************************************
void CBigFile::remove (const std::string &sBigFileName)
{
if (_BNPs.find (sBigFileName) != _BNPs.end())
{
map<string, BNP>::iterator it = _BNPs.find (sBigFileName);
BNP &rbnp = it->second;
// Get a ThreadSafe handle on the file
CHandleFile &handle= _ThreadFileArray.get(rbnp.ThreadFileId);
// close it if needed
if (handle.File != NULL)
{
fclose (handle.File);
handle.File= NULL;
}
delete [] rbnp.FileNames;
_BNPs.erase (it);
}
}
//// ***************************************************************************
bool CBigFile::BNP::readHeader()
{
// Only external use
if (InternalUse || BigFileName.empty()) return false;
FILE *f = fopen (BigFileName.c_str(), "rb");
if (f == NULL) return false;
bool res = readHeader(f);
fclose (f);
return res;
}
//// ***************************************************************************
bool CBigFile::BNP::readHeader(FILE *file)
{
if (file == NULL) return false;
uint32 nFileSize=CFile::getFileSize (file);
// Result
if (nlfseek64 (file, nFileSize-4, SEEK_SET) != 0)
{
return false;
}
if (fread (&OffsetFromBeginning, sizeof(uint32), 1, file) != 1)
{
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nOffsetFromBeginning);
NLMISC_BSWAP32(OffsetFromBeginning);
#endif
if (nlfseek64 (handle.File, nOffsetFromBeginning, SEEK_SET) != 0)
if (nlfseek64 (file, OffsetFromBeginning, SEEK_SET) != 0)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
// Read the file count
uint32 nNbFile;
if (fread (&nNbFile, sizeof(uint32), 1, handle.File) != 1)
if (fread (&nNbFile, sizeof(uint32), 1, file) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
@ -182,31 +244,29 @@ bool CBigFile::add (const std::string &sBigFileName, uint32 nOptions)
NLMISC_BSWAP32(nNbFile);
#endif
map<string,BNPFile> tempMap;
map<string, BNPFile> tempMap;
if (!InternalUse) SFiles.clear();
for (uint32 i = 0; i < nNbFile; ++i)
{
char FileName[256];
uint8 nStringSize;
if (fread (&nStringSize, 1, 1, handle.File) != 1)
if (fread (&nStringSize, 1, 1, file) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
if (fread (FileName, nStringSize, 1, handle.File) != 1)
char sFileName[256];
if (fread (sFileName, 1, nStringSize, file) != nStringSize)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
FileName[nStringSize] = 0;
sFileName[nStringSize] = 0;
uint32 nFileSize2;
if (fread (&nFileSize2, sizeof(uint32), 1, handle.File) != 1)
if (fread (&nFileSize2, sizeof(uint32), 1, file) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
@ -215,10 +275,8 @@ bool CBigFile::add (const std::string &sBigFileName, uint32 nOptions)
#endif
uint32 nFilePos;
if (fread (&nFilePos, sizeof(uint32), 1, handle.File) != 1)
if (fread (&nFilePos, sizeof(uint32), 1, file) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
@ -226,21 +284,30 @@ bool CBigFile::add (const std::string &sBigFileName, uint32 nOptions)
NLMISC_BSWAP32(nFilePos);
#endif
BNPFile bnpfTmp;
bnpfTmp.Pos = nFilePos;
bnpfTmp.Size = nFileSize2;
tempMap.insert (make_pair(toLower(string(FileName)), bnpfTmp));
if (InternalUse)
{
BNPFile bnpfTmp;
bnpfTmp.Pos = nFilePos;
bnpfTmp.Size = nFileSize2;
tempMap.insert (make_pair(toLower(string(sFileName)), bnpfTmp));
}
else
{
SBNPFile bnpfTmp;
bnpfTmp.Name = sFileName;
bnpfTmp.Pos = nFilePos;
bnpfTmp.Size = nFileSize2;
SFiles.push_back(bnpfTmp);
}
}
if (nlfseek64 (handle.File, 0, SEEK_SET) != 0)
if (nlfseek64 (file, 0, SEEK_SET) != 0)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
// Convert temp map
if (nNbFile > 0)
if (InternalUse && nNbFile > 0)
{
uint nSize = 0, nNb = 0;
map<string,BNPFile>::iterator it = tempMap.begin();
@ -251,20 +318,20 @@ bool CBigFile::add (const std::string &sBigFileName, uint32 nOptions)
it++;
}
bnp.FileNames = new char[nSize];
memset(bnp.FileNames, 0, nSize);
bnp.Files.resize(nNb);
FileNames = new char[nSize];
memset(FileNames, 0, nSize);
Files.resize(nNb);
it = tempMap.begin();
nSize = 0;
nNb = 0;
while (it != tempMap.end())
{
strcpy(bnp.FileNames+nSize, it->first.c_str());
strcpy(FileNames+nSize, it->first.c_str());
bnp.Files[nNb].Name = bnp.FileNames+nSize;
bnp.Files[nNb].Size = it->second.Size;
bnp.Files[nNb].Pos = it->second.Pos;
Files[nNb].Name = FileNames+nSize;
Files[nNb].Size = it->second.Size;
Files[nNb].Pos = it->second.Pos;
nSize += (uint)it->first.size() + 1;
nNb++;
@ -273,45 +340,218 @@ bool CBigFile::add (const std::string &sBigFileName, uint32 nOptions)
}
// End of temp map conversion
if (nOptions&BF_CACHE_FILE_ON_OPEN)
bnp.CacheFileOnOpen = true;
else
bnp.CacheFileOnOpen = false;
return true;
}
if (!(nOptions&BF_ALWAYS_OPENED))
bool CBigFile::BNP::appendHeader()
{
// Only external use
if (InternalUse || BigFileName.empty()) return false;
FILE *f = fopen (BigFileName.c_str(), "ab");
if (f == NULL) return false;
uint32 nNbFile = (uint32)SFiles.size();
// value to be serialized
uint32 nNbFile2 = nNbFile;
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nNbFile2);
#endif
if (fwrite (&nNbFile2, sizeof(uint32), 1, f) != 1)
{
fclose (handle.File);
handle.File = NULL;
bnp.AlwaysOpened = false;
fclose(f);
return false;
}
else
for (uint32 i = 0; i < nNbFile; ++i)
{
bnp.AlwaysOpened = true;
uint8 nStringSize = (uint8)SFiles[i].Name.length();
if (fwrite (&nStringSize, 1, 1, f) != 1)
{
fclose(f);
return false;
}
if (fwrite (SFiles[i].Name.c_str(), 1, nStringSize, f) != nStringSize)
{
fclose(f);
return false;
}
uint32 nFileSize = SFiles[i].Size;
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFileSize);
#endif
if (fwrite (&nFileSize, sizeof(uint32), 1, f) != 1)
{
fclose(f);
return false;
}
uint32 nFilePos = SFiles[i].Pos;
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFilePos);
#endif
if (fwrite (&nFilePos, sizeof(uint32), 1, f) != 1)
{
fclose(f);
return false;
}
}
//nldebug("BigFile : added bnp '%s' to the collection", bigfilenamealone.c_str());
uint32 nOffsetFromBeginning = OffsetFromBeginning;
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nOffsetFromBeginning);
#endif
if (fwrite (&nOffsetFromBeginning, sizeof(uint32), 1, f) != 1)
{
fclose(f);
return false;
}
fclose (f);
return true;
}
// ***************************************************************************
void CBigFile::remove (const std::string &sBigFileName)
bool CBigFile::BNP::appendFile(const std::string &filename)
{
if (_BNPs.find (sBigFileName) != _BNPs.end())
// Only external use
if (InternalUse || BigFileName.empty()) return false;
// Check if we can read the source file
if (!CFile::fileExists(filename)) return false;
SBNPFile ftmp;
ftmp.Name = CFile::getFilename(filename);
ftmp.Size = CFile::getFileSize(filename);
ftmp.Pos = OffsetFromBeginning;
SFiles.push_back(ftmp);
OffsetFromBeginning += ftmp.Size;
FILE *f1 = fopen(BigFileName.c_str(), "ab");
if (f1 == NULL) return false;
FILE *f2 = fopen(filename.c_str(), "rb");
if (f2 == NULL)
{
map<string, BNP>::iterator it = _BNPs.find (sBigFileName);
BNP &rbnp = it->second;
// Get a ThreadSafe handle on the file
CHandleFile &handle= _ThreadFileArray.get(rbnp.ThreadFileId);
// close it if needed
if (handle.File != NULL)
fclose(f1);
return false;
}
uint8 *ptr = new uint8[ftmp.Size];
if (fread (ptr, ftmp.Size, 1, f2) != 1)
{
nlwarning("%s read error", filename.c_str());
}
else if (fwrite (ptr, ftmp.Size, 1, f1) != 1)
{
nlwarning("%s write error", BigFileName.c_str());
}
delete [] ptr;
fclose(f2);
fclose(f1);
return true;
}
// ***************************************************************************
bool CBigFile::BNP::unpack(const std::string &sDestDir, TUnpackProgressCallback *callback)
{
// Only external use
if (InternalUse || BigFileName.empty()) return false;
FILE *bnp = fopen (BigFileName.c_str(), "rb");
if (bnp == NULL)
return false;
// only read header is not already read
if (SFiles.empty() && !readHeader(bnp))
{
fclose (bnp);
return false;
}
CFile::createDirectory(sDestDir);
uint32 totalUncompressed = 0, total = 0;
for (uint32 i = 0; i < SFiles.size(); ++i)
{
total += SFiles[i].Size;
}
FILE *out = NULL;
for (uint32 i = 0; i < SFiles.size(); ++i)
{
const SBNPFile &rBNPFile = SFiles[i];
string filename = CPath::standardizePath(sDestDir) + rBNPFile.Name;
if (callback && !(*callback)(filename, totalUncompressed, total))
{
fclose (handle.File);
handle.File= NULL;
fclose (bnp);
return false;
}
out = fopen (filename.c_str(), "wb");
if (out != NULL)
{
nlfseek64 (bnp, rBNPFile.Pos, SEEK_SET);
uint8 *ptr = new uint8[rBNPFile.Size];
bool readError = fread (ptr, rBNPFile.Size, 1, bnp) != 1;
if (readError)
{
nlwarning("%s read error errno = %d: %s", filename.c_str(), errno, strerror(errno));
}
bool writeError = fwrite (ptr, rBNPFile.Size, 1, out) != 1;
if (writeError)
{
nlwarning("%s write error errno = %d: %s", filename.c_str(), errno, strerror(errno));
}
bool diskFull = ferror(out) && errno == 28 /* ENOSPC*/;
fclose (out);
delete [] ptr;
if (diskFull)
{
fclose (bnp);
throw NLMISC::EDiskFullError(filename);
}
if (writeError)
{
fclose (bnp);
throw NLMISC::EWriteError(filename);
}
if (readError)
{
fclose (bnp);
throw NLMISC::EReadError(filename);
}
}
totalUncompressed += rBNPFile.Size;
if (callback && !(*callback)(filename, totalUncompressed, total))
{
fclose (bnp);
return false;
}
delete [] rbnp.FileNames;
_BNPs.erase (it);
}
fclose (bnp);
return true;
}
// ***************************************************************************
@ -359,6 +599,14 @@ void CBigFile::removeAll ()
}
}
struct CBNPFileComp
{
bool operator()(const CBigFile::BNPFile &f, const CBigFile::BNPFile &s )
{
return strcmp( f.Name, s.Name ) < 0;
}
};
// ***************************************************************************
bool CBigFile::getFileInternal (const std::string &sFileName, BNP *&zeBnp, BNPFile *&zeBnpFile)
{
@ -504,5 +752,12 @@ void CBigFile::getBigFilePaths(std::vector<std::string> &bigFilePaths)
}
}
// ***************************************************************************
bool CBigFile::unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback)
{
BNP bnpFile;
bnpFile.BigFileName = sBigFileName;
return bnpFile.unpack(sDestDir, callback);
}
} // namespace NLMISC

Write Preview
Loading…
Cancel
Save