From b3309951c44265ff19bb769ea7edbabe50f9dfc0 Mon Sep 17 00:00:00 2001 From: kervala Date: Tue, 2 Feb 2016 10:21:42 +0100 Subject: [PATCH 01/10] Changed: Moved LZMA and 7z functions/methods in seven_zip Changed: Removed old 7z/lzma code since the library has been updated Fixed: Implemented packLZMA function to compress a file with LZMA --HG-- branch : develop --- code/ryzom/client/src/login_patch.cpp | 7 +- code/ryzom/client/src/login_patch.h | 4 - .../client/src/login_patch_seven_zip.cpp | 468 ------------------ code/ryzom/client/src/seven_zip/seven_zip.cpp | 357 +++++++++++++ code/ryzom/client/src/seven_zip/seven_zip.h | 31 ++ 5 files changed, 392 insertions(+), 475 deletions(-) delete mode 100644 code/ryzom/client/src/login_patch_seven_zip.cpp create mode 100644 code/ryzom/client/src/seven_zip/seven_zip.cpp create mode 100644 code/ryzom/client/src/seven_zip/seven_zip.h diff --git a/code/ryzom/client/src/login_patch.cpp b/code/ryzom/client/src/login_patch.cpp index c942071d0..37cbddcbd 100644 --- a/code/ryzom/client/src/login_patch.cpp +++ b/code/ryzom/client/src/login_patch.cpp @@ -54,6 +54,7 @@ #include "login.h" #include "user_agent.h" +#include "seven_zip/seven_zip.h" #ifndef RY_BG_DOWNLOADER #include "client_cfg.h" @@ -2625,7 +2626,7 @@ void CPatchThread::processFile (CPatchManager::SFileToPatch &rFTP) // try to unpack the file try { - if (!CPatchManager::unpackLZMA(pPM->ClientPatchPath+lzmaFile, OutFilename+".tmp")) + if (!unpackLZMA(pPM->ClientPatchPath+lzmaFile, OutFilename+".tmp")) { // fallback to standard patch method usePatchFile = true; @@ -3087,7 +3088,7 @@ bool CPatchManager::download(const std::string& patchFullname, const std::string && patchName.substr(patchName.size() - zsStrLength) == zsStr) { std::string outFilename = patchName.substr(0, patchName.size() - zsStrLength); - CPatchManager::unpack7Zip(patchName, outFilename); + unpack7Zip(patchName, outFilename); pPM->deleteFile(patchName); pPM->renameFile(outFilename, sourceFullname); } @@ -3449,7 +3450,7 @@ void CInstallThread::run() std::string outFilename = patchName.substr(0, patchName.size() - zsStrLength); std::string localOutFilename = CPath::standardizeDosPath(outFilename); - if ( CPatchManager::unpackLZMA(patchName, localOutFilename) ) + if ( unpackLZMA(patchName, localOutFilename) ) { pPM->deleteFile(patchName); pPM->renameFile(outFilename, sourceName); diff --git a/code/ryzom/client/src/login_patch.h b/code/ryzom/client/src/login_patch.h index 9fdf907e8..c7f0f4509 100644 --- a/code/ryzom/client/src/login_patch.h +++ b/code/ryzom/client/src/login_patch.h @@ -331,10 +331,6 @@ private: void clearDataScanLog(); static void getCorruptedFileInfo(const SFileToPatch &ftp, ucstring &sTranslate); - // utility func to decompress a monofile 7zip archive - static bool unpack7Zip(const std::string &sevenZipFile, const std::string &destFileName); - // utility func to decompress a single LZMA packed file - static bool unpackLZMA(const std::string &sevenZipFile, const std::string &destFileName); static bool downloadAndUnpack(const std::string& patchPath, const std::string& sourceFilename, const std::string& extractPath, const std::string& tmpDirectory, uint32 timestamp); // Forward message to Installation Software void onFileInstallFinished(); diff --git a/code/ryzom/client/src/login_patch_seven_zip.cpp b/code/ryzom/client/src/login_patch_seven_zip.cpp deleted file mode 100644 index 306c0607f..000000000 --- a/code/ryzom/client/src/login_patch_seven_zip.cpp +++ /dev/null @@ -1,468 +0,0 @@ -// Ryzom - MMORPG Framework -// Copyright (C) 2010 Winch Gate Property Limited -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -// -// Includes -// - -#include "stdpch.h" -#include "login_patch.h" - -#define RZ_USE_SEVENZIP 1 -#define RZ_USE_NEW_LZMA - -#ifdef RZ_USE_SEVENZIP - -#ifdef RZ_USE_NEW_LZMA - -#include "seven_zip/7z.h" -#include "seven_zip/7zAlloc.h" -#include "seven_zip/7zBuf.h" -#include "seven_zip/7zCrc.h" -#include "seven_zip/7zFile.h" -#include "seven_zip/7zVersion.h" -#include "seven_zip/LzmaDec.h" - -#else - -#include "seven_zip/7zCrc.h" -#include "seven_zip/7zIn.h" -#include "seven_zip/7zExtract.h" -#include "seven_zip/LzmaDecode.h" - -#endif - -// -// Namespaces -// - -using namespace std; -using namespace NLMISC; - - -#ifdef RZ_USE_NEW_LZMA - -/// Input stream class for 7zip archive -class CNel7ZipInStream : public ISeekInStream -{ - NLMISC::IStream *_Stream; - -public: - /// Constructor, only allow file stream because 7zip will 'seek' in the stream - CNel7ZipInStream(NLMISC::IStream *s) - : _Stream(s) - { - Read = readFunc; - Seek = seekFunc; - } - - // the read function called by 7zip to read data - static SRes readFunc(void *object, void *buffer, size_t *size) - { - try - { - CNel7ZipInStream *me = (CNel7ZipInStream*)object; - uint len = (uint)*size; - me->_Stream->serialBuffer((uint8*)buffer, len); - return SZ_OK; - } - catch (...) - { - return SZ_ERROR_READ; - } - } - - // the seek function called by seven zip to seek inside stream - static SRes seekFunc(void *object, Int64 *pos, ESzSeek origin) - { - try - { - CNel7ZipInStream *me = (CNel7ZipInStream*)object; - sint32 offset = (sint32)*pos; - bool ret = me->_Stream->seek(offset, (NLMISC::IStream::TSeekOrigin)origin); - - if (ret) - { - *pos = (Int64)me->_Stream->getPos(); - return SZ_OK; - } - } - catch (...) - { - } - - return SZ_ERROR_READ; - } -}; - -#else - -/// Input stream class for 7zip archive -class CNel7ZipInStream : public _ISzInStream -{ - NLMISC::IStream *_Stream; - -public: - /// Constructor, only allow file stream because 7zip will 'seek' in the stream - CNel7ZipInStream(NLMISC::IStream *s) - : _Stream(s) - { - Read = readFunc; - Seek = seekFunc; - } - - // the read function called by 7zip to read data - static SZ_RESULT readFunc(void *object, void *buffer, size_t size, size_t *processedSize) - { - try - { - CNel7ZipInStream *me = (CNel7ZipInStream*)object; - me->_Stream->serialBuffer((uint8*)buffer, (uint)size); - *processedSize = size; - return SZ_OK; - } - catch (...) - { - return SZE_FAIL; - } - } - - // the seek function called by seven zip to seek inside stream - static SZ_RESULT seekFunc(void *object, CFileSize pos) - { - try - { - CNel7ZipInStream *me = (CNel7ZipInStream*)object; - bool ret= me->_Stream->seek(pos, NLMISC::IStream::begin); - if (ret) - return SZ_OK; - else - return SZE_FAIL; - } - catch (...) - { - return SZE_FAIL; - } - } -}; - -#endif - -#endif - -bool CPatchManager::unpack7Zip(const std::string &sevenZipFile, const std::string &destFileName) -{ -#ifdef RZ_USE_SEVENZIP - nlinfo("Uncompressing 7zip archive '%s' to '%s'", sevenZipFile.c_str(), destFileName.c_str()); - - // init seven zip - ISzAlloc allocImp; - allocImp.Alloc = SzAlloc; - allocImp.Free = SzFree; - - ISzAlloc allocTempImp; - allocTempImp.Alloc = SzAllocTemp; - allocTempImp.Free = SzFreeTemp; - - // wrap file in a CIFile - CIFile input(sevenZipFile); - CNel7ZipInStream inStr(&input); - -#ifdef RZ_USE_NEW_LZMA - - CLookToRead lookStream; - lookStream.realStream = &inStr; - LookToRead_CreateVTable(&lookStream, False); - LookToRead_Init(&lookStream); - - CrcGenerateTable(); - - CSzArEx db; - SzArEx_Init(&db); - - // unpack the file using the 7zip API - SRes res = SzArEx_Open(&db, &lookStream.s, &allocImp, &allocTempImp); - - if (res != SZ_OK) - { - nlerror("Failed to open archive file %s", sevenZipFile.c_str()); - return false; - } - - if (db.NumFiles != 1) - { - nlerror("Seven zip archive with more than 1 file are unsupported"); - return false; - } - - UInt32 blockIndex = 0xFFFFFFFF; /* it can have any value before first call (if outBuffer = 0) */ - Byte *outBuffer = 0; /* it must be 0 before first call for each new archive. */ - size_t outBufferSize = 0; /* it can have any value before first call (if outBuffer = 0) */ - - size_t offset; - size_t outSizeProcessed = 0; - - // get the first file - res = SzArEx_Extract(&db, &lookStream.s, 0, - &blockIndex, &outBuffer, &outBufferSize, - &offset, &outSizeProcessed, - &allocImp, &allocTempImp); - - // get the length of first file - size_t nameLen = SzArEx_GetFileNameUtf16(&db, 0, NULL); - - ucstring filename; - filename.resize(nameLen); - - // write filename into ucstring - SzArEx_GetFileNameUtf16(&db, 0, &filename[0]); - - // write the extracted file - FILE *outputHandle = fopen(destFileName.c_str(), "wb+"); - - if (outputHandle == 0) - { - nlerror("Can not open output file '%s'", destFileName.c_str()); - return false; - } - - UInt32 processedSize = (UInt32)fwrite(outBuffer + offset, 1, outSizeProcessed, outputHandle); - - if (processedSize != outSizeProcessed) - { - nlerror("Failed to write %u char to output file '%s'", outSizeProcessed-processedSize, destFileName.c_str()); - return false; - } - - fclose(outputHandle); - - IAlloc_Free(&allocImp, outBuffer); - - // free 7z context - SzArEx_Free(&db, &allocImp); - -#else - - InitCrcTable(); - - CArchiveDatabaseEx db; - SzArDbExInit(&db); - - // unpack the file using the 7zip API - SZ_RESULT res = SzArchiveOpen(&inStr, &db, &allocImp, &allocTempImp); - - if (res != SZ_OK) - { - nlerror("Failed to open archive file %s", sevenZipFile.c_str()); - return false; - } - - if (db.Database.NumFiles != 1) - { - nlerror("Seven zip archive with more than 1 file are unsupported"); - return false; - } - - UInt32 blockIndex = 0xFFFFFFFF; /* it can have any value before first call (if outBuffer = 0) */ - Byte *outBuffer = 0; /* it must be 0 before first call for each new archive. */ - size_t outBufferSize = 0; /* it can have any value before first call (if outBuffer = 0) */ - - size_t offset = 0; - size_t outSizeProcessed = 0; - - // get the first file - res = SzExtract(&inStr, &db, 0, - &blockIndex, &outBuffer, &outBufferSize, - &offset, &outSizeProcessed, - &allocImp, &allocTempImp); - - // write the extracted file - FILE *outputHandle = fopen(destFileName.c_str(), "wb+"); - - if (outputHandle == 0) - { - nlerror("Can not open output file '%s'", destFileName.c_str()); - return false; - } - - UInt32 processedSize = (UInt32)fwrite(outBuffer + offset, 1, outSizeProcessed, outputHandle); - - if (processedSize != outSizeProcessed) - { - nlerror("Failed to write %u char to output file '%s'", outSizeProcessed-processedSize, destFileName.c_str()); - return false; - } - - fclose(outputHandle); - allocImp.Free(outBuffer); - - // free 7z context - SzArDbExFree(&db, allocImp.Free); - -#endif - - // ok, all is fine, file is unpacked - return true; -#else - return false; -#endif -} - -bool CPatchManager::unpackLZMA(const std::string &lzmaFile, const std::string &destFileName) -{ -#ifdef RZ_USE_SEVENZIP - nldebug("unpackLZMA : decompression the lzma file '%s' into output file '%s", lzmaFile.c_str(), destFileName.c_str()); - - CIFile inStream(lzmaFile); - uint32 inSize = inStream.getFileSize(); - -#ifdef RZ_USE_NEW_LZMA - - if (inSize < LZMA_PROPS_SIZE + 8) - { - nlwarning("Invalid file size, too small file '%s'", lzmaFile.c_str()); - return false; - } - - // read the compressed file in buffer - auto_ptr inBuffer = auto_ptr(new uint8[inSize]); - inStream.serialBuffer(inBuffer.get(), inSize); - - uint8 *pos = inBuffer.get(); - - pos += LZMA_PROPS_SIZE; - - // read the output file size - size_t fileSize = 0; - - for (int i = 0; i < 8; ++i) - { - if (pos >= inBuffer.get()+inSize) - { - nlassert(false); - return false; - } - - fileSize |= ((UInt64)*pos++) << (8 * i); - } - - // allocators - ISzAlloc allocImp; - allocImp.Alloc = SzAlloc; - allocImp.Free = SzFree; - - CLzmaDec state; - LzmaDec_Construct(&state); - - // allocate and decode props and probs - SRes res = LzmaDec_Allocate(&state, inBuffer.get(), LZMA_PROPS_SIZE, &allocImp); - - if (res != 0) - { - nlwarning("Failed to decode lzma properies in file '%s'!", lzmaFile.c_str()); - return false; - } - - // allocate the output buffer - auto_ptr outBuffer = auto_ptr(new uint8[fileSize]); - - // in and out file sizes - SizeT outProcessed = fileSize; - SizeT inProcessed = (SizeT)(inSize-(pos-inBuffer.get())); - - LzmaDec_Init(&state); - - // decompress the file in memory - ELzmaStatus status; - res = LzmaDec_DecodeToBuf(&state, (Byte*)outBuffer.get(), &outProcessed, (Byte*)pos, &inProcessed, LZMA_FINISH_ANY, &status); - - // free memory - LzmaDec_Free(&state, &allocImp); - - if (res != 0 || outProcessed != fileSize) - { - nlwarning("Failed to decode lzma file '%s' with status %d", lzmaFile.c_str(), (sint)status); - return false; - } - -#else - - auto_ptr inBuffer = auto_ptr(new uint8[inSize]); - inStream.serialBuffer(inBuffer.get(), inSize); - - CLzmaDecoderState state; - - uint8 *pos = inBuffer.get(); - - // read the lzma properties - int res = LzmaDecodeProperties(&state.Properties, (unsigned char*) pos, LZMA_PROPERTIES_SIZE); - if (res != 0) - { - nlwarning("Failed to decode lzma properies in file '%s'!", lzmaFile.c_str()); - return false; - } - - if (inSize < LZMA_PROPERTIES_SIZE + 8) - { - nlwarning("Invalid file size, too small file '%s'", lzmaFile.c_str()); - return false; - } - - // alloc the probs, making sure they are deleted in function exit - size_t nbProb = LzmaGetNumProbs(&state.Properties); - auto_ptr probs = auto_ptr(new CProb[nbProb]); - state.Probs = probs.get(); - - pos += LZMA_PROPERTIES_SIZE; - - // read the output file size - size_t fileSize = 0; - for (int i = 0; i < 8; i++) - { - if (pos >= inBuffer.get()+inSize) - { - nlassert(false); - return false; - } - fileSize |= ((UInt64)*pos++) << (8 * i); - } - - SizeT outProcessed = 0; - SizeT inProcessed = 0; - - // allocate the output buffer - auto_ptr outBuffer = auto_ptr(new uint8[fileSize]); - - // decompress the file in memory - res = LzmaDecode(&state, (unsigned char*) pos, (SizeT)(inSize-(pos-inBuffer.get())), &inProcessed, (unsigned char*)outBuffer.get(), (SizeT)fileSize, &outProcessed); - - if (res != 0 || outProcessed != fileSize) - { - nlwarning("Failed to decode lzma file '%s'", lzmaFile.c_str()); - return false; - } - -#endif - - // store on output buffer - COFile outStream(destFileName); - outStream.serialBuffer(outBuffer.get(), (uint)fileSize); - - return true; -#else - return false; -#endif -} diff --git a/code/ryzom/client/src/seven_zip/seven_zip.cpp b/code/ryzom/client/src/seven_zip/seven_zip.cpp new file mode 100644 index 000000000..84a7eaace --- /dev/null +++ b/code/ryzom/client/src/seven_zip/seven_zip.cpp @@ -0,0 +1,357 @@ +// Ryzom - MMORPG Framework +// Copyright (C) 2010 Winch Gate Property Limited +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "seven_zip.h" + +#include "nel/misc/types_nl.h" +#include "nel/misc/file.h" +#include "nel/misc/path.h" + +#include "7z.h" +#include "7zAlloc.h" +#include "7zCrc.h" +#include "7zVersion.h" +#include "LzmaLib.h" + +#include + +// +// Namespaces +// + +using namespace std; +using namespace NLMISC; + + +/// Input stream class for 7zip archive +class CNel7ZipInStream : public ISeekInStream +{ + NLMISC::IStream *_Stream; + +public: + /// Constructor, only allow file stream because 7zip will 'seek' in the stream + CNel7ZipInStream(NLMISC::IStream *s) + : _Stream(s) + { + Read = readFunc; + Seek = seekFunc; + } + + // the read function called by 7zip to read data + static SRes readFunc(void *object, void *buffer, size_t *size) + { + try + { + CNel7ZipInStream *me = (CNel7ZipInStream*)object; + uint len = (uint)*size; + me->_Stream->serialBuffer((uint8*)buffer, len); + return SZ_OK; + } + catch (...) + { + return SZ_ERROR_READ; + } + } + + // the seek function called by seven zip to seek inside stream + static SRes seekFunc(void *object, Int64 *pos, ESzSeek origin) + { + try + { + CNel7ZipInStream *me = (CNel7ZipInStream*)object; + sint32 offset = (sint32)*pos; + bool ret = me->_Stream->seek(offset, (NLMISC::IStream::TSeekOrigin)origin); + + if (ret) + { + *pos = (Int64)me->_Stream->getPos(); + return SZ_OK; + } + } + catch (...) + { + } + + return SZ_ERROR_READ; + } +}; + +bool unpack7Zip(const std::string &sevenZipFile, const std::string &destFileName) +{ + nlinfo("Uncompressing 7zip archive '%s' to '%s'", sevenZipFile.c_str(), destFileName.c_str()); + + // init seven zip + ISzAlloc allocImp; + allocImp.Alloc = SzAlloc; + allocImp.Free = SzFree; + + ISzAlloc allocTempImp; + allocTempImp.Alloc = SzAllocTemp; + allocTempImp.Free = SzFreeTemp; + + // wrap file in a CIFile + CIFile input(sevenZipFile); + CNel7ZipInStream inStr(&input); + + CLookToRead lookStream; + lookStream.realStream = &inStr; + LookToRead_CreateVTable(&lookStream, False); + LookToRead_Init(&lookStream); + + CrcGenerateTable(); + + CSzArEx db; + SzArEx_Init(&db); + + // unpack the file using the 7zip API + SRes res = SzArEx_Open(&db, &lookStream.s, &allocImp, &allocTempImp); + + if (res != SZ_OK) + { + nlerror("Failed to open archive file %s", sevenZipFile.c_str()); + return false; + } + + if (db.NumFiles != 1) + { + nlerror("Seven zip archive with more than 1 file are unsupported"); + return false; + } + + UInt32 blockIndex = 0xFFFFFFFF; /* it can have any value before first call (if outBuffer = 0) */ + Byte *outBuffer = 0; /* it must be 0 before first call for each new archive. */ + size_t outBufferSize = 0; /* it can have any value before first call (if outBuffer = 0) */ + + size_t offset; + size_t outSizeProcessed = 0; + + // get the first file + res = SzArEx_Extract(&db, &lookStream.s, 0, + &blockIndex, &outBuffer, &outBufferSize, + &offset, &outSizeProcessed, + &allocImp, &allocTempImp); + + // get the length of first file + size_t nameLen = SzArEx_GetFileNameUtf16(&db, 0, NULL); + + ucstring filename; + filename.resize(nameLen); + + // write filename into ucstring + SzArEx_GetFileNameUtf16(&db, 0, &filename[0]); + + // write the extracted file + FILE *outputHandle = fopen(destFileName.c_str(), "wb+"); + + if (outputHandle == 0) + { + nlerror("Can not open output file '%s'", destFileName.c_str()); + return false; + } + + UInt32 processedSize = (UInt32)fwrite(outBuffer + offset, 1, outSizeProcessed, outputHandle); + + if (processedSize != outSizeProcessed) + { + nlerror("Failed to write %u char to output file '%s'", outSizeProcessed-processedSize, destFileName.c_str()); + return false; + } + + fclose(outputHandle); + + IAlloc_Free(&allocImp, outBuffer); + + // free 7z context + SzArEx_Free(&db, &allocImp); + + // ok, all is fine, file is unpacked + return true; +} + +bool unpackLZMA(const std::string &lzmaFile, const std::string &destFileName) +{ + nldebug("unpackLZMA: decompress LZMA file '%s' to '%s", lzmaFile.c_str(), destFileName.c_str()); + + // open input file + CIFile inStream(lzmaFile); + uint32 inSize = inStream.getFileSize(); + + if (inSize < LZMA_PROPS_SIZE + 8) + { + nlwarning("unpackLZMA: Invalid file size, too small file '%s'", lzmaFile.c_str()); + return false; + } + + // allocate input buffer for props + auto_ptr propsBuffer = auto_ptr(new uint8[LZMA_PROPS_SIZE]); + + // size of LZMA content + inSize -= LZMA_PROPS_SIZE + 8; + + // allocate input buffer for lzma data + auto_ptr inBuffer = auto_ptr(new uint8[inSize]); + + uint64 fileSize = 0; + + try + { + // read props + inStream.serialBuffer(propsBuffer.get(), LZMA_PROPS_SIZE); + + // read uncompressed size + inStream.serial(fileSize); + + // read lzma content + inStream.serialBuffer(inBuffer.get(), inSize); + } + catch(const EReadError &e) + { + nlwarning("unpackLZMA: Error while reading '%s': %s", lzmaFile.c_str(), e.what()); + return false; + } + + // allocate the output buffer + auto_ptr outBuffer = auto_ptr(new uint8[fileSize]); + + // in and out file sizes + SizeT outProcessed = (SizeT)fileSize; + SizeT inProcessed = (SizeT)inSize; + + // decompress the file in memory + sint res = LzmaUncompress(outBuffer.get(), &outProcessed, inBuffer.get(), &inProcessed, propsBuffer.get(), LZMA_PROPS_SIZE); + + if (res != 0 || outProcessed != fileSize) + { + nlwarning("unpackLZMA: Failed to decode lzma file '%s' with status %d", lzmaFile.c_str(), res); + return false; + } + + // store on output buffer + COFile outStream(destFileName); + + try + { + // write content + outStream.serialBuffer(outBuffer.get(), (uint)fileSize); + } + catch(const EFile &e) + { + nlwarning("unpackLZMA: Error while writing '%s': %s", destFileName.c_str(), e.what()); + CFile::deleteFile(destFileName); + return false; + } + + return true; +} + +bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName) +{ + nldebug("unpackLZMA: compress '%s' to LZMA file '%s", srcFileName.c_str(), lzmaFileName.c_str()); + + // open file + CIFile inStream(srcFileName); + size_t inSize = inStream.getFileSize(); + + // file empty + if (!inSize) + { + nlwarning("packLZMA: File '%s' not found or empty", srcFileName.c_str()); + return false; + } + + // allocate input buffer + auto_ptr inBuffer = auto_ptr(new uint8[inSize]); + + try + { + // read file in buffer + inStream.serialBuffer(inBuffer.get(), inSize); + } + catch(const EReadError &e) + { + nlwarning("packLZMA: Error while reading '%s': %s", srcFileName.c_str(), e.what()); + return false; + } + + // allocate output buffer + size_t outSize = (11 * inSize / 10) + 65536; // worst case = 1.1 * size + 64K + auto_ptr outBuffer = auto_ptr(new uint8[outSize]); + + // allocate buffer for props + size_t outPropsSize = LZMA_PROPS_SIZE; + auto_ptr outProps = auto_ptr(new uint8[outPropsSize]); + + // compress with best compression and other default settings + sint res = LzmaCompress(outBuffer.get(), &outSize, inBuffer.get(), inSize, outProps.get(), &outPropsSize, 9, 1 << 24, 3, 0, 2, 32, 1); + + switch(res) + { + case SZ_OK: + { + // store on output buffer + COFile outStream(lzmaFileName); + + // unable to create file + if (!outStream.isOpen()) + { + nlwarning("packLZMA: Unable to create '%s'", srcFileName.c_str()); + return false; + } + + try + { + // write props + outStream.serialBuffer(outProps.get(), (uint)outPropsSize); + + // write uncompressed size + uint64 uncompressSize = inSize; + outStream.serial(uncompressSize); + + // write content + outStream.serialBuffer(outBuffer.get(), (uint)outSize); + } + catch(const EFile &e) + { + nlwarning("packLZMA: Error while writing '%s': %s", lzmaFileName.c_str(), e.what()); + CFile::deleteFile(lzmaFileName); + return false; + } + + return true; + } + + case SZ_ERROR_MEM: + nlwarning("packLZMA: Memory allocation error while compressing '%s' (input buffer size: %u, output buffer size: %u)", srcFileName.c_str(), (uint)inSize, (uint)outSize); + break; + + case SZ_ERROR_PARAM: + nlwarning("packLZMA: Incorrect parameter while compressing '%s'", srcFileName.c_str()); + break; + + case SZ_ERROR_OUTPUT_EOF: + nlwarning("packLZMA: Output buffer overflow while compressing '%s' (input buffer size: %u, output buffer size: %u)", srcFileName.c_str(), (uint)inSize, (uint)outSize); + break; + + case SZ_ERROR_THREAD: + nlwarning("packLZMA: Errors in multithreading functions (only for Mt version)"); + break; + + default: + nlwarning("packLZMA: Unknown error (%d) while compressing '%s'", res, srcFileName.c_str()); + } + + return false; +} diff --git a/code/ryzom/client/src/seven_zip/seven_zip.h b/code/ryzom/client/src/seven_zip/seven_zip.h new file mode 100644 index 000000000..e471a9ac0 --- /dev/null +++ b/code/ryzom/client/src/seven_zip/seven_zip.h @@ -0,0 +1,31 @@ +// Ryzom - MMORPG Framework +// Copyright (C) 2010 Winch Gate Property Limited +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef SEVEN_ZIP_H +#define SEVEN_ZIP_H + +#include + +// utility func to decompress a monofile 7zip archive +bool unpack7Zip(const std::string &sevenZipFileName, const std::string &destFileName); + +// utility func to decompress a single LZMA packed file +bool unpackLZMA(const std::string &lzmaFileName, const std::string &destFileName); + +// utility func to compress a single file to LZMA packed file +bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName); + +#endif From 31e6bd1bb1831d1ed6a2c06c603598e03803206c Mon Sep 17 00:00:00 2001 From: kervala Date: Tue, 2 Feb 2016 10:22:47 +0100 Subject: [PATCH 02/10] Fixed: patch_gen is now using packLZMA function to compress a file instead of using lzma executable --HG-- branch : develop --- code/ryzom/tools/patch_gen/CMakeLists.txt | 6 ++-- .../tools/patch_gen/patch_gen_common.cpp | 34 ++++--------------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/code/ryzom/tools/patch_gen/CMakeLists.txt b/code/ryzom/tools/patch_gen/CMakeLists.txt index d7d864f9a..6cde9810b 100644 --- a/code/ryzom/tools/patch_gen/CMakeLists.txt +++ b/code/ryzom/tools/patch_gen/CMakeLists.txt @@ -1,13 +1,15 @@ +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/ryzom/client/src/seven_zip) + SET(MAIN_SRC patch_gen_common.cpp patch_gen_main.cpp patch_gen_main.h) SET(SERVICE_SRC patch_gen_common.cpp patch_gen_service.cpp patch_gen_service.h) ADD_EXECUTABLE(patch_gen ${MAIN_SRC}) -TARGET_LINK_LIBRARIES(patch_gen ryzom_gameshare nelmisc nelnet nelligo nelgeorges) +TARGET_LINK_LIBRARIES(patch_gen ryzom_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges) NL_DEFAULT_PROPS(patch_gen "Ryzom, Tools: Patch Generator") NL_ADD_RUNTIME_FLAGS(patch_gen) ADD_EXECUTABLE(patch_gen_service WIN32 ${SERVICE_SRC}) -TARGET_LINK_LIBRARIES(patch_gen_service ryzom_gameshare nelmisc nelnet nelligo nelgeorges) +TARGET_LINK_LIBRARIES(patch_gen_service ryzom_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges) NL_DEFAULT_PROPS(patch_gen_service "Ryzom, Tools: Patch Generator Service") NL_ADD_RUNTIME_FLAGS(patch_gen_service) diff --git a/code/ryzom/tools/patch_gen/patch_gen_common.cpp b/code/ryzom/tools/patch_gen/patch_gen_common.cpp index dd4bf4211..c1290cfd5 100644 --- a/code/ryzom/tools/patch_gen/patch_gen_common.cpp +++ b/code/ryzom/tools/patch_gen/patch_gen_common.cpp @@ -27,6 +27,7 @@ #include "nel/misc/command.h" #include "nel/misc/sstring.h" #include "game_share/singleton_registry.h" +#include "seven_zip.h" using namespace std; using namespace NLMISC; @@ -74,37 +75,16 @@ void ApplyPatch(const std::string& srcFileName,const std::string& destFileName,c #endif // NL_OS_WINDOWS } -void GenerateLZMA(const std::string sourceFile, const std::string &outputFile) +void GenerateLZMA(const std::string &sourceFile, const std::string &outputFile) { { - // old syntax incompatible with new versions - std::string cmd = "lzma e"; - cmd += " " + sourceFile + " " + outputFile; - nlinfo("Executing system command: %s", cmd.c_str()); + nlinfo("Compressing %s to %s using LZMA...", sourceFile.c_str(), outputFile.c_str()); } -#ifdef NL_OS_WINDOWS - _spawnlp(_P_WAIT, "lzma.exe","lzma.exe", "e", sourceFile.c_str(), outputFile.c_str(), NULL); -#else // NL_OS_WINDOWS - // new lzma only supports one file name on command line, so make a copy - CFile::copyFile(outputFile, sourceFile); - - // new lzma syntax, -z = compress, -9 = best compression - std::string cmd = NLMISC::toString("lzma -z -9 %s", outputFile.c_str()); - - sint error = system(cmd.c_str()); - - if (error) - { - nlwarning("'%s' failed with error code %d", cmd.c_str(), error); - CFile::deleteFile(outputFile); - } - else + if (!packLZMA(sourceFile, outputFile)) { - // lzma always append a .lzma extension, so rename compressed file to wanted one - CFile::moveFile(outputFile, outputFile + ".lzma"); + nlwarning("LZMA compress failed"); } -#endif // NL_OS_WINDOWS } @@ -230,7 +210,7 @@ void CPackageDescription::storeToPdr(CPersistentDataRecord& pdr) const void CPackageDescription::readIndex(CBNPFileSet& packageIndex) const { std::string indexPath = _RootDirectory + _IndexFileName; - + nlinfo("Reading history file: %s ...", indexPath.c_str()); // clear out old contents before reading from input file @@ -294,7 +274,7 @@ void CPackageDescription::generateClientIndex(CProductDescriptionForClient& theC std::string patchNumber = toString("%05u", packageIndex.getVersionNumber()); std::string patchDirectory = _PatchDirectory + patchNumber; std::string patchFile = patchDirectory + "/" + _ClientIndexFileName; - + nlinfo("Generating client index: %s...", patchFile.c_str()); // make sure the version sub directory exist From 0979b19e8edf397550c445c1bde17767d93ed675 Mon Sep 17 00:00:00 2001 From: kervala Date: Tue, 2 Feb 2016 10:23:57 +0100 Subject: [PATCH 03/10] Fixed: Use absolute path of current directory instead of relative path (avoid to check more than once the same location) --HG-- branch : develop --- code/ryzom/client/src/init.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/ryzom/client/src/init.cpp b/code/ryzom/client/src/init.cpp index 8daabdaa5..faa83b51c 100644 --- a/code/ryzom/client/src/init.cpp +++ b/code/ryzom/client/src/init.cpp @@ -659,7 +659,7 @@ static void addPaths(IProgressCallback &progress, const std::vector std::vector directoryPrefixes; // current directory has priority everywhere - directoryPrefixes.push_back(""); + directoryPrefixes.push_back(CPath::standardizePath(CPath::getCurrentPath())); #if defined(NL_OS_WINDOWS) // check in same directory as executable From 0919fc66e49ba605f82768e3cb70d93328644514 Mon Sep 17 00:00:00 2001 From: kervala Date: Tue, 2 Feb 2016 11:32:20 +0100 Subject: [PATCH 04/10] Changed: Support more values for fromString with a boolean --HG-- branch : develop --- code/nel/include/nel/misc/string_common.h | 29 ++++++++++++++++------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/code/nel/include/nel/misc/string_common.h b/code/nel/include/nel/misc/string_common.h index e6d08bdeb..b8a12ad77 100644 --- a/code/nel/include/nel/misc/string_common.h +++ b/code/nel/include/nel/misc/string_common.h @@ -246,27 +246,38 @@ inline bool fromString(const std::string &str, bool &val) { if (str.length() == 1) { - if (str[0] == '1') + const char c = str[0]; + + switch(c) { + case '1': + case 't': + case 'T': + case 'y': + case 'Y': val = true; - } - else if (str[0] == '0') - { + break; + + case '0': + case 'f': + case 'F': + case 'n': + case 'N': val = false; - } - else - { + break; + + default: val = false; return false; } } else { - if (str == "true") + if (str == "true" || str == "yes") { val = true; } - else if (str == "false") + else if (str == "false" || str == "no") { val = false; } From 7b1ab1592ecd503e5f16589af162f8700d91b754 Mon Sep 17 00:00:00 2001 From: kervala Date: Tue, 2 Feb 2016 11:33:44 +0100 Subject: [PATCH 05/10] Fixed: Support new ISO units (KiB, MiB, etc...) in humanReadableToBytes --HG-- branch : develop --- code/nel/src/misc/common.cpp | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/code/nel/src/misc/common.cpp b/code/nel/src/misc/common.cpp index 4c7d4fe02..5115547a5 100644 --- a/code/nel/src/misc/common.cpp +++ b/code/nel/src/misc/common.cpp @@ -394,7 +394,8 @@ uint32 humanReadableToBytes (const string &str) if(str[0]<'0' || str[0]>'9') return 0; - res = atoi (str.c_str()); + if (!fromString(str, res)) + return 0; if(str[str.size()-1] == 'B') { @@ -404,10 +405,28 @@ uint32 humanReadableToBytes (const string &str) // there's no break and it's **normal** switch (str[str.size()-2]) { - case 'G': res *= 1024; - case 'M': res *= 1024; - case 'K': res *= 1024; - default: ; + // kB/KB, MB, GB and TB are 1000 multiples + case 'T': res *= 1000; + case 'G': res *= 1000; + case 'M': res *= 1000; + case 'k': res *= 1000; break; // kilo symbol should be a lowercase K + case 'K': res *= 1000; break; + case 'i': + { + // KiB, MiB, GiB and TiB are 1024 multiples + if (str.size()<4) + return res; + + switch (str[str.size()-3]) + { + case 'T': res *= 1024; + case 'G': res *= 1024; + case 'M': res *= 1024; + case 'K': res *= 1024; + default: ; + } + } + default: ; } } From 20c7c342ca76579a30d7969c342715f256d55275 Mon Sep 17 00:00:00 2001 From: kervala Date: Tue, 2 Feb 2016 11:34:16 +0100 Subject: [PATCH 06/10] Fixed: Don't display logs in console for unit tests --HG-- branch : develop --- code/nel/tools/nel_unit_test/nel_unit_test.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/code/nel/tools/nel_unit_test/nel_unit_test.cpp b/code/nel/tools/nel_unit_test/nel_unit_test.cpp index 8289ea490..753dfd46a 100644 --- a/code/nel/tools/nel_unit_test/nel_unit_test.cpp +++ b/code/nel/tools/nel_unit_test/nel_unit_test.cpp @@ -114,6 +114,20 @@ int main(int argc, char *argv[]) // init Nel context new NLMISC::CApplicationContext; + // disable nldebug messages in logs in Release +#ifdef NL_RELEASE + NLMISC::DisableNLDebug = true; +#endif + + NLMISC::createDebug(NULL); + +#ifndef NL_DEBUG + NLMISC::INelContext::getInstance().getDebugLog()->removeDisplayer("DEFAULT_SD"); + NLMISC::INelContext::getInstance().getInfoLog()->removeDisplayer("DEFAULT_SD"); + NLMISC::INelContext::getInstance().getWarningLog()->removeDisplayer("DEFAULT_SD"); + NLMISC::INelContext::getInstance().getErrorLog()->removeDisplayer("DEFAULT_SD"); +#endif // NL_DEBUG + bool noerrors = false; try From 285965fe76753d450fae534cdc7a8f9566ae5ad5 Mon Sep 17 00:00:00 2001 From: kervala Date: Tue, 2 Feb 2016 11:37:46 +0100 Subject: [PATCH 07/10] Added: New unit tests for common.h --HG-- branch : develop --- code/nel/tools/nel_unit_test/ut_misc.h | 2 + code/nel/tools/nel_unit_test/ut_misc_common.h | 106 ++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 code/nel/tools/nel_unit_test/ut_misc_common.h diff --git a/code/nel/tools/nel_unit_test/ut_misc.h b/code/nel/tools/nel_unit_test/ut_misc.h index 8538d3f53..e7a3cc8c6 100644 --- a/code/nel/tools/nel_unit_test/ut_misc.h +++ b/code/nel/tools/nel_unit_test/ut_misc.h @@ -19,6 +19,7 @@ #include "ut_misc_co_task.h" #include "ut_misc_command.h" +#include "ut_misc_common.h" #include "ut_misc_config_file.h" #include "ut_misc_debug.h" #include "ut_misc_dynlibload.h" @@ -38,6 +39,7 @@ struct CUTMisc : public Test::Suite { add(auto_ptr(new CUTMiscCoTask)); add(auto_ptr(new CUTMiscCommand)); + add(auto_ptr(new CUTMiscCommon)); add(auto_ptr(new CUTMiscConfigFile)); add(auto_ptr(new CUTMiscDebug)); add(auto_ptr(new CUTMiscDynLibLoad)); diff --git a/code/nel/tools/nel_unit_test/ut_misc_common.h b/code/nel/tools/nel_unit_test/ut_misc_common.h new file mode 100644 index 000000000..e915a4225 --- /dev/null +++ b/code/nel/tools/nel_unit_test/ut_misc_common.h @@ -0,0 +1,106 @@ +// NeL - MMORPG Framework +// Copyright (C) 2010 Winch Gate Property Limited +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef UT_MISC_COMMON +#define UT_MISC_COMMON + +#include +#include + +struct CUTMiscCommon : public Test::Suite +{ + CUTMiscCommon() + { + TEST_ADD(CUTMiscCommon::humanReadableToBytes); + + // Add a line here when adding a new test METHOD + } + + void humanReadableToBytes() + { + uint64 bytes = 0; + + // kiB is a wrong unit + bytes = NLMISC::humanReadableToBytes("1kiB"); + TEST_ASSERT(bytes == 1); + + // 1 kibibyte + bytes = NLMISC::humanReadableToBytes("1KiB"); + TEST_ASSERT(bytes == 1024); + + // 1 mebibyte + bytes = NLMISC::humanReadableToBytes("1MiB"); + TEST_ASSERT(bytes == 1024*1024); + + // 1 kilobyte + bytes = NLMISC::humanReadableToBytes("1KB"); + TEST_ASSERT(bytes == 1000); + + // 1 kilobyte + bytes = NLMISC::humanReadableToBytes("1kB"); + TEST_ASSERT(bytes == 1000); + + // 1 megabyte + bytes = NLMISC::humanReadableToBytes("1MB"); + TEST_ASSERT(bytes == 1000*1000); + + // 1 byte + bytes = NLMISC::humanReadableToBytes("1B"); + TEST_ASSERT(bytes == 1); + + // 1 byte + bytes = NLMISC::humanReadableToBytes("1"); + TEST_ASSERT(bytes == 1); + + // kiB is a wrong unit + bytes = NLMISC::humanReadableToBytes("1 kiB"); + TEST_ASSERT(bytes == 1); + + // 1 kibibyte + bytes = NLMISC::humanReadableToBytes("1 KiB"); + TEST_ASSERT(bytes == 1024); + + // 1 mebibyte + bytes = NLMISC::humanReadableToBytes("1 MiB"); + TEST_ASSERT(bytes == 1024*1024); + + // 1 kilobyte + bytes = NLMISC::humanReadableToBytes("1 KB"); + TEST_ASSERT(bytes == 1000); + + // 1 kilobyte + bytes = NLMISC::humanReadableToBytes("1 kB"); + TEST_ASSERT(bytes == 1000); + + // 1 Megabyte + bytes = NLMISC::humanReadableToBytes("1 MB"); + TEST_ASSERT(bytes == 1000*1000); + + // 1 byte + bytes = NLMISC::humanReadableToBytes("1 B"); + TEST_ASSERT(bytes == 1); + + // not a number + bytes = NLMISC::humanReadableToBytes("AB"); + TEST_ASSERT(bytes == 0); + + // not a positive number + bytes = NLMISC::humanReadableToBytes("-1 B"); + TEST_ASSERT(bytes == 0); + } +}; + +#endif From b692b78718663a01f961d812ba01014a649d0e48 Mon Sep 17 00:00:00 2001 From: kervala Date: Tue, 2 Feb 2016 11:38:08 +0100 Subject: [PATCH 08/10] Fixed: Tests for fromStringwith a boolean --HG-- branch : develop --- .../nel_unit_test/ut_misc_string_common.h | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/code/nel/tools/nel_unit_test/ut_misc_string_common.h b/code/nel/tools/nel_unit_test/ut_misc_string_common.h index 2fc646a36..33378a8ec 100644 --- a/code/nel/tools/nel_unit_test/ut_misc_string_common.h +++ b/code/nel/tools/nel_unit_test/ut_misc_string_common.h @@ -682,60 +682,59 @@ struct CUTMiscStringCommon : public Test::Suite // tests for bool bool val; - // true value - val = false; + // true values ret = NLMISC::fromString("1", val); - TEST_ASSERT(val); - TEST_ASSERT_MSG(ret, "should succeed"); + TEST_ASSERT(ret && val); - val = false; - NLMISC::fromString("t", val); - TEST_ASSERT(val); + ret = NLMISC::fromString("t", val); + TEST_ASSERT(ret && val); - val = false; - NLMISC::fromString("y", val); - TEST_ASSERT(val); + ret = NLMISC::fromString("y", val); + TEST_ASSERT(ret && val); - val = false; - NLMISC::fromString("T", val); - TEST_ASSERT(val); + ret = NLMISC::fromString("T", val); + TEST_ASSERT(ret && val); - val = false; - NLMISC::fromString("Y", val); - TEST_ASSERT(val); + ret = NLMISC::fromString("Y", val); + TEST_ASSERT(ret && val); - val = true; + ret = NLMISC::fromString("true", val); + TEST_ASSERT(ret && val); + + ret = NLMISC::fromString("yes", val); + TEST_ASSERT(ret && val); + + // false values ret = NLMISC::fromString("0", val); - TEST_ASSERT(!val); - TEST_ASSERT_MSG(ret, "should succeed"); + TEST_ASSERT(ret && !val); - val = true; - NLMISC::fromString("f", val); - TEST_ASSERT(!val); + ret = NLMISC::fromString("f", val); + TEST_ASSERT(ret && !val); - val = true; - NLMISC::fromString("n", val); - TEST_ASSERT(!val); + ret = NLMISC::fromString("n", val); + TEST_ASSERT(ret && !val); - val = true; - NLMISC::fromString("F", val); - TEST_ASSERT(!val); + ret = NLMISC::fromString("F", val); + TEST_ASSERT(ret && !val); - val = true; - NLMISC::fromString("N", val); - TEST_ASSERT(!val); + ret = NLMISC::fromString("N", val); + TEST_ASSERT(ret && !val); - // bad character - ret = NLMISC::fromString("a", val); - TEST_ASSERT_MSG(!ret, "should not succeed"); + ret = NLMISC::fromString("false", val); + TEST_ASSERT(ret && !val); - val = true; - NLMISC::fromString("a", val); - TEST_ASSERT_MSG(val, "should not modify the value"); + ret = NLMISC::fromString("no", val); + TEST_ASSERT(ret && !val); - val = false; - NLMISC::fromString("a", val); - TEST_ASSERT_MSG(!val, "should not modify the value"); + // wrong values + ret = NLMISC::fromString("YES", val); + TEST_ASSERT(!ret && !val); + + ret = NLMISC::fromString("foo", val); + TEST_ASSERT(!ret && !val); + + ret = NLMISC::fromString("a", val); + TEST_ASSERT(!ret && !val); } }; From 092a3226bd3ebd3972a1f1d8b989f0abc88d205a Mon Sep 17 00:00:00 2001 From: kervala Date: Tue, 2 Feb 2016 12:37:06 +0100 Subject: [PATCH 09/10] Changed: New function bytesToHumanReadableUnits to use specific strings for units --HG-- branch : develop --- code/nel/include/nel/misc/common.h | 4 ++++ code/nel/src/misc/common.cpp | 21 ++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/code/nel/include/nel/misc/common.h b/code/nel/include/nel/misc/common.h index 710a9e6df..3b5465eaa 100644 --- a/code/nel/include/nel/misc/common.h +++ b/code/nel/include/nel/misc/common.h @@ -336,6 +336,10 @@ void itoaInt64 (sint64 number, char *str, sint64 base = 10); std::string bytesToHumanReadable (const std::string &bytes); std::string bytesToHumanReadable (uint64 bytes); +/// Convert a number in bytes into a string that is easily readable by an human, for example 105123 -> "102kb" +/// Using units array as string: 0 => B, 1 => KiB, 2 => MiB, 3 => GiB, etc... +std::string bytesToHumanReadableUnits (uint64 bytes, const std::vector &units); + /// Convert a human readable into a bytes, for example "102kb" -> 105123 uint32 humanReadableToBytes (const std::string &str); diff --git a/code/nel/src/misc/common.cpp b/code/nel/src/misc/common.cpp index 5115547a5..119323331 100644 --- a/code/nel/src/misc/common.cpp +++ b/code/nel/src/misc/common.cpp @@ -380,7 +380,26 @@ string bytesToHumanReadable (uint64 bytes) div++; res = newres; } - return toString ("%" NL_I64 "u%s", res, divTable[div]); + return toString ("%" NL_I64 "u %s", res, divTable[div]); +} + +std::string bytesToHumanReadableUnits (uint64 bytes, const std::vector &units) +{ + if (units.empty()) return ""; + + uint div = 0; + uint last = units.size()-1; + uint64 res = bytes; + uint64 newres = res; + for(;;) + { + newres /= 1024; + if(newres < 8 || div > 3 || div == last) + break; + ++div; + res = newres; + } + return toString ("%" NL_I64 "u %s", res, units[div].c_str()); } uint32 humanReadableToBytes (const string &str) From e5a601c1090c697392a26399a114f338d9e4fd89 Mon Sep 17 00:00:00 2001 From: kervala Date: Tue, 2 Feb 2016 12:37:20 +0100 Subject: [PATCH 10/10] Changed: Unit tests for bytesToHumanReadableUnits --HG-- branch : develop --- code/nel/tools/nel_unit_test/ut_misc_common.h | 67 ++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/code/nel/tools/nel_unit_test/ut_misc_common.h b/code/nel/tools/nel_unit_test/ut_misc_common.h index e915a4225..d33d65064 100644 --- a/code/nel/tools/nel_unit_test/ut_misc_common.h +++ b/code/nel/tools/nel_unit_test/ut_misc_common.h @@ -24,11 +24,76 @@ struct CUTMiscCommon : public Test::Suite { CUTMiscCommon() { + TEST_ADD(CUTMiscCommon::bytesToHumanReadableUnits); TEST_ADD(CUTMiscCommon::humanReadableToBytes); // Add a line here when adding a new test METHOD } + void bytesToHumanReadableUnits() + { + std::vector units; + + std::string res; + + // no unit, returns an empty string + res = NLMISC::bytesToHumanReadableUnits(0, units); + TEST_ASSERT(res.empty()); + + // support bytes + units.push_back("B"); + + // 0 bytes + res = NLMISC::bytesToHumanReadableUnits(0, units); + TEST_ASSERT(res == "0 B"); + + // 1000 bytes in B + res = NLMISC::bytesToHumanReadableUnits(1000, units); + TEST_ASSERT(res == "1000 B"); + + // 1024 bytes in B + res = NLMISC::bytesToHumanReadableUnits(1024, units); + TEST_ASSERT(res == "1024 B"); + + // support kibibytes + units.push_back("KiB"); + + // 1000 bytes in B or KiB + res = NLMISC::bytesToHumanReadableUnits(1000, units); + TEST_ASSERT(res == "1000 B"); + + // 1024 bytes in B or KiB + res = NLMISC::bytesToHumanReadableUnits(1024, units); + TEST_ASSERT(res == "1024 B"); + + // 1 MB in B or KiB + res = NLMISC::bytesToHumanReadableUnits(1000 * 1000, units); + TEST_ASSERT(res == "976 KiB"); + + // 1 MiB in B or KiB + res = NLMISC::bytesToHumanReadableUnits(1024 * 1024, units); + TEST_ASSERT(res == "1024 KiB"); + + // 1 GB in B or KiB + res = NLMISC::bytesToHumanReadableUnits(1000 * 1000 * 1000, units); + TEST_ASSERT(res == "976562 KiB"); + + // 1 GiB in B or KiB + res = NLMISC::bytesToHumanReadableUnits(1024 * 1024 * 1024, units); + TEST_ASSERT(res == "1048576 KiB"); + + // support mebibytes + units.push_back("MiB"); + + // 1 GB in B, KiB or MiB + res = NLMISC::bytesToHumanReadableUnits(1000 * 1000 * 1000, units); + TEST_ASSERT(res == "953 MiB"); + + // 1 GiB in B, KiB or MiB + res = NLMISC::bytesToHumanReadableUnits(1024 * 1024 * 1024, units); + TEST_ASSERT(res == "1024 MiB"); + } + void humanReadableToBytes() { uint64 bytes = 0; @@ -85,7 +150,7 @@ struct CUTMiscCommon : public Test::Suite bytes = NLMISC::humanReadableToBytes("1 kB"); TEST_ASSERT(bytes == 1000); - // 1 Megabyte + // 1 megabyte bytes = NLMISC::humanReadableToBytes("1 MB"); TEST_ASSERT(bytes == 1000*1000);