diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index 46900daa6..e263da274 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -48,15 +48,15 @@ CHECK_OUT_OF_SOURCE() IF(CMAKE_VERSION VERSION_GREATER "2.8.10") STRING(TIMESTAMP CURRENT_YEAR "%Y") ELSE() - SET(CURRENT_YEAR "2019") + SET(CURRENT_YEAR "2016") ENDIF() CMAKE_MINIMUM_REQUIRED(VERSION 2.6) PROJECT(RyzomCore CXX C) SET(NL_VERSION_MAJOR 1) SET(NL_VERSION_MINOR 0) -SET(NL_VERSION_PATCH 2) -SET(YEAR "2001-${CURRENT_YEAR}") +SET(NL_VERSION_PATCH 1) +SET(YEAR "2004-${CURRENT_YEAR}") SET(AUTHOR "Winch Gate and The Ryzom Core Community") SET(RYZOM_VERSION_MAJOR 3) diff --git a/code/nel/3rdparty/CMakeLists.txt b/code/nel/3rdparty/CMakeLists.txt new file mode 100644 index 000000000..bdd3e5fcd --- /dev/null +++ b/code/nel/3rdparty/CMakeLists.txt @@ -0,0 +1,2 @@ +SET(SEVENZIP_LIBRARY "nel_sevenzip") +ADD_SUBDIRECTORY(seven_zip) diff --git a/code/ryzom/client/src/seven_zip/7zAlloc.cpp b/code/nel/3rdparty/seven_zip/7zAlloc.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zAlloc.cpp rename to code/nel/3rdparty/seven_zip/7zAlloc.cpp diff --git a/code/ryzom/client/src/seven_zip/7zAlloc.h b/code/nel/3rdparty/seven_zip/7zAlloc.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zAlloc.h rename to code/nel/3rdparty/seven_zip/7zAlloc.h diff --git a/code/nel/3rdparty/seven_zip/7zBuffer.cpp b/code/nel/3rdparty/seven_zip/7zBuffer.cpp new file mode 100644 index 000000000..3c4b71e80 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zBuffer.cpp @@ -0,0 +1,29 @@ +/* 7zBuffer.c */ + +#include "7zBuffer.h" +#include "7zAlloc.h" + +void SzByteBufferInit(CSzByteBuffer *buffer) +{ + buffer->Capacity = 0; + buffer->Items = 0; +} + +int SzByteBufferCreate(CSzByteBuffer *buffer, size_t newCapacity, void * (*allocFunc)(size_t size)) +{ + buffer->Capacity = newCapacity; + if (newCapacity == 0) + { + buffer->Items = 0; + return 1; + } + buffer->Items = (Byte *)allocFunc(newCapacity); + return (buffer->Items != 0); +} + +void SzByteBufferFree(CSzByteBuffer *buffer, void (*freeFunc)(void *)) +{ + freeFunc(buffer->Items); + buffer->Items = 0; + buffer->Capacity = 0; +} diff --git a/code/nel/3rdparty/seven_zip/7zBuffer.h b/code/nel/3rdparty/seven_zip/7zBuffer.h new file mode 100644 index 000000000..17e59060f --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zBuffer.h @@ -0,0 +1,19 @@ +/* 7zBuffer.h */ + +#ifndef __7Z_BUFFER_H +#define __7Z_BUFFER_H + +#include +#include "7zTypes.h" + +typedef struct _CSzByteBuffer +{ + size_t Capacity; + Byte *Items; +}CSzByteBuffer; + +void SzByteBufferInit(CSzByteBuffer *buffer); +int SzByteBufferCreate(CSzByteBuffer *buffer, size_t newCapacity, void * (*allocFunc)(size_t size)); +void SzByteBufferFree(CSzByteBuffer *buffer, void (*freeFunc)(void *)); + +#endif diff --git a/code/ryzom/client/src/seven_zip/7zCrc.cpp b/code/nel/3rdparty/seven_zip/7zCrc.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zCrc.cpp rename to code/nel/3rdparty/seven_zip/7zCrc.cpp diff --git a/code/ryzom/client/src/seven_zip/7zCrc.h b/code/nel/3rdparty/seven_zip/7zCrc.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zCrc.h rename to code/nel/3rdparty/seven_zip/7zCrc.h diff --git a/code/nel/3rdparty/seven_zip/7zDecode.cpp b/code/nel/3rdparty/seven_zip/7zDecode.cpp new file mode 100644 index 000000000..50d6b8a1f --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zDecode.cpp @@ -0,0 +1,150 @@ +/* 7zDecode.c */ + +#include "7zDecode.h" +#ifdef _SZ_ONE_DIRECTORY +#include "LzmaDecode.h" +#else +#include "../../Compress/LZMA_C/LzmaDecode.h" +#endif + +CMethodID k_Copy = { { 0x0 }, 1 }; +CMethodID k_LZMA = { { 0x3, 0x1, 0x1 }, 3 }; + +#ifdef _LZMA_IN_CB + +typedef struct _CLzmaInCallbackImp +{ + ILzmaInCallback InCallback; + ISzInStream *InStream; + size_t Size; +} CLzmaInCallbackImp; + +int LzmaReadImp(void *object, const unsigned char **buffer, SizeT *size) +{ + CLzmaInCallbackImp *cb = (CLzmaInCallbackImp *)object; + size_t processedSize; + SZ_RESULT res; + *size = 0; + res = cb->InStream->Read((void *)cb->InStream, (void **)buffer, cb->Size, &processedSize); + *size = (SizeT)processedSize; + if (processedSize > cb->Size) + return (int)SZE_FAIL; + cb->Size -= processedSize; + if (res == SZ_OK) + return 0; + return (int)res; +} + +#endif + +SZ_RESULT SzDecode(const CFileSize *packSizes, const CFolder *folder, + #ifdef _LZMA_IN_CB + ISzInStream *inStream, + #else + const Byte *inBuffer, + #endif + Byte *outBuffer, size_t outSize, + size_t *outSizeProcessed, ISzAlloc *allocMain) +{ + UInt32 si; + size_t inSize = 0; + CCoderInfo *coder; + if (folder->NumPackStreams != 1) + return SZE_NOTIMPL; + if (folder->NumCoders != 1) + return SZE_NOTIMPL; + coder = folder->Coders; + *outSizeProcessed = 0; + + for (si = 0; si < folder->NumPackStreams; si++) + inSize += (size_t)packSizes[si]; + + if (AreMethodsEqual(&coder->MethodID, &k_Copy)) + { + size_t i; + if (inSize != outSize) + return SZE_DATA_ERROR; + #ifdef _LZMA_IN_CB + for (i = 0; i < inSize;) + { + size_t j; + Byte *inBuffer; + size_t bufferSize; + RINOK(inStream->Read((void *)inStream, (void **)&inBuffer, inSize - i, &bufferSize)); + if (bufferSize == 0) + return SZE_DATA_ERROR; + if (bufferSize > inSize - i) + return SZE_FAIL; + *outSizeProcessed += bufferSize; + for (j = 0; j < bufferSize && i < inSize; j++, i++) + outBuffer[i] = inBuffer[j]; + } + #else + for (i = 0; i < inSize; i++) + outBuffer[i] = inBuffer[i]; + *outSizeProcessed = inSize; + #endif + return SZ_OK; + } + + if (AreMethodsEqual(&coder->MethodID, &k_LZMA)) + { + #ifdef _LZMA_IN_CB + CLzmaInCallbackImp lzmaCallback; + #else + SizeT inProcessed; + #endif + + CLzmaDecoderState state; /* it's about 24-80 bytes structure, if int is 32-bit */ + int result; + SizeT outSizeProcessedLoc; + + #ifdef _LZMA_IN_CB + lzmaCallback.Size = inSize; + lzmaCallback.InStream = inStream; + lzmaCallback.InCallback.Read = LzmaReadImp; + #endif + + if (LzmaDecodeProperties(&state.Properties, coder->Properties.Items, + (SizeT)coder->Properties.Capacity) != LZMA_RESULT_OK) + return SZE_FAIL; + + state.Probs = (CProb *)allocMain->Alloc(LzmaGetNumProbs(&state.Properties) * sizeof(CProb)); + if (state.Probs == 0) + return SZE_OUTOFMEMORY; + + #ifdef _LZMA_OUT_READ + if (state.Properties.DictionarySize == 0) + state.Dictionary = 0; + else + { + state.Dictionary = (unsigned char *)allocMain->Alloc(state.Properties.DictionarySize); + if (state.Dictionary == 0) + { + allocMain->Free(state.Probs); + return SZE_OUTOFMEMORY; + } + } + LzmaDecoderInit(&state); + #endif + + result = LzmaDecode(&state, + #ifdef _LZMA_IN_CB + &lzmaCallback.InCallback, + #else + inBuffer, (SizeT)inSize, &inProcessed, + #endif + outBuffer, (SizeT)outSize, &outSizeProcessedLoc); + *outSizeProcessed = (size_t)outSizeProcessedLoc; + allocMain->Free(state.Probs); + #ifdef _LZMA_OUT_READ + allocMain->Free(state.Dictionary); + #endif + if (result == LZMA_RESULT_DATA_ERROR) + return SZE_DATA_ERROR; + if (result != LZMA_RESULT_OK) + return SZE_FAIL; + return SZ_OK; + } + return SZE_NOTIMPL; +} diff --git a/code/nel/3rdparty/seven_zip/7zDecode.h b/code/nel/3rdparty/seven_zip/7zDecode.h new file mode 100644 index 000000000..74bb180fd --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zDecode.h @@ -0,0 +1,21 @@ +/* 7zDecode.h */ + +#ifndef __7Z_DECODE_H +#define __7Z_DECODE_H + +#include "7zItem.h" +#include "7zAlloc.h" +#ifdef _LZMA_IN_CB +#include "7zIn.h" +#endif + +SZ_RESULT SzDecode(const CFileSize *packSizes, const CFolder *folder, + #ifdef _LZMA_IN_CB + ISzInStream *stream, + #else + const Byte *inBuffer, + #endif + Byte *outBuffer, size_t outSize, + size_t *outSizeProcessed, ISzAlloc *allocMain); + +#endif diff --git a/code/nel/3rdparty/seven_zip/7zExtract.cpp b/code/nel/3rdparty/seven_zip/7zExtract.cpp new file mode 100644 index 000000000..6ef872c31 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zExtract.cpp @@ -0,0 +1,116 @@ +/* 7zExtract.c */ + +#include "7zExtract.h" +#include "7zDecode.h" +#include "7zCrc.h" + +SZ_RESULT SzExtract( + ISzInStream *inStream, + CArchiveDatabaseEx *db, + UInt32 fileIndex, + UInt32 *blockIndex, + Byte **outBuffer, + size_t *outBufferSize, + size_t *offset, + size_t *outSizeProcessed, + ISzAlloc *allocMain, + ISzAlloc *allocTemp) +{ + UInt32 folderIndex = db->FileIndexToFolderIndexMap[fileIndex]; + SZ_RESULT res = SZ_OK; + *offset = 0; + *outSizeProcessed = 0; + if (folderIndex == (UInt32)-1) + { + allocMain->Free(*outBuffer); + *blockIndex = folderIndex; + *outBuffer = 0; + *outBufferSize = 0; + return SZ_OK; + } + + if (*outBuffer == 0 || *blockIndex != folderIndex) + { + CFolder *folder = db->Database.Folders + folderIndex; + CFileSize unPackSize = SzFolderGetUnPackSize(folder); + #ifndef _LZMA_IN_CB + CFileSize packSize = SzArDbGetFolderFullPackSize(db, folderIndex); + Byte *inBuffer = 0; + size_t processedSize; + #endif + *blockIndex = folderIndex; + allocMain->Free(*outBuffer); + *outBuffer = 0; + + RINOK(inStream->Seek(inStream, SzArDbGetFolderStreamPos(db, folderIndex, 0))); + + #ifndef _LZMA_IN_CB + if (packSize != 0) + { + inBuffer = (Byte *)allocTemp->Alloc((size_t)packSize); + if (inBuffer == 0) + return SZE_OUTOFMEMORY; + } + res = inStream->Read(inStream, inBuffer, (size_t)packSize, &processedSize); + if (res == SZ_OK && processedSize != (size_t)packSize) + res = SZE_FAIL; + #endif + if (res == SZ_OK) + { + *outBufferSize = (size_t)unPackSize; + if (unPackSize != 0) + { + *outBuffer = (Byte *)allocMain->Alloc((size_t)unPackSize); + if (*outBuffer == 0) + res = SZE_OUTOFMEMORY; + } + if (res == SZ_OK) + { + size_t outRealSize; + res = SzDecode(db->Database.PackSizes + + db->FolderStartPackStreamIndex[folderIndex], folder, + #ifdef _LZMA_IN_CB + inStream, + #else + inBuffer, + #endif + *outBuffer, (size_t)unPackSize, &outRealSize, allocTemp); + if (res == SZ_OK) + { + if (outRealSize == (size_t)unPackSize) + { + if (folder->UnPackCRCDefined) + { + if (!CrcVerifyDigest(folder->UnPackCRC, *outBuffer, (size_t)unPackSize)) + res = SZE_FAIL; + } + } + else + res = SZE_FAIL; + } + } + } + #ifndef _LZMA_IN_CB + allocTemp->Free(inBuffer); + #endif + } + if (res == SZ_OK) + { + UInt32 i; + CFileItem *fileItem = db->Database.Files + fileIndex; + *offset = 0; + for(i = db->FolderStartFileIndex[folderIndex]; i < fileIndex; i++) + *offset += (UInt32)db->Database.Files[i].Size; + *outSizeProcessed = (size_t)fileItem->Size; + if (*offset + *outSizeProcessed > *outBufferSize) + return SZE_FAIL; + { + if (fileItem->IsFileCRCDefined) + { + if (!CrcVerifyDigest(fileItem->FileCRC, *outBuffer + *offset, *outSizeProcessed)) + res = SZE_FAIL; + } + } + } + return res; +} diff --git a/code/nel/3rdparty/seven_zip/7zExtract.h b/code/nel/3rdparty/seven_zip/7zExtract.h new file mode 100644 index 000000000..e9a4fb4e3 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zExtract.h @@ -0,0 +1,40 @@ +/* 7zExtract.h */ + +#ifndef __7Z_EXTRACT_H +#define __7Z_EXTRACT_H + +#include "7zIn.h" + +/* + SzExtract extracts file from archive + + *outBuffer must be 0 before first call for each new archive. + + Extracting cache: + If you need to decompress more than one file, you can send + these values from previous call: + *blockIndex, + *outBuffer, + *outBufferSize + You can consider "*outBuffer" as cache of solid block. If your archive is solid, + it will increase decompression speed. + + If you use external function, you can declare these 3 cache variables + (blockIndex, outBuffer, outBufferSize) as static in that external function. + + Free *outBuffer and set *outBuffer to 0, if you want to flush cache. +*/ + +SZ_RESULT SzExtract( + ISzInStream *inStream, + CArchiveDatabaseEx *db, + UInt32 fileIndex, /* index of file */ + UInt32 *blockIndex, /* index of solid block */ + Byte **outBuffer, /* pointer to pointer to output buffer (allocated with allocMain) */ + size_t *outBufferSize, /* buffer size for output buffer */ + size_t *offset, /* offset of stream for required file in *outBuffer */ + size_t *outSizeProcessed, /* size of file in *outBuffer */ + ISzAlloc *allocMain, + ISzAlloc *allocTemp); + +#endif diff --git a/code/nel/3rdparty/seven_zip/7zHeader.cpp b/code/nel/3rdparty/seven_zip/7zHeader.cpp new file mode 100644 index 000000000..3be4bc27e --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zHeader.cpp @@ -0,0 +1,5 @@ +/* 7zHeader.c */ + +#include "7zHeader.h" + +Byte k7zSignature[k7zSignatureSize] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C}; diff --git a/code/nel/3rdparty/seven_zip/7zHeader.h b/code/nel/3rdparty/seven_zip/7zHeader.h new file mode 100644 index 000000000..0356aaa6b --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zHeader.h @@ -0,0 +1,55 @@ +/* 7zHeader.h */ + +#ifndef __7Z_HEADER_H +#define __7Z_HEADER_H + +#include "7zTypes.h" + +#define k7zSignatureSize 6 +extern Byte k7zSignature[k7zSignatureSize]; + +#define k7zMajorVersion 0 + +#define k7zStartHeaderSize 0x20 + +enum EIdEnum +{ + k7zIdEnd, + + k7zIdHeader, + + k7zIdArchiveProperties, + + k7zIdAdditionalStreamsInfo, + k7zIdMainStreamsInfo, + k7zIdFilesInfo, + + k7zIdPackInfo, + k7zIdUnPackInfo, + k7zIdSubStreamsInfo, + + k7zIdSize, + k7zIdCRC, + + k7zIdFolder, + + k7zIdCodersUnPackSize, + k7zIdNumUnPackStream, + + k7zIdEmptyStream, + k7zIdEmptyFile, + k7zIdAnti, + + k7zIdName, + k7zIdCreationTime, + k7zIdLastAccessTime, + k7zIdLastWriteTime, + k7zIdWinAttributes, + k7zIdComment, + + k7zIdEncodedHeader, + + k7zIdStartPos +}; + +#endif diff --git a/code/nel/3rdparty/seven_zip/7zIn.cpp b/code/nel/3rdparty/seven_zip/7zIn.cpp new file mode 100644 index 000000000..b9dbf4695 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zIn.cpp @@ -0,0 +1,1281 @@ +/* 7zIn.c */ + +#include "7zIn.h" +#include "7zCrc.h" +#include "7zDecode.h" + +#define RINOM(x) { if((x) == 0) return SZE_OUTOFMEMORY; } + +void SzArDbExInit(CArchiveDatabaseEx *db) +{ + SzArchiveDatabaseInit(&db->Database); + db->FolderStartPackStreamIndex = 0; + db->PackStreamStartPositions = 0; + db->FolderStartFileIndex = 0; + db->FileIndexToFolderIndexMap = 0; +} + +void SzArDbExFree(CArchiveDatabaseEx *db, void (*freeFunc)(void *)) +{ + freeFunc(db->FolderStartPackStreamIndex); + freeFunc(db->PackStreamStartPositions); + freeFunc(db->FolderStartFileIndex); + freeFunc(db->FileIndexToFolderIndexMap); + SzArchiveDatabaseFree(&db->Database, freeFunc); + SzArDbExInit(db); +} + +/* +CFileSize GetFolderPackStreamSize(int folderIndex, int streamIndex) const +{ + return PackSizes[FolderStartPackStreamIndex[folderIndex] + streamIndex]; +} + +CFileSize GetFilePackSize(int fileIndex) const +{ + int folderIndex = FileIndexToFolderIndexMap[fileIndex]; + if (folderIndex >= 0) + { + const CFolder &folderInfo = Folders[folderIndex]; + if (FolderStartFileIndex[folderIndex] == fileIndex) + return GetFolderFullPackSize(folderIndex); + } + return 0; +} +*/ + +#define MY_ALLOC(T, p, size, allocFunc) { if ((size) == 0) p = 0; else \ + if ((p = (T *)allocFunc((size) * sizeof(T))) == 0) return SZE_OUTOFMEMORY; } + +SZ_RESULT SzArDbExFill(CArchiveDatabaseEx *db, void * (*allocFunc)(size_t size)) +{ + UInt32 startPos = 0; + CFileSize startPosSize = 0; + UInt32 i; + UInt32 folderIndex = 0; + UInt32 indexInFolder = 0; + MY_ALLOC(UInt32, db->FolderStartPackStreamIndex, db->Database.NumFolders, allocFunc); + for(i = 0; i < db->Database.NumFolders; i++) + { + db->FolderStartPackStreamIndex[i] = startPos; + startPos += db->Database.Folders[i].NumPackStreams; + } + + MY_ALLOC(CFileSize, db->PackStreamStartPositions, db->Database.NumPackStreams, allocFunc); + + for(i = 0; i < db->Database.NumPackStreams; i++) + { + db->PackStreamStartPositions[i] = startPosSize; + startPosSize += db->Database.PackSizes[i]; + } + + MY_ALLOC(UInt32, db->FolderStartFileIndex, db->Database.NumFolders, allocFunc); + MY_ALLOC(UInt32, db->FileIndexToFolderIndexMap, db->Database.NumFiles, allocFunc); + + for (i = 0; i < db->Database.NumFiles; i++) + { + CFileItem *file = db->Database.Files + i; + int emptyStream = !file->HasStream; + if (emptyStream && indexInFolder == 0) + { + db->FileIndexToFolderIndexMap[i] = (UInt32)-1; + continue; + } + if (indexInFolder == 0) + { + /* + v3.13 incorrectly worked with empty folders + v4.07: Loop for skipping empty folders + */ + for(;;) + { + if (folderIndex >= db->Database.NumFolders) + return SZE_ARCHIVE_ERROR; + db->FolderStartFileIndex[folderIndex] = i; + if (db->Database.Folders[folderIndex].NumUnPackStreams != 0) + break; + folderIndex++; + } + } + db->FileIndexToFolderIndexMap[i] = folderIndex; + if (emptyStream) + continue; + indexInFolder++; + if (indexInFolder >= db->Database.Folders[folderIndex].NumUnPackStreams) + { + folderIndex++; + indexInFolder = 0; + } + } + return SZ_OK; +} + + +CFileSize SzArDbGetFolderStreamPos(CArchiveDatabaseEx *db, UInt32 folderIndex, UInt32 indexInFolder) +{ + return db->ArchiveInfo.DataStartPosition + + db->PackStreamStartPositions[db->FolderStartPackStreamIndex[folderIndex] + indexInFolder]; +} + +CFileSize SzArDbGetFolderFullPackSize(CArchiveDatabaseEx *db, UInt32 folderIndex) +{ + UInt32 packStreamIndex = db->FolderStartPackStreamIndex[folderIndex]; + CFolder *folder = db->Database.Folders + folderIndex; + CFileSize size = 0; + UInt32 i; + for (i = 0; i < folder->NumPackStreams; i++) + size += db->Database.PackSizes[packStreamIndex + i]; + return size; +} + + +/* +SZ_RESULT SzReadTime(const CObjectVector &dataVector, + CObjectVector &files, UInt64 type) +{ + CBoolVector boolVector; + RINOK(ReadBoolVector2(files.Size(), boolVector)) + + CStreamSwitch streamSwitch; + RINOK(streamSwitch.Set(this, &dataVector)); + + for(int i = 0; i < files.Size(); i++) + { + CFileItem &file = files[i]; + CArchiveFileTime fileTime; + bool defined = boolVector[i]; + if (defined) + { + UInt32 low, high; + RINOK(SzReadUInt32(low)); + RINOK(SzReadUInt32(high)); + fileTime.dwLowDateTime = low; + fileTime.dwHighDateTime = high; + } + switch(type) + { + case k7zIdCreationTime: + file.IsCreationTimeDefined = defined; + if (defined) + file.CreationTime = fileTime; + break; + case k7zIdLastWriteTime: + file.IsLastWriteTimeDefined = defined; + if (defined) + file.LastWriteTime = fileTime; + break; + case k7zIdLastAccessTime: + file.IsLastAccessTimeDefined = defined; + if (defined) + file.LastAccessTime = fileTime; + break; + } + } + return SZ_OK; +} +*/ + +SZ_RESULT SafeReadDirect(ISzInStream *inStream, Byte *data, size_t size) +{ + #ifdef _LZMA_IN_CB + while (size > 0) + { + Byte *inBuffer; + size_t processedSize; + RINOK(inStream->Read(inStream, (void **)&inBuffer, size, &processedSize)); + if (processedSize == 0 || processedSize > size) + return SZE_FAIL; + size -= processedSize; + do + { + *data++ = *inBuffer++; + } + while (--processedSize != 0); + } + #else + size_t processedSize; + RINOK(inStream->Read(inStream, data, size, &processedSize)); + if (processedSize != size) + return SZE_FAIL; + #endif + return SZ_OK; +} + +SZ_RESULT SafeReadDirectByte(ISzInStream *inStream, Byte *data) +{ + return SafeReadDirect(inStream, data, 1); +} + +SZ_RESULT SafeReadDirectUInt32(ISzInStream *inStream, UInt32 *value) +{ + int i; + *value = 0; + for (i = 0; i < 4; i++) + { + Byte b; + RINOK(SafeReadDirectByte(inStream, &b)); + *value |= ((UInt32)b << (8 * i)); + } + return SZ_OK; +} + +SZ_RESULT SafeReadDirectUInt64(ISzInStream *inStream, UInt64 *value) +{ + int i; + *value = 0; + for (i = 0; i < 8; i++) + { + Byte b; + RINOK(SafeReadDirectByte(inStream, &b)); + *value |= ((UInt32)b << (8 * i)); + } + return SZ_OK; +} + +int TestSignatureCandidate(Byte *testBytes) +{ + size_t i; + for (i = 0; i < k7zSignatureSize; i++) + if (testBytes[i] != k7zSignature[i]) + return 0; + return 1; +} + +typedef struct _CSzState +{ + Byte *Data; + size_t Size; +}CSzData; + +SZ_RESULT SzReadByte(CSzData *sd, Byte *b) +{ + if (sd->Size == 0) + return SZE_ARCHIVE_ERROR; + sd->Size--; + *b = *sd->Data++; + return SZ_OK; +} + +SZ_RESULT SzReadBytes(CSzData *sd, Byte *data, size_t size) +{ + size_t i; + for (i = 0; i < size; i++) + { + RINOK(SzReadByte(sd, data + i)); + } + return SZ_OK; +} + +SZ_RESULT SzReadUInt32(CSzData *sd, UInt32 *value) +{ + int i; + *value = 0; + for (i = 0; i < 4; i++) + { + Byte b; + RINOK(SzReadByte(sd, &b)); + *value |= ((UInt32)(b) << (8 * i)); + } + return SZ_OK; +} + +SZ_RESULT SzReadNumber(CSzData *sd, UInt64 *value) +{ + Byte firstByte; + Byte mask = 0x80; + int i; + RINOK(SzReadByte(sd, &firstByte)); + *value = 0; + for (i = 0; i < 8; i++) + { + Byte b; + if ((firstByte & mask) == 0) + { + UInt64 highPart = firstByte & (mask - 1); + *value += (highPart << (8 * i)); + return SZ_OK; + } + RINOK(SzReadByte(sd, &b)); + *value |= ((UInt64)b << (8 * i)); + mask >>= 1; + } + return SZ_OK; +} + +SZ_RESULT SzReadSize(CSzData *sd, CFileSize *value) +{ + UInt64 value64; + RINOK(SzReadNumber(sd, &value64)); + *value = (CFileSize)value64; + return SZ_OK; +} + +SZ_RESULT SzReadNumber32(CSzData *sd, UInt32 *value) +{ + UInt64 value64; + RINOK(SzReadNumber(sd, &value64)); + if (value64 >= 0x80000000) + return SZE_NOTIMPL; + if (value64 >= ((UInt64)(1) << ((sizeof(size_t) - 1) * 8 + 2))) + return SZE_NOTIMPL; + *value = (UInt32)value64; + return SZ_OK; +} + +SZ_RESULT SzReadID(CSzData *sd, UInt64 *value) +{ + return SzReadNumber(sd, value); +} + +SZ_RESULT SzSkeepDataSize(CSzData *sd, UInt64 size) +{ + if (size > sd->Size) + return SZE_ARCHIVE_ERROR; + sd->Size -= (size_t)size; + sd->Data += (size_t)size; + return SZ_OK; +} + +SZ_RESULT SzSkeepData(CSzData *sd) +{ + UInt64 size; + RINOK(SzReadNumber(sd, &size)); + return SzSkeepDataSize(sd, size); +} + +SZ_RESULT SzReadArchiveProperties(CSzData *sd) +{ + for(;;) + { + UInt64 type; + RINOK(SzReadID(sd, &type)); + if (type == k7zIdEnd) + break; + SzSkeepData(sd); + } + return SZ_OK; +} + +SZ_RESULT SzWaitAttribute(CSzData *sd, UInt64 attribute) +{ + for(;;) + { + UInt64 type; + RINOK(SzReadID(sd, &type)); + if (type == attribute) + return SZ_OK; + if (type == k7zIdEnd) + return SZE_ARCHIVE_ERROR; + RINOK(SzSkeepData(sd)); + } +} + +SZ_RESULT SzReadBoolVector(CSzData *sd, size_t numItems, Byte **v, void * (*allocFunc)(size_t size)) +{ + Byte b = 0; + Byte mask = 0; + size_t i; + MY_ALLOC(Byte, *v, numItems, allocFunc); + for(i = 0; i < numItems; i++) + { + if (mask == 0) + { + RINOK(SzReadByte(sd, &b)); + mask = 0x80; + } + (*v)[i] = (Byte)(((b & mask) != 0) ? 1 : 0); + mask >>= 1; + } + return SZ_OK; +} + +SZ_RESULT SzReadBoolVector2(CSzData *sd, size_t numItems, Byte **v, void * (*allocFunc)(size_t size)) +{ + Byte allAreDefined; + size_t i; + RINOK(SzReadByte(sd, &allAreDefined)); + if (allAreDefined == 0) + return SzReadBoolVector(sd, numItems, v, allocFunc); + MY_ALLOC(Byte, *v, numItems, allocFunc); + for(i = 0; i < numItems; i++) + (*v)[i] = 1; + return SZ_OK; +} + +SZ_RESULT SzReadHashDigests( + CSzData *sd, + size_t numItems, + Byte **digestsDefined, + UInt32 **digests, + void * (*allocFunc)(size_t size)) +{ + size_t i; + RINOK(SzReadBoolVector2(sd, numItems, digestsDefined, allocFunc)); + MY_ALLOC(UInt32, *digests, numItems, allocFunc); + for(i = 0; i < numItems; i++) + if ((*digestsDefined)[i]) + { + RINOK(SzReadUInt32(sd, (*digests) + i)); + } + return SZ_OK; +} + +SZ_RESULT SzReadPackInfo( + CSzData *sd, + CFileSize *dataOffset, + UInt32 *numPackStreams, + CFileSize **packSizes, + Byte **packCRCsDefined, + UInt32 **packCRCs, + void * (*allocFunc)(size_t size)) +{ + UInt32 i; + RINOK(SzReadSize(sd, dataOffset)); + RINOK(SzReadNumber32(sd, numPackStreams)); + + RINOK(SzWaitAttribute(sd, k7zIdSize)); + + MY_ALLOC(CFileSize, *packSizes, (size_t)*numPackStreams, allocFunc); + + for(i = 0; i < *numPackStreams; i++) + { + RINOK(SzReadSize(sd, (*packSizes) + i)); + } + + for(;;) + { + UInt64 type; + RINOK(SzReadID(sd, &type)); + if (type == k7zIdEnd) + break; + if (type == k7zIdCRC) + { + RINOK(SzReadHashDigests(sd, (size_t)*numPackStreams, packCRCsDefined, packCRCs, allocFunc)); + continue; + } + RINOK(SzSkeepData(sd)); + } + if (*packCRCsDefined == 0) + { + MY_ALLOC(Byte, *packCRCsDefined, (size_t)*numPackStreams, allocFunc); + MY_ALLOC(UInt32, *packCRCs, (size_t)*numPackStreams, allocFunc); + for(i = 0; i < *numPackStreams; i++) + { + (*packCRCsDefined)[i] = 0; + (*packCRCs)[i] = 0; + } + } + return SZ_OK; +} + +SZ_RESULT SzReadSwitch(CSzData *sd) +{ + Byte external; + RINOK(SzReadByte(sd, &external)); + return (external == 0) ? SZ_OK: SZE_ARCHIVE_ERROR; +} + +SZ_RESULT SzGetNextFolderItem(CSzData *sd, CFolder *folder, void * (*allocFunc)(size_t size)) +{ + UInt32 numCoders; + UInt32 numBindPairs; + UInt32 numPackedStreams; + UInt32 i; + UInt32 numInStreams = 0; + UInt32 numOutStreams = 0; + RINOK(SzReadNumber32(sd, &numCoders)); + folder->NumCoders = numCoders; + + MY_ALLOC(CCoderInfo, folder->Coders, (size_t)numCoders, allocFunc); + + for (i = 0; i < numCoders; i++) + SzCoderInfoInit(folder->Coders + i); + + for (i = 0; i < numCoders; i++) + { + Byte mainByte; + CCoderInfo *coder = folder->Coders + i; + { + RINOK(SzReadByte(sd, &mainByte)); + coder->MethodID.IDSize = (Byte)(mainByte & 0xF); + RINOK(SzReadBytes(sd, coder->MethodID.ID, coder->MethodID.IDSize)); + if ((mainByte & 0x10) != 0) + { + RINOK(SzReadNumber32(sd, &coder->NumInStreams)); + RINOK(SzReadNumber32(sd, &coder->NumOutStreams)); + } + else + { + coder->NumInStreams = 1; + coder->NumOutStreams = 1; + } + if ((mainByte & 0x20) != 0) + { + UInt64 propertiesSize = 0; + RINOK(SzReadNumber(sd, &propertiesSize)); + if (!SzByteBufferCreate(&coder->Properties, (size_t)propertiesSize, allocFunc)) + return SZE_OUTOFMEMORY; + RINOK(SzReadBytes(sd, coder->Properties.Items, (size_t)propertiesSize)); + } + } + while ((mainByte & 0x80) != 0) + { + RINOK(SzReadByte(sd, &mainByte)); + RINOK(SzSkeepDataSize(sd, (mainByte & 0xF))); + if ((mainByte & 0x10) != 0) + { + UInt32 n; + RINOK(SzReadNumber32(sd, &n)); + RINOK(SzReadNumber32(sd, &n)); + } + if ((mainByte & 0x20) != 0) + { + UInt64 propertiesSize = 0; + RINOK(SzReadNumber(sd, &propertiesSize)); + RINOK(SzSkeepDataSize(sd, propertiesSize)); + } + } + numInStreams += (UInt32)coder->NumInStreams; + numOutStreams += (UInt32)coder->NumOutStreams; + } + + numBindPairs = numOutStreams - 1; + folder->NumBindPairs = numBindPairs; + + + MY_ALLOC(CBindPair, folder->BindPairs, (size_t)numBindPairs, allocFunc); + + for (i = 0; i < numBindPairs; i++) + { + CBindPair *bindPair = folder->BindPairs + i;; + RINOK(SzReadNumber32(sd, &bindPair->InIndex)); + RINOK(SzReadNumber32(sd, &bindPair->OutIndex)); + } + + numPackedStreams = numInStreams - (UInt32)numBindPairs; + + folder->NumPackStreams = numPackedStreams; + MY_ALLOC(UInt32, folder->PackStreams, (size_t)numPackedStreams, allocFunc); + + if (numPackedStreams == 1) + { + UInt32 j; + UInt32 pi = 0; + for (j = 0; j < numInStreams; j++) + if (SzFolderFindBindPairForInStream(folder, j) < 0) + { + folder->PackStreams[pi++] = j; + break; + } + } + else + for(i = 0; i < numPackedStreams; i++) + { + RINOK(SzReadNumber32(sd, folder->PackStreams + i)); + } + return SZ_OK; +} + +SZ_RESULT SzReadUnPackInfo( + CSzData *sd, + UInt32 *numFolders, + CFolder **folders, /* for allocFunc */ + void * (*allocFunc)(size_t size), + ISzAlloc *allocTemp) +{ + UInt32 i; + RINOK(SzWaitAttribute(sd, k7zIdFolder)); + RINOK(SzReadNumber32(sd, numFolders)); + { + RINOK(SzReadSwitch(sd)); + + MY_ALLOC(CFolder, *folders, (size_t)*numFolders, allocFunc); + + for(i = 0; i < *numFolders; i++) + SzFolderInit((*folders) + i); + + for(i = 0; i < *numFolders; i++) + { + RINOK(SzGetNextFolderItem(sd, (*folders) + i, allocFunc)); + } + } + + RINOK(SzWaitAttribute(sd, k7zIdCodersUnPackSize)); + + for(i = 0; i < *numFolders; i++) + { + UInt32 j; + CFolder *folder = (*folders) + i; + UInt32 numOutStreams = SzFolderGetNumOutStreams(folder); + + MY_ALLOC(CFileSize, folder->UnPackSizes, (size_t)numOutStreams, allocFunc); + + for(j = 0; j < numOutStreams; j++) + { + RINOK(SzReadSize(sd, folder->UnPackSizes + j)); + } + } + + for(;;) + { + UInt64 type; + RINOK(SzReadID(sd, &type)); + if (type == k7zIdEnd) + return SZ_OK; + if (type == k7zIdCRC) + { + SZ_RESULT res; + Byte *crcsDefined = 0; + UInt32 *crcs = 0; + res = SzReadHashDigests(sd, *numFolders, &crcsDefined, &crcs, allocTemp->Alloc); + if (res == SZ_OK) + { + for(i = 0; i < *numFolders; i++) + { + CFolder *folder = (*folders) + i; + folder->UnPackCRCDefined = crcsDefined[i]; + folder->UnPackCRC = crcs[i]; + } + } + allocTemp->Free(crcs); + allocTemp->Free(crcsDefined); + RINOK(res); + continue; + } + RINOK(SzSkeepData(sd)); + } +} + +SZ_RESULT SzReadSubStreamsInfo( + CSzData *sd, + UInt32 numFolders, + CFolder *folders, + UInt32 *numUnPackStreams, + CFileSize **unPackSizes, + Byte **digestsDefined, + UInt32 **digests, + ISzAlloc *allocTemp) +{ + UInt64 type = 0; + UInt32 i; + UInt32 si = 0; + UInt32 numDigests = 0; + + for(i = 0; i < numFolders; i++) + folders[i].NumUnPackStreams = 1; + *numUnPackStreams = numFolders; + + for(;;) + { + RINOK(SzReadID(sd, &type)); + if (type == k7zIdNumUnPackStream) + { + *numUnPackStreams = 0; + for(i = 0; i < numFolders; i++) + { + UInt32 numStreams; + RINOK(SzReadNumber32(sd, &numStreams)); + folders[i].NumUnPackStreams = numStreams; + *numUnPackStreams += numStreams; + } + continue; + } + if (type == k7zIdCRC || type == k7zIdSize) + break; + if (type == k7zIdEnd) + break; + RINOK(SzSkeepData(sd)); + } + + if (*numUnPackStreams == 0) + { + *unPackSizes = 0; + *digestsDefined = 0; + *digests = 0; + } + else + { + *unPackSizes = (CFileSize *)allocTemp->Alloc((size_t)*numUnPackStreams * sizeof(CFileSize)); + RINOM(*unPackSizes); + *digestsDefined = (Byte *)allocTemp->Alloc((size_t)*numUnPackStreams * sizeof(Byte)); + RINOM(*digestsDefined); + *digests = (UInt32 *)allocTemp->Alloc((size_t)*numUnPackStreams * sizeof(UInt32)); + RINOM(*digests); + } + + for(i = 0; i < numFolders; i++) + { + /* + v3.13 incorrectly worked with empty folders + v4.07: we check that folder is empty + */ + CFileSize sum = 0; + UInt32 j; + UInt32 numSubstreams = folders[i].NumUnPackStreams; + if (numSubstreams == 0) + continue; + if (type == k7zIdSize) + for (j = 1; j < numSubstreams; j++) + { + CFileSize size; + RINOK(SzReadSize(sd, &size)); + (*unPackSizes)[si++] = size; + sum += size; + } + (*unPackSizes)[si++] = SzFolderGetUnPackSize(folders + i) - sum; + } + if (type == k7zIdSize) + { + RINOK(SzReadID(sd, &type)); + } + + for(i = 0; i < *numUnPackStreams; i++) + { + (*digestsDefined)[i] = 0; + (*digests)[i] = 0; + } + + + for(i = 0; i < numFolders; i++) + { + UInt32 numSubstreams = folders[i].NumUnPackStreams; + if (numSubstreams != 1 || !folders[i].UnPackCRCDefined) + numDigests += numSubstreams; + } + + + si = 0; + for(;;) + { + if (type == k7zIdCRC) + { + int digestIndex = 0; + Byte *digestsDefined2 = 0; + UInt32 *digests2 = 0; + SZ_RESULT res = SzReadHashDigests(sd, numDigests, &digestsDefined2, &digests2, allocTemp->Alloc); + if (res == SZ_OK) + { + for (i = 0; i < numFolders; i++) + { + CFolder *folder = folders + i; + UInt32 numSubstreams = folder->NumUnPackStreams; + if (numSubstreams == 1 && folder->UnPackCRCDefined) + { + (*digestsDefined)[si] = 1; + (*digests)[si] = folder->UnPackCRC; + si++; + } + else + { + UInt32 j; + for (j = 0; j < numSubstreams; j++, digestIndex++) + { + (*digestsDefined)[si] = digestsDefined2[digestIndex]; + (*digests)[si] = digests2[digestIndex]; + si++; + } + } + } + } + allocTemp->Free(digestsDefined2); + allocTemp->Free(digests2); + RINOK(res); + } + else if (type == k7zIdEnd) + return SZ_OK; + else + { + RINOK(SzSkeepData(sd)); + } + RINOK(SzReadID(sd, &type)); + } +} + + +SZ_RESULT SzReadStreamsInfo( + CSzData *sd, + CFileSize *dataOffset, + CArchiveDatabase *db, + UInt32 *numUnPackStreams, + CFileSize **unPackSizes, /* allocTemp */ + Byte **digestsDefined, /* allocTemp */ + UInt32 **digests, /* allocTemp */ + void * (*allocFunc)(size_t size), + ISzAlloc *allocTemp) +{ + for(;;) + { + UInt64 type; + RINOK(SzReadID(sd, &type)); + if ((UInt64)(int)type != type) + return SZE_FAIL; + switch((int)type) + { + case k7zIdEnd: + return SZ_OK; + case k7zIdPackInfo: + { + RINOK(SzReadPackInfo(sd, dataOffset, &db->NumPackStreams, + &db->PackSizes, &db->PackCRCsDefined, &db->PackCRCs, allocFunc)); + break; + } + case k7zIdUnPackInfo: + { + RINOK(SzReadUnPackInfo(sd, &db->NumFolders, &db->Folders, allocFunc, allocTemp)); + break; + } + case k7zIdSubStreamsInfo: + { + RINOK(SzReadSubStreamsInfo(sd, db->NumFolders, db->Folders, + numUnPackStreams, unPackSizes, digestsDefined, digests, allocTemp)); + break; + } + default: + return SZE_FAIL; + } + } +} + +Byte kUtf8Limits[5] = { 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; + +SZ_RESULT SzReadFileNames(CSzData *sd, UInt32 numFiles, CFileItem *files, + void * (*allocFunc)(size_t size)) +{ + UInt32 i; + for(i = 0; i < numFiles; i++) + { + UInt32 len = 0; + UInt32 pos = 0; + CFileItem *file = files + i; + while(pos + 2 <= sd->Size) + { + int numAdds; + UInt32 value = (UInt32)(sd->Data[pos] | (((UInt32)sd->Data[pos + 1]) << 8)); + pos += 2; + len++; + if (value == 0) + break; + if (value < 0x80) + continue; + if (value >= 0xD800 && value < 0xE000) + { + UInt32 c2; + if (value >= 0xDC00) + return SZE_ARCHIVE_ERROR; + if (pos + 2 > sd->Size) + return SZE_ARCHIVE_ERROR; + c2 = (UInt32)(sd->Data[pos] | (((UInt32)sd->Data[pos + 1]) << 8)); + pos += 2; + if (c2 < 0xDC00 || c2 >= 0xE000) + return SZE_ARCHIVE_ERROR; + value = ((value - 0xD800) << 10) | (c2 - 0xDC00); + } + for (numAdds = 1; numAdds < 5; numAdds++) + if (value < (((UInt32)1) << (numAdds * 5 + 6))) + break; + len += numAdds; + } + + MY_ALLOC(char, file->Name, (size_t)len, allocFunc); + + len = 0; + while(2 <= sd->Size) + { + int numAdds; + UInt32 value = (UInt32)(sd->Data[0] | (((UInt32)sd->Data[1]) << 8)); + SzSkeepDataSize(sd, 2); + if (value < 0x80) + { + file->Name[len++] = (char)value; + if (value == 0) + break; + continue; + } + if (value >= 0xD800 && value < 0xE000) + { + UInt32 c2 = (UInt32)(sd->Data[0] | (((UInt32)sd->Data[1]) << 8)); + SzSkeepDataSize(sd, 2); + value = ((value - 0xD800) << 10) | (c2 - 0xDC00); + } + for (numAdds = 1; numAdds < 5; numAdds++) + if (value < (((UInt32)1) << (numAdds * 5 + 6))) + break; + file->Name[len++] = (char)(kUtf8Limits[numAdds - 1] + (value >> (6 * numAdds))); + do + { + numAdds--; + file->Name[len++] = (char)(0x80 + ((value >> (6 * numAdds)) & 0x3F)); + } + while(numAdds > 0); + + len += numAdds; + } + } + return SZ_OK; +} + +SZ_RESULT SzReadHeader2( + CSzData *sd, + CArchiveDatabaseEx *db, /* allocMain */ + CFileSize **unPackSizes, /* allocTemp */ + Byte **digestsDefined, /* allocTemp */ + UInt32 **digests, /* allocTemp */ + Byte **emptyStreamVector, /* allocTemp */ + Byte **emptyFileVector, /* allocTemp */ + ISzAlloc *allocMain, + ISzAlloc *allocTemp) +{ + UInt64 type; + UInt32 numUnPackStreams = 0; + UInt32 numFiles = 0; + CFileItem *files = 0; + UInt32 numEmptyStreams = 0; + UInt32 i; + + RINOK(SzReadID(sd, &type)); + + if (type == k7zIdArchiveProperties) + { + RINOK(SzReadArchiveProperties(sd)); + RINOK(SzReadID(sd, &type)); + } + + + if (type == k7zIdMainStreamsInfo) + { + RINOK(SzReadStreamsInfo(sd, + &db->ArchiveInfo.DataStartPosition, + &db->Database, + &numUnPackStreams, + unPackSizes, + digestsDefined, + digests, allocMain->Alloc, allocTemp)); + db->ArchiveInfo.DataStartPosition += db->ArchiveInfo.StartPositionAfterHeader; + RINOK(SzReadID(sd, &type)); + } + + if (type == k7zIdEnd) + return SZ_OK; + if (type != k7zIdFilesInfo) + return SZE_ARCHIVE_ERROR; + + RINOK(SzReadNumber32(sd, &numFiles)); + db->Database.NumFiles = numFiles; + + MY_ALLOC(CFileItem, files, (size_t)numFiles, allocMain->Alloc); + + db->Database.Files = files; + for(i = 0; i < numFiles; i++) + SzFileInit(files + i); + + for(;;) + { + UInt64 type; + UInt64 size; + RINOK(SzReadID(sd, &type)); + if (type == k7zIdEnd) + break; + RINOK(SzReadNumber(sd, &size)); + + if ((UInt64)(int)type != type) + { + RINOK(SzSkeepDataSize(sd, size)); + } + else + switch((int)type) + { + case k7zIdName: + { + RINOK(SzReadSwitch(sd)); + RINOK(SzReadFileNames(sd, numFiles, files, allocMain->Alloc)) + break; + } + case k7zIdEmptyStream: + { + RINOK(SzReadBoolVector(sd, numFiles, emptyStreamVector, allocTemp->Alloc)); + numEmptyStreams = 0; + for (i = 0; i < numFiles; i++) + if ((*emptyStreamVector)[i]) + numEmptyStreams++; + break; + } + case k7zIdEmptyFile: + { + RINOK(SzReadBoolVector(sd, numEmptyStreams, emptyFileVector, allocTemp->Alloc)); + break; + } + default: + { + RINOK(SzSkeepDataSize(sd, size)); + } + } + } + + { + UInt32 emptyFileIndex = 0; + UInt32 sizeIndex = 0; + for(i = 0; i < numFiles; i++) + { + CFileItem *file = files + i; + file->IsAnti = 0; + if (*emptyStreamVector == 0) + file->HasStream = 1; + else + file->HasStream = (Byte)((*emptyStreamVector)[i] ? 0 : 1); + if(file->HasStream) + { + file->IsDirectory = 0; + file->Size = (*unPackSizes)[sizeIndex]; + file->FileCRC = (*digests)[sizeIndex]; + file->IsFileCRCDefined = (Byte)(*digestsDefined)[sizeIndex]; + sizeIndex++; + } + else + { + if (*emptyFileVector == 0) + file->IsDirectory = 1; + else + file->IsDirectory = (Byte)((*emptyFileVector)[emptyFileIndex] ? 0 : 1); + emptyFileIndex++; + file->Size = 0; + file->IsFileCRCDefined = 0; + } + } + } + return SzArDbExFill(db, allocMain->Alloc); +} + +SZ_RESULT SzReadHeader( + CSzData *sd, + CArchiveDatabaseEx *db, + ISzAlloc *allocMain, + ISzAlloc *allocTemp) +{ + CFileSize *unPackSizes = 0; + Byte *digestsDefined = 0; + UInt32 *digests = 0; + Byte *emptyStreamVector = 0; + Byte *emptyFileVector = 0; + SZ_RESULT res = SzReadHeader2(sd, db, + &unPackSizes, &digestsDefined, &digests, + &emptyStreamVector, &emptyFileVector, + allocMain, allocTemp); + allocTemp->Free(unPackSizes); + allocTemp->Free(digestsDefined); + allocTemp->Free(digests); + allocTemp->Free(emptyStreamVector); + allocTemp->Free(emptyFileVector); + return res; +} + +SZ_RESULT SzReadAndDecodePackedStreams2( + ISzInStream *inStream, + CSzData *sd, + CSzByteBuffer *outBuffer, + CFileSize baseOffset, + CArchiveDatabase *db, + CFileSize **unPackSizes, + Byte **digestsDefined, + UInt32 **digests, + #ifndef _LZMA_IN_CB + Byte **inBuffer, + #endif + ISzAlloc *allocTemp) +{ + + UInt32 numUnPackStreams = 0; + CFileSize dataStartPos; + CFolder *folder; + #ifndef _LZMA_IN_CB + CFileSize packSize = 0; + UInt32 i = 0; + #endif + CFileSize unPackSize; + size_t outRealSize; + SZ_RESULT res; + + RINOK(SzReadStreamsInfo(sd, &dataStartPos, db, + &numUnPackStreams, unPackSizes, digestsDefined, digests, + allocTemp->Alloc, allocTemp)); + + dataStartPos += baseOffset; + if (db->NumFolders != 1) + return SZE_ARCHIVE_ERROR; + + folder = db->Folders; + unPackSize = SzFolderGetUnPackSize(folder); + + RINOK(inStream->Seek(inStream, dataStartPos)); + + #ifndef _LZMA_IN_CB + for (i = 0; i < db->NumPackStreams; i++) + packSize += db->PackSizes[i]; + + MY_ALLOC(Byte, *inBuffer, (size_t)packSize, allocTemp->Alloc); + + RINOK(SafeReadDirect(inStream, *inBuffer, (size_t)packSize)); + #endif + + if (!SzByteBufferCreate(outBuffer, (size_t)unPackSize, allocTemp->Alloc)) + return SZE_OUTOFMEMORY; + + res = SzDecode(db->PackSizes, folder, + #ifdef _LZMA_IN_CB + inStream, + #else + *inBuffer, + #endif + outBuffer->Items, (size_t)unPackSize, + &outRealSize, allocTemp); + RINOK(res) + if (outRealSize != (UInt32)unPackSize) + return SZE_FAIL; + if (folder->UnPackCRCDefined) + if (!CrcVerifyDigest(folder->UnPackCRC, outBuffer->Items, (size_t)unPackSize)) + return SZE_FAIL; + return SZ_OK; +} + +SZ_RESULT SzReadAndDecodePackedStreams( + ISzInStream *inStream, + CSzData *sd, + CSzByteBuffer *outBuffer, + CFileSize baseOffset, + ISzAlloc *allocTemp) +{ + CArchiveDatabase db; + CFileSize *unPackSizes = 0; + Byte *digestsDefined = 0; + UInt32 *digests = 0; + #ifndef _LZMA_IN_CB + Byte *inBuffer = 0; + #endif + SZ_RESULT res; + SzArchiveDatabaseInit(&db); + res = SzReadAndDecodePackedStreams2(inStream, sd, outBuffer, baseOffset, + &db, &unPackSizes, &digestsDefined, &digests, + #ifndef _LZMA_IN_CB + &inBuffer, + #endif + allocTemp); + SzArchiveDatabaseFree(&db, allocTemp->Free); + allocTemp->Free(unPackSizes); + allocTemp->Free(digestsDefined); + allocTemp->Free(digests); + #ifndef _LZMA_IN_CB + allocTemp->Free(inBuffer); + #endif + return res; +} + +SZ_RESULT SzArchiveOpen2( + ISzInStream *inStream, + CArchiveDatabaseEx *db, + ISzAlloc *allocMain, + ISzAlloc *allocTemp) +{ + Byte signature[k7zSignatureSize]; + Byte version; + UInt32 crcFromArchive; + UInt64 nextHeaderOffset; + UInt64 nextHeaderSize; + UInt32 nextHeaderCRC; + UInt32 crc; + CFileSize pos = 0; + CSzByteBuffer buffer; + CSzData sd; + SZ_RESULT res; + + RINOK(SafeReadDirect(inStream, signature, k7zSignatureSize)); + + if (!TestSignatureCandidate(signature)) + return SZE_ARCHIVE_ERROR; + + /* + db.Clear(); + db.ArchiveInfo.StartPosition = _arhiveBeginStreamPosition; + */ + RINOK(SafeReadDirectByte(inStream, &version)); + if (version != k7zMajorVersion) + return SZE_ARCHIVE_ERROR; + RINOK(SafeReadDirectByte(inStream, &version)); + + RINOK(SafeReadDirectUInt32(inStream, &crcFromArchive)); + + CrcInit(&crc); + RINOK(SafeReadDirectUInt64(inStream, &nextHeaderOffset)); + CrcUpdateUInt64(&crc, nextHeaderOffset); + RINOK(SafeReadDirectUInt64(inStream, &nextHeaderSize)); + CrcUpdateUInt64(&crc, nextHeaderSize); + RINOK(SafeReadDirectUInt32(inStream, &nextHeaderCRC)); + CrcUpdateUInt32(&crc, nextHeaderCRC); + + pos = k7zStartHeaderSize; + db->ArchiveInfo.StartPositionAfterHeader = pos; + + if (CrcGetDigest(&crc) != crcFromArchive) + return SZE_ARCHIVE_ERROR; + + if (nextHeaderSize == 0) + return SZ_OK; + + RINOK(inStream->Seek(inStream, (CFileSize)(pos + nextHeaderOffset))); + + if (!SzByteBufferCreate(&buffer, (size_t)nextHeaderSize, allocTemp->Alloc)) + return SZE_OUTOFMEMORY; + + res = SafeReadDirect(inStream, buffer.Items, (size_t)nextHeaderSize); + if (res == SZ_OK) + { + if (CrcVerifyDigest(nextHeaderCRC, buffer.Items, (UInt32)nextHeaderSize)) + { + for(;;) + { + UInt64 type; + sd.Data = buffer.Items; + sd.Size = buffer.Capacity; + res = SzReadID(&sd, &type); + if (res != SZ_OK) + break; + if (type == k7zIdHeader) + { + res = SzReadHeader(&sd, db, allocMain, allocTemp); + break; + } + if (type != k7zIdEncodedHeader) + { + res = SZE_ARCHIVE_ERROR; + break; + } + { + CSzByteBuffer outBuffer; + res = SzReadAndDecodePackedStreams(inStream, &sd, &outBuffer, + db->ArchiveInfo.StartPositionAfterHeader, + allocTemp); + if (res != SZ_OK) + { + SzByteBufferFree(&outBuffer, allocTemp->Free); + break; + } + SzByteBufferFree(&buffer, allocTemp->Free); + buffer.Items = outBuffer.Items; + buffer.Capacity = outBuffer.Capacity; + } + } + } + } + SzByteBufferFree(&buffer, allocTemp->Free); + return res; +} + +SZ_RESULT SzArchiveOpen( + ISzInStream *inStream, + CArchiveDatabaseEx *db, + ISzAlloc *allocMain, + ISzAlloc *allocTemp) +{ + SZ_RESULT res = SzArchiveOpen2(inStream, db, allocMain, allocTemp); + if (res != SZ_OK) + SzArDbExFree(db, allocMain->Free); + return res; +} diff --git a/code/nel/3rdparty/seven_zip/7zIn.h b/code/nel/3rdparty/seven_zip/7zIn.h new file mode 100644 index 000000000..6bfa2a709 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zIn.h @@ -0,0 +1,55 @@ +/* 7zIn.h */ + +#ifndef __7Z_IN_H +#define __7Z_IN_H + +#include "7zHeader.h" +#include "7zItem.h" +#include "7zAlloc.h" + +typedef struct _CInArchiveInfo +{ + CFileSize StartPositionAfterHeader; + CFileSize DataStartPosition; +}CInArchiveInfo; + +typedef struct _CArchiveDatabaseEx +{ + CArchiveDatabase Database; + CInArchiveInfo ArchiveInfo; + UInt32 *FolderStartPackStreamIndex; + CFileSize *PackStreamStartPositions; + UInt32 *FolderStartFileIndex; + UInt32 *FileIndexToFolderIndexMap; +}CArchiveDatabaseEx; + +void SzArDbExInit(CArchiveDatabaseEx *db); +void SzArDbExFree(CArchiveDatabaseEx *db, void (*freeFunc)(void *)); +CFileSize SzArDbGetFolderStreamPos(CArchiveDatabaseEx *db, UInt32 folderIndex, UInt32 indexInFolder); +CFileSize SzArDbGetFolderFullPackSize(CArchiveDatabaseEx *db, UInt32 folderIndex); + +typedef struct _ISzInStream +{ + #ifdef _LZMA_IN_CB + SZ_RESULT (*Read)( + void *object, /* pointer to ISzInStream itself */ + void **buffer, /* out: pointer to buffer with data */ + size_t maxRequiredSize, /* max required size to read */ + size_t *processedSize); /* real processed size. + processedSize can be less than maxRequiredSize. + If processedSize == 0, then there are no more + bytes in stream. */ + #else + SZ_RESULT (*Read)(void *object, void *buffer, size_t size, size_t *processedSize); + #endif + SZ_RESULT (*Seek)(void *object, CFileSize pos); +} ISzInStream; + + +int SzArchiveOpen( + ISzInStream *inStream, + CArchiveDatabaseEx *db, + ISzAlloc *allocMain, + ISzAlloc *allocTemp); + +#endif diff --git a/code/nel/3rdparty/seven_zip/7zItem.cpp b/code/nel/3rdparty/seven_zip/7zItem.cpp new file mode 100644 index 000000000..2a408050f --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zItem.cpp @@ -0,0 +1,133 @@ +/* 7zItem.c */ + +#include "7zItem.h" +#include "7zAlloc.h" + +void SzCoderInfoInit(CCoderInfo *coder) +{ + SzByteBufferInit(&coder->Properties); +} + +void SzCoderInfoFree(CCoderInfo *coder, void (*freeFunc)(void *p)) +{ + SzByteBufferFree(&coder->Properties, freeFunc); + SzCoderInfoInit(coder); +} + +void SzFolderInit(CFolder *folder) +{ + folder->NumCoders = 0; + folder->Coders = 0; + folder->NumBindPairs = 0; + folder->BindPairs = 0; + folder->NumPackStreams = 0; + folder->PackStreams = 0; + folder->UnPackSizes = 0; + folder->UnPackCRCDefined = 0; + folder->UnPackCRC = 0; + folder->NumUnPackStreams = 0; +} + +void SzFolderFree(CFolder *folder, void (*freeFunc)(void *p)) +{ + UInt32 i; + for (i = 0; i < folder->NumCoders; i++) + SzCoderInfoFree(&folder->Coders[i], freeFunc); + freeFunc(folder->Coders); + freeFunc(folder->BindPairs); + freeFunc(folder->PackStreams); + freeFunc(folder->UnPackSizes); + SzFolderInit(folder); +} + +UInt32 SzFolderGetNumOutStreams(CFolder *folder) +{ + UInt32 result = 0; + UInt32 i; + for (i = 0; i < folder->NumCoders; i++) + result += folder->Coders[i].NumOutStreams; + return result; +} + +int SzFolderFindBindPairForInStream(CFolder *folder, UInt32 inStreamIndex) +{ + UInt32 i; + for(i = 0; i < folder->NumBindPairs; i++) + if (folder->BindPairs[i].InIndex == inStreamIndex) + return i; + return -1; +} + + +int SzFolderFindBindPairForOutStream(CFolder *folder, UInt32 outStreamIndex) +{ + UInt32 i; + for(i = 0; i < folder->NumBindPairs; i++) + if (folder->BindPairs[i].OutIndex == outStreamIndex) + return i; + return -1; +} + +CFileSize SzFolderGetUnPackSize(CFolder *folder) +{ + int i = (int)SzFolderGetNumOutStreams(folder); + if (i == 0) + return 0; + for (i--; i >= 0; i--) + if (SzFolderFindBindPairForOutStream(folder, i) < 0) + return folder->UnPackSizes[i]; + /* throw 1; */ + return 0; +} + +/* +int FindPackStreamArrayIndex(int inStreamIndex) const +{ + for(int i = 0; i < PackStreams.Size(); i++) + if (PackStreams[i] == inStreamIndex) + return i; + return -1; +} +*/ + +void SzFileInit(CFileItem *fileItem) +{ + fileItem->IsFileCRCDefined = 0; + fileItem->HasStream = 1; + fileItem->IsDirectory = 0; + fileItem->IsAnti = 0; + fileItem->Name = 0; +} + +void SzFileFree(CFileItem *fileItem, void (*freeFunc)(void *p)) +{ + freeFunc(fileItem->Name); + SzFileInit(fileItem); +} + +void SzArchiveDatabaseInit(CArchiveDatabase *db) +{ + db->NumPackStreams = 0; + db->PackSizes = 0; + db->PackCRCsDefined = 0; + db->PackCRCs = 0; + db->NumFolders = 0; + db->Folders = 0; + db->NumFiles = 0; + db->Files = 0; +} + +void SzArchiveDatabaseFree(CArchiveDatabase *db, void (*freeFunc)(void *)) +{ + UInt32 i; + for (i = 0; i < db->NumFolders; i++) + SzFolderFree(&db->Folders[i], freeFunc); + for (i = 0; i < db->NumFiles; i++) + SzFileFree(&db->Files[i], freeFunc); + freeFunc(db->PackSizes); + freeFunc(db->PackCRCsDefined); + freeFunc(db->PackCRCs); + freeFunc(db->Folders); + freeFunc(db->Files); + SzArchiveDatabaseInit(db); +} diff --git a/code/nel/3rdparty/seven_zip/7zItem.h b/code/nel/3rdparty/seven_zip/7zItem.h new file mode 100644 index 000000000..876539a98 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zItem.h @@ -0,0 +1,90 @@ +/* 7zItem.h */ + +#ifndef __7Z_ITEM_H +#define __7Z_ITEM_H + +#include "7zMethodID.h" +#include "7zHeader.h" +#include "7zBuffer.h" + +typedef struct _CCoderInfo +{ + UInt32 NumInStreams; + UInt32 NumOutStreams; + CMethodID MethodID; + CSzByteBuffer Properties; +}CCoderInfo; + +void SzCoderInfoInit(CCoderInfo *coder); +void SzCoderInfoFree(CCoderInfo *coder, void (*freeFunc)(void *p)); + +typedef struct _CBindPair +{ + UInt32 InIndex; + UInt32 OutIndex; +}CBindPair; + +typedef struct _CFolder +{ + UInt32 NumCoders; + CCoderInfo *Coders; + UInt32 NumBindPairs; + CBindPair *BindPairs; + UInt32 NumPackStreams; + UInt32 *PackStreams; + CFileSize *UnPackSizes; + int UnPackCRCDefined; + UInt32 UnPackCRC; + + UInt32 NumUnPackStreams; +}CFolder; + +void SzFolderInit(CFolder *folder); +CFileSize SzFolderGetUnPackSize(CFolder *folder); +int SzFolderFindBindPairForInStream(CFolder *folder, UInt32 inStreamIndex); +UInt32 SzFolderGetNumOutStreams(CFolder *folder); +CFileSize SzFolderGetUnPackSize(CFolder *folder); + +/* #define CArchiveFileTime UInt64 */ + +typedef struct _CFileItem +{ + /* + CArchiveFileTime LastWriteTime; + CFileSize StartPos; + UInt32 Attributes; + */ + CFileSize Size; + UInt32 FileCRC; + char *Name; + + Byte IsFileCRCDefined; + Byte HasStream; + Byte IsDirectory; + Byte IsAnti; + /* + int AreAttributesDefined; + int IsLastWriteTimeDefined; + int IsStartPosDefined; + */ +}CFileItem; + +void SzFileInit(CFileItem *fileItem); + +typedef struct _CArchiveDatabase +{ + UInt32 NumPackStreams; + CFileSize *PackSizes; + Byte *PackCRCsDefined; + UInt32 *PackCRCs; + UInt32 NumFolders; + CFolder *Folders; + UInt32 NumFiles; + CFileItem *Files; +}CArchiveDatabase; + +void SzArchiveDatabaseInit(CArchiveDatabase *db); +void SzArchiveDatabaseFree(CArchiveDatabase *db, void (*freeFunc)(void *)); + + +#endif diff --git a/code/ryzom/client/src/seven_zip/7zMain.cpp b/code/nel/3rdparty/seven_zip/7zMain.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zMain.cpp rename to code/nel/3rdparty/seven_zip/7zMain.cpp diff --git a/code/nel/3rdparty/seven_zip/7zMethodID.cpp b/code/nel/3rdparty/seven_zip/7zMethodID.cpp new file mode 100644 index 000000000..5047359f3 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zMethodID.cpp @@ -0,0 +1,14 @@ +/* 7zMethodID.c */ + +#include "7zMethodID.h" + +int AreMethodsEqual(CMethodID *a1, CMethodID *a2) +{ + int i; + if (a1->IDSize != a2->IDSize) + return 0; + for (i = 0; i < a1->IDSize; i++) + if (a1->ID[i] != a2->ID[i]) + return 0; + return 1; +} diff --git a/code/nel/3rdparty/seven_zip/7zMethodID.h b/code/nel/3rdparty/seven_zip/7zMethodID.h new file mode 100644 index 000000000..162fcd15b --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zMethodID.h @@ -0,0 +1,18 @@ +/* 7zMethodID.h */ + +#ifndef __7Z_METHOD_ID_H +#define __7Z_METHOD_ID_H + +#include "7zTypes.h" + +#define kMethodIDSize 15 + +typedef struct _CMethodID +{ + Byte ID[kMethodIDSize]; + Byte IDSize; +} CMethodID; + +int AreMethodsEqual(CMethodID *a1, CMethodID *a2); + +#endif diff --git a/code/ryzom/client/src/seven_zip/7zTypes.h b/code/nel/3rdparty/seven_zip/7zTypes.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zTypes.h rename to code/nel/3rdparty/seven_zip/7zTypes.h diff --git a/code/nel/3rdparty/seven_zip/BranchTypes.h b/code/nel/3rdparty/seven_zip/BranchTypes.h new file mode 100644 index 000000000..f7ad3abc5 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/BranchTypes.h @@ -0,0 +1,25 @@ +/* BranchTypes.h */ + +#ifndef __BRANCHTYPES_H +#define __BRANCHTYPES_H + +#ifndef _7ZIP_BYTE_DEFINED +#define _7ZIP_BYTE_DEFINED +typedef unsigned char Byte; +#endif + +#ifndef _7ZIP_UINT16_DEFINED +#define _7ZIP_UINT16_DEFINED +typedef unsigned short UInt16; +#endif + +#ifndef _7ZIP_UINT32_DEFINED +#define _7ZIP_UINT32_DEFINED +#ifdef _LZMA_UINT32_IS_ULONG +typedef unsigned long UInt32; +#else +typedef unsigned int UInt32; +#endif +#endif + +#endif diff --git a/code/nel/3rdparty/seven_zip/BranchX86.cpp b/code/nel/3rdparty/seven_zip/BranchX86.cpp new file mode 100644 index 000000000..1be3149b8 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/BranchX86.cpp @@ -0,0 +1,101 @@ +/* BranchX86.c */ + +#include "BranchX86.h" + +/* +static int inline Test86MSByte(Byte b) +{ + return (b == 0 || b == 0xFF); +} +*/ +#define Test86MSByte(b) ((b) == 0 || (b) == 0xFF) + +const int kMaskToAllowedStatus[8] = {1, 1, 1, 0, 1, 0, 0, 0}; +const Byte kMaskToBitNumber[8] = {0, 1, 2, 2, 3, 3, 3, 3}; + +/* +void x86_Convert_Init(UInt32 *prevMask, UInt32 *prevPos) +{ + *prevMask = 0; + *prevPos = (UInt32)(-5); +} +*/ + +UInt32 x86_Convert(Byte *buffer, UInt32 endPos, UInt32 nowPos, + UInt32 *prevMask, UInt32 *prevPos, int encoding) +{ + UInt32 bufferPos = 0; + UInt32 limit; + + if (endPos < 5) + return 0; + + if (nowPos - *prevPos > 5) + *prevPos = nowPos - 5; + + limit = endPos - 5; + while(bufferPos <= limit) + { + Byte b = buffer[bufferPos]; + UInt32 offset; + if (b != 0xE8 && b != 0xE9) + { + bufferPos++; + continue; + } + offset = (nowPos + bufferPos - *prevPos); + *prevPos = (nowPos + bufferPos); + if (offset > 5) + *prevMask = 0; + else + { + UInt32 i; + for (i = 0; i < offset; i++) + { + *prevMask &= 0x77; + *prevMask <<= 1; + } + } + b = buffer[bufferPos + 4]; + if (Test86MSByte(b) && kMaskToAllowedStatus[(*prevMask >> 1) & 0x7] && + (*prevMask >> 1) < 0x10) + { + UInt32 src = + ((UInt32)(b) << 24) | + ((UInt32)(buffer[bufferPos + 3]) << 16) | + ((UInt32)(buffer[bufferPos + 2]) << 8) | + (buffer[bufferPos + 1]); + + UInt32 dest; + for(;;) + { + UInt32 index; + if (encoding) + dest = (nowPos + bufferPos + 5) + src; + else + dest = src - (nowPos + bufferPos + 5); + if (*prevMask == 0) + break; + index = kMaskToBitNumber[*prevMask >> 1]; + b = (Byte)(dest >> (24 - index * 8)); + if (!Test86MSByte(b)) + break; + src = dest ^ ((1 << (32 - index * 8)) - 1); + } + buffer[bufferPos + 4] = (Byte)(~(((dest >> 24) & 1) - 1)); + buffer[bufferPos + 3] = (Byte)(dest >> 16); + buffer[bufferPos + 2] = (Byte)(dest >> 8); + buffer[bufferPos + 1] = (Byte)dest; + bufferPos += 5; + *prevMask = 0; + } + else + { + bufferPos++; + *prevMask |= 1; + if (Test86MSByte(b)) + *prevMask |= 0x10; + } + } + return bufferPos; +} diff --git a/code/nel/3rdparty/seven_zip/BranchX86.h b/code/nel/3rdparty/seven_zip/BranchX86.h new file mode 100644 index 000000000..25c1ae51b --- /dev/null +++ b/code/nel/3rdparty/seven_zip/BranchX86.h @@ -0,0 +1,13 @@ +/* BranchX86.h */ + +#ifndef __BRANCHX86_H +#define __BRANCHX86_H + +#include "BranchTypes.h" + +#define x86_Convert_Init(prevMask, prevPos) { prevMask = 0; prevPos = (UInt32)(-5); } + +UInt32 x86_Convert(Byte *buffer, UInt32 endPos, UInt32 nowPos, + UInt32 *prevMask, UInt32 *prevPos, int encoding); + +#endif diff --git a/code/nel/3rdparty/seven_zip/CMakeLists.txt b/code/nel/3rdparty/seven_zip/CMakeLists.txt new file mode 100644 index 000000000..d2d2e690d --- /dev/null +++ b/code/nel/3rdparty/seven_zip/CMakeLists.txt @@ -0,0 +1,27 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +FILE(GLOB LIB_SRC *.cpp *.h) + +LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) + +NL_TARGET_LIB(nel_sevenzip ${LIB_SRC}) +# TARGET_LINK_LIBRARIES(nel_sevenzip ${PLATFORM_LINKFLAGS}) +NL_DEFAULT_PROPS(nel_sevenzip "NeL, 3rd Party: Seven Zip") +NL_ADD_RUNTIME_FLAGS(nel_sevenzip) +NL_ADD_LIB_SUFFIX(nel_sevenzip) + +ADD_DEFINITIONS(-D_SZ_ONE_DIRECTORY) + +IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) + INSTALL(TARGETS nel_sevenzip LIBRARY DESTINATION ${NL_LIB_PREFIX} ARCHIVE DESTINATION ${NL_LIB_PREFIX} COMPONENT libraries) +ENDIF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) + +IF(WITH_TOOLS) + ADD_EXECUTABLE(7zDec ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) + + TARGET_LINK_LIBRARIES(7zDec nel_sevenzip) + NL_DEFAULT_PROPS(7zDec "NeL, 3rd Party: Seven Zip Decoder") + NL_ADD_RUNTIME_FLAGS(7zDec) + + INSTALL(TARGETS 7zDec RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT tools) +ENDIF(WITH_TOOLS) diff --git a/code/nel/3rdparty/seven_zip/LzmaDecode.cpp b/code/nel/3rdparty/seven_zip/LzmaDecode.cpp new file mode 100644 index 000000000..c148a5a65 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaDecode.cpp @@ -0,0 +1,584 @@ +/* + LzmaDecode.c + LZMA Decoder (optimized for Speed version) + + LZMA SDK 4.40 Copyright (c) 1999-2006 Igor Pavlov (2006-05-01) + http://www.7-zip.org/ + + LZMA SDK is licensed under two licenses: + 1) GNU Lesser General Public License (GNU LGPL) + 2) Common Public License (CPL) + It means that you can select one of these two licenses and + follow rules of that license. + + SPECIAL EXCEPTION: + Igor Pavlov, as the author of this Code, expressly permits you to + statically or dynamically link your Code (or bind by name) to the + interfaces of this file without subjecting your linked Code to the + terms of the CPL or GNU LGPL. Any modifications or additions + to this file, however, are subject to the LGPL or CPL terms. +*/ + +#include "LzmaDecode.h" + +#define kNumTopBits 24 +#define kTopValue ((UInt32)1 << kNumTopBits) + +#define kNumBitModelTotalBits 11 +#define kBitModelTotal (1 << kNumBitModelTotalBits) +#define kNumMoveBits 5 + +#define RC_READ_BYTE (*Buffer++) + +#define RC_INIT2 Code = 0; Range = 0xFFFFFFFF; \ + { int i; for(i = 0; i < 5; i++) { RC_TEST; Code = (Code << 8) | RC_READ_BYTE; }} + +#ifdef _LZMA_IN_CB + +#define RC_TEST { if (Buffer == BufferLim) \ + { SizeT size; int result = InCallback->Read(InCallback, &Buffer, &size); if (result != LZMA_RESULT_OK) return result; \ + BufferLim = Buffer + size; if (size == 0) return LZMA_RESULT_DATA_ERROR; }} + +#define RC_INIT Buffer = BufferLim = 0; RC_INIT2 + +#else + +#define RC_TEST { if (Buffer == BufferLim) return LZMA_RESULT_DATA_ERROR; } + +#define RC_INIT(buffer, bufferSize) Buffer = buffer; BufferLim = buffer + bufferSize; RC_INIT2 + +#endif + +#define RC_NORMALIZE if (Range < kTopValue) { RC_TEST; Range <<= 8; Code = (Code << 8) | RC_READ_BYTE; } + +#define IfBit0(p) RC_NORMALIZE; bound = (Range >> kNumBitModelTotalBits) * *(p); if (Code < bound) +#define UpdateBit0(p) Range = bound; *(p) += (kBitModelTotal - *(p)) >> kNumMoveBits; +#define UpdateBit1(p) Range -= bound; Code -= bound; *(p) -= (*(p)) >> kNumMoveBits; + +#define RC_GET_BIT2(p, mi, A0, A1) IfBit0(p) \ + { UpdateBit0(p); mi <<= 1; A0; } else \ + { UpdateBit1(p); mi = (mi + mi) + 1; A1; } + +#define RC_GET_BIT(p, mi) RC_GET_BIT2(p, mi, ; , ;) + +#define RangeDecoderBitTreeDecode(probs, numLevels, res) \ + { int i = numLevels; res = 1; \ + do { CProb *p = probs + res; RC_GET_BIT(p, res) } while(--i != 0); \ + res -= (1 << numLevels); } + + +#define kNumPosBitsMax 4 +#define kNumPosStatesMax (1 << kNumPosBitsMax) + +#define kLenNumLowBits 3 +#define kLenNumLowSymbols (1 << kLenNumLowBits) +#define kLenNumMidBits 3 +#define kLenNumMidSymbols (1 << kLenNumMidBits) +#define kLenNumHighBits 8 +#define kLenNumHighSymbols (1 << kLenNumHighBits) + +#define LenChoice 0 +#define LenChoice2 (LenChoice + 1) +#define LenLow (LenChoice2 + 1) +#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) +#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define kNumLenProbs (LenHigh + kLenNumHighSymbols) + + +#define kNumStates 12 +#define kNumLitStates 7 + +#define kStartPosModelIndex 4 +#define kEndPosModelIndex 14 +#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) + +#define kNumPosSlotBits 6 +#define kNumLenToPosStates 4 + +#define kNumAlignBits 4 +#define kAlignTableSize (1 << kNumAlignBits) + +#define kMatchMinLen 2 + +#define IsMatch 0 +#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +#define IsRepG0 (IsRep + kNumStates) +#define IsRepG1 (IsRepG0 + kNumStates) +#define IsRepG2 (IsRepG1 + kNumStates) +#define IsRep0Long (IsRepG2 + kNumStates) +#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) +#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) +#define LenCoder (Align + kAlignTableSize) +#define RepLenCoder (LenCoder + kNumLenProbs) +#define Literal (RepLenCoder + kNumLenProbs) + +#if Literal != LZMA_BASE_SIZE +StopCompilingDueBUG +#endif + +int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, SizeT size) +{ + unsigned char prop0; + if (size < LZMA_PROPERTIES_SIZE) + return LZMA_RESULT_DATA_ERROR; + prop0 = propsData[0]; + if (prop0 >= (9 * 5 * 5)) + return LZMA_RESULT_DATA_ERROR; + { + for (propsRes->pb = 0; prop0 >= (9 * 5); propsRes->pb++, prop0 -= (9 * 5)) ; + for (propsRes->lp = 0; prop0 >= 9; propsRes->lp++, prop0 -= 9) ; + propsRes->lc = prop0; + /* + unsigned char remainder = (unsigned char)(prop0 / 9); + propsRes->lc = prop0 % 9; + propsRes->pb = remainder / 5; + propsRes->lp = remainder % 5; + */ + } + + #ifdef _LZMA_OUT_READ + { + int i; + propsRes->DictionarySize = 0; + for (i = 0; i < 4; i++) + propsRes->DictionarySize += (UInt32)(propsData[1 + i]) << (i * 8); + if (propsRes->DictionarySize == 0) + propsRes->DictionarySize = 1; + } + #endif + return LZMA_RESULT_OK; +} + +#define kLzmaStreamWasFinishedId (-1) + +int LzmaDecode(CLzmaDecoderState *vs, + #ifdef _LZMA_IN_CB + ILzmaInCallback *InCallback, + #else + const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, + #endif + unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed) +{ + CProb *p = vs->Probs; + SizeT nowPos = 0; + Byte previousByte = 0; + UInt32 posStateMask = (1 << (vs->Properties.pb)) - 1; + UInt32 literalPosMask = (1 << (vs->Properties.lp)) - 1; + int lc = vs->Properties.lc; + + #ifdef _LZMA_OUT_READ + + UInt32 Range = vs->Range; + UInt32 Code = vs->Code; + #ifdef _LZMA_IN_CB + const Byte *Buffer = vs->Buffer; + const Byte *BufferLim = vs->BufferLim; + #else + const Byte *Buffer = inStream; + const Byte *BufferLim = inStream + inSize; + #endif + int state = vs->State; + UInt32 rep0 = vs->Reps[0], rep1 = vs->Reps[1], rep2 = vs->Reps[2], rep3 = vs->Reps[3]; + int len = vs->RemainLen; + UInt32 globalPos = vs->GlobalPos; + UInt32 distanceLimit = vs->DistanceLimit; + + Byte *dictionary = vs->Dictionary; + UInt32 dictionarySize = vs->Properties.DictionarySize; + UInt32 dictionaryPos = vs->DictionaryPos; + + Byte tempDictionary[4]; + + #ifndef _LZMA_IN_CB + *inSizeProcessed = 0; + #endif + *outSizeProcessed = 0; + if (len == kLzmaStreamWasFinishedId) + return LZMA_RESULT_OK; + + if (dictionarySize == 0) + { + dictionary = tempDictionary; + dictionarySize = 1; + tempDictionary[0] = vs->TempDictionary[0]; + } + + if (len == kLzmaNeedInitId) + { + { + UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); + UInt32 i; + for (i = 0; i < numProbs; i++) + p[i] = kBitModelTotal >> 1; + rep0 = rep1 = rep2 = rep3 = 1; + state = 0; + globalPos = 0; + distanceLimit = 0; + dictionaryPos = 0; + dictionary[dictionarySize - 1] = 0; + #ifdef _LZMA_IN_CB + RC_INIT; + #else + RC_INIT(inStream, inSize); + #endif + } + len = 0; + } + while(len != 0 && nowPos < outSize) + { + UInt32 pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + outStream[nowPos++] = dictionary[dictionaryPos] = dictionary[pos]; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + len--; + } + if (dictionaryPos == 0) + previousByte = dictionary[dictionarySize - 1]; + else + previousByte = dictionary[dictionaryPos - 1]; + + #else /* if !_LZMA_OUT_READ */ + + int state = 0; + UInt32 rep0 = 1, rep1 = 1, rep2 = 1, rep3 = 1; + int len = 0; + const Byte *Buffer; + const Byte *BufferLim; + UInt32 Range; + UInt32 Code; + + #ifndef _LZMA_IN_CB + *inSizeProcessed = 0; + #endif + *outSizeProcessed = 0; + + { + UInt32 i; + UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); + for (i = 0; i < numProbs; i++) + p[i] = kBitModelTotal >> 1; + } + + #ifdef _LZMA_IN_CB + RC_INIT; + #else + RC_INIT(inStream, inSize); + #endif + + #endif /* _LZMA_OUT_READ */ + + while(nowPos < outSize) + { + CProb *prob; + UInt32 bound; + int posState = (int)( + (nowPos + #ifdef _LZMA_OUT_READ + + globalPos + #endif + ) + & posStateMask); + + prob = p + IsMatch + (state << kNumPosBitsMax) + posState; + IfBit0(prob) + { + int symbol = 1; + UpdateBit0(prob) + prob = p + Literal + (LZMA_LIT_SIZE * + ((( + (nowPos + #ifdef _LZMA_OUT_READ + + globalPos + #endif + ) + & literalPosMask) << lc) + (previousByte >> (8 - lc)))); + + if (state >= kNumLitStates) + { + int matchByte; + #ifdef _LZMA_OUT_READ + UInt32 pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + matchByte = dictionary[pos]; + #else + matchByte = outStream[nowPos - rep0]; + #endif + do + { + int bit; + CProb *probLit; + matchByte <<= 1; + bit = (matchByte & 0x100); + probLit = prob + 0x100 + bit + symbol; + RC_GET_BIT2(probLit, symbol, if (bit != 0) break, if (bit == 0) break) + } + while (symbol < 0x100); + } + while (symbol < 0x100) + { + CProb *probLit = prob + symbol; + RC_GET_BIT(probLit, symbol) + } + previousByte = (Byte)(symbol & 0xff); + + outStream[nowPos++] = previousByte; + #ifdef _LZMA_OUT_READ + if (distanceLimit < dictionarySize) + distanceLimit++; + + dictionary[dictionaryPos] = previousByte; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + #endif + if (state < 4) state = 0; + else if (state < 10) state -= 3; + else state -= 6; + } + else + { + UpdateBit1(prob); + prob = p + IsRep + state; + IfBit0(prob) + { + UpdateBit0(prob); + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + state = state < kNumLitStates ? 0 : 3; + prob = p + LenCoder; + } + else + { + UpdateBit1(prob); + prob = p + IsRepG0 + state; + IfBit0(prob) + { + UpdateBit0(prob); + prob = p + IsRep0Long + (state << kNumPosBitsMax) + posState; + IfBit0(prob) + { + #ifdef _LZMA_OUT_READ + UInt32 pos; + #endif + UpdateBit0(prob); + + #ifdef _LZMA_OUT_READ + if (distanceLimit == 0) + #else + if (nowPos == 0) + #endif + return LZMA_RESULT_DATA_ERROR; + + state = state < kNumLitStates ? 9 : 11; + #ifdef _LZMA_OUT_READ + pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + previousByte = dictionary[pos]; + dictionary[dictionaryPos] = previousByte; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + #else + previousByte = outStream[nowPos - rep0]; + #endif + outStream[nowPos++] = previousByte; + #ifdef _LZMA_OUT_READ + if (distanceLimit < dictionarySize) + distanceLimit++; + #endif + + continue; + } + else + { + UpdateBit1(prob); + } + } + else + { + UInt32 distance; + UpdateBit1(prob); + prob = p + IsRepG1 + state; + IfBit0(prob) + { + UpdateBit0(prob); + distance = rep1; + } + else + { + UpdateBit1(prob); + prob = p + IsRepG2 + state; + IfBit0(prob) + { + UpdateBit0(prob); + distance = rep2; + } + else + { + UpdateBit1(prob); + distance = rep3; + rep3 = rep2; + } + rep2 = rep1; + } + rep1 = rep0; + rep0 = distance; + } + state = state < kNumLitStates ? 8 : 11; + prob = p + RepLenCoder; + } + { + int numBits, offset; + CProb *probLen = prob + LenChoice; + IfBit0(probLen) + { + UpdateBit0(probLen); + probLen = prob + LenLow + (posState << kLenNumLowBits); + offset = 0; + numBits = kLenNumLowBits; + } + else + { + UpdateBit1(probLen); + probLen = prob + LenChoice2; + IfBit0(probLen) + { + UpdateBit0(probLen); + probLen = prob + LenMid + (posState << kLenNumMidBits); + offset = kLenNumLowSymbols; + numBits = kLenNumMidBits; + } + else + { + UpdateBit1(probLen); + probLen = prob + LenHigh; + offset = kLenNumLowSymbols + kLenNumMidSymbols; + numBits = kLenNumHighBits; + } + } + RangeDecoderBitTreeDecode(probLen, numBits, len); + len += offset; + } + + if (state < 4) + { + int posSlot; + state += kNumLitStates; + prob = p + PosSlot + + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << + kNumPosSlotBits); + RangeDecoderBitTreeDecode(prob, kNumPosSlotBits, posSlot); + if (posSlot >= kStartPosModelIndex) + { + int numDirectBits = ((posSlot >> 1) - 1); + rep0 = (2 | ((UInt32)posSlot & 1)); + if (posSlot < kEndPosModelIndex) + { + rep0 <<= numDirectBits; + prob = p + SpecPos + rep0 - posSlot - 1; + } + else + { + numDirectBits -= kNumAlignBits; + do + { + RC_NORMALIZE + Range >>= 1; + rep0 <<= 1; + if (Code >= Range) + { + Code -= Range; + rep0 |= 1; + } + } + while (--numDirectBits != 0); + prob = p + Align; + rep0 <<= kNumAlignBits; + numDirectBits = kNumAlignBits; + } + { + int i = 1; + int mi = 1; + do + { + CProb *prob3 = prob + mi; + RC_GET_BIT2(prob3, mi, ; , rep0 |= i); + i <<= 1; + } + while(--numDirectBits != 0); + } + } + else + rep0 = posSlot; + if (++rep0 == (UInt32)(0)) + { + /* it's for stream version */ + len = kLzmaStreamWasFinishedId; + break; + } + } + + len += kMatchMinLen; + #ifdef _LZMA_OUT_READ + if (rep0 > distanceLimit) + #else + if (rep0 > nowPos) + #endif + return LZMA_RESULT_DATA_ERROR; + + #ifdef _LZMA_OUT_READ + if (dictionarySize - distanceLimit > (UInt32)len) + distanceLimit += len; + else + distanceLimit = dictionarySize; + #endif + + do + { + #ifdef _LZMA_OUT_READ + UInt32 pos = dictionaryPos - rep0; + if (pos >= dictionarySize) + pos += dictionarySize; + previousByte = dictionary[pos]; + dictionary[dictionaryPos] = previousByte; + if (++dictionaryPos == dictionarySize) + dictionaryPos = 0; + #else + previousByte = outStream[nowPos - rep0]; + #endif + len--; + outStream[nowPos++] = previousByte; + } + while(len != 0 && nowPos < outSize); + } + } + RC_NORMALIZE; + + #ifdef _LZMA_OUT_READ + vs->Range = Range; + vs->Code = Code; + vs->DictionaryPos = dictionaryPos; + vs->GlobalPos = globalPos + (UInt32)nowPos; + vs->DistanceLimit = distanceLimit; + vs->Reps[0] = rep0; + vs->Reps[1] = rep1; + vs->Reps[2] = rep2; + vs->Reps[3] = rep3; + vs->State = state; + vs->RemainLen = len; + vs->TempDictionary[0] = tempDictionary[0]; + #endif + + #ifdef _LZMA_IN_CB + vs->Buffer = Buffer; + vs->BufferLim = BufferLim; + #else + *inSizeProcessed = (SizeT)(Buffer - inStream); + #endif + *outSizeProcessed = nowPos; + return LZMA_RESULT_OK; +} diff --git a/code/nel/3rdparty/seven_zip/LzmaDecode.h b/code/nel/3rdparty/seven_zip/LzmaDecode.h new file mode 100644 index 000000000..748fd0a68 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaDecode.h @@ -0,0 +1,113 @@ +/* + LzmaDecode.h + LZMA Decoder interface + + LZMA SDK 4.40 Copyright (c) 1999-2006 Igor Pavlov (2006-05-01) + http://www.7-zip.org/ + + LZMA SDK is licensed under two licenses: + 1) GNU Lesser General Public License (GNU LGPL) + 2) Common Public License (CPL) + It means that you can select one of these two licenses and + follow rules of that license. + + SPECIAL EXCEPTION: + Igor Pavlov, as the author of this code, expressly permits you to + statically or dynamically link your code (or bind by name) to the + interfaces of this file without subjecting your linked code to the + terms of the CPL or GNU LGPL. Any modifications or additions + to this file, however, are subject to the LGPL or CPL terms. +*/ + +#ifndef __LZMADECODE_H +#define __LZMADECODE_H + +#include "LzmaTypes.h" + +/* #define _LZMA_IN_CB */ +/* Use callback for input data */ + +/* #define _LZMA_OUT_READ */ +/* Use read function for output data */ + +/* #define _LZMA_PROB32 */ +/* It can increase speed on some 32-bit CPUs, + but memory usage will be doubled in that case */ + +/* #define _LZMA_LOC_OPT */ +/* Enable local speed optimizations inside code */ + +#ifdef _LZMA_PROB32 +#define CProb UInt32 +#else +#define CProb UInt16 +#endif + +#define LZMA_RESULT_OK 0 +#define LZMA_RESULT_DATA_ERROR 1 + +#ifdef _LZMA_IN_CB +typedef struct _ILzmaInCallback +{ + int (*Read)(void *object, const unsigned char **buffer, SizeT *bufferSize); +} ILzmaInCallback; +#endif + +#define LZMA_BASE_SIZE 1846 +#define LZMA_LIT_SIZE 768 + +#define LZMA_PROPERTIES_SIZE 5 + +typedef struct _CLzmaProperties +{ + int lc; + int lp; + int pb; + #ifdef _LZMA_OUT_READ + UInt32 DictionarySize; + #endif +}CLzmaProperties; + +int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, SizeT size); + +#define LzmaGetNumProbs(Properties) (LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((Properties)->lc + (Properties)->lp))) + +#define kLzmaNeedInitId (-2) + +typedef struct _CLzmaDecoderState +{ + CLzmaProperties Properties; + CProb *Probs; + + #ifdef _LZMA_IN_CB + const unsigned char *Buffer; + const unsigned char *BufferLim; + #endif + + #ifdef _LZMA_OUT_READ + unsigned char *Dictionary; + UInt32 Range; + UInt32 Code; + UInt32 DictionaryPos; + UInt32 GlobalPos; + UInt32 DistanceLimit; + UInt32 Reps[4]; + int State; + int RemainLen; + unsigned char TempDictionary[4]; + #endif +} CLzmaDecoderState; + +#ifdef _LZMA_OUT_READ +#define LzmaDecoderInit(vs) { (vs)->RemainLen = kLzmaNeedInitId; } +#endif + +int LzmaDecode(CLzmaDecoderState *vs, + #ifdef _LZMA_IN_CB + ILzmaInCallback *inCallback, + #else + const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, + #endif + unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed); + +#endif diff --git a/code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp b/code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp new file mode 100644 index 000000000..ed1784d6d --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp @@ -0,0 +1,79 @@ +/* LzmaRamDecode.c */ + +#include "LzmaRamDecode.h" +#ifdef _SZ_ONE_DIRECTORY +#include "LzmaDecode.h" +#include "BranchX86.h" +#else +#include "../LZMA_C/LzmaDecode.h" +#include "../Branch/BranchX86.h" +#endif + +#define LZMA_PROPS_SIZE 14 +#define LZMA_SIZE_OFFSET 6 + +int LzmaRamGetUncompressedSize( + const unsigned char *inBuffer, + size_t inSize, + size_t *outSize) +{ + unsigned int i; + if (inSize < LZMA_PROPS_SIZE) + return 1; + *outSize = 0; + for(i = 0; i < sizeof(size_t); i++) + *outSize += ((size_t)inBuffer[LZMA_SIZE_OFFSET + i]) << (8 * i); + for(; i < 8; i++) + if (inBuffer[LZMA_SIZE_OFFSET + i] != 0) + return 1; + return 0; +} + +#define SZE_DATA_ERROR (1) +#define SZE_OUTOFMEMORY (2) + +int LzmaRamDecompress( + const unsigned char *inBuffer, + size_t inSize, + unsigned char *outBuffer, + size_t outSize, + size_t *outSizeProcessed, + void * (*allocFunc)(size_t size), + void (*freeFunc)(void *)) +{ + CLzmaDecoderState state; /* it's about 24 bytes structure, if int is 32-bit */ + int result; + SizeT outSizeProcessedLoc; + SizeT inProcessed; + int useFilter; + + if (inSize < LZMA_PROPS_SIZE) + return 1; + useFilter = inBuffer[0]; + + *outSizeProcessed = 0; + if (useFilter > 1) + return 1; + + if (LzmaDecodeProperties(&state.Properties, inBuffer + 1, LZMA_PROPERTIES_SIZE) != LZMA_RESULT_OK) + return 1; + state.Probs = (CProb *)allocFunc(LzmaGetNumProbs(&state.Properties) * sizeof(CProb)); + if (state.Probs == 0) + return SZE_OUTOFMEMORY; + + result = LzmaDecode(&state, + inBuffer + LZMA_PROPS_SIZE, (SizeT)inSize - LZMA_PROPS_SIZE, &inProcessed, + outBuffer, (SizeT)outSize, &outSizeProcessedLoc); + freeFunc(state.Probs); + if (result != LZMA_RESULT_OK) + return 1; + *outSizeProcessed = (size_t)outSizeProcessedLoc; + if (useFilter == 1) + { + UInt32 _prevMask; + UInt32 _prevPos; + x86_Convert_Init(_prevMask, _prevPos); + x86_Convert(outBuffer, (UInt32)outSizeProcessedLoc, 0, &_prevMask, &_prevPos, 0); + } + return 0; +} diff --git a/code/nel/3rdparty/seven_zip/LzmaRamDecode.h b/code/nel/3rdparty/seven_zip/LzmaRamDecode.h new file mode 100644 index 000000000..7e641c553 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaRamDecode.h @@ -0,0 +1,55 @@ +/* LzmaRamDecode.h */ + +#ifndef __LzmaRamDecode_h +#define __LzmaRamDecode_h + +#include + +/* +LzmaRamGetUncompressedSize: + In: + inBuffer - input data + inSize - input data size + Out: + outSize - uncompressed size + Return code: + 0 - OK + 1 - Error in headers +*/ + +int LzmaRamGetUncompressedSize( + const unsigned char *inBuffer, + size_t inSize, + size_t *outSize); + + +/* +LzmaRamDecompress: + In: + inBuffer - input data + inSize - input data size + outBuffer - output data + outSize - output size + allocFunc - alloc function (can be malloc) + freeFunc - free function (can be free) + Out: + outSizeProcessed - processed size + Return code: + 0 - OK + 1 - Error in headers / data stream + 2 - Memory allocating error + +Memory requirements depend from properties of LZMA stream. +With default lzma settings it's about 16 KB. +*/ + +int LzmaRamDecompress( + const unsigned char *inBuffer, + size_t inSize, + unsigned char *outBuffer, + size_t outSize, + size_t *outSizeProcessed, + void * (*allocFunc)(size_t size), + void (*freeFunc)(void *)); + +#endif diff --git a/code/nel/3rdparty/seven_zip/LzmaTypes.h b/code/nel/3rdparty/seven_zip/LzmaTypes.h new file mode 100644 index 000000000..288c5e45d --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaTypes.h @@ -0,0 +1,45 @@ +/* +LzmaTypes.h + +Types for LZMA Decoder + +This file written and distributed to public domain by Igor Pavlov. +This file is part of LZMA SDK 4.40 (2006-05-01) +*/ + +#ifndef __LZMATYPES_H +#define __LZMATYPES_H + +#ifndef _7ZIP_BYTE_DEFINED +#define _7ZIP_BYTE_DEFINED +typedef unsigned char Byte; +#endif + +#ifndef _7ZIP_UINT16_DEFINED +#define _7ZIP_UINT16_DEFINED +typedef unsigned short UInt16; +#endif + +#ifndef _7ZIP_UINT32_DEFINED +#define _7ZIP_UINT32_DEFINED +#ifdef _LZMA_UINT32_IS_ULONG +typedef unsigned long UInt32; +#else +typedef unsigned int UInt32; +#endif +#endif + +/* #define _LZMA_SYSTEM_SIZE_T */ +/* Use system's size_t. You can use it to enable 64-bit sizes supporting */ + +#ifndef _7ZIP_SIZET_DEFINED +#define _7ZIP_SIZET_DEFINED +#ifdef _LZMA_SYSTEM_SIZE_T +#include +typedef size_t SizeT; +#else +typedef UInt32 SizeT; +#endif +#endif + +#endif diff --git a/code/ryzom/client/src/seven_zip/readme.txt b/code/nel/3rdparty/seven_zip/readme.txt similarity index 100% rename from code/ryzom/client/src/seven_zip/readme.txt rename to code/nel/3rdparty/seven_zip/readme.txt diff --git a/code/nel/CMakeLists.txt b/code/nel/CMakeLists.txt index 3470fcebb..1530e75e3 100644 --- a/code/nel/CMakeLists.txt +++ b/code/nel/CMakeLists.txt @@ -56,6 +56,7 @@ IF(WITH_INSTALL_LIBRARIES) ADD_SUBDIRECTORY(include) ENDIF() +ADD_SUBDIRECTORY(3rdparty) ADD_SUBDIRECTORY(src) IF(WITH_NEL_SAMPLES) diff --git a/code/nel/include/nel/misc/app_context.h b/code/nel/include/nel/misc/app_context.h index bf73ddb83..6afe62c79 100644 --- a/code/nel/include/nel/misc/app_context.h +++ b/code/nel/include/nel/misc/app_context.h @@ -237,6 +237,22 @@ namespace NLMISC } \ private: +#define NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(className) \ + NLMISC_SAFE_SINGLETON_DECL(className); \ + \ + public: \ + static void releaseInstance() \ + { \ + if (_Instance) \ + { \ + NLMISC::INelContext::getInstance().releaseSingletonPointer(#className, _Instance); \ + delete _Instance; \ + _Instance = NULL; \ + } \ + } \ + private: + + /** The same as above, but generate a getInstance method that * return a pointer instead of a reference */ @@ -304,5 +320,6 @@ void initNelLibrary(CLibrary &lib); } // namespace NLMISC +#include #endif //APP_CONTEXT_H diff --git a/code/nel/include/nel/misc/path.h b/code/nel/include/nel/misc/path.h index f42acc91b..de46532b7 100644 --- a/code/nel/include/nel/misc/path.h +++ b/code/nel/include/nel/misc/path.h @@ -74,6 +74,9 @@ public: /** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included (the extention (Nel Big File) is used to know that it's a big file) */ void addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); + + /** Sale but for .snp (Streamed NeL Package) */ + void addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); /** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included */ void addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); @@ -276,8 +279,8 @@ private: struct CMCFileEntry { char *Name; // Normal case (the search is done by using nlstricmp) - uint32 idPath : 16; // Path (not with file at the end) - look in the SSMpath (65536 different path allowed) - uint32 idExt : 15; // real extension of the file if remapped - look in the SSMext (32768 different extension allowed) + uint32 idPath : 20; // Path (not with file at the end) - look in the SSMpath (1048576 different path allowed) + uint32 idExt : 11; // real extension of the file if remapped - look in the SSMext (2048 different extension allowed) uint32 Remapped : 1; // true if the file is remapped }; @@ -371,6 +374,9 @@ public: /** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included (the extention (Nel Big File) is used to know that it's a big file) */ static void addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); + /** Same but Streamed Package */ + static void addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); + /** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included */ static void addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); diff --git a/code/nel/include/nel/misc/streamed_package.h b/code/nel/include/nel/misc/streamed_package.h new file mode 100644 index 000000000..c777d8a02 --- /dev/null +++ b/code/nel/include/nel/misc/streamed_package.h @@ -0,0 +1,57 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_H +#define NLMISC_STREAMED_PACKAGE_H + +#include +#include + +namespace NLMISC { + +class CStreamedPackage +{ +public: + struct CEntry + { + std::string Name; + CHashKey Hash; + uint32 Size; + uint32 LastModified; + + void serial(NLMISC::IStream &f) throw(NLMISC::EStream); + + }; + +public: + CStreamedPackage(); + ~CStreamedPackage(); + + void serial(NLMISC::IStream &f) throw(NLMISC::EStream); + + static void makePath(std::string &result, const CHashKey &hash); + +public: + typedef std::vector TEntries; + TEntries Entries; + +}; /* class CStreamedPackage */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */ + +/* end of file */ diff --git a/code/nel/include/nel/misc/streamed_package_manager.h b/code/nel/include/nel/misc/streamed_package_manager.h new file mode 100644 index 000000000..e23344a70 --- /dev/null +++ b/code/nel/include/nel/misc/streamed_package_manager.h @@ -0,0 +1,72 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H +#define NLMISC_STREAMED_PACKAGE_MANAGER_H + +#include +#include +#include + +namespace NLMISC { + +class CStreamedPackageManager +{ + NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(CStreamedPackageManager); + +public: + CStreamedPackageManager(); + ~CStreamedPackageManager(); + + /// package: ex. /games/nel/data/characters_maps_hr.snp + bool loadPackage(const std::string &package); + + /// Get a file list + void list(std::vector &fileNames, const std::string &package); + + /// Unload all packages + void unloadAll(); + + /// Get an existing file or download if necessary + /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. + /// fileName: ex. fy_hof_underwear.dds + bool getFile(std::string &filePath, const std::string &fileName); + + /// Get file size + bool getFileSize(uint32 &fileSize, const std::string &fileName); + +public: + /// Set storage path (ex. stream/) + std::string Path; + + /// Loads a package into the package manager (ex. http://patch.live.polyverse.org/stream/) + typedef std::vector THosts; + THosts Hosts; + +private: + typedef std::map TPackages; + TPackages m_Packages; + + typedef std::map TEntries; + TEntries m_Entries; + +}; /* class CStreamedPackageManager */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */ + +/* end of file */ diff --git a/code/nel/src/misc/file.cpp b/code/nel/src/misc/file.cpp index 834b11a8d..5bb82c454 100644 --- a/code/nel/src/misc/file.cpp +++ b/code/nel/src/misc/file.cpp @@ -23,6 +23,7 @@ #include "nel/misc/command.h" #include "nel/misc/sstring.h" #include "nel/misc/xml_pack.h" +#include "nel/misc/streamed_package_manager.h" #ifndef NL_OS_WINDOWS #include @@ -211,6 +212,40 @@ bool CIFile::open(const std::string &path, bool text) _F = CXMLPack::getInstance().getFile (path, _FileSize, _BigFileOffset, dummy, _AlwaysOpened); } } + else if (pos > 3 && path[pos-3] == 's' && path[pos-2] == 'n' && path[pos-1] == 'p') + { + // nldebug("Opening a streamed package file"); + + _IsInXMLPackFile = false; + _IsInBigFile = false; + _BigFileOffset = 0; + _AlwaysOpened = false; + std::string filePath; + if (CStreamedPackageManager::getInstance().getFile (filePath, path.substr(pos+1))) + { + _F = fopen (filePath.c_str(), mode); + if (_F != NULL) + { + _FileSize=CFile::getFileSize(_F); + if (_FileSize == 0) + { + nlwarning("FILE: Size of file '%s' is 0", path.c_str()); + fclose(_F); + _F = NULL; + } + } + else + { + nlwarning("Failed to open file '%s', error %u : %s", path.c_str(), errno, strerror(errno)); + _FileSize = 0; + } + } + else + { + // TEMPORARY ERROR + // nlerror("File '%s' not in streamed package", path.c_str()); + } + } else { // bnp file diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index 0c650918f..93a320715 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -23,6 +23,7 @@ #include "nel/misc/progress_callback.h" #include "nel/misc/file.h" #include "nel/misc/xml_pack.h" +#include "nel/misc/streamed_package_manager.h" #ifdef NL_OS_WINDOWS # include @@ -288,6 +289,7 @@ void CFileContainer::clearMap () nlassert(!_MemoryCompressed); _Files.clear (); CBigFile::getInstance().removeAll (); + CStreamedPackageManager::getInstance().unloadAll (); NL_DISPLAY_PATH("PATH: CPath::clearMap(): map directory cleared"); } @@ -336,7 +338,7 @@ void CFileContainer::remapExtension (const string &ext1, const string &ext2, boo nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): can't remap empty extension", ext1lwr.c_str(), ext2lwr.c_str(), substitute); } - if (ext1lwr == "bnp" || ext2lwr == "bnp") + if (ext1lwr == "bnp" || ext2lwr == "bnp" || ext1lwr == "snp" || ext2lwr == "snp") { nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): you can't remap a big file", ext1lwr.c_str(), ext2lwr.c_str(), substitute); } @@ -1162,16 +1164,26 @@ void CFileContainer::addSearchFile (const string &file, bool remap, const string return; } + std::string fileExtension = CFile::getExtension(newFile); + // check if it s a big file - if (CFile::getExtension(newFile) == "bnp") + if (fileExtension == "bnp") { NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is a big file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str()); addSearchBigFile(file, false, false, progressCallBack); return; } + // check if it s a streamed package + if (fileExtension == "snp") + { + NL_DISPLAY_PATH ("PATH: CPath::addSearchStreamedPackage(%s, %d, '%s'): '%s' is a streamed package, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str()); + addSearchStreamedPackage(file, false, false, progressCallBack); + return; + } + // check if it s an xml pack file - if (CFile::getExtension(newFile) == "xml_pack") + if (fileExtension == "xml_pack") { NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is an xml pack file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str()); addSearchXmlpackFile(file, false, false, progressCallBack); @@ -1392,6 +1404,64 @@ void CFileContainer::addSearchBigFile (const string &sBigFilename, bool recurse, fclose (Handle); } +void CPath::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack) +{ + getInstance()->_FileContainer.addSearchBigFile(filename, recurse, alternative, progressCallBack); +} + +void CFileContainer::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack) +{ + // Check if filename is not empty + if (filename.empty()) + { + nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): can't add empty file, skip it", filename.c_str(), recurse, alternative); + return; + } + // Check if the file exists + if (!CFile::isExists(filename)) + { + nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not found, skip it", filename.c_str(), recurse, alternative, filename.c_str()); + return; + } + // Check if it s a file + if (CFile::isDirectory(filename)) + { + nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not a file, skip it", filename.c_str(), recurse, alternative, filename.c_str()); + return; + } + + // Add the file itself + std::string packname = NLMISC::toLower(CFile::getFilename(filename)); + insertFileInMap(packname, filename, false, CFile::getExtension(filename)); + + // Add the package to the package manager + if (CStreamedPackageManager::getInstance().loadPackage(filename)) + { + std::vector fileNames; + CStreamedPackageManager::getInstance().list(fileNames, packname); + + for (std::vector::iterator it(fileNames.begin()), end(fileNames.end()); it != end; ++it) + { + // Add the file to the lookup + std::string filePackageName = packname + "@" + (*it); + // nldebug("Insert '%s'", filePackageName.c_str()); + insertFileInMap((*it), filePackageName, false, CFile::getExtension(*it)); + + // Remapped extensions + std::string ext = CFile::getExtension(*it); + for (uint j = 0; j < _Extensions.size (); j++) + { + if (_Extensions[j].first == ext) + { + // Need to remap + insertFileInMap(CFile::getFilenameWithoutExtension(*it) + "." + _Extensions[j].second, + filePackageName, true, _Extensions[j].first); + } + } + } + } +} + // WARNING : recurse is not used void CPath::addSearchXmlpackFile (const string &sXmlpackFilename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack) { @@ -1679,7 +1749,7 @@ void CFileContainer::memoryCompress() while (it != _Files.end()) { string sTmp = SSMpath.get(it->second.idPath); - if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && !it->second.Remapped) + if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && (sTmp.find("snp@") == string::npos) && !it->second.Remapped) { // This is a file included in a bigfile (so the name is in the bigfile manager) } @@ -1702,7 +1772,7 @@ void CFileContainer::memoryCompress() { CFileEntry &rFE = it->second; string sTmp = SSMpath.get(rFE.idPath); - if (sTmp.find("@") == string::npos || sTmp.find("@@") != string::npos || rFE.Remapped) + if ((sTmp.find("@") == string::npos) || (sTmp.find("@@") != string::npos) || (sTmp.find("snp@") != string::npos) || rFE.Remapped) { strcpy(_AllFileNames+nSize, rFE.Name.c_str()); _MCFiles[nNb].Name = _AllFileNames+nSize; @@ -1999,6 +2069,7 @@ string CFile::findNewFile (const string &filename) // \warning doesn't work with big file uint32 CFile::getFileSize (const std::string &filename) { + std::string::size_type pos; if (filename.find("@@") != string::npos) { uint32 fs = 0, bfo; @@ -2006,12 +2077,21 @@ uint32 CFile::getFileSize (const std::string &filename) CXMLPack::getInstance().getFile (filename, fs, bfo, c, d); return fs; } - else if (filename.find('@') != string::npos) + else if ((pos = filename.find('@')) != string::npos) { - uint32 fs = 0, bfo; - bool c, d; - CBigFile::getInstance().getFile (filename, fs, bfo, c, d); - return fs; + if (pos > 3 && filename[pos-3] == 's' && filename[pos-2] == 'n' && filename[pos-1] == 'p') + { + uint32 fs = 0; + CStreamedPackageManager::getInstance().getFileSize (fs, filename.substr(pos+1)); + return fs; + } + else + { + uint32 fs = 0, bfo; + bool c, d; + CBigFile::getInstance().getFile (filename, fs, bfo, c, d); + return fs; + } } else { diff --git a/code/nel/src/misc/streamed_package.cpp b/code/nel/src/misc/streamed_package.cpp new file mode 100644 index 000000000..fed7d1dc3 --- /dev/null +++ b/code/nel/src/misc/streamed_package.cpp @@ -0,0 +1,66 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// Project includes +#include +#include + +namespace NLMISC { + +CStreamedPackage::CStreamedPackage() +{ + // init +} + +CStreamedPackage::~CStreamedPackage() +{ + // release +} + +void CStreamedPackage::serial(NLMISC::IStream &f) throw(NLMISC::EStream) +{ + f.serialCheck(NELID("SNPK")); + + uint version = 1; + f.serialVersion(version); + + f.serialCont(Entries); +} + +void CStreamedPackage::CEntry::serial(NLMISC::IStream &f) throw(NLMISC::EStream) +{ + uint version = 1; + f.serialVersion(version); + + f.serial(Name); + f.serial(Hash); + f.serial(Size); + f.serial(LastModified); +} + +void CStreamedPackage::makePath(std::string &result, const CHashKey &hash) +{ + std::string lowerHash = NLMISC::toLower(hash.toString()); + result = std::string("/") + lowerHash.substr(0, 2) + + "/" + lowerHash.substr(2, 2) + + "/" + lowerHash.substr(4); +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp new file mode 100644 index 000000000..23b77def7 --- /dev/null +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -0,0 +1,284 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// 3rd Party includes +#include +#include +#include +#include +#include + +// Project includes +#include +#include +#include +#include + +namespace NLMISC { + +NLMISC_SAFE_SINGLETON_IMPL(CStreamedPackageManager); + +CStreamedPackageManager::CStreamedPackageManager() +{ + // init +} + +CStreamedPackageManager::~CStreamedPackageManager() +{ + // release +} + +bool CStreamedPackageManager::loadPackage(const std::string &package) +{ + nldebug("Load package '%s'", package.c_str()); + + std::string packname = NLMISC::toLower(CFile::getFilename(package)); + m_Packages[packname] = CStreamedPackage(); + std::map::iterator it = m_Packages.find(packname); + CStreamedPackage &p = it->second; + + try + { + CIFile fi; + fi.open(package); + fi.serial(p); + } + catch (Exception &e) + { + nlerror("Package serial exception: '%s'", e.what()); + m_Packages.erase(it); + return false; + } + + for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + m_Entries[entry.Name] = &entry; + } + + return true; +} + +void CStreamedPackageManager::list(std::vector &fileNames, const std::string &package) +{ + // nldebug("List package '%s'", package.c_str()); + + std::map::iterator it = m_Packages.find(package); + CStreamedPackage &p = it->second; + + for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + fileNames.push_back(entry.Name); + } +} + +void CStreamedPackageManager::unloadAll() +{ + m_Packages.clear(); + m_Entries.clear(); +} + +bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName) +{ + // nldebug("Get file path for streamed file '%s'", fileName.c_str()); + + TEntries::iterator it = m_Entries.find(fileName); + if (it == m_Entries.end()) + return false; + const CStreamedPackage::CEntry *entry = it->second; + + CStreamedPackage::makePath(filePath, entry->Hash); + std::string downloadUrlFile = filePath + ".lzma"; + filePath = Path + filePath; + std::string downloadPath = filePath + ".download." + toString(rand()); + + std::string storageDirectory = CFile::getPath(downloadPath); + CFile::createDirectoryTree(storageDirectory); + /*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory)) + { + nldebug("Unable to create directory '%s'", storageDirectory.c_str()); + return false; + }*/ + + // download + for (; ; ) + { + if (CFile::fileExists(filePath)) + return true; + + std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile; + nldebug("Download streamed package '%s' from '%s'", fileName.c_str(), downloadUrl.c_str()); + + FILE *fp = fopen(downloadPath.c_str(), "wb"); + if (fp == NULL) + { + nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), fileName.c_str()); + return false; + } + + CURL *curl; + CURLcode res; + curl = curl_easy_init(); + if (curl) + { + curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_FILE, fp); + res = curl_easy_perform(curl); + long r; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r); + curl_easy_cleanup(curl); + + bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/; + fclose(fp); + + if (diskFull) + { + CFile::deleteFile(downloadPath); + throw EDiskFullError(downloadPath); + } + + if (res != CURLE_OK || r < 200 || r >= 300) + { + CFile::deleteFile(downloadPath); + nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str()); + nlSleep(1000); + continue; + } + } + else + { + nldebug("Curl initialize failed"); + fclose(fp); + CFile::deleteFile(downloadPath); + return false; + } + + // ok! + break; + } + + // read into memory :( + std::string unpackPath = filePath + ".extract." + toString(rand()); + + std::vector inBuffer; + { + CIFile inStream(downloadPath); + uint32 inSize = inStream.getFileSize(); + inBuffer.resize(inSize); + inStream.serialBuffer(&inBuffer[0], inSize); + } + CFile::deleteFile(downloadPath); + + if (inBuffer.size() < LZMA_PROPERTIES_SIZE + 8) + { + nlwarning("Invalid file size %u, too small file '%s'", inBuffer.size(), downloadPath.c_str()); + return false; + } + + // extract + { + CLzmaDecoderState state; + uint8 *pos = &inBuffer[0]; + int ret = LzmaDecodeProperties(&state.Properties, (unsigned char *)pos, LZMA_PROPERTIES_SIZE); + if (ret != 0) + { + nlwarning("Failed to decode lzma properies in file '%s'", downloadPath.c_str()); + return false; + } + + // FROM login_patch.cpp + // alloc the probs, making sure they are deleted in function exit + size_t nbProb = LzmaGetNumProbs(&state.Properties); + std::vector probs; + probs.resize(nbProb); + state.Probs = &probs[0]; + + pos += LZMA_PROPERTIES_SIZE; + + // read the output file size + size_t fileSize = 0; + for (int i = 0; i < 8; i++) + { + //Byte b; + if (pos >= &inBuffer[0] + inBuffer.size()) + { + nlerror("pos >= inBuffer.get() + inSize"); + return false; + } + fileSize |= ((UInt64)*pos++) << (8 * i); + } + + nlassert(fileSize == entry->Size); + + SizeT outProcessed = 0; + SizeT inProcessed = 0; + // allocate the output buffer :( + std::vector outBuffer; + outBuffer.resize(fileSize); + if (fileSize) + { + // decompress the file in memory + ret = LzmaDecode(&state, (unsigned char *)pos, (SizeT)(inBuffer.size() - (pos - &inBuffer[0])), &inProcessed, (unsigned char*)&outBuffer[0], (SizeT)fileSize, &outProcessed); + if (ret != 0 || outProcessed != fileSize) + { + nlwarning("Failed to decode lzma file '%s'", downloadPath.c_str()); + return false; + } + } + + CHashKey outHash = outBuffer.size() ? getSHA1(&outBuffer[0], outBuffer.size()) : CHashKey(); + if (!(outHash == entry->Hash)) + { + std::string wantHashS = entry->Hash.toString(); + std::string outHashS = outHash.toString(); + nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str()); + return false; + } + + { + COFile outStream(unpackPath); + if (fileSize) + outStream.serialBuffer(&outBuffer[0], (uint)fileSize); + } + + if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str())) + { + nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str()); + // in case downloaded from another thread + return CFile::fileExists(filePath); + } + + return true; + } +} + +bool CStreamedPackageManager::getFileSize(uint32 &fileSize, const std::string &fileName) +{ + // nldebug("Get file size for streamed file '%s'", fileName.c_str()); + TEntries::iterator it = m_Entries.find(fileName); + if (it == m_Entries.end()) + return false; + fileSize = it->second->Size; + return true; +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/tools/build_gamedata/configuration/tools.py b/code/nel/tools/build_gamedata/configuration/tools.py index 67cefdd5b..b9a74820c 100755 --- a/code/nel/tools/build_gamedata/configuration/tools.py +++ b/code/nel/tools/build_gamedata/configuration/tools.py @@ -90,6 +90,7 @@ BuildClodBankTool = "build_clod_bank" SheetsPackerTool = "sheets_packer" SheetsPackerShardTool = "sheets_packer_shard" BnpMakeTool = "bnp_make" +SnpMakeTool = "snp_make" AiBuildWmapTool = "ai_build_wmap" TgaCutTool = "tga_cut" PatchGenTool = "patch_gen" diff --git a/code/nel/tools/build_gamedata/d1_client_patch.py b/code/nel/tools/build_gamedata/d1_client_patch.py index 449d6e0b7..00eaf1aaa 100755 --- a/code/nel/tools/build_gamedata/d1_client_patch.py +++ b/code/nel/tools/build_gamedata/d1_client_patch.py @@ -51,6 +51,7 @@ printLog(log, "") # Find tools BnpMake = findTool(log, ToolDirectories, BnpMakeTool, ToolSuffix) +SnpMake = findTool(log, ToolDirectories, SnpMakeTool, ToolSuffix); PatchGen = findTool(log, ToolDirectories, PatchGenTool, ToolSuffix) printLog(log, "") @@ -90,20 +91,23 @@ else: inCategories = 1 cfg.write("\t<_Categories>\n") for category in InstallClientData: + packExt = ".bnp" + if (category["StreamedPackages"]): + packExt = ".snp" cfg.write("\t\t<_Category>\n") cfg.write("\t\t\t<_Name type=\"STRING\" value=\"" + category["Name"] + "\"/>\n") if category["UnpackTo"] != None: if category["UnpackTo"] != "": cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./" + category["UnpackTo"] + "/\"/>\n") else: - cfg.write("\t\t\t<_UnpackTo type=\"SINT32\" value=\"./\"/>\n") + cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./\"/>\n") cfg.write("\t\t\t<_IsOptional type=\"SINT32\" value=\"" + str(category["IsOptional"]) + "\"/>\n") cfg.write("\t\t\t<_IsIncremental type=\"SINT32\" value=\"" + str(category["IsIncremental"]) + "\"/>\n") for package in category["Packages"]: if (len(package[1]) > 0): cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[1][0] + "\"/>\n") else: - cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[0] + ".bnp\"/>\n") + cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[0] + packExt + "\"/>\n") for ref in category["Refs"]: cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + ref + "_.ref\"/>\n") cfg.write("\t\t\n") @@ -121,11 +125,14 @@ else: targetPath = ClientPatchDirectory + "/bnp" mkPath(log, targetPath) for category in InstallClientData: + packExt = ".bnp" + if (category["StreamedPackages"]): + packExt = ".snp" for package in category["Packages"]: printLog(log, "PACKAGE " + package[0]) sourcePath = InstallDirectory + "/" + package[0] mkPath(log, sourcePath) - targetBnp = targetPath + "/" + package[0] + ".bnp" + targetBnp = targetPath + "/" + package[0] + packExt if (len(package[1]) > 0): targetBnp = targetPath + "/" + package[1][0] printLog(log, "TARGET " + package[1][0]) @@ -135,8 +142,16 @@ else: else: needUpdateBnp = needUpdateDirNoSubdirFile(log, sourcePath, targetBnp) if (needUpdateBnp): - printLog(log, "BNP " + targetBnp) - subprocess.call([ BnpMake, "-p", sourcePath, "-o", targetBnp ] + package[1][1:]) + if (category["StreamedPackages"]): + printLog(log, "SNP " + targetBnp) + cwDir = os.getcwd().replace("\\", "/") + toolDir = os.path.dirname(Lzma).replace("\\", "/") + os.chdir(toolDir) + subprocess.call([ SnpMake, "/p", sourcePath, targetBnp, ClientPatchDirectory + "/stream" ] + package[1][1:]) + os.chdir(cwDir) + else: + printLog(log, "BNP " + targetBnp) + subprocess.call([ BnpMake, "-p", sourcePath, "-o", targetPath ] + package[1][1:]) else: printLog(log, "SKIP " + targetBnp) printLog(log, "") diff --git a/code/nel/tools/build_gamedata/d2_client_install.py b/code/nel/tools/build_gamedata/d2_client_install.py index febaef656..bb399447f 100755 --- a/code/nel/tools/build_gamedata/d2_client_install.py +++ b/code/nel/tools/build_gamedata/d2_client_install.py @@ -47,6 +47,9 @@ printLog(log, "") for category in InstallClientData: printLog(log, "CATEGORY " + category["Name"]) + packExt = ".bnp" + if (category["StreamedPackages"]): + packExt = ".snp" if (category["UnpackTo"] != None): targetPath = ClientInstallDirectory if (category["UnpackTo"] != ""): @@ -62,8 +65,8 @@ for category in InstallClientData: mkPath(log, targetPath) for package in category["Packages"]: printLog(log, "PACKAGE " + package[0]) - sourceBnp = sourcePath + "/" + package[0] + ".bnp" - targetBnp = targetPath + "/" + package[0] + ".bnp" + sourceBnp = sourcePath + "/" + package[0] + packExt + targetBnp = targetPath + "/" + package[0] + packExt if (len(package[1]) > 0): sourceBnp = sourcePath + "/" + package[1][0] targetBnp = targetPath + "/" + package[1][0] diff --git a/code/nel/tools/misc/snp_make/CMakeLists.txt b/code/nel/tools/misc/snp_make/CMakeLists.txt new file mode 100644 index 000000000..a1b5f388d --- /dev/null +++ b/code/nel/tools/misc/snp_make/CMakeLists.txt @@ -0,0 +1,9 @@ +FILE(GLOB SRC *.cpp *.h) + +ADD_EXECUTABLE(snp_make ${SRC}) + +TARGET_LINK_LIBRARIES(snp_make nelmisc) +NL_DEFAULT_PROPS(snp_make "NeL, Tools, Misc: snp_make") +NL_ADD_RUNTIME_FLAGS(snp_make) + +INSTALL(TARGETS snp_make RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT toolsmisc) diff --git a/code/nel/tools/misc/snp_make/main.cpp b/code/nel/tools/misc/snp_make/main.cpp new file mode 100644 index 000000000..61fc11810 --- /dev/null +++ b/code/nel/tools/misc/snp_make/main.cpp @@ -0,0 +1,355 @@ +// NeL - MMORPG Framework +// Copyright (C) 2010 Winch Gate Property Limited +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "nel/misc/types_nl.h" + +#include +#include + +#ifdef NL_OS_WINDOWS +# include +# include +#endif + +#include +#include + +#include "nel/misc/debug.h" +#include "nel/misc/file.h" +#include "nel/misc/path.h" +#include "nel/misc/algo.h" +#include "nel/misc/common.h" +#include "nel/misc/streamed_package.h" + +using namespace std; +using namespace NLMISC; + +// --------------------------------------------------------------------------- + +class CWildCard +{ +public: + string Expression; + bool Not; +}; +std::vector WildCards; + +std::string SourceDirectory; +std::string PackageFileName; +std::string StreamDirectory; + +CStreamedPackage Package; + +// --------------------------------------------------------------------------- + +bool keepFile (const char *fileName) +{ + uint i; + bool ifPresent = false; + bool ifTrue = false; + string file = toLower(CFile::getFilename (fileName)); + for (i=0; i [option] ... [option]\n"); + printf (" option : \n"); + printf (" -if wildcard : add the file if it matches the wilcard (at least one 'if' conditions must be met for a file to be adding)\n"); + printf (" -ifnot wildcard : add the file if it doesn't match the wilcard (all the 'ifnot' conditions must be met for a file to be adding)\n"); + printf (" Pack the directory to a snp file\n"); + printf (" snp_make /l \n"); + printf (" List the files contained in the snp file\n"); +} + +// --------------------------------------------------------------------------- + +void generateLZMA(const std::string sourceFile, const std::string &outputFile) +{ + std::string cmd="lzma e "; + cmd+=" "+sourceFile+" "+outputFile; + nlinfo("executing system command: %s",cmd.c_str()); +#ifdef NL_OS_WINDOWS + _spawnlp(_P_WAIT, "lzma.exe","lzma.exe", "e", sourceFile.c_str(), outputFile.c_str(), NULL); +#else // NL_OS_WINDOWS + sint error = system (cmd.c_str()); + if (error) + nlwarning("'%s' failed with error code %d", cmd.c_str(), error); +#endif // NL_OS_WINDOWS +} + +// --------------------------------------------------------------------------- + +uint readOptions (int nNbArg, char **ppArgs) +{ + uint i; + uint optionCount = 0; + for (i=0; i<(uint)nNbArg; i++) + { + // If ? + if ((strcmp (ppArgs[i], "-if") == 0) && ((i+1)<(uint)nNbArg)) + { + CWildCard card; + card.Expression = toLower(string(ppArgs[i+1])); + card.Not = false; + WildCards.push_back (card); + optionCount += 2; + } + // If not ? + if ((strcmp (ppArgs[i], "-ifnot") == 0) && ((i+1)<(uint)nNbArg)) + { + CWildCard card; + card.Expression = toLower(string(ppArgs[i+1])); + card.Not = true; + WildCards.push_back (card); + optionCount += 2; + } + } + return optionCount; +} + +// --------------------------------------------------------------------------- +int main (int nNbArg, char **ppArgs) +{ + NLMISC::CApplicationContext myApplicationContext; + + if (nNbArg < 3) + { + usage(); + return -1; + } + + if ((strcmp(ppArgs[1], "/p") == 0) || (strcmp(ppArgs[1], "/P") == 0) || + (strcmp(ppArgs[1], "-p") == 0) || (strcmp(ppArgs[1], "-P") == 0)) + { + if (nNbArg < 5) + { + usage(); + return -1; + } + + SourceDirectory = ppArgs[2]; + PackageFileName = ppArgs[3]; + StreamDirectory = ppArgs[4]; + readOptions(nNbArg, ppArgs); + + nldebug("Make streamed package: '%s'", PackageFileName.c_str()); + + if (CFile::fileExists(PackageFileName)) + { + nldebug("Update existing package"); + try + { + CIFile fi; + fi.open(PackageFileName); + fi.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + } + else + { + nldebug("New package"); + } + + std::vector pathContent; // contains full pathnames + std::vector nameContent; // only filename + CPath::getPathContent(SourceDirectory, true, false, true, pathContent); + nameContent.reserve(pathContent.size()); + for (std::vector::size_type i = 0; i < pathContent.size(); ++i) + { + const std::string &file = pathContent[i]; + if (keepFile(file.c_str())) + { + std::string fileName = NLMISC::toLower(CFile::getFilename(file)); + // nldebug("File: '%s' ('%s')", file.c_str(), fileName.c_str()); + nameContent.push_back(fileName); + nlassert(nameContent.size() == (i + 1)); + } + else + { + // Not included in this package + pathContent.erase(pathContent.begin() + i); + --i; + } + } + + std::vector packageIndex; // index of file in package + packageIndex.resize(pathContent.size(), -1); + + for (CStreamedPackage::TEntries::size_type i = 0; i < Package.Entries.size(); ++i) + { + const CStreamedPackage::CEntry &entry = Package.Entries[i]; + + sint foundIndex = -1; // find index in found file list + for (std::vector::size_type j = 0; j < pathContent.size(); ++j) + { + if (nameContent[j] == entry.Name) + { + foundIndex = j; + break; + } + } + + if (foundIndex < 0) + { + nlinfo("File no longer exists: '%s'", entry.Name.c_str()); + Package.Entries.erase(Package.Entries.begin() + i); + --i; + } + else + { + // File still exists, map it + packageIndex[foundIndex] = i; + } + } + + for (std::vector::size_type i = 0; i < pathContent.size(); ++i) + { + sint pidx = packageIndex[i]; + const std::string &name = nameContent[i]; + const std::string &path = pathContent[i]; + + if (pidx < 0) + { + nlinfo("File added: '%s'", name.c_str()); + pidx = Package.Entries.size(); + Package.Entries.push_back(CStreamedPackage::CEntry()); + Package.Entries[pidx].Name = name; + Package.Entries[pidx].LastModified = 0; + Package.Entries[pidx].Size = 0; + } + else + { + nlinfo("File check for changes: '%s'", name.c_str()); + } + + CStreamedPackage::CEntry &entry = Package.Entries[pidx]; + + std::string targetLzmaOld; // in case lzma wasn't made make sure it exists a second run + CStreamedPackage::makePath(targetLzmaOld, entry.Hash); + targetLzmaOld = StreamDirectory + targetLzmaOld + ".lzma"; + + uint32 lastModified = CFile::getFileModificationDate(path); + uint32 fileSize = CFile::getFileSize(path); + if (lastModified > entry.LastModified || fileSize != entry.Size || !CFile::fileExists(targetLzmaOld)) + { + entry.LastModified = lastModified; + + nlinfo("Calculate file hash"); + CHashKey hash = getSHA1(path, true); + /*nldebug("%s", hash.toString().c_str()); + std::string hashPath; + CStreamedPackage::makePath(hashPath, hash); + nldebug("%s", hashPath.c_str());*/ + + if (hash == entry.Hash && fileSize == entry.Size) + { + // File has not changed + } + else + { + nlinfo("File changed"); + entry.Hash = hash; + entry.Size = fileSize; + } + + std::string targetLzma; // in case lzma wasn't made make sure it exists a second run + CStreamedPackage::makePath(targetLzma, entry.Hash); + targetLzma = StreamDirectory + targetLzma + ".lzma"; + + if (!CFile::fileExists(targetLzma)) + { + // make the compressed file + nlinfo("%s -> %s", path.c_str(), targetLzma.c_str()); + CFile::createDirectoryTree(CFile::getPath(targetLzma)); + generateLZMA(path, targetLzma); + } + } + } + + try + { + nldebug("Store package '%s'", PackageFileName.c_str()); + COFile fo; + fo.open(PackageFileName); + fo.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + + return 0; + } + + if ((strcmp(ppArgs[1], "/l") == 0) || (strcmp(ppArgs[1], "/L") == 0) || + (strcmp(ppArgs[1], "-l") == 0) || (strcmp(ppArgs[1], "-L") == 0)) + { + PackageFileName = ppArgs[2]; + if (!CFile::fileExists(PackageFileName)) + { + nlwarning("ERROR (snp_make) : package doesn't exist: '%s'", PackageFileName.c_str()); + return -1; + } + + try + { + CIFile fi; + fi.open(PackageFileName); + fi.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + + for (CStreamedPackage::TEntries::const_iterator it(Package.Entries.begin()), end(Package.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + + printf("List files in '%s'", PackageFileName.c_str()); + printf("%s { Hash: '%s', Size: '%u', LastModified: '%u' }", entry.Name.c_str(), entry.Hash.toString().c_str(), entry.Size, entry.LastModified); + } + + return 0; + } + + usage (); + return -1; +} diff --git a/code/ryzom/client/src/client_cfg.cpp b/code/ryzom/client/src/client_cfg.cpp index 7f0222088..132b8bef9 100644 --- a/code/ryzom/client/src/client_cfg.cpp +++ b/code/ryzom/client/src/client_cfg.cpp @@ -468,6 +468,8 @@ CClientConfig::CClientConfig() ColorShout = CRGBA(150,0,0,255); // Default Shout color. ColorTalk = CRGBA(255,255,255,255); // Default Talk color. + StreamedPackagePath = "stream"; + // MP3 player MediaPlayerDirectory = "music"; MediaPlayerAutoPlay = false; @@ -1281,18 +1283,25 @@ void CClientConfig::setValues() ////////// // MISC // + // Pre Data Path. READ_STRINGVECTOR_FV(PreDataPath); // Data Path. READ_STRINGVECTOR_FV(DataPath); - // List of files that trigger R2ED reload when touched - READ_STRINGVECTOR_FV(R2EDReloadFiles); - // Data Path no recurse. READ_STRINGVECTOR_FV(DataPathNoRecurse); + // Streamed package path + READ_STRING_FV(StreamedPackagePath); + + // Streamed package hosts + READ_STRINGVECTOR_FV(StreamedPackageHosts); + + // List of files that trigger R2ED reload when touched + READ_STRINGVECTOR_FV(R2EDReloadFiles); + // Update packed sheet Path READ_STRINGVECTOR_FV(UpdatePackedSheetPath); diff --git a/code/ryzom/client/src/client_cfg.h b/code/ryzom/client/src/client_cfg.h index 7c1fa2ad6..7d2dbac37 100644 --- a/code/ryzom/client/src/client_cfg.h +++ b/code/ryzom/client/src/client_cfg.h @@ -376,6 +376,10 @@ struct CClientConfig std::vector DataPath; /// Data Path no recurse. std::vector DataPathNoRecurse; + /// Streamed package path + std::string StreamedPackagePath; + /// Streamed package hosts + std::vector StreamedPackageHosts; // TODO: From 'domain' SQL table /// Update packed sheet Path. std::vector UpdatePackedSheetPath; /// True if we want the packed sheet to be updated if needed diff --git a/code/ryzom/client/src/init.cpp b/code/ryzom/client/src/init.cpp index abbf8b6b4..1f9c10b34 100644 --- a/code/ryzom/client/src/init.cpp +++ b/code/ryzom/client/src/init.cpp @@ -34,6 +34,7 @@ #include "nel/misc/system_info.h" #include "nel/misc/block_memory.h" #include "nel/misc/system_utils.h" +#include "nel/misc/streamed_package_manager.h" #include "nel/misc/cmd_args.h" // 3D Interface. #include "nel/3d/bloom_effect.h" @@ -157,6 +158,8 @@ bool LastScreenSaverEnabled = false; extern void registerInterfaceElements(); extern CContinentManager ContinentMngr; +extern NLMISC::CCmdArgs Args; + // Tips of the day count #define RZ_NUM_TIPS 17 ucstring TipsOfTheDay; @@ -660,8 +663,6 @@ void initStereoDisplayDevice() } // we want to get executable directory -extern NLMISC::CCmdArgs Args; - static void addPaths(IProgressCallback &progress, const std::vector &paths, bool recurse) { // all prefixes for paths @@ -736,6 +737,14 @@ static void addPaths(IProgressCallback &progress, const std::vector } } +void initStreamedPackageManager(NLMISC::IProgressCallback &progress) +{ + CStreamedPackageManager &spm = CStreamedPackageManager::getInstance(); + spm.Path = ClientCfg.StreamedPackagePath; + for (uint i = 0; i < ClientCfg.StreamedPackageHosts.size(); i++) + spm.Hosts.push_back(ClientCfg.StreamedPackageHosts[i]); +} + void addSearchPaths(IProgressCallback &progress) { // Add search path of UI addon. Allow only a subset of files. @@ -976,6 +985,7 @@ void prelogInit() CPath::remapExtension ("png", "tga", true); FPU_CHECKER_ONCE + initStreamedPackageManager(ProgressBar); addPreDataPaths(ProgressBar); FPU_CHECKER_ONCE diff --git a/code/ryzom/client/src/init.h b/code/ryzom/client/src/init.h index f179bf13d..4ef738828 100644 --- a/code/ryzom/client/src/init.h +++ b/code/ryzom/client/src/init.h @@ -36,6 +36,7 @@ void prelogInit(); // Initialize the application after login step void postlogInit(); +void initStreamedPackageManager(NLMISC::IProgressCallback &progress); void addSearchPaths(NLMISC::IProgressCallback &progress); void addPreDataPaths(NLMISC::IProgressCallback &progress); diff --git a/code/ryzom/client/src/login.cpp b/code/ryzom/client/src/login.cpp index 1eb6ec867..5265f801e 100644 --- a/code/ryzom/client/src/login.cpp +++ b/code/ryzom/client/src/login.cpp @@ -28,6 +28,7 @@ #include "nel/misc/thread.h" #include "nel/misc/big_file.h" #include "nel/misc/system_utils.h" +#include "nel/misc/streamed_package_manager.h" #include "nel/net/tcp_sock.h" #include "nel/3d/u_driver.h" @@ -1685,6 +1686,7 @@ void initPatch() CBGDownloaderAccess::getInstance().startTask(taskDesc, string(), true /* showDownloader */); // no command line since bg downloader should already be started // release lock on bnp, so that they can be written NLMISC::CBigFile::getInstance().removeAll(); + NLMISC::CStreamedPackageManager::getInstance().unloadAll(); } NLGUI::CDBManager::getInstance()->getDbProp("UI:VARIABLES:SCREEN")->setValue32(UI_VARIABLES_SCREEN_PATCHING); diff --git a/code/ryzom/client/src/login_patch.cpp b/code/ryzom/client/src/login_patch.cpp index f4e7e237f..84d5247fc 100644 --- a/code/ryzom/client/src/login_patch.cpp +++ b/code/ryzom/client/src/login_patch.cpp @@ -1882,7 +1882,6 @@ bool CPatchManager::bnpUnpack(const string &srcBigfile, const string &dstPath, v return false; } - // Unpack if (!bnpFile.unpack(DestPath)) return false; @@ -2495,7 +2494,7 @@ void CPatchThread::run() CPatchManager::SFileToPatch &rFTP = AllFilesToPatch[i]; string ext = NLMISC::CFile::getExtension(rFTP.FileName); - if (ext == "bnp") + if (ext == "bnp" || ext == "snp") { float oldCurrentFilePatched = CurrentFilePatched; processFile (rFTP); diff --git a/code/ryzom/client/src/release.cpp b/code/ryzom/client/src/release.cpp index 97265dcaa..be34d9ba8 100644 --- a/code/ryzom/client/src/release.cpp +++ b/code/ryzom/client/src/release.cpp @@ -25,6 +25,7 @@ #include "nel/misc/debug.h" #include "nel/misc/async_file_manager.h" #include "nel/misc/system_utils.h" +#include "nel/misc/streamed_package_manager.h" // 3D Interface. #include "nel/3d/bloom_effect.h" #include "nel/3d/fxaa.h" @@ -666,6 +667,7 @@ void release() R2::CObjectSerializer::releaseInstance(); NLMISC::CBigFile::getInstance().removeAll(); NLMISC::CBigFile::releaseInstance(); + NLMISC::CStreamedPackageManager::releaseInstance(); NL3D::CFastHLSModifier::releaseInstance(); CLandscapePolyDrawer::releaseInstance(); NL3D::CParticleSystemShape::releaseInstance(); diff --git a/code/ryzom/client/src/streamable_ig.cpp b/code/ryzom/client/src/streamable_ig.cpp index eaaea24f5..3216b2ee1 100644 --- a/code/ryzom/client/src/streamable_ig.cpp +++ b/code/ryzom/client/src/streamable_ig.cpp @@ -93,8 +93,10 @@ CStreamableIG::~CStreamableIG() H_AUTO_USE(RZ_StremableIG) if (!_Linked) { + #ifdef NL_DEBUG if(!ClientCfg.Light) nlwarning("Loading async %p", this); + #endif #ifdef NL_DEBUG //nlinfo("Loading async : %s", Name.c_str()); #endif diff --git a/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp b/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp index db8481453..b176b496c 100644 --- a/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp +++ b/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp @@ -296,7 +296,7 @@ void CScreenshotIslands::searchIslandsBorders() zonelFiles.clear(); string bnpFileName = itCont->first + ".bnp"; - CBigFile::getInstance().list(bnpFileName.c_str(), filenames); + CBigFile::getInstance().list(bnpFileName.c_str(), filenames); // FIXME FIXME NOT READING FROM BNP! for(uint i=0; i fileList; NLMISC::CPath::getPathContent(_BnpDirectory,false,false,true,fileList); for (uint32 i=0;i