Merge branch 'feature/streamed-package' into feature/streamed-package-update

code/CMakeLists.txt

@@ -48,15 +48,15 @@ CHECK_OUT_OF_SOURCE()
 IF(CMAKE_VERSION VERSION_GREATER "2.8.10")
   STRING(TIMESTAMP CURRENT_YEAR "%Y")
 ELSE()
-  SET(CURRENT_YEAR "2019")
+  SET(CURRENT_YEAR "2016")
 ENDIF()
 
 CMAKE_MINIMUM_REQUIRED(VERSION 2.6)
 PROJECT(RyzomCore CXX C)
 SET(NL_VERSION_MAJOR 1)
 SET(NL_VERSION_MINOR 0)
-SET(NL_VERSION_PATCH 2)
+SET(NL_VERSION_PATCH 1)
-SET(YEAR "2001-${CURRENT_YEAR}")
+SET(YEAR "2004-${CURRENT_YEAR}")
 SET(AUTHOR "Winch Gate and The Ryzom Core Community")
 
 SET(RYZOM_VERSION_MAJOR 3)

code/nel/3rdparty/CMakeLists.txt

@@ -0,0 +1,2 @@
+SET(SEVENZIP_LIBRARY "nel_sevenzip")
+ADD_SUBDIRECTORY(seven_zip)

code/nel/3rdparty/seven_zip/7zBuffer.cpp

@@ -0,0 +1,29 @@
+/* 7zBuffer.c */
+
+#include "7zBuffer.h"
+#include "7zAlloc.h"
+
+void SzByteBufferInit(CSzByteBuffer *buffer)
+{
+  buffer->Capacity = 0;
+  buffer->Items = 0;
+}
+
+int SzByteBufferCreate(CSzByteBuffer *buffer, size_t newCapacity, void * (*allocFunc)(size_t size))
+{
+  buffer->Capacity = newCapacity;
+  if (newCapacity == 0)
+  {
+    buffer->Items = 0;
+    return 1;
+  }
+  buffer->Items = (Byte *)allocFunc(newCapacity);
+  return (buffer->Items != 0);
+}
+
+void SzByteBufferFree(CSzByteBuffer *buffer, void (*freeFunc)(void *))
+{
+  freeFunc(buffer->Items);
+  buffer->Items = 0;
+  buffer->Capacity = 0;
+}

code/nel/3rdparty/seven_zip/7zBuffer.h

@@ -0,0 +1,19 @@
+/* 7zBuffer.h */
+
+#ifndef __7Z_BUFFER_H
+#define __7Z_BUFFER_H
+
+#include <stddef.h>
+#include "7zTypes.h"
+
+typedef struct _CSzByteBuffer
+{    
+	size_t Capacity;
+  Byte *Items;
+}CSzByteBuffer;
+
+void SzByteBufferInit(CSzByteBuffer *buffer);
+int SzByteBufferCreate(CSzByteBuffer *buffer, size_t newCapacity, void * (*allocFunc)(size_t size));
+void SzByteBufferFree(CSzByteBuffer *buffer, void (*freeFunc)(void *));
+
+#endif

code/nel/3rdparty/seven_zip/7zDecode.cpp

@@ -0,0 +1,150 @@
+/* 7zDecode.c */
+
+#include "7zDecode.h"
+#ifdef _SZ_ONE_DIRECTORY
+#include "LzmaDecode.h"
+#else
+#include "../../Compress/LZMA_C/LzmaDecode.h"
+#endif
+
+CMethodID k_Copy = { { 0x0 }, 1 };
+CMethodID k_LZMA = { { 0x3, 0x1, 0x1 }, 3 };
+
+#ifdef _LZMA_IN_CB
+
+typedef struct _CLzmaInCallbackImp
+{
+  ILzmaInCallback InCallback;
+  ISzInStream *InStream;
+  size_t Size;
+} CLzmaInCallbackImp;
+
+int LzmaReadImp(void *object, const unsigned char **buffer, SizeT *size)
+{
+  CLzmaInCallbackImp *cb = (CLzmaInCallbackImp *)object;
+  size_t processedSize;
+  SZ_RESULT res;
+  *size = 0;
+  res = cb->InStream->Read((void *)cb->InStream, (void **)buffer, cb->Size, &processedSize);
+  *size = (SizeT)processedSize;
+  if (processedSize > cb->Size)
+    return (int)SZE_FAIL;
+  cb->Size -= processedSize;
+  if (res == SZ_OK)
+    return 0;
+  return (int)res;
+}
+
+#endif
+
+SZ_RESULT SzDecode(const CFileSize *packSizes, const CFolder *folder,
+    #ifdef _LZMA_IN_CB
+    ISzInStream *inStream,
+    #else
+    const Byte *inBuffer,
+    #endif
+    Byte *outBuffer, size_t outSize, 
+    size_t *outSizeProcessed, ISzAlloc *allocMain)
+{
+  UInt32 si;
+  size_t inSize = 0;
+  CCoderInfo *coder;
+  if (folder->NumPackStreams != 1)
+    return SZE_NOTIMPL;
+  if (folder->NumCoders != 1)
+    return SZE_NOTIMPL;
+  coder = folder->Coders;
+  *outSizeProcessed = 0;
+
+  for (si = 0; si < folder->NumPackStreams; si++)
+    inSize += (size_t)packSizes[si];
+
+  if (AreMethodsEqual(&coder->MethodID, &k_Copy))
+  {
+    size_t i;
+    if (inSize != outSize)
+      return SZE_DATA_ERROR;
+    #ifdef _LZMA_IN_CB
+    for (i = 0; i < inSize;)
+    {
+      size_t j;
+      Byte *inBuffer;
+      size_t bufferSize;
+      RINOK(inStream->Read((void *)inStream,  (void **)&inBuffer, inSize - i, &bufferSize));
+      if (bufferSize == 0)
+        return SZE_DATA_ERROR;
+      if (bufferSize > inSize - i)
+        return SZE_FAIL;
+      *outSizeProcessed += bufferSize;
+      for (j = 0; j < bufferSize && i < inSize; j++, i++)
+        outBuffer[i] = inBuffer[j];
+    }
+    #else
+    for (i = 0; i < inSize; i++)
+      outBuffer[i] = inBuffer[i];
+    *outSizeProcessed = inSize;
+    #endif
+    return SZ_OK;
+  }
+
+  if (AreMethodsEqual(&coder->MethodID, &k_LZMA))
+  {
+    #ifdef _LZMA_IN_CB
+    CLzmaInCallbackImp lzmaCallback;
+    #else
+    SizeT inProcessed;
+    #endif
+
+    CLzmaDecoderState state;  /* it's about 24-80 bytes structure, if int is 32-bit */
+    int result;
+    SizeT outSizeProcessedLoc;
+
+    #ifdef _LZMA_IN_CB
+    lzmaCallback.Size = inSize;
+    lzmaCallback.InStream = inStream;
+    lzmaCallback.InCallback.Read = LzmaReadImp;
+    #endif
+
+    if (LzmaDecodeProperties(&state.Properties, coder->Properties.Items, 
+        (SizeT)coder->Properties.Capacity) != LZMA_RESULT_OK)
+      return SZE_FAIL;
+
+    state.Probs = (CProb *)allocMain->Alloc(LzmaGetNumProbs(&state.Properties) * sizeof(CProb));
+    if (state.Probs == 0)
+      return SZE_OUTOFMEMORY;
+
+    #ifdef _LZMA_OUT_READ
+    if (state.Properties.DictionarySize == 0)
+      state.Dictionary = 0;
+    else
+    {
+      state.Dictionary = (unsigned char *)allocMain->Alloc(state.Properties.DictionarySize);
+      if (state.Dictionary == 0)
+      {
+        allocMain->Free(state.Probs);
+        return SZE_OUTOFMEMORY;
+      }
+    }
+    LzmaDecoderInit(&state);
+    #endif
+
+    result = LzmaDecode(&state,
+        #ifdef _LZMA_IN_CB
+        &lzmaCallback.InCallback,
+        #else
+        inBuffer, (SizeT)inSize, &inProcessed,
+        #endif
+        outBuffer, (SizeT)outSize, &outSizeProcessedLoc);
+    *outSizeProcessed = (size_t)outSizeProcessedLoc;
+    allocMain->Free(state.Probs);
+    #ifdef _LZMA_OUT_READ
+    allocMain->Free(state.Dictionary);
+    #endif
+    if (result == LZMA_RESULT_DATA_ERROR)
+      return SZE_DATA_ERROR;
+    if (result != LZMA_RESULT_OK)
+      return SZE_FAIL;
+    return SZ_OK;
+  }
+  return SZE_NOTIMPL;
+}

code/nel/3rdparty/seven_zip/7zDecode.h

@@ -0,0 +1,21 @@
+/* 7zDecode.h */
+
+#ifndef __7Z_DECODE_H
+#define __7Z_DECODE_H
+
+#include "7zItem.h"
+#include "7zAlloc.h"
+#ifdef _LZMA_IN_CB
+#include "7zIn.h"
+#endif
+
+SZ_RESULT SzDecode(const CFileSize *packSizes, const CFolder *folder,
+    #ifdef _LZMA_IN_CB
+    ISzInStream *stream,
+    #else
+    const Byte *inBuffer,
+    #endif
+    Byte *outBuffer, size_t outSize, 
+    size_t *outSizeProcessed, ISzAlloc *allocMain);
+
+#endif

code/nel/3rdparty/seven_zip/7zExtract.cpp

@@ -0,0 +1,116 @@
+/* 7zExtract.c */
+
+#include "7zExtract.h"
+#include "7zDecode.h"
+#include "7zCrc.h"
+
+SZ_RESULT SzExtract(
+    ISzInStream *inStream, 
+    CArchiveDatabaseEx *db,
+    UInt32 fileIndex,
+    UInt32 *blockIndex,
+    Byte **outBuffer, 
+    size_t *outBufferSize,
+    size_t *offset, 
+    size_t *outSizeProcessed, 
+    ISzAlloc *allocMain,
+    ISzAlloc *allocTemp)
+{
+  UInt32 folderIndex = db->FileIndexToFolderIndexMap[fileIndex];
+  SZ_RESULT res = SZ_OK;
+  *offset = 0;
+  *outSizeProcessed = 0;
+  if (folderIndex == (UInt32)-1)
+  {
+    allocMain->Free(*outBuffer);
+    *blockIndex = folderIndex;
+    *outBuffer = 0;
+    *outBufferSize = 0;
+    return SZ_OK;
+  }
+
+  if (*outBuffer == 0 || *blockIndex != folderIndex)
+  {
+    CFolder *folder = db->Database.Folders + folderIndex;
+    CFileSize unPackSize = SzFolderGetUnPackSize(folder);
+    #ifndef _LZMA_IN_CB
+    CFileSize packSize = SzArDbGetFolderFullPackSize(db, folderIndex);
+    Byte *inBuffer = 0;
+    size_t processedSize;
+    #endif
+    *blockIndex = folderIndex;
+    allocMain->Free(*outBuffer);
+    *outBuffer = 0;
+    
+    RINOK(inStream->Seek(inStream, SzArDbGetFolderStreamPos(db, folderIndex, 0)));
+    
+    #ifndef _LZMA_IN_CB
+    if (packSize != 0)
+    {
+      inBuffer = (Byte *)allocTemp->Alloc((size_t)packSize);
+      if (inBuffer == 0)
+        return SZE_OUTOFMEMORY;
+    }
+    res = inStream->Read(inStream, inBuffer, (size_t)packSize, &processedSize);
+    if (res == SZ_OK && processedSize != (size_t)packSize)
+      res = SZE_FAIL;
+    #endif
+    if (res == SZ_OK)
+    {
+      *outBufferSize = (size_t)unPackSize;
+      if (unPackSize != 0)
+      {
+        *outBuffer = (Byte *)allocMain->Alloc((size_t)unPackSize);
+        if (*outBuffer == 0)
+          res = SZE_OUTOFMEMORY;
+      }
+      if (res == SZ_OK)
+      {
+        size_t outRealSize;
+        res = SzDecode(db->Database.PackSizes + 
+          db->FolderStartPackStreamIndex[folderIndex], folder, 
+          #ifdef _LZMA_IN_CB
+          inStream,
+          #else
+          inBuffer, 
+          #endif
+          *outBuffer, (size_t)unPackSize, &outRealSize, allocTemp);
+        if (res == SZ_OK)
+        {
+          if (outRealSize == (size_t)unPackSize)
+          {
+            if (folder->UnPackCRCDefined)
+            {
+              if (!CrcVerifyDigest(folder->UnPackCRC, *outBuffer, (size_t)unPackSize))
+                res = SZE_FAIL;
+            }
+          }
+          else
+            res = SZE_FAIL;
+        }
+      }
+    }
+    #ifndef _LZMA_IN_CB
+    allocTemp->Free(inBuffer);
+    #endif
+  }
+  if (res == SZ_OK)
+  {
+    UInt32 i; 
+    CFileItem *fileItem = db->Database.Files + fileIndex;
+    *offset = 0;
+    for(i = db->FolderStartFileIndex[folderIndex]; i < fileIndex; i++)
+      *offset += (UInt32)db->Database.Files[i].Size;
+    *outSizeProcessed = (size_t)fileItem->Size;
+    if (*offset + *outSizeProcessed > *outBufferSize)
+      return SZE_FAIL;
+    {
+      if (fileItem->IsFileCRCDefined)
+      {
+        if (!CrcVerifyDigest(fileItem->FileCRC, *outBuffer + *offset, *outSizeProcessed))
+          res = SZE_FAIL;
+      }
+    }
+  }
+  return res;
+}

code/nel/3rdparty/seven_zip/7zExtract.h

@@ -0,0 +1,40 @@
+/* 7zExtract.h */
+
+#ifndef __7Z_EXTRACT_H
+#define __7Z_EXTRACT_H
+
+#include "7zIn.h"
+
+/*
+  SzExtract extracts file from archive
+
+  *outBuffer must be 0 before first call for each new archive. 
+
+  Extracting cache:
+    If you need to decompress more than one file, you can send 
+    these values from previous call:
+      *blockIndex, 
+      *outBuffer, 
+      *outBufferSize
+    You can consider "*outBuffer" as cache of solid block. If your archive is solid, 
+    it will increase decompression speed.
+  
+    If you use external function, you can declare these 3 cache variables 
+    (blockIndex, outBuffer, outBufferSize) as static in that external function.
+    
+    Free *outBuffer and set *outBuffer to 0, if you want to flush cache.
+*/
+
+SZ_RESULT SzExtract(
+    ISzInStream *inStream, 
+    CArchiveDatabaseEx *db,
+    UInt32 fileIndex,         /* index of file */
+    UInt32 *blockIndex,       /* index of solid block */
+    Byte **outBuffer,         /* pointer to pointer to output buffer (allocated with allocMain) */
+    size_t *outBufferSize,    /* buffer size for output buffer */
+    size_t *offset,           /* offset of stream for required file in *outBuffer */
+    size_t *outSizeProcessed, /* size of file in *outBuffer */
+    ISzAlloc *allocMain,
+    ISzAlloc *allocTemp);
+
+#endif

code/nel/3rdparty/seven_zip/7zHeader.cpp

@@ -0,0 +1,5 @@
+/*  7zHeader.c */
+
+#include "7zHeader.h"
+
+Byte k7zSignature[k7zSignatureSize] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C};

code/nel/3rdparty/seven_zip/7zHeader.h

@@ -0,0 +1,55 @@
+/* 7zHeader.h */
+
+#ifndef __7Z_HEADER_H
+#define __7Z_HEADER_H
+
+#include "7zTypes.h"
+
+#define k7zSignatureSize 6
+extern Byte k7zSignature[k7zSignatureSize];
+
+#define k7zMajorVersion 0
+
+#define k7zStartHeaderSize 0x20
+
+enum EIdEnum
+{
+  k7zIdEnd,
+    
+  k7zIdHeader,
+    
+  k7zIdArchiveProperties,
+    
+  k7zIdAdditionalStreamsInfo,
+  k7zIdMainStreamsInfo,
+  k7zIdFilesInfo,
+  
+  k7zIdPackInfo,
+  k7zIdUnPackInfo,
+  k7zIdSubStreamsInfo,
+  
+  k7zIdSize,
+  k7zIdCRC,
+  
+  k7zIdFolder,
+  
+  k7zIdCodersUnPackSize,
+  k7zIdNumUnPackStream,
+  
+  k7zIdEmptyStream,
+  k7zIdEmptyFile,
+  k7zIdAnti,
+  
+  k7zIdName,
+  k7zIdCreationTime,
+  k7zIdLastAccessTime,
+  k7zIdLastWriteTime,
+  k7zIdWinAttributes,
+  k7zIdComment,
+  
+  k7zIdEncodedHeader,
+  
+  k7zIdStartPos
+};
+
+#endif

code/nel/3rdparty/seven_zip/7zIn.h

@@ -0,0 +1,55 @@
+/* 7zIn.h */
+
+#ifndef __7Z_IN_H
+#define __7Z_IN_H
+
+#include "7zHeader.h"
+#include "7zItem.h"
+#include "7zAlloc.h"
+ 
+typedef struct _CInArchiveInfo
+{
+  CFileSize StartPositionAfterHeader; 
+  CFileSize DataStartPosition;
+}CInArchiveInfo;
+
+typedef struct _CArchiveDatabaseEx
+{
+  CArchiveDatabase Database;
+  CInArchiveInfo ArchiveInfo;
+  UInt32 *FolderStartPackStreamIndex;
+  CFileSize *PackStreamStartPositions;
+  UInt32 *FolderStartFileIndex;
+  UInt32 *FileIndexToFolderIndexMap;
+}CArchiveDatabaseEx;
+
+void SzArDbExInit(CArchiveDatabaseEx *db);
+void SzArDbExFree(CArchiveDatabaseEx *db, void (*freeFunc)(void *));
+CFileSize SzArDbGetFolderStreamPos(CArchiveDatabaseEx *db, UInt32 folderIndex, UInt32 indexInFolder);
+CFileSize SzArDbGetFolderFullPackSize(CArchiveDatabaseEx *db, UInt32 folderIndex);
+
+typedef struct _ISzInStream
+{
+  #ifdef _LZMA_IN_CB
+  SZ_RESULT (*Read)(
+      void *object,           /* pointer to ISzInStream itself */
+      void **buffer,          /* out: pointer to buffer with data */
+      size_t maxRequiredSize, /* max required size to read */
+      size_t *processedSize); /* real processed size. 
+                                 processedSize can be less than maxRequiredSize.
+                                 If processedSize == 0, then there are no more 
+                                 bytes in stream. */
+  #else
+  SZ_RESULT (*Read)(void *object, void *buffer, size_t size, size_t *processedSize);
+  #endif
+  SZ_RESULT (*Seek)(void *object, CFileSize pos);
+} ISzInStream;
+
+ 
+int SzArchiveOpen(
+    ISzInStream *inStream, 
+    CArchiveDatabaseEx *db,
+    ISzAlloc *allocMain, 
+    ISzAlloc *allocTemp);
+ 
+#endif

code/nel/3rdparty/seven_zip/7zItem.cpp

@@ -0,0 +1,133 @@
+/* 7zItem.c */
+
+#include "7zItem.h"
+#include "7zAlloc.h"
+
+void SzCoderInfoInit(CCoderInfo *coder)
+{
+  SzByteBufferInit(&coder->Properties);
+}
+
+void SzCoderInfoFree(CCoderInfo *coder, void (*freeFunc)(void *p))
+{
+  SzByteBufferFree(&coder->Properties, freeFunc);
+  SzCoderInfoInit(coder);
+}
+
+void SzFolderInit(CFolder *folder)
+{
+  folder->NumCoders = 0;
+  folder->Coders = 0;
+  folder->NumBindPairs = 0;
+  folder->BindPairs = 0;
+  folder->NumPackStreams = 0;
+  folder->PackStreams = 0;
+  folder->UnPackSizes = 0;
+  folder->UnPackCRCDefined = 0;
+  folder->UnPackCRC = 0;
+  folder->NumUnPackStreams = 0;
+}
+
+void SzFolderFree(CFolder *folder, void (*freeFunc)(void *p))
+{
+  UInt32 i;
+  for (i = 0; i < folder->NumCoders; i++)
+    SzCoderInfoFree(&folder->Coders[i], freeFunc);
+  freeFunc(folder->Coders);
+  freeFunc(folder->BindPairs);
+  freeFunc(folder->PackStreams);
+  freeFunc(folder->UnPackSizes);
+  SzFolderInit(folder);
+}
+
+UInt32 SzFolderGetNumOutStreams(CFolder *folder)
+{
+  UInt32 result = 0;
+  UInt32 i;
+  for (i = 0; i < folder->NumCoders; i++)
+    result += folder->Coders[i].NumOutStreams;
+  return result;
+}
+
+int SzFolderFindBindPairForInStream(CFolder *folder, UInt32 inStreamIndex)
+{
+  UInt32 i;
+  for(i = 0; i < folder->NumBindPairs; i++)
+    if (folder->BindPairs[i].InIndex == inStreamIndex)
+      return i;
+  return -1;
+}
+
+
+int SzFolderFindBindPairForOutStream(CFolder *folder, UInt32 outStreamIndex)
+{
+  UInt32 i;
+  for(i = 0; i < folder->NumBindPairs; i++)
+    if (folder->BindPairs[i].OutIndex == outStreamIndex)
+      return i;
+  return -1;
+}
+
+CFileSize SzFolderGetUnPackSize(CFolder *folder)
+{ 
+  int i = (int)SzFolderGetNumOutStreams(folder);
+  if (i == 0)
+    return 0;
+  for (i--; i >= 0; i--)
+    if (SzFolderFindBindPairForOutStream(folder, i) < 0)
+      return folder->UnPackSizes[i];
+  /* throw 1; */
+  return 0;
+}
+
+/*
+int FindPackStreamArrayIndex(int inStreamIndex) const
+{
+  for(int i = 0; i < PackStreams.Size(); i++)
+  if (PackStreams[i] == inStreamIndex)
+    return i;
+  return -1;
+}
+*/
+
+void SzFileInit(CFileItem *fileItem)
+{
+  fileItem->IsFileCRCDefined = 0;
+  fileItem->HasStream = 1;
+  fileItem->IsDirectory = 0;
+  fileItem->IsAnti = 0;
+  fileItem->Name = 0;
+}
+
+void SzFileFree(CFileItem *fileItem, void (*freeFunc)(void *p))
+{
+  freeFunc(fileItem->Name);
+  SzFileInit(fileItem);
+}
+
+void SzArchiveDatabaseInit(CArchiveDatabase *db)
+{
+  db->NumPackStreams = 0;
+  db->PackSizes = 0;
+  db->PackCRCsDefined = 0;
+  db->PackCRCs = 0;
+  db->NumFolders = 0;
+  db->Folders = 0;
+  db->NumFiles = 0;
+  db->Files = 0;
+}
+
+void SzArchiveDatabaseFree(CArchiveDatabase *db, void (*freeFunc)(void *))
+{
+  UInt32 i;
+  for (i = 0; i < db->NumFolders; i++)
+    SzFolderFree(&db->Folders[i], freeFunc);
+  for (i = 0; i < db->NumFiles; i++)
+    SzFileFree(&db->Files[i], freeFunc);
+  freeFunc(db->PackSizes);
+  freeFunc(db->PackCRCsDefined);
+  freeFunc(db->PackCRCs);
+  freeFunc(db->Folders);
+  freeFunc(db->Files);
+  SzArchiveDatabaseInit(db);
+}

code/nel/3rdparty/seven_zip/7zItem.h

@@ -0,0 +1,90 @@
+/* 7zItem.h */
+
+#ifndef __7Z_ITEM_H
+#define __7Z_ITEM_H
+
+#include "7zMethodID.h"
+#include "7zHeader.h"
+#include "7zBuffer.h"
+
+typedef struct _CCoderInfo
+{
+  UInt32 NumInStreams;
+  UInt32 NumOutStreams;
+  CMethodID MethodID;
+  CSzByteBuffer Properties;
+}CCoderInfo;
+
+void SzCoderInfoInit(CCoderInfo *coder);
+void SzCoderInfoFree(CCoderInfo *coder, void (*freeFunc)(void *p));
+
+typedef struct _CBindPair
+{
+  UInt32 InIndex;
+  UInt32 OutIndex;
+}CBindPair;
+
+typedef struct _CFolder
+{
+  UInt32 NumCoders;
+  CCoderInfo *Coders;
+  UInt32 NumBindPairs;
+  CBindPair *BindPairs;
+  UInt32 NumPackStreams; 
+  UInt32 *PackStreams;
+  CFileSize *UnPackSizes;
+  int UnPackCRCDefined;
+  UInt32 UnPackCRC;
+
+  UInt32 NumUnPackStreams;
+}CFolder;
+
+void SzFolderInit(CFolder *folder);
+CFileSize SzFolderGetUnPackSize(CFolder *folder);
+int SzFolderFindBindPairForInStream(CFolder *folder, UInt32 inStreamIndex);
+UInt32 SzFolderGetNumOutStreams(CFolder *folder);
+CFileSize SzFolderGetUnPackSize(CFolder *folder);
+
+/* #define CArchiveFileTime UInt64 */
+
+typedef struct _CFileItem
+{
+  /*
+  CArchiveFileTime LastWriteTime;
+  CFileSize StartPos;
+  UInt32 Attributes; 
+  */
+  CFileSize Size;
+  UInt32 FileCRC;
+  char *Name;
+
+  Byte IsFileCRCDefined;
+  Byte HasStream;
+  Byte IsDirectory;
+  Byte IsAnti;
+  /*
+  int AreAttributesDefined;
+  int IsLastWriteTimeDefined;
+  int IsStartPosDefined;
+  */
+}CFileItem;
+
+void SzFileInit(CFileItem *fileItem);
+
+typedef struct _CArchiveDatabase
+{
+  UInt32 NumPackStreams;
+  CFileSize *PackSizes;
+  Byte *PackCRCsDefined;
+  UInt32 *PackCRCs;
+  UInt32 NumFolders;
+  CFolder *Folders;
+  UInt32 NumFiles;
+  CFileItem *Files;
+}CArchiveDatabase;
+
+void SzArchiveDatabaseInit(CArchiveDatabase *db);
+void SzArchiveDatabaseFree(CArchiveDatabase *db, void (*freeFunc)(void *));
+
+
+#endif

code/nel/3rdparty/seven_zip/7zMethodID.cpp

@@ -0,0 +1,14 @@
+/* 7zMethodID.c */
+
+#include "7zMethodID.h"
+
+int AreMethodsEqual(CMethodID *a1, CMethodID *a2)
+{
+  int i;
+  if (a1->IDSize != a2->IDSize)
+    return 0;
+  for (i = 0; i < a1->IDSize; i++)
+    if (a1->ID[i] != a2->ID[i])
+      return 0;
+  return 1;
+}

code/nel/3rdparty/seven_zip/7zMethodID.h

@@ -0,0 +1,18 @@
+/* 7zMethodID.h */
+
+#ifndef __7Z_METHOD_ID_H
+#define __7Z_METHOD_ID_H
+
+#include "7zTypes.h"
+
+#define kMethodIDSize 15
+  
+typedef struct _CMethodID
+{
+  Byte ID[kMethodIDSize];
+  Byte IDSize;
+} CMethodID;
+
+int AreMethodsEqual(CMethodID *a1, CMethodID *a2);
+
+#endif

code/nel/3rdparty/seven_zip/BranchTypes.h

@@ -0,0 +1,25 @@
+/* BranchTypes.h */
+
+#ifndef __BRANCHTYPES_H
+#define __BRANCHTYPES_H
+
+#ifndef _7ZIP_BYTE_DEFINED
+#define _7ZIP_BYTE_DEFINED
+typedef unsigned char Byte;
+#endif 
+
+#ifndef _7ZIP_UINT16_DEFINED
+#define _7ZIP_UINT16_DEFINED
+typedef unsigned short UInt16;
+#endif 
+
+#ifndef _7ZIP_UINT32_DEFINED
+#define _7ZIP_UINT32_DEFINED
+#ifdef _LZMA_UINT32_IS_ULONG
+typedef unsigned long UInt32;
+#else
+typedef unsigned int UInt32;
+#endif
+#endif
+
+#endif

code/nel/3rdparty/seven_zip/BranchX86.cpp

@@ -0,0 +1,101 @@
+/* BranchX86.c */
+
+#include "BranchX86.h"
+
+/*
+static int inline Test86MSByte(Byte b)
+{
+  return (b == 0 || b == 0xFF);
+}
+*/
+#define Test86MSByte(b) ((b) == 0 || (b) == 0xFF)
+
+const int kMaskToAllowedStatus[8] = {1, 1, 1, 0, 1, 0, 0, 0};
+const Byte kMaskToBitNumber[8] = {0, 1, 2, 2, 3, 3, 3, 3};
+
+/*
+void x86_Convert_Init(UInt32 *prevMask, UInt32 *prevPos)
+{
+  *prevMask = 0;
+  *prevPos = (UInt32)(-5);
+}
+*/
+
+UInt32 x86_Convert(Byte *buffer, UInt32 endPos, UInt32 nowPos, 
+    UInt32 *prevMask, UInt32 *prevPos, int encoding)
+{
+  UInt32 bufferPos = 0;
+  UInt32 limit;
+
+  if (endPos < 5)
+    return 0;
+  
+  if (nowPos - *prevPos > 5)
+    *prevPos = nowPos - 5;
+  
+  limit = endPos - 5;
+  while(bufferPos <= limit)
+  {
+    Byte b = buffer[bufferPos];
+    UInt32 offset;
+    if (b != 0xE8 && b != 0xE9)
+    {
+      bufferPos++;
+      continue;
+    }
+    offset = (nowPos + bufferPos - *prevPos);
+    *prevPos = (nowPos + bufferPos);
+    if (offset > 5)
+      *prevMask = 0;
+    else
+    {
+      UInt32 i;
+      for (i = 0; i < offset; i++)
+      {
+        *prevMask &= 0x77;
+        *prevMask <<= 1;
+      }
+    }
+    b = buffer[bufferPos + 4];
+    if (Test86MSByte(b) && kMaskToAllowedStatus[(*prevMask >> 1) & 0x7] && 
+      (*prevMask >> 1) < 0x10)
+    {
+      UInt32 src = 
+        ((UInt32)(b) << 24) |
+        ((UInt32)(buffer[bufferPos + 3]) << 16) |
+        ((UInt32)(buffer[bufferPos + 2]) << 8) |
+        (buffer[bufferPos + 1]);
+      
+      UInt32 dest;
+      for(;;)
+      {
+        UInt32 index;
+        if (encoding)
+          dest = (nowPos + bufferPos + 5) + src;
+        else
+          dest = src - (nowPos + bufferPos + 5);
+        if (*prevMask == 0)
+          break;
+        index = kMaskToBitNumber[*prevMask >> 1];
+        b = (Byte)(dest >> (24 - index * 8));
+        if (!Test86MSByte(b))
+          break;
+        src = dest ^ ((1 << (32 - index * 8)) - 1);
+      }
+      buffer[bufferPos + 4] = (Byte)(~(((dest >> 24) & 1) - 1));
+      buffer[bufferPos + 3] = (Byte)(dest >> 16);
+      buffer[bufferPos + 2] = (Byte)(dest >> 8);
+      buffer[bufferPos + 1] = (Byte)dest;
+      bufferPos += 5;
+      *prevMask = 0;
+    }
+    else
+    {
+      bufferPos++;
+      *prevMask |= 1;
+      if (Test86MSByte(b))
+        *prevMask |= 0x10;
+    }
+  }
+  return bufferPos;
+}

code/nel/3rdparty/seven_zip/BranchX86.h

@@ -0,0 +1,13 @@
+/* BranchX86.h */
+
+#ifndef __BRANCHX86_H
+#define __BRANCHX86_H
+
+#include "BranchTypes.h"
+
+#define x86_Convert_Init(prevMask, prevPos) { prevMask = 0; prevPos = (UInt32)(-5); }
+
+UInt32 x86_Convert(Byte *buffer, UInt32 endPos, UInt32 nowPos, 
+    UInt32 *prevMask, UInt32 *prevPos, int encoding);
+
+#endif

code/nel/3rdparty/seven_zip/CMakeLists.txt

@@ -0,0 +1,27 @@
+INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR})
+
+FILE(GLOB LIB_SRC *.cpp *.h)
+
+LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp)
+
+NL_TARGET_LIB(nel_sevenzip ${LIB_SRC})
+# TARGET_LINK_LIBRARIES(nel_sevenzip ${PLATFORM_LINKFLAGS})
+NL_DEFAULT_PROPS(nel_sevenzip "NeL, 3rd Party: Seven Zip")
+NL_ADD_RUNTIME_FLAGS(nel_sevenzip)
+NL_ADD_LIB_SUFFIX(nel_sevenzip)
+
+ADD_DEFINITIONS(-D_SZ_ONE_DIRECTORY)
+
+IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC)
+  INSTALL(TARGETS nel_sevenzip LIBRARY DESTINATION ${NL_LIB_PREFIX} ARCHIVE DESTINATION ${NL_LIB_PREFIX} COMPONENT libraries)
+ENDIF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC)
+
+IF(WITH_TOOLS)
+  ADD_EXECUTABLE(7zDec ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp)
+
+  TARGET_LINK_LIBRARIES(7zDec nel_sevenzip)
+  NL_DEFAULT_PROPS(7zDec "NeL, 3rd Party: Seven Zip Decoder")
+  NL_ADD_RUNTIME_FLAGS(7zDec)
+
+  INSTALL(TARGETS 7zDec RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT tools)
+ENDIF(WITH_TOOLS)

code/nel/3rdparty/seven_zip/LzmaDecode.cpp

@@ -0,0 +1,584 @@
+/*
+  LzmaDecode.c
+  LZMA Decoder (optimized for Speed version)
+  
+  LZMA SDK 4.40 Copyright (c) 1999-2006 Igor Pavlov (2006-05-01)
+  http://www.7-zip.org/
+
+  LZMA SDK is licensed under two licenses:
+  1) GNU Lesser General Public License (GNU LGPL)
+  2) Common Public License (CPL)
+  It means that you can select one of these two licenses and 
+  follow rules of that license.
+
+  SPECIAL EXCEPTION:
+  Igor Pavlov, as the author of this Code, expressly permits you to 
+  statically or dynamically link your Code (or bind by name) to the 
+  interfaces of this file without subjecting your linked Code to the 
+  terms of the CPL or GNU LGPL. Any modifications or additions 
+  to this file, however, are subject to the LGPL or CPL terms.
+*/
+
+#include "LzmaDecode.h"
+
+#define kNumTopBits 24
+#define kTopValue ((UInt32)1 << kNumTopBits)
+
+#define kNumBitModelTotalBits 11
+#define kBitModelTotal (1 << kNumBitModelTotalBits)
+#define kNumMoveBits 5
+
+#define RC_READ_BYTE (*Buffer++)
+
+#define RC_INIT2 Code = 0; Range = 0xFFFFFFFF; \
+  { int i; for(i = 0; i < 5; i++) { RC_TEST; Code = (Code << 8) | RC_READ_BYTE; }}
+
+#ifdef _LZMA_IN_CB
+
+#define RC_TEST { if (Buffer == BufferLim) \
+  { SizeT size; int result = InCallback->Read(InCallback, &Buffer, &size); if (result != LZMA_RESULT_OK) return result; \
+  BufferLim = Buffer + size; if (size == 0) return LZMA_RESULT_DATA_ERROR; }}
+
+#define RC_INIT Buffer = BufferLim = 0; RC_INIT2
+
+#else
+
+#define RC_TEST { if (Buffer == BufferLim) return LZMA_RESULT_DATA_ERROR; }
+
+#define RC_INIT(buffer, bufferSize) Buffer = buffer; BufferLim = buffer + bufferSize; RC_INIT2
+ 
+#endif
+
+#define RC_NORMALIZE if (Range < kTopValue) { RC_TEST; Range <<= 8; Code = (Code << 8) | RC_READ_BYTE; }
+
+#define IfBit0(p) RC_NORMALIZE; bound = (Range >> kNumBitModelTotalBits) * *(p); if (Code < bound)
+#define UpdateBit0(p) Range = bound; *(p) += (kBitModelTotal - *(p)) >> kNumMoveBits;
+#define UpdateBit1(p) Range -= bound; Code -= bound; *(p) -= (*(p)) >> kNumMoveBits;
+
+#define RC_GET_BIT2(p, mi, A0, A1) IfBit0(p) \
+  { UpdateBit0(p); mi <<= 1; A0; } else \
+  { UpdateBit1(p); mi = (mi + mi) + 1; A1; } 
+  
+#define RC_GET_BIT(p, mi) RC_GET_BIT2(p, mi, ; , ;)               
+
+#define RangeDecoderBitTreeDecode(probs, numLevels, res) \
+  { int i = numLevels; res = 1; \
+  do { CProb *p = probs + res; RC_GET_BIT(p, res) } while(--i != 0); \
+  res -= (1 << numLevels); }
+
+
+#define kNumPosBitsMax 4
+#define kNumPosStatesMax (1 << kNumPosBitsMax)
+
+#define kLenNumLowBits 3
+#define kLenNumLowSymbols (1 << kLenNumLowBits)
+#define kLenNumMidBits 3
+#define kLenNumMidSymbols (1 << kLenNumMidBits)
+#define kLenNumHighBits 8
+#define kLenNumHighSymbols (1 << kLenNumHighBits)
+
+#define LenChoice 0
+#define LenChoice2 (LenChoice + 1)
+#define LenLow (LenChoice2 + 1)
+#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits))
+#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits))
+#define kNumLenProbs (LenHigh + kLenNumHighSymbols) 
+
+
+#define kNumStates 12
+#define kNumLitStates 7
+
+#define kStartPosModelIndex 4
+#define kEndPosModelIndex 14
+#define kNumFullDistances (1 << (kEndPosModelIndex >> 1))
+
+#define kNumPosSlotBits 6
+#define kNumLenToPosStates 4
+
+#define kNumAlignBits 4
+#define kAlignTableSize (1 << kNumAlignBits)
+
+#define kMatchMinLen 2
+
+#define IsMatch 0
+#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax))
+#define IsRepG0 (IsRep + kNumStates)
+#define IsRepG1 (IsRepG0 + kNumStates)
+#define IsRepG2 (IsRepG1 + kNumStates)
+#define IsRep0Long (IsRepG2 + kNumStates)
+#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax))
+#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits))
+#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex)
+#define LenCoder (Align + kAlignTableSize)
+#define RepLenCoder (LenCoder + kNumLenProbs)
+#define Literal (RepLenCoder + kNumLenProbs)
+
+#if Literal != LZMA_BASE_SIZE
+StopCompilingDueBUG
+#endif
+
+int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, SizeT size)
+{
+  unsigned char prop0;
+  if (size < LZMA_PROPERTIES_SIZE)
+    return LZMA_RESULT_DATA_ERROR;
+  prop0 = propsData[0];
+  if (prop0 >= (9 * 5 * 5))
+    return LZMA_RESULT_DATA_ERROR;
+  {
+    for (propsRes->pb = 0; prop0 >= (9 * 5); propsRes->pb++, prop0 -= (9 * 5)) ;
+    for (propsRes->lp = 0; prop0 >= 9; propsRes->lp++, prop0 -= 9) ;
+    propsRes->lc = prop0;
+    /*
+    unsigned char remainder = (unsigned char)(prop0 / 9);
+    propsRes->lc = prop0 % 9;
+    propsRes->pb = remainder / 5;
+    propsRes->lp = remainder % 5;
+    */
+  }
+
+  #ifdef _LZMA_OUT_READ
+  {
+    int i;
+    propsRes->DictionarySize = 0;
+    for (i = 0; i < 4; i++)
+      propsRes->DictionarySize += (UInt32)(propsData[1 + i]) << (i * 8);
+    if (propsRes->DictionarySize == 0)
+      propsRes->DictionarySize = 1;
+  }
+  #endif
+  return LZMA_RESULT_OK;
+}
+
+#define kLzmaStreamWasFinishedId (-1)
+
+int LzmaDecode(CLzmaDecoderState *vs,
+    #ifdef _LZMA_IN_CB
+    ILzmaInCallback *InCallback,
+    #else
+    const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed,
+    #endif
+    unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed)
+{
+  CProb *p = vs->Probs;
+  SizeT nowPos = 0;
+  Byte previousByte = 0;
+  UInt32 posStateMask = (1 << (vs->Properties.pb)) - 1;
+  UInt32 literalPosMask = (1 << (vs->Properties.lp)) - 1;
+  int lc = vs->Properties.lc;
+
+  #ifdef _LZMA_OUT_READ
+  
+  UInt32 Range = vs->Range;
+  UInt32 Code = vs->Code;
+  #ifdef _LZMA_IN_CB
+  const Byte *Buffer = vs->Buffer;
+  const Byte *BufferLim = vs->BufferLim;
+  #else
+  const Byte *Buffer = inStream;
+  const Byte *BufferLim = inStream + inSize;
+  #endif
+  int state = vs->State;
+  UInt32 rep0 = vs->Reps[0], rep1 = vs->Reps[1], rep2 = vs->Reps[2], rep3 = vs->Reps[3];
+  int len = vs->RemainLen;
+  UInt32 globalPos = vs->GlobalPos;
+  UInt32 distanceLimit = vs->DistanceLimit;
+
+  Byte *dictionary = vs->Dictionary;
+  UInt32 dictionarySize = vs->Properties.DictionarySize;
+  UInt32 dictionaryPos = vs->DictionaryPos;
+
+  Byte tempDictionary[4];
+
+  #ifndef _LZMA_IN_CB
+  *inSizeProcessed = 0;
+  #endif
+  *outSizeProcessed = 0;
+  if (len == kLzmaStreamWasFinishedId)
+    return LZMA_RESULT_OK;
+
+  if (dictionarySize == 0)
+  {
+    dictionary = tempDictionary;
+    dictionarySize = 1;
+    tempDictionary[0] = vs->TempDictionary[0];
+  }
+
+  if (len == kLzmaNeedInitId)
+  {
+    {
+      UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp));
+      UInt32 i;
+      for (i = 0; i < numProbs; i++)
+        p[i] = kBitModelTotal >> 1; 
+      rep0 = rep1 = rep2 = rep3 = 1;
+      state = 0;
+      globalPos = 0;
+      distanceLimit = 0;
+      dictionaryPos = 0;
+      dictionary[dictionarySize - 1] = 0;
+      #ifdef _LZMA_IN_CB
+      RC_INIT;
+      #else
+      RC_INIT(inStream, inSize);
+      #endif
+    }
+    len = 0;
+  }
+  while(len != 0 && nowPos < outSize)
+  {
+    UInt32 pos = dictionaryPos - rep0;
+    if (pos >= dictionarySize)
+      pos += dictionarySize;
+    outStream[nowPos++] = dictionary[dictionaryPos] = dictionary[pos];
+    if (++dictionaryPos == dictionarySize)
+      dictionaryPos = 0;
+    len--;
+  }
+  if (dictionaryPos == 0)
+    previousByte = dictionary[dictionarySize - 1];
+  else
+    previousByte = dictionary[dictionaryPos - 1];
+
+  #else /* if !_LZMA_OUT_READ */
+
+  int state = 0;
+  UInt32 rep0 = 1, rep1 = 1, rep2 = 1, rep3 = 1;
+  int len = 0;
+  const Byte *Buffer;
+  const Byte *BufferLim;
+  UInt32 Range;
+  UInt32 Code;
+
+  #ifndef _LZMA_IN_CB
+  *inSizeProcessed = 0;
+  #endif
+  *outSizeProcessed = 0;
+
+  {
+    UInt32 i;
+    UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp));
+    for (i = 0; i < numProbs; i++)
+      p[i] = kBitModelTotal >> 1;
+  }
+  
+  #ifdef _LZMA_IN_CB
+  RC_INIT;
+  #else
+  RC_INIT(inStream, inSize);
+  #endif
+
+  #endif /* _LZMA_OUT_READ */
+
+  while(nowPos < outSize)
+  {
+    CProb *prob;
+    UInt32 bound;
+    int posState = (int)(
+        (nowPos 
+        #ifdef _LZMA_OUT_READ
+        + globalPos
+        #endif
+        )
+        & posStateMask);
+
+    prob = p + IsMatch + (state << kNumPosBitsMax) + posState;
+    IfBit0(prob)
+    {
+      int symbol = 1;
+      UpdateBit0(prob)
+      prob = p + Literal + (LZMA_LIT_SIZE * 
+        (((
+        (nowPos 
+        #ifdef _LZMA_OUT_READ
+        + globalPos
+        #endif
+        )
+        & literalPosMask) << lc) + (previousByte >> (8 - lc))));
+
+      if (state >= kNumLitStates)
+      {
+        int matchByte;
+        #ifdef _LZMA_OUT_READ
+        UInt32 pos = dictionaryPos - rep0;
+        if (pos >= dictionarySize)
+          pos += dictionarySize;
+        matchByte = dictionary[pos];
+        #else
+        matchByte = outStream[nowPos - rep0];
+        #endif
+        do
+        {
+          int bit;
+          CProb *probLit;
+          matchByte <<= 1;
+          bit = (matchByte & 0x100);
+          probLit = prob + 0x100 + bit + symbol;
+          RC_GET_BIT2(probLit, symbol, if (bit != 0) break, if (bit == 0) break)
+        }
+        while (symbol < 0x100);
+      }
+      while (symbol < 0x100)
+      {
+        CProb *probLit = prob + symbol;
+        RC_GET_BIT(probLit, symbol)
+      }
+      previousByte = (Byte)(symbol & 0xff);
+
+      outStream[nowPos++] = previousByte;
+      #ifdef _LZMA_OUT_READ
+      if (distanceLimit < dictionarySize)
+        distanceLimit++;
+
+      dictionary[dictionaryPos] = previousByte;
+      if (++dictionaryPos == dictionarySize)
+        dictionaryPos = 0;
+      #endif
+      if (state < 4) state = 0;
+      else if (state < 10) state -= 3;
+      else state -= 6;
+    }
+    else             
+    {
+      UpdateBit1(prob);
+      prob = p + IsRep + state;
+      IfBit0(prob)
+      {
+        UpdateBit0(prob);
+        rep3 = rep2;
+        rep2 = rep1;
+        rep1 = rep0;
+        state = state < kNumLitStates ? 0 : 3;
+        prob = p + LenCoder;
+      }
+      else
+      {
+        UpdateBit1(prob);
+        prob = p + IsRepG0 + state;
+        IfBit0(prob)
+        {
+          UpdateBit0(prob);
+          prob = p + IsRep0Long + (state << kNumPosBitsMax) + posState;
+          IfBit0(prob)
+          {
+            #ifdef _LZMA_OUT_READ
+            UInt32 pos;
+            #endif
+            UpdateBit0(prob);
+            
+            #ifdef _LZMA_OUT_READ
+            if (distanceLimit == 0)
+            #else
+            if (nowPos == 0)
+            #endif
+              return LZMA_RESULT_DATA_ERROR;
+            
+            state = state < kNumLitStates ? 9 : 11;
+            #ifdef _LZMA_OUT_READ
+            pos = dictionaryPos - rep0;
+            if (pos >= dictionarySize)
+              pos += dictionarySize;
+            previousByte = dictionary[pos];
+            dictionary[dictionaryPos] = previousByte;
+            if (++dictionaryPos == dictionarySize)
+              dictionaryPos = 0;
+            #else
+            previousByte = outStream[nowPos - rep0];
+            #endif
+            outStream[nowPos++] = previousByte;
+            #ifdef _LZMA_OUT_READ
+            if (distanceLimit < dictionarySize)
+              distanceLimit++;
+            #endif
+
+            continue;
+          }
+          else
+          {
+            UpdateBit1(prob);
+          }
+        }
+        else
+        {
+          UInt32 distance;
+          UpdateBit1(prob);
+          prob = p + IsRepG1 + state;
+          IfBit0(prob)
+          {
+            UpdateBit0(prob);
+            distance = rep1;
+          }
+          else 
+          {
+            UpdateBit1(prob);
+            prob = p + IsRepG2 + state;
+            IfBit0(prob)
+            {
+              UpdateBit0(prob);
+              distance = rep2;
+            }
+            else
+            {
+              UpdateBit1(prob);
+              distance = rep3;
+              rep3 = rep2;
+            }
+            rep2 = rep1;
+          }
+          rep1 = rep0;
+          rep0 = distance;
+        }
+        state = state < kNumLitStates ? 8 : 11;
+        prob = p + RepLenCoder;
+      }
+      {
+        int numBits, offset;
+        CProb *probLen = prob + LenChoice;
+        IfBit0(probLen)
+        {
+          UpdateBit0(probLen);
+          probLen = prob + LenLow + (posState << kLenNumLowBits);
+          offset = 0;
+          numBits = kLenNumLowBits;
+        }
+        else
+        {
+          UpdateBit1(probLen);
+          probLen = prob + LenChoice2;
+          IfBit0(probLen)
+          {
+            UpdateBit0(probLen);
+            probLen = prob + LenMid + (posState << kLenNumMidBits);
+            offset = kLenNumLowSymbols;
+            numBits = kLenNumMidBits;
+          }
+          else
+          {
+            UpdateBit1(probLen);
+            probLen = prob + LenHigh;
+            offset = kLenNumLowSymbols + kLenNumMidSymbols;
+            numBits = kLenNumHighBits;
+          }
+        }
+        RangeDecoderBitTreeDecode(probLen, numBits, len);
+        len += offset;
+      }
+
+      if (state < 4)
+      {
+        int posSlot;
+        state += kNumLitStates;
+        prob = p + PosSlot +
+            ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << 
+            kNumPosSlotBits);
+        RangeDecoderBitTreeDecode(prob, kNumPosSlotBits, posSlot);
+        if (posSlot >= kStartPosModelIndex)
+        {
+          int numDirectBits = ((posSlot >> 1) - 1);
+          rep0 = (2 | ((UInt32)posSlot & 1));
+          if (posSlot < kEndPosModelIndex)
+          {
+            rep0 <<= numDirectBits;
+            prob = p + SpecPos + rep0 - posSlot - 1;
+          }
+          else
+          {
+            numDirectBits -= kNumAlignBits;
+            do
+            {
+              RC_NORMALIZE
+              Range >>= 1;
+              rep0 <<= 1;
+              if (Code >= Range)
+              {
+                Code -= Range;
+                rep0 |= 1;
+              }
+            }
+            while (--numDirectBits != 0);
+            prob = p + Align;
+            rep0 <<= kNumAlignBits;
+            numDirectBits = kNumAlignBits;
+          }
+          {
+            int i = 1;
+            int mi = 1;
+            do
+            {
+              CProb *prob3 = prob + mi;
+              RC_GET_BIT2(prob3, mi, ; , rep0 |= i);
+              i <<= 1;
+            }
+            while(--numDirectBits != 0);
+          }
+        }
+        else
+          rep0 = posSlot;
+        if (++rep0 == (UInt32)(0))
+        {
+          /* it's for stream version */
+          len = kLzmaStreamWasFinishedId;
+          break;
+        }
+      }
+
+      len += kMatchMinLen;
+      #ifdef _LZMA_OUT_READ
+      if (rep0 > distanceLimit) 
+      #else
+      if (rep0 > nowPos)
+      #endif
+        return LZMA_RESULT_DATA_ERROR;
+
+      #ifdef _LZMA_OUT_READ
+      if (dictionarySize - distanceLimit > (UInt32)len)
+        distanceLimit += len;
+      else
+        distanceLimit = dictionarySize;
+      #endif
+
+      do
+      {
+        #ifdef _LZMA_OUT_READ
+        UInt32 pos = dictionaryPos - rep0;
+        if (pos >= dictionarySize)
+          pos += dictionarySize;
+        previousByte = dictionary[pos];
+        dictionary[dictionaryPos] = previousByte;
+        if (++dictionaryPos == dictionarySize)
+          dictionaryPos = 0;
+        #else
+        previousByte = outStream[nowPos - rep0];
+        #endif
+        len--;
+        outStream[nowPos++] = previousByte;
+      }
+      while(len != 0 && nowPos < outSize);
+    }
+  }
+  RC_NORMALIZE;
+
+  #ifdef _LZMA_OUT_READ
+  vs->Range = Range;
+  vs->Code = Code;
+  vs->DictionaryPos = dictionaryPos;
+  vs->GlobalPos = globalPos + (UInt32)nowPos;
+  vs->DistanceLimit = distanceLimit;
+  vs->Reps[0] = rep0;
+  vs->Reps[1] = rep1;
+  vs->Reps[2] = rep2;
+  vs->Reps[3] = rep3;
+  vs->State = state;
+  vs->RemainLen = len;
+  vs->TempDictionary[0] = tempDictionary[0];
+  #endif
+
+  #ifdef _LZMA_IN_CB
+  vs->Buffer = Buffer;
+  vs->BufferLim = BufferLim;
+  #else
+  *inSizeProcessed = (SizeT)(Buffer - inStream);
+  #endif
+  *outSizeProcessed = nowPos;
+  return LZMA_RESULT_OK;
+}

code/nel/3rdparty/seven_zip/LzmaDecode.h

@@ -0,0 +1,113 @@
+/* 
+  LzmaDecode.h
+  LZMA Decoder interface
+
+  LZMA SDK 4.40 Copyright (c) 1999-2006 Igor Pavlov (2006-05-01)
+  http://www.7-zip.org/
+
+  LZMA SDK is licensed under two licenses:
+  1) GNU Lesser General Public License (GNU LGPL)
+  2) Common Public License (CPL)
+  It means that you can select one of these two licenses and 
+  follow rules of that license.
+
+  SPECIAL EXCEPTION:
+  Igor Pavlov, as the author of this code, expressly permits you to 
+  statically or dynamically link your code (or bind by name) to the 
+  interfaces of this file without subjecting your linked code to the 
+  terms of the CPL or GNU LGPL. Any modifications or additions 
+  to this file, however, are subject to the LGPL or CPL terms.
+*/
+
+#ifndef __LZMADECODE_H
+#define __LZMADECODE_H
+
+#include "LzmaTypes.h"
+
+/* #define _LZMA_IN_CB */
+/* Use callback for input data */
+
+/* #define _LZMA_OUT_READ */
+/* Use read function for output data */
+
+/* #define _LZMA_PROB32 */
+/* It can increase speed on some 32-bit CPUs, 
+   but memory usage will be doubled in that case */
+
+/* #define _LZMA_LOC_OPT */
+/* Enable local speed optimizations inside code */
+
+#ifdef _LZMA_PROB32
+#define CProb UInt32
+#else
+#define CProb UInt16
+#endif
+
+#define LZMA_RESULT_OK 0
+#define LZMA_RESULT_DATA_ERROR 1
+
+#ifdef _LZMA_IN_CB
+typedef struct _ILzmaInCallback
+{
+  int (*Read)(void *object, const unsigned char **buffer, SizeT *bufferSize);
+} ILzmaInCallback;
+#endif
+
+#define LZMA_BASE_SIZE 1846
+#define LZMA_LIT_SIZE 768
+
+#define LZMA_PROPERTIES_SIZE 5
+
+typedef struct _CLzmaProperties
+{
+  int lc;
+  int lp;
+  int pb;
+  #ifdef _LZMA_OUT_READ
+  UInt32 DictionarySize;
+  #endif
+}CLzmaProperties;
+
+int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, SizeT size);
+
+#define LzmaGetNumProbs(Properties) (LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((Properties)->lc + (Properties)->lp)))
+
+#define kLzmaNeedInitId (-2)
+
+typedef struct _CLzmaDecoderState
+{
+  CLzmaProperties Properties;
+  CProb *Probs;
+
+  #ifdef _LZMA_IN_CB
+  const unsigned char *Buffer;
+  const unsigned char *BufferLim;
+  #endif
+
+  #ifdef _LZMA_OUT_READ
+  unsigned char *Dictionary;
+  UInt32 Range;
+  UInt32 Code;
+  UInt32 DictionaryPos;
+  UInt32 GlobalPos;
+  UInt32 DistanceLimit;
+  UInt32 Reps[4];
+  int State;
+  int RemainLen;
+  unsigned char TempDictionary[4];
+  #endif
+} CLzmaDecoderState;
+
+#ifdef _LZMA_OUT_READ
+#define LzmaDecoderInit(vs) { (vs)->RemainLen = kLzmaNeedInitId; }
+#endif
+
+int LzmaDecode(CLzmaDecoderState *vs,
+    #ifdef _LZMA_IN_CB
+    ILzmaInCallback *inCallback,
+    #else
+    const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed,
+    #endif
+    unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed);
+
+#endif

code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp

@@ -0,0 +1,79 @@
+/* LzmaRamDecode.c */
+
+#include "LzmaRamDecode.h"
+#ifdef _SZ_ONE_DIRECTORY
+#include "LzmaDecode.h"
+#include "BranchX86.h"
+#else
+#include "../LZMA_C/LzmaDecode.h"
+#include "../Branch/BranchX86.h"
+#endif
+
+#define LZMA_PROPS_SIZE 14
+#define LZMA_SIZE_OFFSET 6
+
+int LzmaRamGetUncompressedSize(
+    const unsigned char *inBuffer, 
+    size_t inSize, 
+    size_t *outSize)
+{
+  unsigned int i;
+  if (inSize < LZMA_PROPS_SIZE)
+    return 1;
+  *outSize = 0;
+  for(i = 0; i < sizeof(size_t); i++)
+    *outSize += ((size_t)inBuffer[LZMA_SIZE_OFFSET + i]) << (8 * i);
+  for(; i < 8; i++)
+    if (inBuffer[LZMA_SIZE_OFFSET + i] != 0)
+      return 1;
+  return 0;
+}
+
+#define SZE_DATA_ERROR (1)
+#define SZE_OUTOFMEMORY (2)
+
+int LzmaRamDecompress(
+    const unsigned char *inBuffer, 
+    size_t inSize,
+    unsigned char *outBuffer,
+    size_t outSize,
+    size_t *outSizeProcessed,
+    void * (*allocFunc)(size_t size), 
+    void (*freeFunc)(void *))
+{
+  CLzmaDecoderState state;  /* it's about 24 bytes structure, if int is 32-bit */
+  int result;
+  SizeT outSizeProcessedLoc;
+  SizeT inProcessed;
+  int useFilter;
+  
+  if (inSize < LZMA_PROPS_SIZE)
+    return 1;
+  useFilter = inBuffer[0];
+
+  *outSizeProcessed = 0;
+  if (useFilter > 1)
+    return 1;
+
+  if (LzmaDecodeProperties(&state.Properties, inBuffer + 1, LZMA_PROPERTIES_SIZE) != LZMA_RESULT_OK)
+    return 1;
+  state.Probs = (CProb *)allocFunc(LzmaGetNumProbs(&state.Properties) * sizeof(CProb));
+  if (state.Probs == 0)
+    return SZE_OUTOFMEMORY;
+  
+  result = LzmaDecode(&state,
+    inBuffer + LZMA_PROPS_SIZE, (SizeT)inSize - LZMA_PROPS_SIZE, &inProcessed,
+    outBuffer, (SizeT)outSize, &outSizeProcessedLoc);
+  freeFunc(state.Probs);
+  if (result != LZMA_RESULT_OK)
+    return 1;
+  *outSizeProcessed = (size_t)outSizeProcessedLoc;
+  if (useFilter == 1)
+  {
+    UInt32 _prevMask;
+    UInt32 _prevPos;
+    x86_Convert_Init(_prevMask, _prevPos);
+    x86_Convert(outBuffer, (UInt32)outSizeProcessedLoc, 0, &_prevMask, &_prevPos, 0);
+  }
+  return 0;
+}

code/nel/3rdparty/seven_zip/LzmaRamDecode.h

@@ -0,0 +1,55 @@
+/* LzmaRamDecode.h */
+
+#ifndef __LzmaRamDecode_h
+#define __LzmaRamDecode_h
+
+#include <stdlib.h>
+
+/*
+LzmaRamGetUncompressedSize:
+  In: 
+    inBuffer - input data
+    inSize   - input data size
+  Out: 
+    outSize  - uncompressed size
+  Return code:
+    0 - OK
+    1 - Error in headers
+*/
+
+int LzmaRamGetUncompressedSize(
+    const unsigned char *inBuffer, 
+    size_t inSize,
+    size_t *outSize);
+
+
+/*
+LzmaRamDecompress:
+  In: 
+    inBuffer  - input data
+    inSize    - input data size
+    outBuffer - output data
+    outSize   - output size
+    allocFunc - alloc function (can be malloc)
+    freeFunc  - free function (can be free)
+  Out: 
+    outSizeProcessed - processed size
+  Return code:
+    0 - OK
+    1 - Error in headers / data stream
+    2 - Memory allocating error
+
+Memory requirements depend from properties of LZMA stream.
+With default lzma settings it's about 16 KB.
+*/
+
+int LzmaRamDecompress(
+    const unsigned char *inBuffer, 
+    size_t inSize,
+    unsigned char *outBuffer,
+    size_t outSize,
+    size_t *outSizeProcessed,
+    void * (*allocFunc)(size_t size), 
+    void (*freeFunc)(void *));
+
+#endif

code/nel/3rdparty/seven_zip/LzmaTypes.h

@@ -0,0 +1,45 @@
+/* 
+LzmaTypes.h 
+
+Types for LZMA Decoder
+
+This file written and distributed to public domain by Igor Pavlov.
+This file is part of LZMA SDK 4.40 (2006-05-01)
+*/
+
+#ifndef __LZMATYPES_H
+#define __LZMATYPES_H
+
+#ifndef _7ZIP_BYTE_DEFINED
+#define _7ZIP_BYTE_DEFINED
+typedef unsigned char Byte;
+#endif 
+
+#ifndef _7ZIP_UINT16_DEFINED
+#define _7ZIP_UINT16_DEFINED
+typedef unsigned short UInt16;
+#endif 
+
+#ifndef _7ZIP_UINT32_DEFINED
+#define _7ZIP_UINT32_DEFINED
+#ifdef _LZMA_UINT32_IS_ULONG
+typedef unsigned long UInt32;
+#else
+typedef unsigned int UInt32;
+#endif
+#endif 
+
+/* #define _LZMA_SYSTEM_SIZE_T */
+/* Use system's size_t. You can use it to enable 64-bit sizes supporting */
+
+#ifndef _7ZIP_SIZET_DEFINED
+#define _7ZIP_SIZET_DEFINED
+#ifdef _LZMA_SYSTEM_SIZE_T
+#include <stddef.h>
+typedef size_t SizeT;
+#else
+typedef UInt32 SizeT;
+#endif
+#endif
+
+#endif

code/nel/CMakeLists.txt

@@ -56,6 +56,7 @@ IF(WITH_INSTALL_LIBRARIES)
   ADD_SUBDIRECTORY(include)
 ENDIF()
 
+ADD_SUBDIRECTORY(3rdparty)
 ADD_SUBDIRECTORY(src)
 
 IF(WITH_NEL_SAMPLES)

code/nel/include/nel/misc/app_context.h

@@ -237,6 +237,22 @@ namespace NLMISC
 		} \
 	private:
 
+#define NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(className) \
+		NLMISC_SAFE_SINGLETON_DECL(className); \
+		\
+	public: \
+		static void releaseInstance() \
+		{ \
+			if (_Instance) \
+			{ \
+				NLMISC::INelContext::getInstance().releaseSingletonPointer(#className, _Instance); \
+				delete _Instance; \
+				_Instance = NULL; \
+			} \
+		} \
+	private:
+
+
 	/** The same as above, but generate a getInstance method that
 	 *	return a pointer instead of a reference
 	 */
@@ -304,5 +320,6 @@ void initNelLibrary(CLibrary &lib);
 
 } // namespace NLMISC
 
+#include <nel/misc/debug.h>
 
 #endif //APP_CONTEXT_H

code/nel/include/nel/misc/path.h

@@ -74,6 +74,9 @@ public:
 
 	/** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included  (the extention (Nel Big File) is used to know that it's a big file) */
 	void			addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
+	
+	/** Sale but for .snp (Streamed NeL Package) */
+	void			addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
 
 	/** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included   */
 	void			addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
@@ -276,8 +279,8 @@ private:
 	struct CMCFileEntry
 	{
 		char *Name;				// Normal case (the search is done by using nlstricmp)
-		uint32	idPath	 : 16;	// Path (not with file at the end) - look in the SSMpath (65536 different path allowed)
+		uint32	idPath	 : 20;	// Path (not with file at the end) - look in the SSMpath (1048576 different path allowed)
-		uint32	idExt	 : 15;	// real extension of the file if remapped - look in the SSMext (32768 different extension allowed)
+		uint32	idExt	 : 11;	// real extension of the file if remapped - look in the SSMext (2048 different extension allowed)
 		uint32	Remapped : 1;	// true if the file is remapped
 	};
 
@@ -371,6 +374,9 @@ public:
 	/** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included  (the extention (Nel Big File) is used to know that it's a big file) */
 	static void			addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
 
+	/** Same but Streamed Package */
+	static void			addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
+
 	/** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included   */
 	static void			addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
 

code/nel/include/nel/misc/streamed_package.h

@@ -0,0 +1,57 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2014  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef NLMISC_STREAMED_PACKAGE_H
+#define NLMISC_STREAMED_PACKAGE_H
+
+#include <nel/misc/types_nl.h>
+#include <nel/misc/sha1.h>
+
+namespace NLMISC {
+
+class CStreamedPackage
+{
+public:
+	struct CEntry
+	{
+		std::string Name;
+		CHashKey Hash;
+		uint32 Size;
+		uint32 LastModified;
+
+		void serial(NLMISC::IStream &f) throw(NLMISC::EStream);
+
+	};
+
+public:
+	CStreamedPackage();
+	~CStreamedPackage();
+
+	void serial(NLMISC::IStream &f) throw(NLMISC::EStream);
+
+	static void makePath(std::string &result, const CHashKey &hash);
+
+public:
+	typedef std::vector<CEntry> TEntries;
+	TEntries Entries;
+	
+}; /* class CStreamedPackage */
+
+} /* namespace NLMISC */
+
+#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */
+
+/* end of file */

code/nel/include/nel/misc/streamed_package_manager.h

@@ -0,0 +1,72 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2014  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H
+#define NLMISC_STREAMED_PACKAGE_MANAGER_H
+
+#include <nel/misc/types_nl.h>
+#include <nel/misc/app_context.h>
+#include <nel/misc/streamed_package.h>
+
+namespace NLMISC {
+
+class CStreamedPackageManager
+{
+	NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(CStreamedPackageManager);
+
+public:
+	CStreamedPackageManager();
+	~CStreamedPackageManager();
+
+	/// package: ex. /games/nel/data/characters_maps_hr.snp
+	bool loadPackage(const std::string &package);
+
+	/// Get a file list
+	void list(std::vector<std::string> &fileNames, const std::string &package);
+
+	/// Unload all packages
+	void unloadAll();
+
+	/// Get an existing file or download if necessary
+	/// filePath: [out] ex. /games/nel/stream/00/00/000000000..
+	/// fileName: ex. fy_hof_underwear.dds
+	bool getFile(std::string &filePath, const std::string &fileName);
+
+	/// Get file size
+	bool getFileSize(uint32 &fileSize, const std::string &fileName);
+
+public:
+	/// Set storage path (ex. stream/)
+	std::string Path;
+
+	/// Loads a package into the package manager (ex. http://patch.live.polyverse.org/stream/)
+	typedef std::vector<std::string> THosts;
+	THosts Hosts;
+
+private:
+	typedef std::map<std::string, CStreamedPackage> TPackages;
+	TPackages m_Packages;
+
+	typedef std::map<std::string, const CStreamedPackage::CEntry *> TEntries;
+	TEntries m_Entries;
+	
+}; /* class CStreamedPackageManager */
+
+} /* namespace NLMISC */
+
+#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */
+
+/* end of file */

code/nel/src/misc/file.cpp

@@ -23,6 +23,7 @@
 #include "nel/misc/command.h"
 #include "nel/misc/sstring.h"
 #include "nel/misc/xml_pack.h"
+#include "nel/misc/streamed_package_manager.h"
 
 #ifndef NL_OS_WINDOWS
 #include <errno.h>
@@ -211,6 +212,40 @@ bool		CIFile::open(const std::string &path, bool text)
 				_F = CXMLPack::getInstance().getFile (path, _FileSize, _BigFileOffset, dummy, _AlwaysOpened);
 			}
 		}
+		else if (pos > 3 && path[pos-3] == 's' && path[pos-2] == 'n' && path[pos-1] == 'p')
+		{
+			// nldebug("Opening a streamed package file");
+
+			_IsInXMLPackFile = false;
+			_IsInBigFile = false;
+			_BigFileOffset = 0;
+			_AlwaysOpened = false;
+			std::string filePath;
+			if (CStreamedPackageManager::getInstance().getFile (filePath, path.substr(pos+1)))
+			{
+				_F = fopen (filePath.c_str(), mode);
+				if (_F != NULL)
+				{
+					_FileSize=CFile::getFileSize(_F);
+					if (_FileSize == 0)
+					{
+						nlwarning("FILE: Size of file '%s' is 0", path.c_str());
+						fclose(_F);
+						_F = NULL;
+					}
+				}
+				else
+				{
+					nlwarning("Failed to open file '%s', error %u : %s", path.c_str(), errno, strerror(errno));
+					_FileSize = 0;
+				}
+			}
+			else
+			{
+				// TEMPORARY ERROR
+				// nlerror("File '%s' not in streamed package", path.c_str());
+			}
+		}
 		else
 		{
 			// bnp file

code/nel/src/misc/path.cpp

@@ -23,6 +23,7 @@
 #include "nel/misc/progress_callback.h"
 #include "nel/misc/file.h"
 #include "nel/misc/xml_pack.h"
+#include "nel/misc/streamed_package_manager.h"
 
 #ifdef NL_OS_WINDOWS
 #	include <sys/types.h>
@@ -288,6 +289,7 @@ void CFileContainer::clearMap ()
 	nlassert(!_MemoryCompressed);
 	_Files.clear ();
 	CBigFile::getInstance().removeAll ();
+	CStreamedPackageManager::getInstance().unloadAll ();
 	NL_DISPLAY_PATH("PATH: CPath::clearMap(): map directory cleared");
 }
 
@@ -336,7 +338,7 @@ void CFileContainer::remapExtension (const string &ext1, const string &ext2, boo
 		nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): can't remap empty extension", ext1lwr.c_str(), ext2lwr.c_str(), substitute);
 	}
 
-	if (ext1lwr == "bnp" || ext2lwr == "bnp")
+	if (ext1lwr == "bnp" || ext2lwr == "bnp" || ext1lwr == "snp" || ext2lwr == "snp")
 	{
 		nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): you can't remap a big file", ext1lwr.c_str(), ext2lwr.c_str(), substitute);
 	}
@@ -1162,16 +1164,26 @@ void CFileContainer::addSearchFile (const string &file, bool remap, const string
 		return;
 	}
 
+	std::string fileExtension = CFile::getExtension(newFile);
+
 	// check if it s a big file
-	if (CFile::getExtension(newFile) == "bnp")
+	if (fileExtension == "bnp")
 	{
 		NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is a big file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str());
 		addSearchBigFile(file, false, false, progressCallBack);
 		return;
 	}
 
+	// check if it s a streamed package
+	if (fileExtension == "snp")
+	{
+		NL_DISPLAY_PATH ("PATH: CPath::addSearchStreamedPackage(%s, %d, '%s'): '%s' is a streamed package, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str());
+		addSearchStreamedPackage(file, false, false, progressCallBack);
+		return;
+	}
+
 	// check if it s an xml pack file
-	if (CFile::getExtension(newFile) == "xml_pack")
+	if (fileExtension == "xml_pack")
 	{
 		NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is an xml pack file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str());
 		addSearchXmlpackFile(file, false, false, progressCallBack);
@@ -1392,6 +1404,64 @@ void CFileContainer::addSearchBigFile (const string &sBigFilename, bool recurse,
 	fclose (Handle);
 }
 
+void CPath::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack)
+{
+	getInstance()->_FileContainer.addSearchBigFile(filename, recurse, alternative, progressCallBack);
+}
+
+void CFileContainer::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack)
+{
+	// Check if filename is not empty
+	if (filename.empty())
+	{
+		nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): can't add empty file, skip it", filename.c_str(), recurse, alternative);
+		return;
+	}
+	// Check if the file exists
+	if (!CFile::isExists(filename))
+	{
+		nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not found, skip it", filename.c_str(), recurse, alternative, filename.c_str());
+		return;
+	}
+	// Check if it s a file
+	if (CFile::isDirectory(filename))
+	{
+		nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not a file, skip it", filename.c_str(), recurse, alternative, filename.c_str());
+		return;
+	}
+
+	// Add the file itself
+	std::string packname = NLMISC::toLower(CFile::getFilename(filename));
+	insertFileInMap(packname, filename, false, CFile::getExtension(filename));
+
+	// Add the package to the package manager
+	if (CStreamedPackageManager::getInstance().loadPackage(filename))
+	{
+		std::vector<std::string> fileNames;
+		CStreamedPackageManager::getInstance().list(fileNames, packname);
+
+		for (std::vector<std::string>::iterator it(fileNames.begin()), end(fileNames.end()); it != end; ++it)
+		{
+			// Add the file to the lookup
+			std::string filePackageName = packname + "@" + (*it);
+			// nldebug("Insert '%s'", filePackageName.c_str());
+			insertFileInMap((*it), filePackageName, false, CFile::getExtension(*it));
+
+			// Remapped extensions
+			std::string ext = CFile::getExtension(*it);
+			for (uint j = 0; j < _Extensions.size (); j++)
+			{
+				if (_Extensions[j].first == ext)
+				{
+					// Need to remap
+					insertFileInMap(CFile::getFilenameWithoutExtension(*it) + "." + _Extensions[j].second,
+						filePackageName, true, _Extensions[j].first);
+				}
+			}
+		}
+	}
+}
+
 // WARNING : recurse is not used
 void CPath::addSearchXmlpackFile (const string &sXmlpackFilename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack)
 {
@@ -1679,7 +1749,7 @@ void CFileContainer::memoryCompress()
 	while (it != _Files.end())
 	{
 		string sTmp = SSMpath.get(it->second.idPath);
-		if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && !it->second.Remapped)
+		if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && (sTmp.find("snp@") == string::npos) && !it->second.Remapped)
 		{
 			// This is a file included in a bigfile (so the name is in the bigfile manager)
 		}
@@ -1702,7 +1772,7 @@ void CFileContainer::memoryCompress()
 	{
 		CFileEntry &rFE = it->second;
 		string sTmp = SSMpath.get(rFE.idPath);
-		if (sTmp.find("@") == string::npos || sTmp.find("@@") != string::npos || rFE.Remapped)
+		if ((sTmp.find("@") == string::npos) || (sTmp.find("@@") != string::npos) || (sTmp.find("snp@") != string::npos) || rFE.Remapped)
 		{
 			strcpy(_AllFileNames+nSize, rFE.Name.c_str());
 			_MCFiles[nNb].Name = _AllFileNames+nSize;
@@ -1999,6 +2069,7 @@ string CFile::findNewFile (const string &filename)
 // \warning doesn't work with big file
 uint32	CFile::getFileSize (const std::string &filename)
 {
+	std::string::size_type pos;
 	if (filename.find("@@") != string::npos)
 	{
 		uint32 fs = 0, bfo;
@@ -2006,12 +2077,21 @@ uint32	CFile::getFileSize (const std::string &filename)
 		CXMLPack::getInstance().getFile (filename, fs, bfo, c, d);
 		return fs;
 	}
-	else if (filename.find('@') != string::npos)
+	else if ((pos = filename.find('@')) != string::npos)
 	{
-		uint32 fs = 0, bfo;
+		if (pos > 3 && filename[pos-3] == 's' && filename[pos-2] == 'n' && filename[pos-1] == 'p')
-		bool c, d;
+		{
-		CBigFile::getInstance().getFile (filename, fs, bfo, c, d);
+			uint32 fs = 0;
-		return fs;
+			CStreamedPackageManager::getInstance().getFileSize (fs, filename.substr(pos+1));
+			return fs;
+		}
+		else
+		{
+			uint32 fs = 0, bfo;
+			bool c, d;
+			CBigFile::getInstance().getFile (filename, fs, bfo, c, d);
+			return fs;
+		}
 	}
 	else
 	{

code/nel/src/misc/streamed_package.cpp

@@ -0,0 +1,66 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2014  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "stdmisc.h"
+
+// Project includes
+#include <nel/misc/streamed_package.h>
+#include <nel/misc/stream.h>
+
+namespace NLMISC {
+
+CStreamedPackage::CStreamedPackage()
+{
+	// init
+}
+
+CStreamedPackage::~CStreamedPackage()
+{
+	// release
+}
+
+void CStreamedPackage::serial(NLMISC::IStream &f) throw(NLMISC::EStream)
+{
+	f.serialCheck(NELID("SNPK"));
+
+	uint version = 1;
+	f.serialVersion(version);
+
+	f.serialCont(Entries);
+}
+
+void CStreamedPackage::CEntry::serial(NLMISC::IStream &f) throw(NLMISC::EStream)
+{
+	uint version = 1;
+	f.serialVersion(version);
+
+	f.serial(Name);
+	f.serial(Hash);
+	f.serial(Size);
+	f.serial(LastModified);
+}
+
+void CStreamedPackage::makePath(std::string &result, const CHashKey &hash)
+{
+	std::string lowerHash = NLMISC::toLower(hash.toString());
+	result = std::string("/") + lowerHash.substr(0, 2)
+		+ "/" + lowerHash.substr(2, 2)
+		+ "/" + lowerHash.substr(4);
+}
+
+} /* namespace NLMISC */
+
+/* end of file */

code/nel/src/misc/streamed_package_manager.cpp

@@ -0,0 +1,284 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2014  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "stdmisc.h"
+
+// 3rd Party includes
+#include <curl/curl.h>
+#include <seven_zip/7zCrc.h>
+#include <seven_zip/7zIn.h>
+#include <seven_zip/7zExtract.h>
+#include <seven_zip/LzmaDecode.h>
+
+// Project includes
+#include <nel/misc/streamed_package_manager.h>
+#include <nel/misc/file.h>
+#include <nel/misc/path.h>
+#include <nel/misc/sha1.h>
+
+namespace NLMISC {
+
+NLMISC_SAFE_SINGLETON_IMPL(CStreamedPackageManager);
+
+CStreamedPackageManager::CStreamedPackageManager()
+{
+	// init
+}
+
+CStreamedPackageManager::~CStreamedPackageManager()
+{
+	// release
+}
+
+bool CStreamedPackageManager::loadPackage(const std::string &package)
+{
+	nldebug("Load package '%s'", package.c_str());
+
+	std::string packname = NLMISC::toLower(CFile::getFilename(package));
+	m_Packages[packname] = CStreamedPackage();
+	std::map<std::string, CStreamedPackage>::iterator it = m_Packages.find(packname);
+	CStreamedPackage &p = it->second;
+
+	try
+	{
+		CIFile fi;
+		fi.open(package);
+		fi.serial(p);
+	}
+	catch (Exception &e)
+	{
+		nlerror("Package serial exception: '%s'", e.what());
+		m_Packages.erase(it);
+		return false;
+	}
+
+	for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it)
+	{
+		const CStreamedPackage::CEntry &entry = (*it);
+		m_Entries[entry.Name] = &entry;
+	}
+
+	return true;
+}
+
+void CStreamedPackageManager::list(std::vector<std::string> &fileNames, const std::string &package)
+{
+	// nldebug("List package '%s'", package.c_str());
+
+	std::map<std::string, CStreamedPackage>::iterator it = m_Packages.find(package);
+	CStreamedPackage &p = it->second;
+
+	for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it)
+	{
+		const CStreamedPackage::CEntry &entry = (*it);
+		fileNames.push_back(entry.Name);
+	}
+}
+
+void CStreamedPackageManager::unloadAll()
+{
+	m_Packages.clear();
+	m_Entries.clear();
+}
+
+bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName)
+{
+	// nldebug("Get file path for streamed file '%s'", fileName.c_str());
+
+	TEntries::iterator it = m_Entries.find(fileName);
+	if (it == m_Entries.end())
+		return false;
+	const CStreamedPackage::CEntry *entry = it->second;
+
+	CStreamedPackage::makePath(filePath, entry->Hash);
+	std::string downloadUrlFile = filePath + ".lzma";
+	filePath = Path + filePath;
+	std::string downloadPath = filePath + ".download." + toString(rand());
+
+	std::string storageDirectory = CFile::getPath(downloadPath);
+	CFile::createDirectoryTree(storageDirectory);
+	/*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory))
+	{
+		nldebug("Unable to create directory '%s'", storageDirectory.c_str());
+		return false;
+	}*/
+
+	// download
+	for (; ; )
+	{
+		if (CFile::fileExists(filePath))
+			return true;
+
+		std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile;
+		nldebug("Download streamed package '%s' from '%s'", fileName.c_str(), downloadUrl.c_str());
+
+		FILE *fp = fopen(downloadPath.c_str(), "wb");
+		if (fp == NULL)
+		{
+			nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), fileName.c_str());
+			return false;
+		}
+
+		CURL *curl;
+		CURLcode res;
+		curl = curl_easy_init();
+		if (curl)
+		{
+			curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str());
+			curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+			curl_easy_setopt(curl, CURLOPT_FILE, fp);
+			res = curl_easy_perform(curl);
+			long r;
+			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r);
+			curl_easy_cleanup(curl);
+
+			bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/;
+			fclose(fp);
+
+			if (diskFull)
+			{
+				CFile::deleteFile(downloadPath);
+				throw EDiskFullError(downloadPath);
+			}
+
+			if (res != CURLE_OK || r < 200 || r >= 300)
+			{
+				CFile::deleteFile(downloadPath);
+				nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str());
+				nlSleep(1000);
+				continue;
+			}
+		}
+		else
+		{
+			nldebug("Curl initialize failed");
+			fclose(fp);
+			CFile::deleteFile(downloadPath);
+			return false;
+		}
+
+		// ok!
+		break;
+	}
+
+	// read into memory :(
+	std::string unpackPath = filePath + ".extract." + toString(rand());
+
+	std::vector<uint8> inBuffer;
+	{
+		CIFile inStream(downloadPath);
+		uint32 inSize = inStream.getFileSize();
+		inBuffer.resize(inSize);
+		inStream.serialBuffer(&inBuffer[0], inSize);
+	}
+	CFile::deleteFile(downloadPath);
+
+	if (inBuffer.size() < LZMA_PROPERTIES_SIZE + 8)
+	{
+		nlwarning("Invalid file size %u, too small file '%s'", inBuffer.size(), downloadPath.c_str());
+		return false;
+	}
+
+	// extract
+	{
+		CLzmaDecoderState state;
+		uint8 *pos = &inBuffer[0];
+		int ret = LzmaDecodeProperties(&state.Properties, (unsigned char *)pos, LZMA_PROPERTIES_SIZE);
+		if (ret != 0)
+		{
+			nlwarning("Failed to decode lzma properies in file '%s'", downloadPath.c_str());
+			return false;
+		}
+
+		// FROM login_patch.cpp
+		// alloc the probs, making sure they are deleted in function exit
+		size_t nbProb = LzmaGetNumProbs(&state.Properties);
+		std::vector<CProb> probs;
+		probs.resize(nbProb);
+		state.Probs = &probs[0];
+
+		pos += LZMA_PROPERTIES_SIZE;
+
+		// read the output file size
+		size_t fileSize = 0;
+		for (int i = 0; i < 8; i++)
+		{
+			//Byte b;
+			if (pos >= &inBuffer[0] + inBuffer.size())
+			{
+				nlerror("pos >= inBuffer.get() + inSize");
+				return false;
+			}
+			fileSize |= ((UInt64)*pos++) << (8 * i);
+		}
+
+		nlassert(fileSize == entry->Size);
+
+		SizeT outProcessed = 0;
+		SizeT inProcessed = 0;
+		// allocate the output buffer :(
+		std::vector<uint8> outBuffer;
+		outBuffer.resize(fileSize);
+		if (fileSize)
+		{
+			// decompress the file in memory
+			ret = LzmaDecode(&state, (unsigned char *)pos, (SizeT)(inBuffer.size() - (pos - &inBuffer[0])), &inProcessed, (unsigned char*)&outBuffer[0], (SizeT)fileSize, &outProcessed);
+			if (ret != 0 || outProcessed != fileSize)
+			{
+				nlwarning("Failed to decode lzma file '%s'", downloadPath.c_str());
+				return false;
+			}
+		}
+
+		CHashKey outHash = outBuffer.size() ? getSHA1(&outBuffer[0], outBuffer.size()) : CHashKey();
+		if (!(outHash == entry->Hash))
+		{
+			std::string wantHashS = entry->Hash.toString();
+			std::string outHashS = outHash.toString();
+			nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str());
+			return false;
+		}
+
+		{
+			COFile outStream(unpackPath);
+			if (fileSize)
+				outStream.serialBuffer(&outBuffer[0], (uint)fileSize);
+		}
+
+		if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str()))
+		{
+			nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str());
+			// in case downloaded from another thread
+			return CFile::fileExists(filePath);
+		}
+
+		return true;
+	}
+}
+
+bool CStreamedPackageManager::getFileSize(uint32 &fileSize, const std::string &fileName)
+{
+	// nldebug("Get file size for streamed file '%s'", fileName.c_str());
+	TEntries::iterator it = m_Entries.find(fileName);
+	if (it == m_Entries.end())
+		return false;
+	fileSize = it->second->Size;
+	return true;
+}
+
+} /* namespace NLMISC */
+
+/* end of file */

code/nel/tools/build_gamedata/configuration/tools.py

@@ -90,6 +90,7 @@ BuildClodBankTool = "build_clod_bank"
 SheetsPackerTool = "sheets_packer"
 SheetsPackerShardTool = "sheets_packer_shard"
 BnpMakeTool = "bnp_make"
+SnpMakeTool = "snp_make"
 AiBuildWmapTool = "ai_build_wmap"
 TgaCutTool = "tga_cut"
 PatchGenTool = "patch_gen"

code/nel/tools/build_gamedata/d1_client_patch.py

@@ -51,6 +51,7 @@ printLog(log, "")
 
 # Find tools
 BnpMake = findTool(log, ToolDirectories, BnpMakeTool, ToolSuffix)
+SnpMake = findTool(log, ToolDirectories, SnpMakeTool, ToolSuffix);
 PatchGen = findTool(log, ToolDirectories, PatchGenTool, ToolSuffix)
 printLog(log, "")
 
@@ -90,20 +91,23 @@ else:
 					inCategories = 1
 					cfg.write("\t<_Categories>\n")
 					for category in InstallClientData:
+						packExt = ".bnp"
+						if (category["StreamedPackages"]):
+							packExt = ".snp"
 						cfg.write("\t\t<_Category>\n")
 						cfg.write("\t\t\t<_Name type=\"STRING\" value=\"" + category["Name"] + "\"/>\n")
 						if category["UnpackTo"] != None:
 							if category["UnpackTo"] != "":
 								cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./" + category["UnpackTo"] + "/\"/>\n")
 							else:
-								cfg.write("\t\t\t<_UnpackTo type=\"SINT32\" value=\"./\"/>\n")
+								cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./\"/>\n")
 						cfg.write("\t\t\t<_IsOptional type=\"SINT32\" value=\"" + str(category["IsOptional"]) + "\"/>\n")
 						cfg.write("\t\t\t<_IsIncremental type=\"SINT32\" value=\"" + str(category["IsIncremental"]) + "\"/>\n")
 						for package in category["Packages"]:
 							if (len(package[1]) > 0):
 								cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[1][0] + "\"/>\n")
 							else:
-								cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[0] + ".bnp\"/>\n")
+								cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[0] + packExt + "\"/>\n")
 						for ref in category["Refs"]:
 							cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + ref + "_.ref\"/>\n")
 						cfg.write("\t\t</_Category>\n")
@@ -121,11 +125,14 @@ else:
 	targetPath = ClientPatchDirectory + "/bnp"
 	mkPath(log, targetPath)
 	for category in InstallClientData:
+		packExt = ".bnp"
+		if (category["StreamedPackages"]):
+			packExt = ".snp"
 		for package in category["Packages"]:
 			printLog(log, "PACKAGE " + package[0])
 			sourcePath = InstallDirectory + "/" + package[0]
 			mkPath(log, sourcePath)
-			targetBnp = targetPath + "/" + package[0] + ".bnp"
+			targetBnp = targetPath + "/" + package[0] + packExt
 			if (len(package[1]) > 0):
 				targetBnp = targetPath + "/" + package[1][0]
 				printLog(log, "TARGET " + package[1][0])
@@ -135,8 +142,16 @@ else:
 			else:
 				needUpdateBnp = needUpdateDirNoSubdirFile(log, sourcePath, targetBnp)
 			if (needUpdateBnp):
-				printLog(log, "BNP " + targetBnp)
+				if (category["StreamedPackages"]):
-				subprocess.call([ BnpMake, "-p", sourcePath, "-o", targetBnp ] + package[1][1:])
+					printLog(log, "SNP " + targetBnp)
+					cwDir = os.getcwd().replace("\\", "/")
+					toolDir = os.path.dirname(Lzma).replace("\\", "/")
+					os.chdir(toolDir)
+					subprocess.call([ SnpMake, "/p", sourcePath, targetBnp, ClientPatchDirectory + "/stream" ] + package[1][1:])
+					os.chdir(cwDir)
+				else:
+					printLog(log, "BNP " + targetBnp)
+					subprocess.call([ BnpMake, "-p", sourcePath, "-o", targetPath ] + package[1][1:])
 			else:
 				printLog(log, "SKIP " + targetBnp)
 	printLog(log, "")

code/nel/tools/build_gamedata/d2_client_install.py

@@ -47,6 +47,9 @@ printLog(log, "")
 
 for category in InstallClientData:
 	printLog(log, "CATEGORY " + category["Name"])
+	packExt = ".bnp"
+	if (category["StreamedPackages"]):
+		packExt = ".snp"
 	if (category["UnpackTo"] != None):
 		targetPath = ClientInstallDirectory
 		if (category["UnpackTo"] != ""):
@@ -62,8 +65,8 @@ for category in InstallClientData:
 		mkPath(log, targetPath)
 		for package in category["Packages"]:
 			printLog(log, "PACKAGE " + package[0])
-			sourceBnp = sourcePath + "/" + package[0] + ".bnp"
+			sourceBnp = sourcePath + "/" + package[0] + packExt
-			targetBnp = targetPath + "/" + package[0] + ".bnp"
+			targetBnp = targetPath + "/" + package[0] + packExt
 			if (len(package[1]) > 0):
 				sourceBnp = sourcePath + "/" + package[1][0]
 				targetBnp = targetPath + "/" + package[1][0]

code/nel/tools/misc/snp_make/CMakeLists.txt

@@ -0,0 +1,9 @@
+FILE(GLOB SRC *.cpp *.h)
+
+ADD_EXECUTABLE(snp_make ${SRC})
+
+TARGET_LINK_LIBRARIES(snp_make nelmisc)
+NL_DEFAULT_PROPS(snp_make "NeL, Tools, Misc: snp_make")
+NL_ADD_RUNTIME_FLAGS(snp_make)
+
+INSTALL(TARGETS snp_make RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT toolsmisc)

code/nel/tools/misc/snp_make/main.cpp

@@ -0,0 +1,355 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2010  Winch Gate Property Limited
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "nel/misc/types_nl.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef NL_OS_WINDOWS
+#	include <io.h>
+#	include <direct.h>
+#endif
+
+#include <vector>
+#include <string>
+
+#include "nel/misc/debug.h"
+#include "nel/misc/file.h"
+#include "nel/misc/path.h"
+#include "nel/misc/algo.h"
+#include "nel/misc/common.h"
+#include "nel/misc/streamed_package.h"
+
+using namespace std;
+using namespace NLMISC;
+
+// ---------------------------------------------------------------------------
+
+class CWildCard
+{
+public:
+	string		Expression;
+	bool		Not;
+};
+std::vector<CWildCard>	WildCards;
+
+std::string SourceDirectory;
+std::string PackageFileName;
+std::string StreamDirectory;
+
+CStreamedPackage Package;
+
+// ---------------------------------------------------------------------------
+
+bool keepFile (const char *fileName)
+{
+	uint i;
+	bool ifPresent = false;
+	bool ifTrue = false;
+	string file = toLower(CFile::getFilename (fileName));
+	for (i=0; i<WildCards.size(); i++)
+	{
+		if (WildCards[i].Not)
+		{
+			// One ifnot condition met and the file is not added
+			if (testWildCard(file.c_str(), WildCards[i].Expression.c_str()))
+				return false;
+		}
+		else
+		{
+			ifPresent = true;
+			ifTrue |= testWildCard(file.c_str(), WildCards[i].Expression.c_str());
+		}
+	}
+
+	return !ifPresent || ifTrue;
+}
+
+// ---------------------------------------------------------------------------
+void usage()
+{
+	printf ("USAGE : \n");
+	printf ("   snp_make /p <directory_name> <package_file> <stream_directory> [option] ... [option]\n");
+	printf ("   option : \n");
+	printf ("      -if wildcard : add the file if it matches the wilcard (at least one 'if' conditions must be met for a file to be adding)\n");
+	printf ("      -ifnot wildcard : add the file if it doesn't match the wilcard (all the 'ifnot' conditions must be met for a file to be adding)\n");
+	printf (" Pack the directory to a snp file\n");
+	printf ("   snp_make /l <package_file>\n");
+	printf (" List the files contained in the snp file\n");
+}
+
+// ---------------------------------------------------------------------------
+
+void generateLZMA(const std::string sourceFile, const std::string &outputFile)
+{
+	std::string cmd="lzma e ";
+	cmd+=" "+sourceFile+" "+outputFile;
+	nlinfo("executing system command: %s",cmd.c_str());
+#ifdef NL_OS_WINDOWS
+	_spawnlp(_P_WAIT, "lzma.exe","lzma.exe", "e", sourceFile.c_str(), outputFile.c_str(), NULL);
+#else // NL_OS_WINDOWS
+	sint error = system (cmd.c_str());
+	if (error)
+		nlwarning("'%s' failed with error code %d", cmd.c_str(), error);
+#endif // NL_OS_WINDOWS
+}
+
+// ---------------------------------------------------------------------------
+
+uint readOptions (int nNbArg, char **ppArgs)
+{
+	uint i;
+	uint optionCount = 0;
+	for (i=0; i<(uint)nNbArg; i++)
+	{
+		// If ?
+		if ((strcmp (ppArgs[i], "-if") == 0) && ((i+1)<(uint)nNbArg))
+		{
+			CWildCard card;
+			card.Expression = toLower(string(ppArgs[i+1]));
+			card.Not = false;
+			WildCards.push_back (card);
+			optionCount += 2;
+		}
+		// If not ?
+		if ((strcmp (ppArgs[i], "-ifnot") == 0) && ((i+1)<(uint)nNbArg))
+		{
+			CWildCard card;
+			card.Expression = toLower(string(ppArgs[i+1]));
+			card.Not = true;
+			WildCards.push_back (card);
+			optionCount += 2;
+		}
+	}
+	return optionCount;
+}
+
+// ---------------------------------------------------------------------------
+int main (int nNbArg, char **ppArgs)
+{
+	NLMISC::CApplicationContext myApplicationContext;
+
+	if (nNbArg < 3)
+	{
+		usage();
+		return -1;
+	}
+
+	if ((strcmp(ppArgs[1], "/p") == 0) || (strcmp(ppArgs[1], "/P") == 0) ||
+		(strcmp(ppArgs[1], "-p") == 0) || (strcmp(ppArgs[1], "-P") == 0))
+	{
+		if (nNbArg < 5)
+		{
+			usage();
+			return -1;
+		}
+
+		SourceDirectory = ppArgs[2];
+		PackageFileName = ppArgs[3];
+		StreamDirectory = ppArgs[4];
+		readOptions(nNbArg, ppArgs);
+		
+		nldebug("Make streamed package: '%s'", PackageFileName.c_str());
+
+		if (CFile::fileExists(PackageFileName))
+		{
+			nldebug("Update existing package");
+			try
+			{
+				CIFile fi;
+				fi.open(PackageFileName);
+				fi.serial(Package);
+			}
+			catch (Exception &e)
+			{
+				nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what());
+				return -1;
+			}
+		}
+		else
+		{
+			nldebug("New package");
+		}
+
+		std::vector<std::string> pathContent; // contains full pathnames
+		std::vector<std::string> nameContent; // only filename
+		CPath::getPathContent(SourceDirectory, true, false, true, pathContent);
+		nameContent.reserve(pathContent.size());
+		for (std::vector<std::string>::size_type i = 0; i < pathContent.size(); ++i)
+		{
+			const std::string &file = pathContent[i];
+			if (keepFile(file.c_str()))
+			{
+				std::string fileName = NLMISC::toLower(CFile::getFilename(file));
+				// nldebug("File: '%s' ('%s')", file.c_str(), fileName.c_str());
+				nameContent.push_back(fileName);
+				nlassert(nameContent.size() == (i + 1));
+			}
+			else
+			{
+				// Not included in this package
+				pathContent.erase(pathContent.begin() + i);
+				--i;
+			}
+		}
+
+		std::vector<sint> packageIndex; // index of file in package
+		packageIndex.resize(pathContent.size(), -1);
+
+		for (CStreamedPackage::TEntries::size_type i = 0; i < Package.Entries.size(); ++i)
+		{
+			const CStreamedPackage::CEntry &entry = Package.Entries[i];
+			
+			sint foundIndex = -1; // find index in found file list
+			for (std::vector<std::string>::size_type j = 0; j < pathContent.size(); ++j)
+			{
+				if (nameContent[j] == entry.Name)
+				{
+					foundIndex = j;
+					break;
+				}
+			}
+
+			if (foundIndex < 0)
+			{
+				nlinfo("File no longer exists: '%s'", entry.Name.c_str());
+				Package.Entries.erase(Package.Entries.begin() + i);
+				--i;
+			}
+			else
+			{
+				// File still exists, map it
+				packageIndex[foundIndex] = i;
+			}
+		}
+
+		for (std::vector<std::string>::size_type i = 0; i < pathContent.size(); ++i)
+		{
+			sint pidx = packageIndex[i];
+			const std::string &name = nameContent[i];
+			const std::string &path = pathContent[i];
+
+			if (pidx < 0)
+			{
+				nlinfo("File added: '%s'", name.c_str());
+				pidx = Package.Entries.size();
+				Package.Entries.push_back(CStreamedPackage::CEntry());
+				Package.Entries[pidx].Name = name;
+				Package.Entries[pidx].LastModified = 0;
+				Package.Entries[pidx].Size = 0;
+			}
+			else
+			{
+				nlinfo("File check for changes: '%s'", name.c_str());
+			}
+
+			CStreamedPackage::CEntry &entry = Package.Entries[pidx];
+
+			std::string targetLzmaOld; // in case lzma wasn't made make sure it exists a second run
+			CStreamedPackage::makePath(targetLzmaOld, entry.Hash);
+			targetLzmaOld = StreamDirectory + targetLzmaOld + ".lzma";
+
+			uint32 lastModified = CFile::getFileModificationDate(path);
+			uint32 fileSize = CFile::getFileSize(path);
+			if (lastModified > entry.LastModified || fileSize != entry.Size || !CFile::fileExists(targetLzmaOld))
+			{
+				entry.LastModified = lastModified;
+
+				nlinfo("Calculate file hash");
+				CHashKey hash = getSHA1(path, true);
+				/*nldebug("%s", hash.toString().c_str());
+				std::string hashPath;
+				CStreamedPackage::makePath(hashPath, hash);
+				nldebug("%s", hashPath.c_str());*/
+
+				if (hash == entry.Hash && fileSize == entry.Size)
+				{
+					// File has not changed
+				}
+				else
+				{
+					nlinfo("File changed");
+					entry.Hash = hash;
+					entry.Size = fileSize;
+				}
+
+				std::string targetLzma; // in case lzma wasn't made make sure it exists a second run
+				CStreamedPackage::makePath(targetLzma, entry.Hash);
+				targetLzma = StreamDirectory + targetLzma + ".lzma";
+
+				if (!CFile::fileExists(targetLzma))
+				{
+					// make the compressed file
+					nlinfo("%s -> %s", path.c_str(), targetLzma.c_str());
+					CFile::createDirectoryTree(CFile::getPath(targetLzma));
+					generateLZMA(path, targetLzma);
+				}
+			}
+		}
+
+		try
+		{
+			nldebug("Store package '%s'", PackageFileName.c_str());
+			COFile fo;
+			fo.open(PackageFileName);
+			fo.serial(Package);
+		}
+		catch (Exception &e)
+		{
+			nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what());
+			return -1;
+		}
+
+		return 0;
+	}	
+
+	if ((strcmp(ppArgs[1], "/l") == 0) || (strcmp(ppArgs[1], "/L") == 0) ||
+		(strcmp(ppArgs[1], "-l") == 0) || (strcmp(ppArgs[1], "-L") == 0))
+	{
+		PackageFileName = ppArgs[2];
+		if (!CFile::fileExists(PackageFileName))
+		{
+			nlwarning("ERROR (snp_make) : package doesn't exist: '%s'", PackageFileName.c_str());
+			return -1;
+		}
+
+		try
+		{
+			CIFile fi;
+			fi.open(PackageFileName);
+			fi.serial(Package);
+		}
+		catch (Exception &e)
+		{
+			nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what());
+			return -1;
+		}
+
+		for (CStreamedPackage::TEntries::const_iterator it(Package.Entries.begin()), end(Package.Entries.end()); it != end; ++it)
+		{
+			const CStreamedPackage::CEntry &entry = (*it);
+
+			printf("List files in '%s'", PackageFileName.c_str());
+			printf("%s { Hash: '%s', Size: '%u', LastModified: '%u' }", entry.Name.c_str(), entry.Hash.toString().c_str(), entry.Size, entry.LastModified);
+		}
+		
+		return 0;
+	}
+
+	usage ();
+	return -1;
+}

code/ryzom/client/src/client_cfg.cpp

@@ -468,6 +468,8 @@ CClientConfig::CClientConfig()
 	ColorShout			= CRGBA(150,0,0,255);		// Default Shout color.
 	ColorTalk			= CRGBA(255,255,255,255);	// Default Talk color.
 
+	StreamedPackagePath = "stream";
+
 	// MP3 player
 	MediaPlayerDirectory	= "music";
 	MediaPlayerAutoPlay		= false;
@@ -1281,18 +1283,25 @@ void CClientConfig::setValues()
 
 	//////////
 	// MISC //
+
 	// Pre Data Path.
 	READ_STRINGVECTOR_FV(PreDataPath);
 
 	// Data Path.
 	READ_STRINGVECTOR_FV(DataPath);
 
-	// List of files that trigger R2ED reload when touched
-	READ_STRINGVECTOR_FV(R2EDReloadFiles);
-
 	// Data Path no recurse.
 	READ_STRINGVECTOR_FV(DataPathNoRecurse);
 
+	// Streamed package path
+	READ_STRING_FV(StreamedPackagePath);
+
+	// Streamed package hosts
+	READ_STRINGVECTOR_FV(StreamedPackageHosts);
+
+	// List of files that trigger R2ED reload when touched
+	READ_STRINGVECTOR_FV(R2EDReloadFiles);
+
 	// Update packed sheet Path
 	READ_STRINGVECTOR_FV(UpdatePackedSheetPath);
 

code/ryzom/client/src/client_cfg.h

@@ -376,6 +376,10 @@ struct CClientConfig
 	std::vector<string>			DataPath;
 	/// Data Path no recurse.
 	std::vector<string>			DataPathNoRecurse;
+	/// Streamed package path
+	std::string					StreamedPackagePath;
+	/// Streamed package hosts
+	std::vector<string>			StreamedPackageHosts; // TODO: From 'domain' SQL table
 	/// Update packed sheet Path.
 	std::vector<string>			UpdatePackedSheetPath;
 	/// True if we want the packed sheet to be updated if needed

code/ryzom/client/src/init.cpp

@@ -34,6 +34,7 @@
 #include "nel/misc/system_info.h"
 #include "nel/misc/block_memory.h"
 #include "nel/misc/system_utils.h"
+#include "nel/misc/streamed_package_manager.h"
 #include "nel/misc/cmd_args.h"
 // 3D Interface.
 #include "nel/3d/bloom_effect.h"
@@ -157,6 +158,8 @@ bool					LastScreenSaverEnabled = false;
 extern void				registerInterfaceElements();
 extern CContinentManager ContinentMngr;
 
+extern NLMISC::CCmdArgs Args;
+
 // Tips of the day count
 #define RZ_NUM_TIPS 17
 ucstring				TipsOfTheDay;
@@ -660,8 +663,6 @@ void initStereoDisplayDevice()
 }
 
 // we want to get executable directory
-extern NLMISC::CCmdArgs Args;
-
 static void addPaths(IProgressCallback &progress, const std::vector<std::string> &paths, bool recurse)
 {
 	// all prefixes for paths
@@ -736,6 +737,14 @@ static void addPaths(IProgressCallback &progress, const std::vector<std::string>
 	}
 }
 
+void initStreamedPackageManager(NLMISC::IProgressCallback &progress)
+{
+	CStreamedPackageManager &spm = CStreamedPackageManager::getInstance();
+	spm.Path = ClientCfg.StreamedPackagePath;
+	for (uint i = 0; i < ClientCfg.StreamedPackageHosts.size(); i++)
+		spm.Hosts.push_back(ClientCfg.StreamedPackageHosts[i]);
+}
+
 void addSearchPaths(IProgressCallback &progress)
 {
 	// Add search path of UI addon. Allow only a subset of files.
@@ -976,6 +985,7 @@ void prelogInit()
 		CPath::remapExtension ("png", "tga", true);
 		FPU_CHECKER_ONCE
 
+		initStreamedPackageManager(ProgressBar);
 		addPreDataPaths(ProgressBar);
 
 		FPU_CHECKER_ONCE

code/ryzom/client/src/init.h

@@ -36,6 +36,7 @@ void prelogInit();
 // Initialize the application after login step
 void postlogInit();
 
+void initStreamedPackageManager(NLMISC::IProgressCallback &progress);
 void addSearchPaths(NLMISC::IProgressCallback &progress);
 void addPreDataPaths(NLMISC::IProgressCallback &progress);
 

code/ryzom/client/src/login.cpp

@@ -28,6 +28,7 @@
 #include "nel/misc/thread.h"
 #include "nel/misc/big_file.h"
 #include "nel/misc/system_utils.h"
+#include "nel/misc/streamed_package_manager.h"
 
 #include "nel/net/tcp_sock.h"
 #include "nel/3d/u_driver.h"
@@ -1685,6 +1686,7 @@ void initPatch()
 		CBGDownloaderAccess::getInstance().startTask(taskDesc, string(), true /* showDownloader */); // no command line since bg downloader should already be started
 		// release lock on bnp, so that they can be written
 		NLMISC::CBigFile::getInstance().removeAll();
+		NLMISC::CStreamedPackageManager::getInstance().unloadAll();
 	}
 	NLGUI::CDBManager::getInstance()->getDbProp("UI:VARIABLES:SCREEN")->setValue32(UI_VARIABLES_SCREEN_PATCHING);
 

code/ryzom/client/src/login_patch.cpp

@@ -1882,7 +1882,6 @@ bool CPatchManager::bnpUnpack(const string &srcBigfile, const string &dstPath, v
 		return false;
 	}
 
-	// Unpack
 	if (!bnpFile.unpack(DestPath))
 		return false;
 
@@ -2495,7 +2494,7 @@ void CPatchThread::run()
 			CPatchManager::SFileToPatch &rFTP = AllFilesToPatch[i];
 
 			string ext = NLMISC::CFile::getExtension(rFTP.FileName);
-			if (ext == "bnp")
+			if (ext == "bnp" || ext == "snp")
 			{
 				float oldCurrentFilePatched = CurrentFilePatched;
 				processFile (rFTP);

code/ryzom/client/src/release.cpp

@@ -25,6 +25,7 @@
 #include "nel/misc/debug.h"
 #include "nel/misc/async_file_manager.h"
 #include "nel/misc/system_utils.h"
+#include "nel/misc/streamed_package_manager.h"
 // 3D Interface.
 #include "nel/3d/bloom_effect.h"
 #include "nel/3d/fxaa.h"
@@ -666,6 +667,7 @@ void release()
 	R2::CObjectSerializer::releaseInstance();
 	NLMISC::CBigFile::getInstance().removeAll();
 	NLMISC::CBigFile::releaseInstance();
+	NLMISC::CStreamedPackageManager::releaseInstance();
 	NL3D::CFastHLSModifier::releaseInstance();
 	CLandscapePolyDrawer::releaseInstance();
 	NL3D::CParticleSystemShape::releaseInstance();

code/ryzom/client/src/streamable_ig.cpp

@@ -93,8 +93,10 @@ CStreamableIG::~CStreamableIG()
 	H_AUTO_USE(RZ_StremableIG)
 	if (!_Linked)
 	{
+		#ifdef NL_DEBUG
 		if(!ClientCfg.Light)
 			nlwarning("Loading async %p", this);
+		#endif
 		#ifdef NL_DEBUG
 				//nlinfo("Loading async : %s", Name.c_str());
 		#endif

code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp

@@ -296,7 +296,7 @@ void CScreenshotIslands::searchIslandsBorders()
 		zonelFiles.clear();
 
 		string bnpFileName = itCont->first + ".bnp";
-		CBigFile::getInstance().list(bnpFileName.c_str(), filenames);
+		CBigFile::getInstance().list(bnpFileName.c_str(), filenames); // FIXME FIXME NOT READING FROM BNP!
 
 		for(uint i=0; i<filenames.size(); i++)
 		{

code/ryzom/tools/patch_gen/patch_gen_common.cpp

@@ -454,7 +454,8 @@ void CPackageDescription::buildDefaultFileList()
 	std::vector<std::string> fileList;
 	NLMISC::CPath::getPathContent(_BnpDirectory,false,false,true,fileList);
 	for (uint32 i=0;i<fileList.size();++i)
-		if (NLMISC::toLower(NLMISC::CFile::getExtension(fileList[i]))=="bnp")
+		if (NLMISC::toLower(NLMISC::CFile::getExtension(fileList[i]))=="bnp"
+			|| NLMISC::toLower(NLMISC::CFile::getExtension(fileList[i]))=="snp")
 			_Categories.addFile("main",NLMISC::toLower(NLMISC::CFile::getFilename(fileList[i])));
 
 	_Categories.addFile("unpacked","root.bnp");

code/snowballs2/client/src/snowballs_client.cpp

@@ -1417,6 +1417,7 @@ public:
 		NLMISC::CAsyncFileManager::terminate();
 		NL3D::CParticleSystemManager::release();
 		NLMISC::CBigFile::releaseInstance();
+		NLMISC::CStreamedPackageManager::releaseInstance();
 		NLMISC::CClassRegistry::release();
 		delete &NLMISC::CObjectArenaAllocator::getDefaultAllocator();
 		cf_delete_buffer(_CfBufferState); _CfBufferState = NULL;