Merge branch 'feature/streamed-package' into ryzomclassic-develop

.gitignore

@@ -142,7 +142,6 @@ moc_*.cpp
 *.cache
 *.patch
 *.7z
-3rdParty
 .svn
 thumbs.db
 Thumbs.db

code/nel/3rdparty/CMakeLists.txt

@@ -0,0 +1,2 @@
+SET(SEVENZIP_LIBRARY "nel_sevenzip")
+ADD_SUBDIRECTORY(seven_zip)

code/nel/3rdparty/seven_zip/7zVersion.rc

@@ -1,55 +1,55 @@
-#define MY_VS_FFI_FILEFLAGSMASK  0x0000003FL
+#define MY_VS_FFI_FILEFLAGSMASK  0x0000003FL
-#define MY_VOS_NT_WINDOWS32  0x00040004L
+#define MY_VOS_NT_WINDOWS32  0x00040004L
-#define MY_VOS_CE_WINDOWS32  0x00050004L
+#define MY_VOS_CE_WINDOWS32  0x00050004L
-
+
-#define MY_VFT_APP  0x00000001L
+#define MY_VFT_APP  0x00000001L
-#define MY_VFT_DLL  0x00000002L
+#define MY_VFT_DLL  0x00000002L
-
+
-// #include <WinVer.h>
+// #include <WinVer.h>
-
+
-#ifndef MY_VERSION
+#ifndef MY_VERSION
-#include "7zVersion.h"
+#include "7zVersion.h"
-#endif
+#endif
-
+
-#define MY_VER MY_VER_MAJOR,MY_VER_MINOR,MY_VER_BUILD,0
+#define MY_VER MY_VER_MAJOR,MY_VER_MINOR,MY_VER_BUILD,0
-
+
-#ifdef DEBUG
+#ifdef DEBUG
-#define DBG_FL VS_FF_DEBUG
+#define DBG_FL VS_FF_DEBUG
-#else
+#else
-#define DBG_FL 0
+#define DBG_FL 0
-#endif
+#endif
-
+
-#define MY_VERSION_INFO(fileType, descr, intName, origName)  \
+#define MY_VERSION_INFO(fileType, descr, intName, origName)  \
-LANGUAGE 9, 1 \
+LANGUAGE 9, 1 \
-1 VERSIONINFO \
+1 VERSIONINFO \
-  FILEVERSION MY_VER \
+  FILEVERSION MY_VER \
-  PRODUCTVERSION MY_VER \
+  PRODUCTVERSION MY_VER \
-  FILEFLAGSMASK MY_VS_FFI_FILEFLAGSMASK \
+  FILEFLAGSMASK MY_VS_FFI_FILEFLAGSMASK \
-  FILEFLAGS DBG_FL \
+  FILEFLAGS DBG_FL \
-  FILEOS MY_VOS_NT_WINDOWS32 \
+  FILEOS MY_VOS_NT_WINDOWS32 \
-  FILETYPE fileType \
+  FILETYPE fileType \
-  FILESUBTYPE 0x0L \
+  FILESUBTYPE 0x0L \
-BEGIN \
+BEGIN \
-    BLOCK "StringFileInfo" \
+    BLOCK "StringFileInfo" \
-    BEGIN  \
+    BEGIN  \
-        BLOCK "040904b0" \
+        BLOCK "040904b0" \
-        BEGIN \
+        BEGIN \
-            VALUE "CompanyName", "Igor Pavlov" \
+            VALUE "CompanyName", "Igor Pavlov" \
-            VALUE "FileDescription", descr \
+            VALUE "FileDescription", descr \
-            VALUE "FileVersion", MY_VERSION  \
+            VALUE "FileVersion", MY_VERSION  \
-            VALUE "InternalName", intName \
+            VALUE "InternalName", intName \
-            VALUE "LegalCopyright", MY_COPYRIGHT \
+            VALUE "LegalCopyright", MY_COPYRIGHT \
-            VALUE "OriginalFilename", origName \
+            VALUE "OriginalFilename", origName \
-            VALUE "ProductName", "7-Zip" \
+            VALUE "ProductName", "7-Zip" \
-            VALUE "ProductVersion", MY_VERSION \
+            VALUE "ProductVersion", MY_VERSION \
-        END \
+        END \
-    END \
+    END \
-    BLOCK "VarFileInfo" \
+    BLOCK "VarFileInfo" \
-    BEGIN \
+    BEGIN \
-        VALUE "Translation", 0x409, 1200 \
+        VALUE "Translation", 0x409, 1200 \
-    END \
+    END \
-END
+END
-
+
-#define MY_VERSION_INFO_APP(descr, intName) MY_VERSION_INFO(MY_VFT_APP, descr, intName, intName ".exe")
+#define MY_VERSION_INFO_APP(descr, intName) MY_VERSION_INFO(MY_VFT_APP, descr, intName, intName ".exe")
-
+
-#define MY_VERSION_INFO_DLL(descr, intName) MY_VERSION_INFO(MY_VFT_DLL, descr, intName, intName ".dll")
+#define MY_VERSION_INFO_DLL(descr, intName) MY_VERSION_INFO(MY_VFT_DLL, descr, intName, intName ".dll")

code/nel/3rdparty/seven_zip/CMakeLists.txt

@@ -0,0 +1,27 @@
+INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR})
+
+FILE(GLOB LIB_SRC *.cpp *.c *.h)
+
+LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/LzmaUtil.c)
+
+NL_TARGET_LIB(nel_sevenzip ${LIB_SRC})
+# TARGET_LINK_LIBRARIES(nel_sevenzip ${PLATFORM_LINKFLAGS})
+NL_DEFAULT_PROPS(nel_sevenzip "NeL, 3rd Party: Seven Zip")
+NL_ADD_RUNTIME_FLAGS(nel_sevenzip)
+NL_ADD_LIB_SUFFIX(nel_sevenzip)
+
+ADD_DEFINITIONS(-D_7ZIP_ST)
+
+IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC)
+  INSTALL(TARGETS nel_sevenzip LIBRARY DESTINATION ${NL_LIB_PREFIX} ARCHIVE DESTINATION ${NL_LIB_PREFIX} COMPONENT libraries)
+ENDIF()
+
+IF(WITH_NEL_TOOLS)
+  ADD_EXECUTABLE(lzma ${CMAKE_CURRENT_SOURCE_DIR}/LzmaUtil.c)
+
+  TARGET_LINK_LIBRARIES(lzma nel_sevenzip)
+  NL_DEFAULT_PROPS(lzma "NeL, 3rd Party: LZMA")
+  NL_ADD_RUNTIME_FLAGS(lzma)
+
+  INSTALL(TARGETS lzma RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT tools)
+ENDIF()

code/nel/3rdparty/seven_zip/DllSecur.c

@@ -0,0 +1,108 @@
+/* DllSecur.c -- DLL loading security
+2018-02-21 : Igor Pavlov : Public domain */
+
+#include "Precomp.h"
+
+#ifdef _WIN32
+
+#include <windows.h>
+
+#include "DllSecur.h"
+
+#ifndef UNDER_CE
+
+typedef BOOL (WINAPI *Func_SetDefaultDllDirectories)(DWORD DirectoryFlags);
+
+#define MY_LOAD_LIBRARY_SEARCH_USER_DIRS 0x400
+#define MY_LOAD_LIBRARY_SEARCH_SYSTEM32  0x800
+
+static const char * const g_Dlls =
+  #ifndef _CONSOLE
+  "UXTHEME\0"
+  #endif
+  "USERENV\0"
+  "SETUPAPI\0"
+  "APPHELP\0"
+  "PROPSYS\0"
+  "DWMAPI\0"
+  "CRYPTBASE\0"
+  "OLEACC\0"
+  "CLBCATQ\0"
+  "VERSION\0"
+  ;
+
+#endif
+
+void My_SetDefaultDllDirectories()
+{
+  #ifndef UNDER_CE
+  
+    OSVERSIONINFO vi;
+    vi.dwOSVersionInfoSize = sizeof(vi);
+    GetVersionEx(&vi);
+    if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0)
+    {
+      Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories)
+          GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories");
+      if (setDllDirs)
+        if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS))
+          return;
+    }
+
+  #endif
+}
+
+
+void LoadSecurityDlls()
+{
+  #ifndef UNDER_CE
+  
+  wchar_t buf[MAX_PATH + 100];
+
+  {
+    // at Vista (ver 6.0) : CoCreateInstance(CLSID_ShellLink, ...) doesn't work after SetDefaultDllDirectories() : Check it ???
+    OSVERSIONINFO vi;
+    vi.dwOSVersionInfoSize = sizeof(vi);
+    if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0)
+    {
+      Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories)
+          GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories");
+      if (setDllDirs)
+        if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS))
+          return;
+    }
+  }
+
+  {
+    unsigned len = GetSystemDirectoryW(buf, MAX_PATH + 2);
+    if (len == 0 || len > MAX_PATH)
+      return;
+  }
+  {
+    const char *dll;
+    unsigned pos = (unsigned)lstrlenW(buf);
+
+    if (buf[pos - 1] != '\\')
+      buf[pos++] = '\\';
+    
+    for (dll = g_Dlls; dll[0] != 0;)
+    {
+      unsigned k = 0;
+      for (;;)
+      {
+        char c = *dll++;
+        buf[pos + k] = (Byte)c;
+        k++;
+        if (c == 0)
+          break;
+      }
+
+      lstrcatW(buf, L".dll");
+      LoadLibraryExW(buf, NULL, LOAD_WITH_ALTERED_SEARCH_PATH);
+    }
+  }
+  
+  #endif
+}
+
+#endif

code/nel/3rdparty/seven_zip/DllSecur.h

@@ -0,0 +1,20 @@
+/* DllSecur.h -- DLL loading for security
+2018-02-19 : Igor Pavlov : Public domain */
+
+#ifndef __DLL_SECUR_H
+#define __DLL_SECUR_H
+
+#include "7zTypes.h"
+
+EXTERN_C_BEGIN
+
+#ifdef _WIN32
+
+void My_SetDefaultDllDirectories();
+void LoadSecurityDlls();
+
+#endif
+
+EXTERN_C_END
+
+#endif

code/nel/3rdparty/seven_zip/Lzma2DecMt.h

@@ -0,0 +1,79 @@
+/* Lzma2DecMt.h -- LZMA2 Decoder Multi-thread
+2018-02-17 : Igor Pavlov : Public domain */
+
+#ifndef __LZMA2_DEC_MT_H
+#define __LZMA2_DEC_MT_H
+
+#include "7zTypes.h"
+
+EXTERN_C_BEGIN
+
+typedef struct
+{
+  size_t inBufSize_ST;
+  size_t outStep_ST;
+  
+  #ifndef _7ZIP_ST
+  unsigned numThreads;
+  size_t inBufSize_MT;
+  size_t outBlockMax;
+  size_t inBlockMax;
+  #endif
+} CLzma2DecMtProps;
+
+/* init to single-thread mode */
+void Lzma2DecMtProps_Init(CLzma2DecMtProps *p);
+
+
+/* ---------- CLzma2DecMtHandle Interface ---------- */
+
+/* Lzma2DecMt_ * functions can return the following exit codes:
+SRes:
+  SZ_OK           - OK
+  SZ_ERROR_MEM    - Memory allocation error
+  SZ_ERROR_PARAM  - Incorrect paramater in props
+  SZ_ERROR_WRITE  - ISeqOutStream write callback error
+  // SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output
+  SZ_ERROR_PROGRESS - some break from progress callback
+  SZ_ERROR_THREAD - error in multithreading functions (only for Mt version)
+*/
+
+typedef void * CLzma2DecMtHandle;
+
+CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid);
+void Lzma2DecMt_Destroy(CLzma2DecMtHandle p);
+
+SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p,
+    Byte prop,
+    const CLzma2DecMtProps *props,
+    ISeqOutStream *outStream,
+    const UInt64 *outDataSize, // NULL means undefined
+    int finishMode,            // 0 - partial unpacking is allowed, 1 - if lzma2 stream must be finished
+    // Byte *outBuf, size_t *outBufSize,
+    ISeqInStream *inStream,
+    // const Byte *inData, size_t inDataSize,
+    
+    // out variables:
+    UInt64 *inProcessed,
+    int *isMT,  /* out: (*isMT == 0), if single thread decoding was used */
+
+    // UInt64 *outProcessed,
+    ICompressProgress *progress);
+
+
+/* ---------- Read from CLzma2DecMtHandle Interface ---------- */
+
+SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp,
+    Byte prop,
+    const CLzma2DecMtProps *props,
+    const UInt64 *outDataSize, int finishMode,
+    ISeqInStream *inStream);
+
+SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp,
+    Byte *data, size_t *outSize,
+    UInt64 *inStreamProcessed);
+
+
+EXTERN_C_END
+
+#endif

code/nel/3rdparty/seven_zip/Lzma2Enc.c

@@ -5,7 +5,7 @@
 
 #include <string.h>
 
-#define _7ZIP_ST
+/* #define _7ZIP_ST */
 
 #include "Lzma2Enc.h"
 

code/nel/3rdparty/seven_zip/LzmaEnc.c

@@ -5,8 +5,6 @@
 
 #include <string.h>
 
-#define _7ZIP_ST
-
 /* #define SHOW_STAT */
 /* #define SHOW_STAT2 */
 

code/nel/3rdparty/seven_zip/LzmaUtil.c

@@ -0,0 +1,258 @@
+/* LzmaUtil.c -- Test application for LZMA compression
+2018-07-04 : Igor Pavlov : Public domain */
+
+#include "Precomp.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "CpuArch.h"
+
+#include "Alloc.h"
+#include "7zFile.h"
+#include "7zVersion.h"
+#include "LzmaDec.h"
+#include "LzmaEnc.h"
+
+static const char * const kCantReadMessage = "Can not read input file";
+static const char * const kCantWriteMessage = "Can not write output file";
+static const char * const kCantAllocateMessage = "Can not allocate memory";
+static const char * const kDataErrorMessage = "Data error";
+
+static void PrintHelp(char *buffer)
+{
+  strcat(buffer,
+    "\nLZMA-C " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n"
+    "Usage:  lzma <e|d> inputFile outputFile\n"
+    "  e: encode file\n"
+    "  d: decode file\n");
+}
+
+static int PrintError(char *buffer, const char *message)
+{
+  strcat(buffer, "\nError: ");
+  strcat(buffer, message);
+  strcat(buffer, "\n");
+  return 1;
+}
+
+static int PrintErrorNumber(char *buffer, SRes val)
+{
+  sprintf(buffer + strlen(buffer), "\nError code: %x\n", (unsigned)val);
+  return 1;
+}
+
+static int PrintUserError(char *buffer)
+{
+  return PrintError(buffer, "Incorrect command");
+}
+
+
+#define IN_BUF_SIZE (1 << 16)
+#define OUT_BUF_SIZE (1 << 16)
+
+
+static SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inStream,
+    UInt64 unpackSize)
+{
+  int thereIsSize = (unpackSize != (UInt64)(Int64)-1);
+  Byte inBuf[IN_BUF_SIZE];
+  Byte outBuf[OUT_BUF_SIZE];
+  size_t inPos = 0, inSize = 0, outPos = 0;
+  LzmaDec_Init(state);
+  for (;;)
+  {
+    if (inPos == inSize)
+    {
+      inSize = IN_BUF_SIZE;
+      RINOK(inStream->Read(inStream, inBuf, &inSize));
+      inPos = 0;
+    }
+    {
+      SRes res;
+      SizeT inProcessed = inSize - inPos;
+      SizeT outProcessed = OUT_BUF_SIZE - outPos;
+      ELzmaFinishMode finishMode = LZMA_FINISH_ANY;
+      ELzmaStatus status;
+      if (thereIsSize && outProcessed > unpackSize)
+      {
+        outProcessed = (SizeT)unpackSize;
+        finishMode = LZMA_FINISH_END;
+      }
+      
+      res = LzmaDec_DecodeToBuf(state, outBuf + outPos, &outProcessed,
+        inBuf + inPos, &inProcessed, finishMode, &status);
+      inPos += inProcessed;
+      outPos += outProcessed;
+      unpackSize -= outProcessed;
+      
+      if (outStream)
+        if (outStream->Write(outStream, outBuf, outPos) != outPos)
+          return SZ_ERROR_WRITE;
+        
+      outPos = 0;
+      
+      if (res != SZ_OK || (thereIsSize && unpackSize == 0))
+        return res;
+      
+      if (inProcessed == 0 && outProcessed == 0)
+      {
+        if (thereIsSize || status != LZMA_STATUS_FINISHED_WITH_MARK)
+          return SZ_ERROR_DATA;
+        return res;
+      }
+    }
+  }
+}
+
+
+static SRes Decode(ISeqOutStream *outStream, ISeqInStream *inStream)
+{
+  UInt64 unpackSize;
+  int i;
+  SRes res = 0;
+
+  CLzmaDec state;
+
+  /* header: 5 bytes of LZMA properties and 8 bytes of uncompressed size */
+  unsigned char header[LZMA_PROPS_SIZE + 8];
+
+  /* Read and parse header */
+
+  RINOK(SeqInStream_Read(inStream, header, sizeof(header)));
+
+  unpackSize = 0;
+  for (i = 0; i < 8; i++)
+    unpackSize += (UInt64)header[LZMA_PROPS_SIZE + i] << (i * 8);
+
+  LzmaDec_Construct(&state);
+  RINOK(LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &g_Alloc));
+  res = Decode2(&state, outStream, inStream, unpackSize);
+  LzmaDec_Free(&state, &g_Alloc);
+  return res;
+}
+
+static SRes Encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize, char *rs)
+{
+  CLzmaEncHandle enc;
+  SRes res;
+  CLzmaEncProps props;
+
+  UNUSED_VAR(rs);
+
+  enc = LzmaEnc_Create(&g_Alloc);
+  if (enc == 0)
+    return SZ_ERROR_MEM;
+
+  LzmaEncProps_Init(&props);
+  res = LzmaEnc_SetProps(enc, &props);
+
+  if (res == SZ_OK)
+  {
+    Byte header[LZMA_PROPS_SIZE + 8];
+    size_t headerSize = LZMA_PROPS_SIZE;
+    int i;
+
+    res = LzmaEnc_WriteProperties(enc, header, &headerSize);
+    for (i = 0; i < 8; i++)
+      header[headerSize++] = (Byte)(fileSize >> (8 * i));
+    if (outStream->Write(outStream, header, headerSize) != headerSize)
+      res = SZ_ERROR_WRITE;
+    else
+    {
+      if (res == SZ_OK)
+        res = LzmaEnc_Encode(enc, outStream, inStream, NULL, &g_Alloc, &g_Alloc);
+    }
+  }
+  LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc);
+  return res;
+}
+
+
+static int main2(int numArgs, const char *args[], char *rs)
+{
+  CFileSeqInStream inStream;
+  CFileOutStream outStream;
+  char c;
+  int res;
+  int encodeMode;
+  BoolInt useOutFile = False;
+
+  FileSeqInStream_CreateVTable(&inStream);
+  File_Construct(&inStream.file);
+
+  FileOutStream_CreateVTable(&outStream);
+  File_Construct(&outStream.file);
+
+  if (numArgs == 1)
+  {
+    PrintHelp(rs);
+    return 0;
+  }
+
+  if (numArgs < 3 || numArgs > 4 || strlen(args[1]) != 1)
+    return PrintUserError(rs);
+
+  c = args[1][0];
+  encodeMode = (c == 'e' || c == 'E');
+  if (!encodeMode && c != 'd' && c != 'D')
+    return PrintUserError(rs);
+
+  {
+    size_t t4 = sizeof(UInt32);
+    size_t t8 = sizeof(UInt64);
+    if (t4 != 4 || t8 != 8)
+      return PrintError(rs, "Incorrect UInt32 or UInt64");
+  }
+
+  if (InFile_Open(&inStream.file, args[2]) != 0)
+    return PrintError(rs, "Can not open input file");
+
+  if (numArgs > 3)
+  {
+    useOutFile = True;
+    if (OutFile_Open(&outStream.file, args[3]) != 0)
+      return PrintError(rs, "Can not open output file");
+  }
+  else if (encodeMode)
+    PrintUserError(rs);
+
+  if (encodeMode)
+  {
+    UInt64 fileSize;
+    File_GetLength(&inStream.file, &fileSize);
+    res = Encode(&outStream.vt, &inStream.vt, fileSize, rs);
+  }
+  else
+  {
+    res = Decode(&outStream.vt, useOutFile ? &inStream.vt : NULL);
+  }
+
+  if (useOutFile)
+    File_Close(&outStream.file);
+  File_Close(&inStream.file);
+
+  if (res != SZ_OK)
+  {
+    if (res == SZ_ERROR_MEM)
+      return PrintError(rs, kCantAllocateMessage);
+    else if (res == SZ_ERROR_DATA)
+      return PrintError(rs, kDataErrorMessage);
+    else if (res == SZ_ERROR_WRITE)
+      return PrintError(rs, kCantWriteMessage);
+    else if (res == SZ_ERROR_READ)
+      return PrintError(rs, kCantReadMessage);
+    return PrintErrorNumber(rs, res);
+  }
+  return 0;
+}
+
+
+int MY_CDECL main(int numArgs, const char *args[])
+{
+  char rs[800] = { 0 };
+  int res = main2(numArgs, args, rs);
+  fputs(rs, stdout);
+  return res;
+}

code/nel/3rdparty/seven_zip/Precomp.h

@@ -0,0 +1,10 @@
+/* Precomp.h -- StdAfx
+2013-11-12 : Igor Pavlov : Public domain */
+
+#ifndef __7Z_PRECOMP_H
+#define __7Z_PRECOMP_H
+
+#include "Compiler.h"
+/* #include "7zTypes.h" */
+
+#endif

code/nel/CMakeLists.txt

@@ -56,6 +56,7 @@ IF(WITH_INSTALL_LIBRARIES)
   ADD_SUBDIRECTORY(include)
 ENDIF()
 
+ADD_SUBDIRECTORY(3rdparty)
 ADD_SUBDIRECTORY(src)
 
 IF(WITH_NEL_SAMPLES)

code/nel/include/nel/misc/app_context.h

@@ -237,6 +237,22 @@ namespace NLMISC
 		} \
 	private:
 
+#define NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(className) \
+		NLMISC_SAFE_SINGLETON_DECL(className); \
+		\
+	public: \
+		static void releaseInstance() \
+		{ \
+			if (_Instance) \
+			{ \
+				NLMISC::INelContext::getInstance().releaseSingletonPointer(#className, _Instance); \
+				delete _Instance; \
+				_Instance = NULL; \
+			} \
+		} \
+	private:
+
+
 	/** The same as above, but generate a getInstance method that
 	 *	return a pointer instead of a reference
 	 */
@@ -304,5 +320,6 @@ void initNelLibrary(CLibrary &lib);
 
 } // namespace NLMISC
 
+#include <nel/misc/debug.h>
 
 #endif //APP_CONTEXT_H

code/nel/include/nel/misc/path.h

@@ -74,6 +74,9 @@ public:
 
 	/** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included  (the extention (Nel Big File) is used to know that it's a big file) */
 	void			addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
+	
+	/** Sale but for .snp (Streamed NeL Package) */
+	void			addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
 
 	/** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included   */
 	void			addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
@@ -276,8 +279,8 @@ private:
 	struct CMCFileEntry
 	{
 		char *Name;				// Normal case (the search is done by using nlstricmp)
-		uint32	idPath	 : 16;	// Path (not with file at the end) - look in the SSMpath (65536 different path allowed)
+		uint32	idPath	 : 20;	// Path (not with file at the end) - look in the SSMpath (1048576 different path allowed)
-		uint32	idExt	 : 15;	// real extension of the file if remapped - look in the SSMext (32768 different extension allowed)
+		uint32	idExt	 : 11;	// real extension of the file if remapped - look in the SSMext (2048 different extension allowed)
 		uint32	Remapped : 1;	// true if the file is remapped
 	};
 
@@ -371,6 +374,9 @@ public:
 	/** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included  (the extention (Nel Big File) is used to know that it's a big file) */
 	static void			addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
 
+	/** Same but Streamed Package */
+	static void			addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
+
 	/** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included   */
 	static void			addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
 

code/nel/include/nel/misc/seven_zip.h

@@ -18,6 +18,9 @@
 #define SEVEN_ZIP_H
 
 #include <string>
+#include <nel/misc/sha1.h>
+
+namespace NLMISC {
 
 // utility func to decompress a monofile 7zip archive
 bool unpack7Zip(const std::string &sevenZipFileName, const std::string &destFileName);
@@ -25,7 +28,12 @@ bool unpack7Zip(const std::string &sevenZipFileName, const std::string &destFile
 // utility func to decompress a single LZMA packed file
 bool unpackLZMA(const std::string &lzmaFileName, const std::string &destFileName);
 
+// utility func to decompress a single LZMA packed file
+bool unpackLZMA(const std::string &lzmaFileName, const std::string &destFileName, CHashKey &sha1);
+
 // utility func to compress a single file to LZMA packed file
 bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName);
 
+}
+
 #endif

code/nel/include/nel/misc/sha1.h

@@ -23,6 +23,8 @@
 
 #include <string>
 
+namespace NLMISC {
+
 struct CHashKey
 {
 	CHashKey () { HashKeyString.resize(20); }
@@ -109,4 +111,6 @@ CHashKey getSHA1(const uint8 *buffer, uint32 size);
 // This function get a buffer and key with size and returns his HMAC-SHA1 hash key
 CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint32 key_len);
 
+}
+
 #endif // NL_SHA1_H

code/nel/include/nel/misc/streamed_package.h

@@ -0,0 +1,57 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2014  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef NLMISC_STREAMED_PACKAGE_H
+#define NLMISC_STREAMED_PACKAGE_H
+
+#include <nel/misc/types_nl.h>
+#include <nel/misc/sha1.h>
+
+namespace NLMISC {
+
+class CStreamedPackage
+{
+public:
+	struct CEntry
+	{
+		std::string Name;
+		CHashKey Hash;
+		uint32 Size;
+		uint32 LastModified;
+
+		void serial(NLMISC::IStream &f) throw(NLMISC::EStream);
+
+	};
+
+public:
+	CStreamedPackage();
+	~CStreamedPackage();
+
+	void serial(NLMISC::IStream &f) throw(NLMISC::EStream);
+
+	static void makePath(std::string &result, const CHashKey &hash);
+
+public:
+	typedef std::vector<CEntry> TEntries;
+	TEntries Entries;
+	
+}; /* class CStreamedPackage */
+
+} /* namespace NLMISC */
+
+#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */
+
+/* end of file */

code/nel/include/nel/misc/streamed_package_manager.h

@@ -0,0 +1,72 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2014  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H
+#define NLMISC_STREAMED_PACKAGE_MANAGER_H
+
+#include <nel/misc/types_nl.h>
+#include <nel/misc/app_context.h>
+#include <nel/misc/streamed_package.h>
+
+namespace NLMISC {
+
+class CStreamedPackageManager
+{
+	NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(CStreamedPackageManager);
+
+public:
+	CStreamedPackageManager();
+	~CStreamedPackageManager();
+
+	/// package: ex. /games/nel/data/characters_maps_hr.snp
+	bool loadPackage(const std::string &package);
+
+	/// Get a file list
+	void list(std::vector<std::string> &fileNames, const std::string &package);
+
+	/// Unload all packages
+	void unloadAll();
+
+	/// Get an existing file or download if necessary
+	/// filePath: [out] ex. /games/nel/stream/00/00/000000000..
+	/// fileName: ex. fy_hof_underwear.dds
+	bool getFile(std::string &filePath, const std::string &fileName);
+
+	/// Get file size
+	bool getFileSize(uint32 &fileSize, const std::string &fileName);
+
+public:
+	/// Set storage path (ex. stream/)
+	std::string Path;
+
+	/// Loads a package into the package manager (ex. http://patch.live.polyverse.org/stream/)
+	typedef std::vector<std::string> THosts;
+	THosts Hosts;
+
+private:
+	typedef std::map<std::string, CStreamedPackage> TPackages;
+	TPackages m_Packages;
+
+	typedef std::map<std::string, const CStreamedPackage::CEntry *> TEntries;
+	TEntries m_Entries;
+	
+}; /* class CStreamedPackageManager */
+
+} /* namespace NLMISC */
+
+#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */
+
+/* end of file */

code/nel/src/misc/CMakeLists.txt

@@ -25,6 +25,8 @@ FILE(GLOB NLMISC_FILESYSTEM
 	big_file.cpp ../../include/nel/misc/big_file.h
 	*_xml.cpp ../../include/nel/misc/*_xml.h
 	xml_*.cpp ../../include/nel/misc/xml_*.h
+	streamed_*.cpp ../../include/nel/misc/streamed_*.h
+	seven_zip.cpp ../../include/nel/misc/seven_zip.h
 )
 
 FILE(GLOB NLMISC_STREAM
@@ -209,15 +211,16 @@ IF(UNIX)
   ENDIF()
 ENDIF()
 
-INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR} ${PNG_INCLUDE_DIR} config_file)
+INCLUDE_DIRECTORIES(../../3rdparty)
+INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR} ${PNG_INCLUDE_DIR} ${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR} config_file)
 
-TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY})
+TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES} nel_sevenzip)
 NL_DEFAULT_PROPS(nelmisc "NeL, Library: NeL Misc")
 NL_ADD_RUNTIME_FLAGS(nelmisc)
 
 NL_ADD_LIB_SUFFIX(nelmisc)
 
-ADD_DEFINITIONS(${LIBXML2_DEFINITIONS})
+ADD_DEFINITIONS(${LIBXML2_DEFINITIONS} ${CURL_DEFINITIONS})
 
 IF(WITH_PCH)
   ADD_NATIVE_PRECOMPILED_HEADER(nelmisc ${CMAKE_CURRENT_SOURCE_DIR}/stdmisc.h ${CMAKE_CURRENT_SOURCE_DIR}/stdmisc.cpp)

code/nel/src/misc/file.cpp

@@ -23,6 +23,7 @@
 #include "nel/misc/command.h"
 #include "nel/misc/sstring.h"
 #include "nel/misc/xml_pack.h"
+#include "nel/misc/streamed_package_manager.h"
 
 #ifndef NL_OS_WINDOWS
 #include <errno.h>
@@ -211,6 +212,40 @@ bool		CIFile::open(const std::string &path, bool text)
 				_F = CXMLPack::getInstance().getFile (path, _FileSize, _BigFileOffset, dummy, _AlwaysOpened);
 			}
 		}
+		else if (pos > 3 && path[pos-3] == 's' && path[pos-2] == 'n' && path[pos-1] == 'p')
+		{
+			// nldebug("Opening a streamed package file");
+
+			_IsInXMLPackFile = false;
+			_IsInBigFile = false;
+			_BigFileOffset = 0;
+			_AlwaysOpened = false;
+			std::string filePath;
+			if (CStreamedPackageManager::getInstance().getFile (filePath, path.substr(pos+1)))
+			{
+				_F = fopen (filePath.c_str(), mode);
+				if (_F != NULL)
+				{
+					_FileSize=CFile::getFileSize(_F);
+					if (_FileSize == 0)
+					{
+						nlwarning("FILE: Size of file '%s' is 0", path.c_str());
+						fclose(_F);
+						_F = NULL;
+					}
+				}
+				else
+				{
+					nlwarning("Failed to open file '%s', error %u : %s", path.c_str(), errno, strerror(errno));
+					_FileSize = 0;
+				}
+			}
+			else
+			{
+				// TEMPORARY ERROR
+				// nlerror("File '%s' not in streamed package", path.c_str());
+			}
+		}
 		else
 		{
 			// bnp file

code/nel/src/misc/path.cpp

@@ -23,6 +23,7 @@
 #include "nel/misc/progress_callback.h"
 #include "nel/misc/file.h"
 #include "nel/misc/xml_pack.h"
+#include "nel/misc/streamed_package_manager.h"
 
 #ifdef NL_OS_WINDOWS
 #	include <sys/types.h>
@@ -288,6 +289,7 @@ void CFileContainer::clearMap ()
 	nlassert(!_MemoryCompressed);
 	_Files.clear ();
 	CBigFile::getInstance().removeAll ();
+	CStreamedPackageManager::getInstance().unloadAll ();
 	NL_DISPLAY_PATH("PATH: CPath::clearMap(): map directory cleared");
 }
 
@@ -336,7 +338,7 @@ void CFileContainer::remapExtension (const string &ext1, const string &ext2, boo
 		nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): can't remap empty extension", ext1lwr.c_str(), ext2lwr.c_str(), substitute);
 	}
 
-	if (ext1lwr == "bnp" || ext2lwr == "bnp")
+	if (ext1lwr == "bnp" || ext2lwr == "bnp" || ext1lwr == "snp" || ext2lwr == "snp")
 	{
 		nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): you can't remap a big file", ext1lwr.c_str(), ext2lwr.c_str(), substitute);
 	}
@@ -1162,16 +1164,26 @@ void CFileContainer::addSearchFile (const string &file, bool remap, const string
 		return;
 	}
 
+	std::string fileExtension = CFile::getExtension(newFile);
+
 	// check if it s a big file
-	if (CFile::getExtension(newFile) == "bnp")
+	if (fileExtension == "bnp")
 	{
 		NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is a big file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str());
 		addSearchBigFile(file, false, false, progressCallBack);
 		return;
 	}
 
+	// check if it s a streamed package
+	if (fileExtension == "snp")
+	{
+		NL_DISPLAY_PATH ("PATH: CPath::addSearchStreamedPackage(%s, %d, '%s'): '%s' is a streamed package, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str());
+		addSearchStreamedPackage(file, false, false, progressCallBack);
+		return;
+	}
+
 	// check if it s an xml pack file
-	if (CFile::getExtension(newFile) == "xml_pack")
+	if (fileExtension == "xml_pack")
 	{
 		NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is an xml pack file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str());
 		addSearchXmlpackFile(file, false, false, progressCallBack);
@@ -1392,6 +1404,64 @@ void CFileContainer::addSearchBigFile (const string &sBigFilename, bool recurse,
 	fclose (Handle);
 }
 
+void CPath::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack)
+{
+	getInstance()->_FileContainer.addSearchBigFile(filename, recurse, alternative, progressCallBack);
+}
+
+void CFileContainer::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack)
+{
+	// Check if filename is not empty
+	if (filename.empty())
+	{
+		nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): can't add empty file, skip it", filename.c_str(), recurse, alternative);
+		return;
+	}
+	// Check if the file exists
+	if (!CFile::isExists(filename))
+	{
+		nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not found, skip it", filename.c_str(), recurse, alternative, filename.c_str());
+		return;
+	}
+	// Check if it s a file
+	if (CFile::isDirectory(filename))
+	{
+		nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not a file, skip it", filename.c_str(), recurse, alternative, filename.c_str());
+		return;
+	}
+
+	// Add the file itself
+	std::string packname = NLMISC::toLower(CFile::getFilename(filename));
+	insertFileInMap(packname, filename, false, CFile::getExtension(filename));
+
+	// Add the package to the package manager
+	if (CStreamedPackageManager::getInstance().loadPackage(filename))
+	{
+		std::vector<std::string> fileNames;
+		CStreamedPackageManager::getInstance().list(fileNames, packname);
+
+		for (std::vector<std::string>::iterator it(fileNames.begin()), end(fileNames.end()); it != end; ++it)
+		{
+			// Add the file to the lookup
+			std::string filePackageName = packname + "@" + (*it);
+			// nldebug("Insert '%s'", filePackageName.c_str());
+			insertFileInMap((*it), filePackageName, false, CFile::getExtension(*it));
+
+			// Remapped extensions
+			std::string ext = CFile::getExtension(*it);
+			for (uint j = 0; j < _Extensions.size (); j++)
+			{
+				if (_Extensions[j].first == ext)
+				{
+					// Need to remap
+					insertFileInMap(CFile::getFilenameWithoutExtension(*it) + "." + _Extensions[j].second,
+						filePackageName, true, _Extensions[j].first);
+				}
+			}
+		}
+	}
+}
+
 // WARNING : recurse is not used
 void CPath::addSearchXmlpackFile (const string &sXmlpackFilename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack)
 {
@@ -1679,7 +1749,7 @@ void CFileContainer::memoryCompress()
 	while (it != _Files.end())
 	{
 		string sTmp = SSMpath.get(it->second.idPath);
-		if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && !it->second.Remapped)
+		if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && (sTmp.find("snp@") == string::npos) && !it->second.Remapped)
 		{
 			// This is a file included in a bigfile (so the name is in the bigfile manager)
 		}
@@ -1702,7 +1772,7 @@ void CFileContainer::memoryCompress()
 	{
 		CFileEntry &rFE = it->second;
 		string sTmp = SSMpath.get(rFE.idPath);
-		if (sTmp.find("@") == string::npos || sTmp.find("@@") != string::npos || rFE.Remapped)
+		if ((sTmp.find("@") == string::npos) || (sTmp.find("@@") != string::npos) || (sTmp.find("snp@") != string::npos) || rFE.Remapped)
 		{
 			strcpy(_AllFileNames+nSize, rFE.Name.c_str());
 			_MCFiles[nNb].Name = _AllFileNames+nSize;
@@ -1999,6 +2069,7 @@ string CFile::findNewFile (const string &filename)
 // \warning doesn't work with big file
 uint32	CFile::getFileSize (const std::string &filename)
 {
+	std::string::size_type pos;
 	if (filename.find("@@") != string::npos)
 	{
 		uint32 fs = 0, bfo;
@@ -2006,12 +2077,21 @@ uint32	CFile::getFileSize (const std::string &filename)
 		CXMLPack::getInstance().getFile (filename, fs, bfo, c, d);
 		return fs;
 	}
-	else if (filename.find('@') != string::npos)
+	else if ((pos = filename.find('@')) != string::npos)
 	{
-		uint32 fs = 0, bfo;
+		if (pos > 3 && filename[pos-3] == 's' && filename[pos-2] == 'n' && filename[pos-1] == 'p')
-		bool c, d;
+		{
-		CBigFile::getInstance().getFile (filename, fs, bfo, c, d);
+			uint32 fs = 0;
-		return fs;
+			CStreamedPackageManager::getInstance().getFileSize (fs, filename.substr(pos+1));
+			return fs;
+		}
+		else
+		{
+			uint32 fs = 0, bfo;
+			bool c, d;
+			CBigFile::getInstance().getFile (filename, fs, bfo, c, d);
+			return fs;
+		}
 	}
 	else
 	{

code/nel/src/misc/seven_zip.cpp

@@ -14,17 +14,18 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-#include "seven_zip.h"
+#include <nel/misc/seven_zip.h>
 
-#include "nel/misc/types_nl.h"
+#include <nel/misc/types_nl.h>
-#include "nel/misc/file.h"
+#include <nel/misc/file.h>
-#include "nel/misc/path.h"
+#include <nel/misc/path.h>
 
-#include "7z.h"
+#include <seven_zip/7z.h>
-#include "7zAlloc.h"
+#include <seven_zip/7zAlloc.h>
-#include "7zCrc.h"
+#include <seven_zip/7zCrc.h>
-#include "7zVersion.h"
+#include <seven_zip/7zVersion.h>
-#include "LzmaLib.h"
+#include <seven_zip/LzmaLib.h>
+#include <seven_zip/LzmaDec.h>
 
 #include <memory>
 
@@ -39,6 +40,7 @@ using namespace NLMISC;
 #define new DEBUG_NEW
 #endif
 
+namespace NLMISC {
 
 /// Input stream class for 7zip archive
 class CNel7ZipInStream : public ISeekInStream
@@ -273,6 +275,85 @@ bool unpackLZMA(const std::string &lzmaFile, const std::string &destFileName)
 	return true;
 }
 
+bool unpackLZMA(const std::string &lzmaFile, const std::string &destFileName, CHashKey &sha1)
+{
+	nldebug("unpackLZMA: decompress LZMA file '%s' to '%s", lzmaFile.c_str(), destFileName.c_str());
+
+	// open input file
+	CIFile inStream(lzmaFile);
+	uint32 inSize = inStream.getFileSize();
+
+	if (inSize < LZMA_PROPS_SIZE + 8)
+	{
+		nlwarning("unpackLZMA: Invalid file size, too small file '%s'", lzmaFile.c_str());
+		return false;
+	}
+
+	// allocate input buffer for props
+	std::vector<uint8> propsBuffer(LZMA_PROPS_SIZE);
+
+	// size of LZMA content
+	inSize -= LZMA_PROPS_SIZE + 8;
+
+	// allocate input buffer for lzma data
+	std::vector<uint8> inBuffer(inSize);
+
+	uint64 fileSize = 0;
+
+	try
+	{
+		// read props
+		inStream.serialBuffer(&propsBuffer[0], LZMA_PROPS_SIZE);
+
+		// read uncompressed size
+		inStream.serial(fileSize);
+
+		// read lzma content
+		inStream.serialBuffer(&inBuffer[0], inSize);
+	}
+	catch(const EReadError &e)
+	{
+		nlwarning("unpackLZMA: Error while reading '%s': %s", lzmaFile.c_str(), e.what());
+		return false;
+	}
+
+	// allocate the output buffer
+	std::vector<uint8> outBuffer(fileSize);
+
+	// in and out file sizes
+	SizeT outProcessed = (SizeT)fileSize;
+	SizeT inProcessed = (SizeT)inSize;
+
+	// decompress the file in memory
+	sint res = LzmaUncompress(&outBuffer[0], &outProcessed, &inBuffer[0], &inProcessed, &propsBuffer[0], LZMA_PROPS_SIZE);
+
+	if (res != 0 || outProcessed != fileSize)
+	{
+		nlwarning("unpackLZMA: Failed to decode lzma file '%s' with status %d", lzmaFile.c_str(), res);
+		return false;
+	}
+
+	// get hash
+	sha1 = getSHA1(&outBuffer[0], outBuffer.size());
+
+	// store on output buffer
+	COFile outStream(destFileName);
+
+	try
+	{
+		// write content
+		outStream.serialBuffer(&outBuffer[0], (uint)fileSize);
+	}
+	catch(const EFile &e)
+	{
+		nlwarning("unpackLZMA: Error while writing '%s': %s", destFileName.c_str(), e.what());
+		CFile::deleteFile(destFileName);
+		return false;
+	}
+
+	return true;
+}
+
 bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName)
 {
 	nldebug("packLZMA: compress '%s' to LZMA file '%s", srcFileName.c_str(), lzmaFileName.c_str());
@@ -371,3 +452,5 @@ bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName)
 
 	return false;
 }
+
+}

code/nel/src/misc/sha1.cpp

@@ -35,67 +35,55 @@ using namespace NLMISC;
 	#define new DEBUG_NEW
 #endif
 
-//
+namespace NLMISC {
-// Types
-//
-
-/*
- * If you do not have the ISO standard stdint.h header file, then you
- * must typedef the following:
- *    name              meaning
- *  uint32_t         unsigned 32 bit integer
- *  uint8_t          unsigned 8 bit integer (i.e., unsigned char)
- *  int_least16_t    integer of >= 16 bits
- *
- */
-
-typedef unsigned int uint32_t;
-typedef unsigned char uint8_t;
-typedef short int_least16_t;
 
 //
 // Constantes
 //
 
 static const int bufferSize = 100000;
-static uint8_t buffer[bufferSize];
+static uint8 buffer[bufferSize];
-
-enum
-{
-    shaSuccess = 0,
-    shaNull,            /* Null pointer parameter */
-    shaInputTooLong,    /* input data too long */
-    shaStateError       /* called Input after Result */
-};
 
 #define SHA1HashSize 20
 
+//
+// Types
+//
+
 /*
- *  This structure will hold context information for the SHA-1
+*  This structure will hold context information for the SHA-1
- *  hashing operation
+*  hashing operation
- */
+*/
 typedef struct SHA1Context
 {
-    uint32_t Intermediate_Hash[SHA1HashSize/4]; /* Message Digest  */
+	uint32 Intermediate_Hash[SHA1HashSize/4]; /* Message Digest  */
 
-    uint32_t Length_Low;            /* Message length in bits      */
+	uint32 Length_Low;            /* Message length in bits      */
-    uint32_t Length_High;           /* Message length in bits      */
+	uint32 Length_High;           /* Message length in bits      */
 
-                               /* Index into message block array   */
+								  /* Index into message block array   */
-    int_least16_t Message_Block_Index;
+	sint16 Message_Block_Index;
-    uint8_t Message_Block[64];      /* 512-bit message blocks      */
+	uint8 Message_Block[64];      /* 512-bit message blocks      */
 
-    int Computed;               /* Is the digest computed?         */
+	int Computed;               /* Is the digest computed?         */
-    int Corrupted;             /* Is the message digest corrupted? */
+	int Corrupted;             /* Is the message digest corrupted? */
 } SHA1Context;
 
+enum
+{
+    shaSuccess = 0,
+    shaNull,            /* Null pointer parameter */
+    shaInputTooLong,    /* input data too long */
+    shaStateError       /* called Input after Result */
+};
+
 //
 //  Function Prototypes
 //
 
 int SHA1Reset (SHA1Context *);
-int SHA1Input (SHA1Context *, const uint8_t *, unsigned int);
+int SHA1Input (SHA1Context *, const uint8 *, unsigned int);
-int SHA1Result (SHA1Context *, uint8_t Message_Digest[SHA1HashSize]);
+int SHA1Result (SHA1Context *, uint8 Message_Digest[SHA1HashSize]);
 
 //
 // Functions
@@ -105,7 +93,7 @@ CHashKey getSHA1(const uint8 *buffer, uint32 size)
 {
     SHA1Context sha;
     int err;
-    uint8_t Message_Digest[20];
+    uint8 Message_Digest[20];
 
 	err = SHA1Reset(&sha);
 	if (err)
@@ -114,7 +102,7 @@ CHashKey getSHA1(const uint8 *buffer, uint32 size)
 		return CHashKey();
 	}
 
-	err = SHA1Input(&sha, (const uint8_t*)buffer, size);
+	err = SHA1Input(&sha, (const uint8*)buffer, size);
 	if (err)
 	{
 		nlwarning ("SHA: SHA1Input Error %d.\n", err);
@@ -136,7 +124,7 @@ CHashKey getSHA1(const string &filename, bool forcePath)
 {
     SHA1Context sha;
     int err;
-    uint8_t Message_Digest[20];
+    uint8 Message_Digest[20];
 
 	//printf("reading '%s'\n", findData.cFileName);
 	CIFile ifile;
@@ -201,9 +189,9 @@ CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint3
 {
 	SHA1Context sha;
 
-	uint8_t SHA1_Key[64];
+	uint8 SHA1_Key[64];
-	uint8_t SHA1_Key1[20];
+	uint8 SHA1_Key1[20];
-	uint8_t SHA1_Key2[20];
+	uint8 SHA1_Key2[20];
 
 	string buffer1;
 	string buffer2;
@@ -214,9 +202,9 @@ CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint3
 
 	// If lenght of key > 64 use sha1 hash
 	if (key_len > 64) {
-		uint8_t SHA1_Key0[20];
+		uint8 SHA1_Key0[20];
 		SHA1Reset(&sha);
-		SHA1Input(&sha, (const uint8_t*)key, key_len);
+		SHA1Input(&sha, (const uint8*)key, key_len);
 		SHA1Result(&sha, SHA1_Key0);
 		CHashKey hk0 (SHA1_Key0);
 		for (uint i = 0; i <  20; i++)
@@ -236,7 +224,7 @@ CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint3
 
 	// Get hash
 	SHA1Reset(&sha);
-	SHA1Input(&sha, (const uint8_t*)buffer1.c_str(), (uint)buffer1.size());
+	SHA1Input(&sha, (const uint8*)buffer1.c_str(), (uint)buffer1.size());
 	SHA1Result(&sha, SHA1_Key1);
 	CHashKey hk1 (SHA1_Key1);
 
@@ -250,7 +238,7 @@ CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint3
 
 	// Get new hash
 	SHA1Reset(&sha);
-	SHA1Input(&sha, (const uint8_t*)buffer2.c_str(), (uint)buffer2.size());
+	SHA1Input(&sha, (const uint8*)buffer2.c_str(), (uint)buffer2.size());
 	SHA1Result(&sha, SHA1_Key2);
 	CHashKey hk (SHA1_Key2);
 
@@ -330,7 +318,7 @@ int SHA1Reset(SHA1Context *context)
  *
  */
 int SHA1Result( SHA1Context *context,
-                uint8_t Message_Digest[SHA1HashSize])
+                uint8 Message_Digest[SHA1HashSize])
 {
     int i;
 
@@ -360,7 +348,7 @@ int SHA1Result( SHA1Context *context,
 
     for(i = 0; i < SHA1HashSize; ++i)
     {
-        Message_Digest[i] = uint8_t((context->Intermediate_Hash[i>>2]
+        Message_Digest[i] = uint8((context->Intermediate_Hash[i>>2]
                             >> 8 * ( 3 - ( i & 0x03 ) ) ) & 0xff );
     }
 
@@ -388,7 +376,7 @@ int SHA1Result( SHA1Context *context,
  *
  */
 int SHA1Input(    SHA1Context    *context,
-                  const uint8_t  *message_array,
+                  const uint8  *message_array,
                   unsigned       length)
 {
     if (!length)
@@ -462,16 +450,16 @@ int SHA1Input(    SHA1Context    *context,
  */
 void SHA1ProcessMessageBlock(SHA1Context *context)
 {
-    const uint32_t K[] =    {       /* Constants defined in SHA-1   */
+    const uint32 K[] =    {       /* Constants defined in SHA-1   */
                             0x5A827999,
                             0x6ED9EBA1,
                             0x8F1BBCDC,
                             0xCA62C1D6
                             };
     int           t;                 /* Loop counter                */
-    uint32_t      temp;              /* Temporary word value        */
+    uint32      temp;              /* Temporary word value        */
-    uint32_t      W[80];             /* Word sequence               */
+    uint32      W[80];             /* Word sequence               */
-    uint32_t      A, B, C, D, E;     /* Word buffers                */
+    uint32      A, B, C, D, E;     /* Word buffers                */
 
     /*
      *  Initialize the first 16 words in the array W
@@ -607,14 +595,17 @@ void SHA1PadMessage(SHA1Context *context)
 	/*
 	*  Store the message length as the last 8 octets
 	*/
-	context->Message_Block[56] = uint8_t((context->Length_High >> 24)&0xff);
+	context->Message_Block[56] = uint8((context->Length_High >> 24)&0xff);
-	context->Message_Block[57] = uint8_t((context->Length_High >> 16)&0xff);
+	context->Message_Block[57] = uint8((context->Length_High >> 16)&0xff);
-	context->Message_Block[58] = uint8_t((context->Length_High >> 8)&0xff);
+	context->Message_Block[58] = uint8((context->Length_High >> 8)&0xff);
-	context->Message_Block[59] = uint8_t((context->Length_High)&0xff);
+	context->Message_Block[59] = uint8((context->Length_High)&0xff);
-	context->Message_Block[60] = uint8_t((context->Length_Low >> 24)&0xff);
+	context->Message_Block[60] = uint8((context->Length_Low >> 24)&0xff);
-	context->Message_Block[61] = uint8_t((context->Length_Low >> 16)&0xff);
+	context->Message_Block[61] = uint8((context->Length_Low >> 16)&0xff);
-	context->Message_Block[62] = uint8_t((context->Length_Low >> 8)&0xff);
+	context->Message_Block[62] = uint8((context->Length_Low >> 8)&0xff);
-	context->Message_Block[63] = uint8_t((context->Length_Low)&0xff);
+	context->Message_Block[63] = uint8((context->Length_Low)&0xff);
 
 	SHA1ProcessMessageBlock(context);
 }
+
+}
+

code/nel/src/misc/streamed_package.cpp

@@ -0,0 +1,66 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2014  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "stdmisc.h"
+
+// Project includes
+#include <nel/misc/streamed_package.h>
+#include <nel/misc/stream.h>
+
+namespace NLMISC {
+
+CStreamedPackage::CStreamedPackage()
+{
+	// init
+}
+
+CStreamedPackage::~CStreamedPackage()
+{
+	// release
+}
+
+void CStreamedPackage::serial(NLMISC::IStream &f) throw(NLMISC::EStream)
+{
+	f.serialCheck(NELID("SNPK"));
+
+	uint version = 1;
+	f.serialVersion(version);
+
+	f.serialCont(Entries);
+}
+
+void CStreamedPackage::CEntry::serial(NLMISC::IStream &f) throw(NLMISC::EStream)
+{
+	uint version = 1;
+	f.serialVersion(version);
+
+	f.serial(Name);
+	f.serial(Hash);
+	f.serial(Size);
+	f.serial(LastModified);
+}
+
+void CStreamedPackage::makePath(std::string &result, const CHashKey &hash)
+{
+	std::string lowerHash = NLMISC::toLower(hash.toString());
+	result = std::string("/") + lowerHash.substr(0, 2)
+		+ "/" + lowerHash.substr(2, 2)
+		+ "/" + lowerHash.substr(4);
+}
+
+} /* namespace NLMISC */
+
+/* end of file */

code/nel/src/misc/streamed_package_manager.cpp

@@ -0,0 +1,214 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2014  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "stdmisc.h"
+
+// 3rd Party includes
+#include <curl/curl.h>
+
+// Project includes
+#include <nel/misc/streamed_package_manager.h>
+#include <nel/misc/file.h>
+#include <nel/misc/path.h>
+#include <nel/misc/seven_zip.h>
+#include <nel/misc/sha1.h>
+
+namespace NLMISC
+{
+
+NLMISC_SAFE_SINGLETON_IMPL(CStreamedPackageManager);
+
+CStreamedPackageManager::CStreamedPackageManager()
+{
+	// init
+}
+
+CStreamedPackageManager::~CStreamedPackageManager()
+{
+	// release
+}
+
+bool CStreamedPackageManager::loadPackage(const std::string &package)
+{
+	nldebug("Load package '%s'", package.c_str());
+
+	std::string packname = NLMISC::toLower(CFile::getFilename(package));
+	m_Packages[packname] = CStreamedPackage();
+	std::map<std::string, CStreamedPackage>::iterator it = m_Packages.find(packname);
+	CStreamedPackage &p = it->second;
+
+	try
+	{
+		CIFile fi;
+		fi.open(package);
+		fi.serial(p);
+	}
+	catch (Exception &e)
+	{
+		nlerror("Package serial exception: '%s'", e.what());
+		m_Packages.erase(it);
+		return false;
+	}
+
+	for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it)
+	{
+		const CStreamedPackage::CEntry &entry = (*it);
+		m_Entries[entry.Name] = &entry;
+	}
+
+	return true;
+}
+
+void CStreamedPackageManager::list(std::vector<std::string> &fileNames, const std::string &package)
+{
+	// nldebug("List package '%s'", package.c_str());
+
+	std::map<std::string, CStreamedPackage>::iterator it = m_Packages.find(package);
+	CStreamedPackage &p = it->second;
+
+	for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it)
+	{
+		const CStreamedPackage::CEntry &entry = (*it);
+		fileNames.push_back(entry.Name);
+	}
+}
+
+void CStreamedPackageManager::unloadAll()
+{
+	m_Packages.clear();
+	m_Entries.clear();
+}
+
+bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName)
+{
+	// nldebug("Get file path for streamed file '%s'", fileName.c_str());
+
+	TEntries::iterator it = m_Entries.find(fileName);
+	if (it == m_Entries.end())
+		return false;
+	const CStreamedPackage::CEntry *entry = it->second;
+
+	CStreamedPackage::makePath(filePath, entry->Hash);
+	std::string downloadUrlFile = filePath + ".lzma";
+	filePath = Path + filePath;
+	std::string downloadPath = filePath + ".download." + toString(rand() * rand());
+
+	std::string storageDirectory = CFile::getPath(downloadPath);
+	CFile::createDirectoryTree(storageDirectory);
+	/*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory))
+	{
+		nldebug("Unable to create directory '%s'", storageDirectory.c_str());
+		return false;
+	}*/
+
+	// download
+	for (;;)
+	{
+		if (CFile::fileExists(filePath))
+			return true;
+
+		std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile;
+		nldebug("Download streamed package '%s' from '%s'", fileName.c_str(), downloadUrl.c_str());
+
+		FILE *fp = fopen(downloadPath.c_str(), "wb");
+		if (fp == NULL)
+		{
+			nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), fileName.c_str());
+			return false;
+		}
+
+		CURL *curl;
+		CURLcode res;
+		curl = curl_easy_init();
+		if (curl)
+		{
+			curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str());
+			curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+			curl_easy_setopt(curl, CURLOPT_FILE, fp);
+			res = curl_easy_perform(curl);
+			long r;
+			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r);
+			curl_easy_cleanup(curl);
+
+			bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/;
+			fclose(fp);
+
+			if (diskFull)
+			{
+				CFile::deleteFile(downloadPath);
+				throw EDiskFullError(downloadPath);
+			}
+
+			if (res != CURLE_OK || r < 200 || r >= 300)
+			{
+				CFile::deleteFile(downloadPath);
+				nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str());
+				nlSleep(1000);
+				continue;
+			}
+		}
+		else
+		{
+			nldebug("Curl initialize failed");
+			fclose(fp);
+			CFile::deleteFile(downloadPath);
+			return false;
+		}
+
+		// ok!
+		break;
+	}
+
+	// extract into file
+	std::string unpackPath = filePath + ".extract." + toString(rand() * rand());
+
+	CHashKey outHash;
+	if (!unpackLZMA(downloadPath, unpackPath, outHash))
+	{
+		return false;
+	}
+
+	if (!(outHash == entry->Hash))
+	{
+		std::string wantHashS = entry->Hash.toString();
+		std::string outHashS = outHash.toString();
+		nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str());
+		return false;
+	}
+
+	if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str()))
+	{
+		nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str());
+		// in case downloaded from another thread
+		return CFile::fileExists(filePath);
+	}
+
+	return true;
+}
+
+bool CStreamedPackageManager::getFileSize(uint32 &fileSize, const std::string &fileName)
+{
+	// nldebug("Get file size for streamed file '%s'", fileName.c_str());
+	TEntries::iterator it = m_Entries.find(fileName);
+	if (it == m_Entries.end())
+		return false;
+	fileSize = it->second->Size;
+	return true;
+}
+
+} /* namespace NLMISC */
+
+/* end of file */

code/nel/tools/build_gamedata/configuration/tools.py

@@ -90,6 +90,7 @@ BuildClodBankTool = "build_clod_bank"
 SheetsPackerTool = "sheets_packer"
 SheetsPackerShardTool = "sheets_packer_shard"
 BnpMakeTool = "bnp_make"
+SnpMakeTool = "snp_make"
 AiBuildWmapTool = "ai_build_wmap"
 TgaCutTool = "tga_cut"
 PatchGenTool = "patch_gen"

code/nel/tools/build_gamedata/d1_client_patch.py

@@ -51,6 +51,7 @@ printLog(log, "")
 
 # Find tools
 BnpMake = findTool(log, ToolDirectories, BnpMakeTool, ToolSuffix)
+SnpMake = findTool(log, ToolDirectories, SnpMakeTool, ToolSuffix);
 PatchGen = findTool(log, ToolDirectories, PatchGenTool, ToolSuffix)
 printLog(log, "")
 
@@ -90,20 +91,23 @@ else:
 					inCategories = 1
 					cfg.write("\t<_Categories>\n")
 					for category in InstallClientData:
+						packExt = ".bnp"
+						if (category["StreamedPackages"]):
+							packExt = ".snp"
 						cfg.write("\t\t<_Category>\n")
 						cfg.write("\t\t\t<_Name type=\"STRING\" value=\"" + category["Name"] + "\"/>\n")
 						if category["UnpackTo"] != None:
 							if category["UnpackTo"] != "":
 								cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./" + category["UnpackTo"] + "/\"/>\n")
 							else:
-								cfg.write("\t\t\t<_UnpackTo type=\"SINT32\" value=\"./\"/>\n")
+								cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./\"/>\n")
 						cfg.write("\t\t\t<_IsOptional type=\"SINT32\" value=\"" + str(category["IsOptional"]) + "\"/>\n")
 						cfg.write("\t\t\t<_IsIncremental type=\"SINT32\" value=\"" + str(category["IsIncremental"]) + "\"/>\n")
 						for package in category["Packages"]:
 							if (len(package[1]) > 0):
 								cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[1][0] + "\"/>\n")
 							else:
-								cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[0] + ".bnp\"/>\n")
+								cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[0] + packExt + "\"/>\n")
 						for ref in category["Refs"]:
 							cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + ref + "_.ref\"/>\n")
 						cfg.write("\t\t</_Category>\n")
@@ -121,11 +125,14 @@ else:
 	targetPath = ClientPatchDirectory + "/bnp"
 	mkPath(log, targetPath)
 	for category in InstallClientData:
+		packExt = ".bnp"
+		if (category["StreamedPackages"]):
+			packExt = ".snp"
 		for package in category["Packages"]:
 			printLog(log, "PACKAGE " + package[0])
 			sourcePath = InstallDirectory + "/" + package[0]
 			mkPath(log, sourcePath)
-			targetBnp = targetPath + "/" + package[0] + ".bnp"
+			targetBnp = targetPath + "/" + package[0] + packExt
 			if (len(package[1]) > 0):
 				targetBnp = targetPath + "/" + package[1][0]
 				printLog(log, "TARGET " + package[1][0])
@@ -135,8 +142,16 @@ else:
 			else:
 				needUpdateBnp = needUpdateDirNoSubdirFile(log, sourcePath, targetBnp)
 			if (needUpdateBnp):
-				printLog(log, "BNP " + targetBnp)
+				if (category["StreamedPackages"]):
-				subprocess.call([ BnpMake, "-p", sourcePath, "-o", targetBnp ] + package[1][1:])
+					printLog(log, "SNP " + targetBnp)
+					# cwDir = os.getcwd().replace("\\", "/")
+					# toolDir = os.path.dirname(Lzma).replace("\\", "/")
+					# os.chdir(toolDir)
+					subprocess.call([ SnpMake, "-p", sourcePath, targetBnp, ClientPatchDirectory + "/stream" ] + package[1][1:])
+					# os.chdir(cwDir)
+				else:
+					printLog(log, "BNP " + targetBnp)
+					subprocess.call([ BnpMake, "-p", sourcePath, "-o", targetBnp ] + package[1][1:])
 			else:
 				printLog(log, "SKIP " + targetBnp)
 	printLog(log, "")

code/nel/tools/build_gamedata/d2_client_install.py

@@ -47,6 +47,9 @@ printLog(log, "")
 
 for category in InstallClientData:
 	printLog(log, "CATEGORY " + category["Name"])
+	packExt = ".bnp"
+	if (category["StreamedPackages"]):
+		packExt = ".snp"
 	if (category["UnpackTo"] != None):
 		targetPath = ClientInstallDirectory
 		if (category["UnpackTo"] != ""):
@@ -62,8 +65,8 @@ for category in InstallClientData:
 		mkPath(log, targetPath)
 		for package in category["Packages"]:
 			printLog(log, "PACKAGE " + package[0])
-			sourceBnp = sourcePath + "/" + package[0] + ".bnp"
+			sourceBnp = sourcePath + "/" + package[0] + packExt
-			targetBnp = targetPath + "/" + package[0] + ".bnp"
+			targetBnp = targetPath + "/" + package[0] + packExt
 			if (len(package[1]) > 0):
 				sourceBnp = sourcePath + "/" + package[1][0]
 				targetBnp = targetPath + "/" + package[1][0]

code/nel/tools/misc/CMakeLists.txt

@@ -4,7 +4,15 @@ IF(WITH_QT OR WITH_QT5)
 ENDIF()
 
 IF(WITH_NEL_TOOLS)
-  SUBDIRS(bnp_make disp_sheet_id extract_filename lock make_sheet_id xml_packer)
+  SUBDIRS(
+    bnp_make
+    snp_make
+    disp_sheet_id
+    extract_filename
+    lock
+    make_sheet_id
+    xml_packer
+  )
 
   IF(WITH_QT OR WITH_QT5)
     ADD_SUBDIRECTORY(words_dic_qt)

code/nel/tools/misc/snp_make/CMakeLists.txt

@@ -0,0 +1,9 @@
+FILE(GLOB SRC *.cpp *.h)
+
+ADD_EXECUTABLE(snp_make ${SRC})
+
+TARGET_LINK_LIBRARIES(snp_make nelmisc)
+NL_DEFAULT_PROPS(snp_make "NeL, Tools, Misc: snp_make")
+NL_ADD_RUNTIME_FLAGS(snp_make)
+
+INSTALL(TARGETS snp_make RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT toolsmisc)

code/nel/tools/misc/snp_make/main.cpp

@@ -0,0 +1,360 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2010  Winch Gate Property Limited
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "nel/misc/types_nl.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef NL_OS_WINDOWS
+#	include <io.h>
+#	include <direct.h>
+#endif
+
+#include <vector>
+#include <string>
+
+#include "nel/misc/debug.h"
+#include "nel/misc/file.h"
+#include "nel/misc/path.h"
+#include "nel/misc/algo.h"
+#include "nel/misc/common.h"
+#include "nel/misc/streamed_package.h"
+#include "nel/misc/seven_zip.h"
+
+using namespace std;
+using namespace NLMISC;
+
+// ---------------------------------------------------------------------------
+
+class CWildCard
+{
+public:
+	string		Expression;
+	bool		Not;
+};
+std::vector<CWildCard>	WildCards;
+
+std::string SourceDirectory;
+std::string PackageFileName;
+std::string StreamDirectory;
+
+CStreamedPackage Package;
+
+// ---------------------------------------------------------------------------
+
+bool keepFile (const char *fileName)
+{
+	uint i;
+	bool ifPresent = false;
+	bool ifTrue = false;
+	string file = toLower(CFile::getFilename (fileName));
+	for (i=0; i<WildCards.size(); i++)
+	{
+		if (WildCards[i].Not)
+		{
+			// One ifnot condition met and the file is not added
+			if (testWildCard(file.c_str(), WildCards[i].Expression.c_str()))
+				return false;
+		}
+		else
+		{
+			ifPresent = true;
+			ifTrue |= testWildCard(file.c_str(), WildCards[i].Expression.c_str());
+		}
+	}
+
+	return !ifPresent || ifTrue;
+}
+
+// ---------------------------------------------------------------------------
+void usage()
+{
+	printf ("USAGE : \n");
+	printf ("   snp_make -p <directory_name> <package_file> <stream_directory> [option] ... [option]\n");
+	printf ("   option : \n");
+	printf ("      -if wildcard : add the file if it matches the wilcard (at least one 'if' conditions must be met for a file to be adding)\n");
+	printf ("      -ifnot wildcard : add the file if it doesn't match the wilcard (all the 'ifnot' conditions must be met for a file to be adding)\n");
+	printf (" Pack the directory to a snp file\n");
+	printf ("   snp_make -l <package_file>\n");
+	printf (" List the files contained in the snp file\n");
+}
+
+// ---------------------------------------------------------------------------
+
+void generateLZMA(const std::string &sourceFile, const std::string &outputFile)
+{
+	NLMISC::packLZMA(sourceFile, outputFile);
+
+	/*
+	std::string cmd="lzma e ";
+	cmd+=" "+sourceFile+" "+outputFile;
+	nlinfo("executing system command: %s",cmd.c_str());
+#ifdef NL_OS_WINDOWS
+	_spawnlp(_P_WAIT, "lzma.exe","lzma.exe", "e", sourceFile.c_str(), outputFile.c_str(), NULL);
+#else // NL_OS_WINDOWS
+	sint error = system (cmd.c_str());
+	if (error)
+		nlwarning("'%s' failed with error code %d", cmd.c_str(), error);
+#endif // NL_OS_WINDOWS
+	*/
+}
+
+// ---------------------------------------------------------------------------
+
+uint readOptions (int nNbArg, char **ppArgs)
+{
+	uint i;
+	uint optionCount = 0;
+	for (i=0; i<(uint)nNbArg; i++)
+	{
+		// If ?
+		if ((strcmp (ppArgs[i], "-if") == 0) && ((i+1)<(uint)nNbArg))
+		{
+			CWildCard card;
+			card.Expression = toLower(string(ppArgs[i+1]));
+			card.Not = false;
+			WildCards.push_back (card);
+			optionCount += 2;
+		}
+		// If not ?
+		if ((strcmp (ppArgs[i], "-ifnot") == 0) && ((i+1)<(uint)nNbArg))
+		{
+			CWildCard card;
+			card.Expression = toLower(string(ppArgs[i+1]));
+			card.Not = true;
+			WildCards.push_back (card);
+			optionCount += 2;
+		}
+	}
+	return optionCount;
+}
+
+// ---------------------------------------------------------------------------
+int main (int nNbArg, char **ppArgs)
+{
+	NLMISC::CApplicationContext myApplicationContext;
+
+	if (nNbArg < 3)
+	{
+		usage();
+		return -1;
+	}
+
+	if ((strcmp(ppArgs[1], "/p") == 0) || (strcmp(ppArgs[1], "/P") == 0) ||
+		(strcmp(ppArgs[1], "-p") == 0) || (strcmp(ppArgs[1], "-P") == 0))
+	{
+		if (nNbArg < 5)
+		{
+			usage();
+			return -1;
+		}
+
+		SourceDirectory = ppArgs[2];
+		PackageFileName = ppArgs[3];
+		StreamDirectory = ppArgs[4];
+		readOptions(nNbArg, ppArgs);
+		
+		nldebug("Make streamed package: '%s'", PackageFileName.c_str());
+
+		if (CFile::fileExists(PackageFileName))
+		{
+			nldebug("Update existing package");
+			try
+			{
+				CIFile fi;
+				fi.open(PackageFileName);
+				fi.serial(Package);
+			}
+			catch (Exception &e)
+			{
+				nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what());
+				return -1;
+			}
+		}
+		else
+		{
+			nldebug("New package");
+		}
+
+		std::vector<std::string> pathContent; // contains full pathnames
+		std::vector<std::string> nameContent; // only filename
+		CPath::getPathContent(SourceDirectory, true, false, true, pathContent);
+		nameContent.reserve(pathContent.size());
+		for (std::vector<std::string>::size_type i = 0; i < pathContent.size(); ++i)
+		{
+			const std::string &file = pathContent[i];
+			if (keepFile(file.c_str()))
+			{
+				std::string fileName = NLMISC::toLower(CFile::getFilename(file));
+				// nldebug("File: '%s' ('%s')", file.c_str(), fileName.c_str());
+				nameContent.push_back(fileName);
+				nlassert(nameContent.size() == (i + 1));
+			}
+			else
+			{
+				// Not included in this package
+				pathContent.erase(pathContent.begin() + i);
+				--i;
+			}
+		}
+
+		std::vector<sint> packageIndex; // index of file in package
+		packageIndex.resize(pathContent.size(), -1);
+
+		for (CStreamedPackage::TEntries::size_type i = 0; i < Package.Entries.size(); ++i)
+		{
+			const CStreamedPackage::CEntry &entry = Package.Entries[i];
+			
+			sint foundIndex = -1; // find index in found file list
+			for (std::vector<std::string>::size_type j = 0; j < pathContent.size(); ++j)
+			{
+				if (nameContent[j] == entry.Name)
+				{
+					foundIndex = j;
+					break;
+				}
+			}
+
+			if (foundIndex < 0)
+			{
+				nlinfo("File no longer exists: '%s'", entry.Name.c_str());
+				Package.Entries.erase(Package.Entries.begin() + i);
+				--i;
+			}
+			else
+			{
+				// File still exists, map it
+				packageIndex[foundIndex] = i;
+			}
+		}
+
+		for (std::vector<std::string>::size_type i = 0; i < pathContent.size(); ++i)
+		{
+			sint pidx = packageIndex[i];
+			const std::string &name = nameContent[i];
+			const std::string &path = pathContent[i];
+
+			if (pidx < 0)
+			{
+				nlinfo("File added: '%s'", name.c_str());
+				pidx = Package.Entries.size();
+				Package.Entries.push_back(CStreamedPackage::CEntry());
+				Package.Entries[pidx].Name = name;
+				Package.Entries[pidx].LastModified = 0;
+				Package.Entries[pidx].Size = 0;
+			}
+			else
+			{
+				nlinfo("File check for changes: '%s'", name.c_str());
+			}
+
+			CStreamedPackage::CEntry &entry = Package.Entries[pidx];
+
+			std::string targetLzmaOld; // in case lzma wasn't made make sure it exists a second run
+			CStreamedPackage::makePath(targetLzmaOld, entry.Hash);
+			targetLzmaOld = StreamDirectory + targetLzmaOld + ".lzma";
+
+			uint32 lastModified = CFile::getFileModificationDate(path);
+			uint32 fileSize = CFile::getFileSize(path);
+			if (lastModified > entry.LastModified || fileSize != entry.Size || !CFile::fileExists(targetLzmaOld))
+			{
+				entry.LastModified = lastModified;
+
+				nlinfo("Calculate file hash");
+				CHashKey hash = getSHA1(path, true);
+				/*nldebug("%s", hash.toString().c_str());
+				std::string hashPath;
+				CStreamedPackage::makePath(hashPath, hash);
+				nldebug("%s", hashPath.c_str());*/
+
+				if (hash == entry.Hash && fileSize == entry.Size)
+				{
+					// File has not changed
+				}
+				else
+				{
+					nlinfo("File changed");
+					entry.Hash = hash;
+					entry.Size = fileSize;
+				}
+
+				std::string targetLzma; // in case lzma wasn't made make sure it exists a second run
+				CStreamedPackage::makePath(targetLzma, entry.Hash);
+				targetLzma = StreamDirectory + targetLzma + ".lzma";
+
+				if (!CFile::fileExists(targetLzma))
+				{
+					// make the compressed file
+					nlinfo("%s -> %s", path.c_str(), targetLzma.c_str());
+					CFile::createDirectoryTree(CFile::getPath(targetLzma));
+					generateLZMA(path, targetLzma);
+				}
+			}
+		}
+
+		try
+		{
+			nldebug("Store package '%s'", PackageFileName.c_str());
+			COFile fo;
+			fo.open(PackageFileName);
+			fo.serial(Package);
+		}
+		catch (Exception &e)
+		{
+			nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what());
+			return -1;
+		}
+
+		return 0;
+	}	
+
+	if ((strcmp(ppArgs[1], "/l") == 0) || (strcmp(ppArgs[1], "/L") == 0) ||
+		(strcmp(ppArgs[1], "-l") == 0) || (strcmp(ppArgs[1], "-L") == 0))
+	{
+		PackageFileName = ppArgs[2];
+		if (!CFile::fileExists(PackageFileName))
+		{
+			nlwarning("ERROR (snp_make) : package doesn't exist: '%s'", PackageFileName.c_str());
+			return -1;
+		}
+
+		try
+		{
+			CIFile fi;
+			fi.open(PackageFileName);
+			fi.serial(Package);
+		}
+		catch (Exception &e)
+		{
+			nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what());
+			return -1;
+		}
+
+		for (CStreamedPackage::TEntries::const_iterator it(Package.Entries.begin()), end(Package.Entries.end()); it != end; ++it)
+		{
+			const CStreamedPackage::CEntry &entry = (*it);
+
+			printf("List files in '%s'", PackageFileName.c_str());
+			printf("%s { Hash: '%s', Size: '%u', LastModified: '%u' }", entry.Name.c_str(), entry.Hash.toString().c_str(), entry.Size, entry.LastModified);
+		}
+		
+		return 0;
+	}
+
+	usage ();
+	return -1;
+}

code/ryzom/client/CMakeLists.txt

@@ -1,5 +1,4 @@
 # Need clientsheets lib for sheets packer tool
-# Need seven_zip for patch_gen and ryzom_installer
 ADD_SUBDIRECTORY(src)
 
 IF(WITH_RYZOM_CLIENT)

code/ryzom/client/src/CMakeLists.txt

@@ -3,11 +3,6 @@ IF(WITH_RYZOM_TOOLS OR WITH_RYZOM_CLIENT)
   ADD_SUBDIRECTORY(client_sheets)
 ENDIF()
 
-IF(WITH_RYZOM_TOOLS OR WITH_RYZOM_CLIENT OR WITH_RYZOM_INSTALLER)
-  # Need seven_zip lib for patch_gen tool
-  ADD_SUBDIRECTORY(seven_zip)
-ENDIF()
-
 IF(WITH_RYZOM_CLIENT)
   IF(NOT WITH_GUI)
     MESSAGE(FATAL_ERROR "The client cannot be built without the NeL GUI Library (WITH_GUI)")
@@ -158,6 +153,8 @@ IF(WITH_RYZOM_CLIENT)
         ${OPENSSL_INCLUDE_DIR}
   )
 
+  INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/nel/3rdparty)
+
   IF(STEAM_FOUND)
     INCLUDE_DIRECTORIES(${STEAM_INCLUDE_DIRS})
   ENDIF()
@@ -170,7 +167,7 @@ IF(WITH_RYZOM_CLIENT)
               nel3d
               nelgui
               nelsound
-              ryzom_sevenzip
+              nel_sevenzip
               ryzom_clientsheets
               ryzom_gameshare
               nelpacs