diff --git a/.gitignore b/.gitignore index 1f3f7abba..22089c19f 100644 --- a/.gitignore +++ b/.gitignore @@ -142,7 +142,6 @@ moc_*.cpp *.cache *.patch *.7z -3rdParty .svn thumbs.db Thumbs.db diff --git a/code/nel/3rdparty/CMakeLists.txt b/code/nel/3rdparty/CMakeLists.txt new file mode 100644 index 000000000..bdd3e5fcd --- /dev/null +++ b/code/nel/3rdparty/CMakeLists.txt @@ -0,0 +1,2 @@ +SET(SEVENZIP_LIBRARY "nel_sevenzip") +ADD_SUBDIRECTORY(seven_zip) diff --git a/code/ryzom/client/src/seven_zip/7z.h b/code/nel/3rdparty/seven_zip/7z.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7z.h rename to code/nel/3rdparty/seven_zip/7z.h diff --git a/code/ryzom/client/src/seven_zip/7zAlloc.cpp b/code/nel/3rdparty/seven_zip/7zAlloc.c similarity index 100% rename from code/ryzom/client/src/seven_zip/7zAlloc.cpp rename to code/nel/3rdparty/seven_zip/7zAlloc.c diff --git a/code/ryzom/client/src/seven_zip/7zAlloc.h b/code/nel/3rdparty/seven_zip/7zAlloc.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zAlloc.h rename to code/nel/3rdparty/seven_zip/7zAlloc.h diff --git a/code/ryzom/client/src/seven_zip/7zArcIn.cpp b/code/nel/3rdparty/seven_zip/7zArcIn.c similarity index 100% rename from code/ryzom/client/src/seven_zip/7zArcIn.cpp rename to code/nel/3rdparty/seven_zip/7zArcIn.c diff --git a/code/ryzom/client/src/seven_zip/7zBuf.cpp b/code/nel/3rdparty/seven_zip/7zBuf.c similarity index 100% rename from code/ryzom/client/src/seven_zip/7zBuf.cpp rename to code/nel/3rdparty/seven_zip/7zBuf.c diff --git a/code/ryzom/client/src/seven_zip/7zBuf.h b/code/nel/3rdparty/seven_zip/7zBuf.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zBuf.h rename to code/nel/3rdparty/seven_zip/7zBuf.h diff --git a/code/ryzom/client/src/seven_zip/7zBuf2.cpp b/code/nel/3rdparty/seven_zip/7zBuf2.c similarity index 100% rename from code/ryzom/client/src/seven_zip/7zBuf2.cpp rename to code/nel/3rdparty/seven_zip/7zBuf2.c diff --git a/code/ryzom/client/src/seven_zip/7zCrc.cpp b/code/nel/3rdparty/seven_zip/7zCrc.c similarity index 100% rename from code/ryzom/client/src/seven_zip/7zCrc.cpp rename to code/nel/3rdparty/seven_zip/7zCrc.c diff --git a/code/ryzom/client/src/seven_zip/7zCrc.h b/code/nel/3rdparty/seven_zip/7zCrc.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zCrc.h rename to code/nel/3rdparty/seven_zip/7zCrc.h diff --git a/code/ryzom/client/src/seven_zip/7zCrcOpt.cpp b/code/nel/3rdparty/seven_zip/7zCrcOpt.c similarity index 100% rename from code/ryzom/client/src/seven_zip/7zCrcOpt.cpp rename to code/nel/3rdparty/seven_zip/7zCrcOpt.c diff --git a/code/ryzom/client/src/seven_zip/7zDec.cpp b/code/nel/3rdparty/seven_zip/7zDec.c similarity index 100% rename from code/ryzom/client/src/seven_zip/7zDec.cpp rename to code/nel/3rdparty/seven_zip/7zDec.c diff --git a/code/ryzom/client/src/seven_zip/7zFile.cpp b/code/nel/3rdparty/seven_zip/7zFile.c similarity index 100% rename from code/ryzom/client/src/seven_zip/7zFile.cpp rename to code/nel/3rdparty/seven_zip/7zFile.c diff --git a/code/ryzom/client/src/seven_zip/7zFile.h b/code/nel/3rdparty/seven_zip/7zFile.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zFile.h rename to code/nel/3rdparty/seven_zip/7zFile.h diff --git a/code/ryzom/client/src/seven_zip/7zStream.cpp b/code/nel/3rdparty/seven_zip/7zStream.c similarity index 100% rename from code/ryzom/client/src/seven_zip/7zStream.cpp rename to code/nel/3rdparty/seven_zip/7zStream.c diff --git a/code/ryzom/client/src/seven_zip/7zTypes.h b/code/nel/3rdparty/seven_zip/7zTypes.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zTypes.h rename to code/nel/3rdparty/seven_zip/7zTypes.h diff --git a/code/ryzom/client/src/seven_zip/7zVersion.h b/code/nel/3rdparty/seven_zip/7zVersion.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zVersion.h rename to code/nel/3rdparty/seven_zip/7zVersion.h diff --git a/code/ryzom/client/src/seven_zip/7zVersion.rc b/code/nel/3rdparty/seven_zip/7zVersion.rc similarity index 96% rename from code/ryzom/client/src/seven_zip/7zVersion.rc rename to code/nel/3rdparty/seven_zip/7zVersion.rc index 6ed26de74..e520995dd 100644 --- a/code/ryzom/client/src/seven_zip/7zVersion.rc +++ b/code/nel/3rdparty/seven_zip/7zVersion.rc @@ -1,55 +1,55 @@ -#define MY_VS_FFI_FILEFLAGSMASK 0x0000003FL -#define MY_VOS_NT_WINDOWS32 0x00040004L -#define MY_VOS_CE_WINDOWS32 0x00050004L - -#define MY_VFT_APP 0x00000001L -#define MY_VFT_DLL 0x00000002L - -// #include - -#ifndef MY_VERSION -#include "7zVersion.h" -#endif - -#define MY_VER MY_VER_MAJOR,MY_VER_MINOR,MY_VER_BUILD,0 - -#ifdef DEBUG -#define DBG_FL VS_FF_DEBUG -#else -#define DBG_FL 0 -#endif - -#define MY_VERSION_INFO(fileType, descr, intName, origName) \ -LANGUAGE 9, 1 \ -1 VERSIONINFO \ - FILEVERSION MY_VER \ - PRODUCTVERSION MY_VER \ - FILEFLAGSMASK MY_VS_FFI_FILEFLAGSMASK \ - FILEFLAGS DBG_FL \ - FILEOS MY_VOS_NT_WINDOWS32 \ - FILETYPE fileType \ - FILESUBTYPE 0x0L \ -BEGIN \ - BLOCK "StringFileInfo" \ - BEGIN \ - BLOCK "040904b0" \ - BEGIN \ - VALUE "CompanyName", "Igor Pavlov" \ - VALUE "FileDescription", descr \ - VALUE "FileVersion", MY_VERSION \ - VALUE "InternalName", intName \ - VALUE "LegalCopyright", MY_COPYRIGHT \ - VALUE "OriginalFilename", origName \ - VALUE "ProductName", "7-Zip" \ - VALUE "ProductVersion", MY_VERSION \ - END \ - END \ - BLOCK "VarFileInfo" \ - BEGIN \ - VALUE "Translation", 0x409, 1200 \ - END \ -END - -#define MY_VERSION_INFO_APP(descr, intName) MY_VERSION_INFO(MY_VFT_APP, descr, intName, intName ".exe") - -#define MY_VERSION_INFO_DLL(descr, intName) MY_VERSION_INFO(MY_VFT_DLL, descr, intName, intName ".dll") +#define MY_VS_FFI_FILEFLAGSMASK 0x0000003FL +#define MY_VOS_NT_WINDOWS32 0x00040004L +#define MY_VOS_CE_WINDOWS32 0x00050004L + +#define MY_VFT_APP 0x00000001L +#define MY_VFT_DLL 0x00000002L + +// #include + +#ifndef MY_VERSION +#include "7zVersion.h" +#endif + +#define MY_VER MY_VER_MAJOR,MY_VER_MINOR,MY_VER_BUILD,0 + +#ifdef DEBUG +#define DBG_FL VS_FF_DEBUG +#else +#define DBG_FL 0 +#endif + +#define MY_VERSION_INFO(fileType, descr, intName, origName) \ +LANGUAGE 9, 1 \ +1 VERSIONINFO \ + FILEVERSION MY_VER \ + PRODUCTVERSION MY_VER \ + FILEFLAGSMASK MY_VS_FFI_FILEFLAGSMASK \ + FILEFLAGS DBG_FL \ + FILEOS MY_VOS_NT_WINDOWS32 \ + FILETYPE fileType \ + FILESUBTYPE 0x0L \ +BEGIN \ + BLOCK "StringFileInfo" \ + BEGIN \ + BLOCK "040904b0" \ + BEGIN \ + VALUE "CompanyName", "Igor Pavlov" \ + VALUE "FileDescription", descr \ + VALUE "FileVersion", MY_VERSION \ + VALUE "InternalName", intName \ + VALUE "LegalCopyright", MY_COPYRIGHT \ + VALUE "OriginalFilename", origName \ + VALUE "ProductName", "7-Zip" \ + VALUE "ProductVersion", MY_VERSION \ + END \ + END \ + BLOCK "VarFileInfo" \ + BEGIN \ + VALUE "Translation", 0x409, 1200 \ + END \ +END + +#define MY_VERSION_INFO_APP(descr, intName) MY_VERSION_INFO(MY_VFT_APP, descr, intName, intName ".exe") + +#define MY_VERSION_INFO_DLL(descr, intName) MY_VERSION_INFO(MY_VFT_DLL, descr, intName, intName ".dll") diff --git a/code/ryzom/client/src/seven_zip/Aes.cpp b/code/nel/3rdparty/seven_zip/Aes.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Aes.cpp rename to code/nel/3rdparty/seven_zip/Aes.c diff --git a/code/ryzom/client/src/seven_zip/Aes.h b/code/nel/3rdparty/seven_zip/Aes.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Aes.h rename to code/nel/3rdparty/seven_zip/Aes.h diff --git a/code/ryzom/client/src/seven_zip/AesOpt.cpp b/code/nel/3rdparty/seven_zip/AesOpt.c similarity index 100% rename from code/ryzom/client/src/seven_zip/AesOpt.cpp rename to code/nel/3rdparty/seven_zip/AesOpt.c diff --git a/code/ryzom/client/src/seven_zip/Alloc.cpp b/code/nel/3rdparty/seven_zip/Alloc.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Alloc.cpp rename to code/nel/3rdparty/seven_zip/Alloc.c diff --git a/code/ryzom/client/src/seven_zip/Alloc.h b/code/nel/3rdparty/seven_zip/Alloc.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Alloc.h rename to code/nel/3rdparty/seven_zip/Alloc.h diff --git a/code/ryzom/client/src/seven_zip/Bcj2.cpp b/code/nel/3rdparty/seven_zip/Bcj2.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Bcj2.cpp rename to code/nel/3rdparty/seven_zip/Bcj2.c diff --git a/code/ryzom/client/src/seven_zip/Bcj2.h b/code/nel/3rdparty/seven_zip/Bcj2.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Bcj2.h rename to code/nel/3rdparty/seven_zip/Bcj2.h diff --git a/code/ryzom/client/src/seven_zip/Bcj2Enc.cpp b/code/nel/3rdparty/seven_zip/Bcj2Enc.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Bcj2Enc.cpp rename to code/nel/3rdparty/seven_zip/Bcj2Enc.c diff --git a/code/ryzom/client/src/seven_zip/Bra.cpp b/code/nel/3rdparty/seven_zip/Bra.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Bra.cpp rename to code/nel/3rdparty/seven_zip/Bra.c diff --git a/code/ryzom/client/src/seven_zip/Bra.h b/code/nel/3rdparty/seven_zip/Bra.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Bra.h rename to code/nel/3rdparty/seven_zip/Bra.h diff --git a/code/ryzom/client/src/seven_zip/Bra86.cpp b/code/nel/3rdparty/seven_zip/Bra86.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Bra86.cpp rename to code/nel/3rdparty/seven_zip/Bra86.c diff --git a/code/ryzom/client/src/seven_zip/BraIA64.cpp b/code/nel/3rdparty/seven_zip/BraIA64.c similarity index 100% rename from code/ryzom/client/src/seven_zip/BraIA64.cpp rename to code/nel/3rdparty/seven_zip/BraIA64.c diff --git a/code/nel/3rdparty/seven_zip/CMakeLists.txt b/code/nel/3rdparty/seven_zip/CMakeLists.txt new file mode 100644 index 000000000..dfbf00aa2 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/CMakeLists.txt @@ -0,0 +1,27 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +FILE(GLOB LIB_SRC *.cpp *.c *.h) + +LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/LzmaUtil.c) + +NL_TARGET_LIB(nel_sevenzip ${LIB_SRC}) +# TARGET_LINK_LIBRARIES(nel_sevenzip ${PLATFORM_LINKFLAGS}) +NL_DEFAULT_PROPS(nel_sevenzip "NeL, 3rd Party: Seven Zip") +NL_ADD_RUNTIME_FLAGS(nel_sevenzip) +NL_ADD_LIB_SUFFIX(nel_sevenzip) + +ADD_DEFINITIONS(-D_7ZIP_ST) + +IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) + INSTALL(TARGETS nel_sevenzip LIBRARY DESTINATION ${NL_LIB_PREFIX} ARCHIVE DESTINATION ${NL_LIB_PREFIX} COMPONENT libraries) +ENDIF() + +IF(WITH_NEL_TOOLS) + ADD_EXECUTABLE(lzma ${CMAKE_CURRENT_SOURCE_DIR}/LzmaUtil.c) + + TARGET_LINK_LIBRARIES(lzma nel_sevenzip) + NL_DEFAULT_PROPS(lzma "NeL, 3rd Party: LZMA") + NL_ADD_RUNTIME_FLAGS(lzma) + + INSTALL(TARGETS lzma RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT tools) +ENDIF() diff --git a/code/ryzom/client/src/seven_zip/Compiler.h b/code/nel/3rdparty/seven_zip/Compiler.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Compiler.h rename to code/nel/3rdparty/seven_zip/Compiler.h diff --git a/code/ryzom/client/src/seven_zip/CpuArch.cpp b/code/nel/3rdparty/seven_zip/CpuArch.c similarity index 100% rename from code/ryzom/client/src/seven_zip/CpuArch.cpp rename to code/nel/3rdparty/seven_zip/CpuArch.c diff --git a/code/ryzom/client/src/seven_zip/CpuArch.h b/code/nel/3rdparty/seven_zip/CpuArch.h similarity index 100% rename from code/ryzom/client/src/seven_zip/CpuArch.h rename to code/nel/3rdparty/seven_zip/CpuArch.h diff --git a/code/ryzom/client/src/seven_zip/Delta.cpp b/code/nel/3rdparty/seven_zip/Delta.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Delta.cpp rename to code/nel/3rdparty/seven_zip/Delta.c diff --git a/code/ryzom/client/src/seven_zip/Delta.h b/code/nel/3rdparty/seven_zip/Delta.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Delta.h rename to code/nel/3rdparty/seven_zip/Delta.h diff --git a/code/nel/3rdparty/seven_zip/DllSecur.c b/code/nel/3rdparty/seven_zip/DllSecur.c new file mode 100644 index 000000000..5ea108ab8 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/DllSecur.c @@ -0,0 +1,108 @@ +/* DllSecur.c -- DLL loading security +2018-02-21 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#ifdef _WIN32 + +#include + +#include "DllSecur.h" + +#ifndef UNDER_CE + +typedef BOOL (WINAPI *Func_SetDefaultDllDirectories)(DWORD DirectoryFlags); + +#define MY_LOAD_LIBRARY_SEARCH_USER_DIRS 0x400 +#define MY_LOAD_LIBRARY_SEARCH_SYSTEM32 0x800 + +static const char * const g_Dlls = + #ifndef _CONSOLE + "UXTHEME\0" + #endif + "USERENV\0" + "SETUPAPI\0" + "APPHELP\0" + "PROPSYS\0" + "DWMAPI\0" + "CRYPTBASE\0" + "OLEACC\0" + "CLBCATQ\0" + "VERSION\0" + ; + +#endif + +void My_SetDefaultDllDirectories() +{ + #ifndef UNDER_CE + + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + GetVersionEx(&vi); + if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) + { + Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + if (setDllDirs) + if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) + return; + } + + #endif +} + + +void LoadSecurityDlls() +{ + #ifndef UNDER_CE + + wchar_t buf[MAX_PATH + 100]; + + { + // at Vista (ver 6.0) : CoCreateInstance(CLSID_ShellLink, ...) doesn't work after SetDefaultDllDirectories() : Check it ??? + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) + { + Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + if (setDllDirs) + if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) + return; + } + } + + { + unsigned len = GetSystemDirectoryW(buf, MAX_PATH + 2); + if (len == 0 || len > MAX_PATH) + return; + } + { + const char *dll; + unsigned pos = (unsigned)lstrlenW(buf); + + if (buf[pos - 1] != '\\') + buf[pos++] = '\\'; + + for (dll = g_Dlls; dll[0] != 0;) + { + unsigned k = 0; + for (;;) + { + char c = *dll++; + buf[pos + k] = (Byte)c; + k++; + if (c == 0) + break; + } + + lstrcatW(buf, L".dll"); + LoadLibraryExW(buf, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); + } + } + + #endif +} + +#endif diff --git a/code/nel/3rdparty/seven_zip/DllSecur.h b/code/nel/3rdparty/seven_zip/DllSecur.h new file mode 100644 index 000000000..e2a049ad2 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/DllSecur.h @@ -0,0 +1,20 @@ +/* DllSecur.h -- DLL loading for security +2018-02-19 : Igor Pavlov : Public domain */ + +#ifndef __DLL_SECUR_H +#define __DLL_SECUR_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#ifdef _WIN32 + +void My_SetDefaultDllDirectories(); +void LoadSecurityDlls(); + +#endif + +EXTERN_C_END + +#endif diff --git a/code/ryzom/client/src/seven_zip/LzFind.cpp b/code/nel/3rdparty/seven_zip/LzFind.c similarity index 100% rename from code/ryzom/client/src/seven_zip/LzFind.cpp rename to code/nel/3rdparty/seven_zip/LzFind.c diff --git a/code/ryzom/client/src/seven_zip/LzFind.h b/code/nel/3rdparty/seven_zip/LzFind.h similarity index 100% rename from code/ryzom/client/src/seven_zip/LzFind.h rename to code/nel/3rdparty/seven_zip/LzFind.h diff --git a/code/ryzom/client/src/seven_zip/LzHash.h b/code/nel/3rdparty/seven_zip/LzHash.h similarity index 100% rename from code/ryzom/client/src/seven_zip/LzHash.h rename to code/nel/3rdparty/seven_zip/LzHash.h diff --git a/code/ryzom/client/src/seven_zip/Lzma2Dec.cpp b/code/nel/3rdparty/seven_zip/Lzma2Dec.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Lzma2Dec.cpp rename to code/nel/3rdparty/seven_zip/Lzma2Dec.c diff --git a/code/ryzom/client/src/seven_zip/Lzma2Dec.h b/code/nel/3rdparty/seven_zip/Lzma2Dec.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Lzma2Dec.h rename to code/nel/3rdparty/seven_zip/Lzma2Dec.h diff --git a/code/nel/3rdparty/seven_zip/Lzma2DecMt.c b/code/nel/3rdparty/seven_zip/Lzma2DecMt.c new file mode 100644 index 000000000..988643d95 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Lzma2DecMt.c @@ -0,0 +1,1082 @@ +/* Lzma2DecMt.c -- LZMA2 Decoder Multi-thread +2019-02-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) +#define PRF_STR_INT_2(s, d1, d2) PRF(printf("\n" s " %d %d\n", (unsigned)d1, (unsigned)d2)) + +// #define _7ZIP_ST + +#include "Alloc.h" + +#include "Lzma2Dec.h" +#include "Lzma2DecMt.h" + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +#define LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT (1 << 28) + +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p) +{ + p->inBufSize_ST = 1 << 20; + p->outStep_ST = 1 << 20; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->outBlockMax = LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT; + p->inBlockMax = p->outBlockMax + p->outBlockMax / 16; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CLzma2DecMtThread ---------- */ + +typedef struct +{ + CLzma2Dec dec; + Byte dec_created; + Byte needInit; + + Byte *outBuf; + size_t outBufSize; + + EMtDecParseState state; + ELzma2ParseStatus parseStatus; + + size_t inPreSize; + size_t outPreSize; + + size_t inCodeSize; + size_t outCodeSize; + SRes codeRes; + + CAlignOffsetAlloc alloc; + + Byte mtPad[1 << 7]; +} CLzma2DecMtThread; + +#endif + + +/* ---------- CLzma2DecMt ---------- */ + +typedef struct +{ + // ISzAllocPtr alloc; + ISzAllocPtr allocMid; + + CAlignOffsetAlloc alignOffsetAlloc; + CLzma2DecMtProps props; + Byte prop; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + + BoolInt finishMode; + BoolInt outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + BoolInt readWasFinished; + SRes readRes; + + Byte *inBuf; + size_t inBufSize; + Byte dec_created; + CLzma2Dec dec; + + size_t inPos; + size_t inLim; + + #ifndef _7ZIP_ST + UInt64 outProcessed_Parse; + BoolInt mtc_WasConstructed; + CMtDec mtc; + CLzma2DecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CLzma2DecMt; + + + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CLzma2DecMt *p = (CLzma2DecMt *)ISzAlloc_Alloc(alloc, sizeof(CLzma2DecMt)); + if (!p) + return NULL; + + // p->alloc = alloc; + p->allocMid = allocMid; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + p->alignOffsetAlloc.baseAlloc = alloc; + + p->inBuf = NULL; + p->inBufSize = 0; + p->dec_created = False; + + // Lzma2DecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + t->dec_created = False; + t->outBuf = NULL; + t->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_FreeOutBufs(CLzma2DecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->outBuf) + { + ISzAlloc_Free(p->allocMid, t->outBuf); + t->outBuf = NULL; + t->outBufSize = 0; + } + } +} + +#endif + + +static void Lzma2DecMt_FreeSt(CLzma2DecMt *p) +{ + if (p->dec_created) + { + Lzma2Dec_Free(&p->dec, &p->alignOffsetAlloc.vt); + p->dec_created = False; + } + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void Lzma2DecMt_Destroy(CLzma2DecMtHandle pp) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + Lzma2DecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + Lzma2Dec_FreeProbs(&t->dec, &t->alloc.vt); // p->alloc !!! + t->dec_created = False; + } + } + } + Lzma2DecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CLzma2DecMt *me = (CLzma2DecMt *)obj; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + PRF_STR_INT_2("Parse", coderIndex, cc->srcSize); + + cc->state = MTDEC_PARSE_CONTINUE; + + if (cc->startCall) + { + if (!t->dec_created) + { + Lzma2Dec_Construct(&t->dec); + t->dec_created = True; + AlignOffsetAlloc_CreateVTable(&t->alloc); + { + /* (1 << 12) is expected size of one way in data cache. + We optimize alignment for cache line size of 128 bytes and smaller */ + const unsigned kNumAlignBits = 12; + const unsigned kNumCacheLineBits = 7; /* <= kNumAlignBits */ + t->alloc.numAlignBits = kNumAlignBits; + t->alloc.offset = ((UInt32)coderIndex * ((1 << 11) + (1 << 8) + (1 << 6))) & ((1 << kNumAlignBits) - (1 << kNumCacheLineBits)); + t->alloc.baseAlloc = me->alignOffsetAlloc.baseAlloc; + } + } + Lzma2Dec_Init(&t->dec); + + t->inPreSize = 0; + t->outPreSize = 0; + // t->blockWasFinished = False; + // t->finishedWithMark = False; + t->parseStatus = (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; + t->state = MTDEC_PARSE_CONTINUE; + + t->inCodeSize = 0; + t->outCodeSize = 0; + t->codeRes = SZ_OK; + + // (cc->srcSize == 0) is allowed + } + + { + ELzma2ParseStatus status; + BoolInt overflow; + UInt32 unpackRem = 0; + + int checkFinishBlock = True; + size_t limit = me->props.outBlockMax; + if (me->outSize_Defined) + { + UInt64 rem = me->outSize - me->outProcessed_Parse; + if (limit >= rem) + { + limit = (size_t)rem; + if (!me->finishMode) + checkFinishBlock = False; + } + } + + // checkFinishBlock = False, if we want to decode partial data + // that must be finished at position <= outBlockMax. + + { + const SizeT srcOrig = cc->srcSize; + SizeT srcSize_Point = 0; + SizeT dicPos_Point = 0; + + cc->srcSize = 0; + overflow = False; + + for (;;) + { + SizeT srcCur = srcOrig - cc->srcSize; + + status = Lzma2Dec_Parse(&t->dec, + limit - t->dec.decoder.dicPos, + cc->src + cc->srcSize, &srcCur, + checkFinishBlock); + + cc->srcSize += srcCur; + + if (status == LZMA2_PARSE_STATUS_NEW_CHUNK) + { + if (t->dec.unpackSize > me->props.outBlockMax - t->dec.decoder.dicPos) + { + overflow = True; + break; + } + continue; + } + + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + if (t->dec.decoder.dicPos == 0) + continue; + // we decode small blocks in one thread + if (t->dec.decoder.dicPos >= (1 << 14)) + break; + dicPos_Point = t->dec.decoder.dicPos; + srcSize_Point = cc->srcSize; + continue; + } + + if ((int)status == LZMA_STATUS_NOT_FINISHED && checkFinishBlock + // && limit == t->dec.decoder.dicPos + // && limit == me->props.outBlockMax + ) + { + overflow = True; + break; + } + + unpackRem = Lzma2Dec_GetUnpackExtra(&t->dec); + break; + } + + if (dicPos_Point != 0 + && (int)status != LZMA2_PARSE_STATUS_NEW_BLOCK + && (int)status != LZMA_STATUS_FINISHED_WITH_MARK + && (int)status != LZMA_STATUS_NOT_SPECIFIED) + { + // we revert to latest newBlock state + status = LZMA2_PARSE_STATUS_NEW_BLOCK; + unpackRem = 0; + t->dec.decoder.dicPos = dicPos_Point; + cc->srcSize = srcSize_Point; + overflow = False; + } + } + + t->inPreSize += cc->srcSize; + t->parseStatus = status; + + if (overflow) + cc->state = MTDEC_PARSE_OVERFLOW; + else + { + size_t dicPos = t->dec.decoder.dicPos; + + if ((int)status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + cc->state = MTDEC_PARSE_NEW; + cc->srcSize--; // we don't need control byte of next block + t->inPreSize--; + } + else + { + cc->state = MTDEC_PARSE_END; + if ((int)status != LZMA_STATUS_FINISHED_WITH_MARK) + { + // (status == LZMA_STATUS_NOT_SPECIFIED) + // (status == LZMA_STATUS_NOT_FINISHED) + if (unpackRem != 0) + { + /* we also reserve space for max possible number of output bytes of current LZMA chunk */ + SizeT rem = limit - dicPos; + if (rem > unpackRem) + rem = unpackRem; + dicPos += rem; + } + } + } + + me->outProcessed_Parse += dicPos; + } + + cc->outPos = dicPos; + t->outPreSize = (size_t)dicPos; + } + + t->state = cc->state; + return; + } +} + + +static SRes Lzma2DecMt_MtCallback_PreCode(void *pp, unsigned coderIndex) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + Byte *dest = t->outBuf; + + if (t->inPreSize == 0) + { + t->codeRes = SZ_ERROR_DATA; + return t->codeRes; + } + + if (!dest || t->outBufSize < t->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + t->outBuf = NULL; + t->outBufSize = 0; + } + + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, t->outPreSize + // + (1 << 28) + ); + // Sleep(200); + if (!dest) + return SZ_ERROR_MEM; + t->outBuf = dest; + t->outBufSize = t->outPreSize; + } + + t->dec.decoder.dic = dest; + t->dec.decoder.dicBufSize = t->outPreSize; + + t->needInit = True; + + return Lzma2Dec_AllocateProbs(&t->dec, me->prop, &t->alloc.vt); // alloc.vt +} + + +static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + UNUSED_VAR(srcFinished) + + PRF_STR_INT_2("Code", coderIndex, srcSize); + + *inCodePos = t->inCodeSize; + *outCodePos = 0; + *stop = True; + + if (t->needInit) + { + Lzma2Dec_Init(&t->dec); + t->needInit = False; + } + + { + ELzmaStatus status; + size_t srcProcessed = srcSize; + BoolInt blockWasFinished = + ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK); + + SRes res = Lzma2Dec_DecodeToDic(&t->dec, + t->outPreSize, + src, &srcProcessed, + blockWasFinished ? LZMA_FINISH_END : LZMA_FINISH_ANY, + &status); + + t->codeRes = res; + + t->inCodeSize += srcProcessed; + *inCodePos = t->inCodeSize; + t->outCodeSize = t->dec.decoder.dicPos; + *outCodePos = t->dec.decoder.dicPos; + + if (res != SZ_OK) + return res; + + if (srcProcessed == srcSize) + *stop = False; + + if (blockWasFinished) + { + if (srcSize != srcProcessed) + return SZ_ERROR_FAIL; + + if (t->inPreSize == t->inCodeSize) + { + if (t->outPreSize != t->outCodeSize) + return SZ_ERROR_FAIL; + *stop = True; + } + } + else + { + if (t->outPreSize == t->outCodeSize) + *stop = True; + } + + return SZ_OK; + } +} + + +#define LZMA2DECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, + BoolInt *needContinue, BoolInt *canRecode) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + const CLzma2DecMtThread *t = &me->coders[coderIndex]; + size_t size = t->outCodeSize; + const Byte *data = t->outBuf; + BoolInt needContinue2 = True; + + PRF_STR_INT_2("Write", coderIndex, srcSize); + + *needContinue = False; + *canRecode = True; + UNUSED_VAR(src) + UNUSED_VAR(srcSize) + + if ( + // t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + t->state == MTDEC_PARSE_OVERFLOW + || t->state == MTDEC_PARSE_END) + needContinue2 = False; + + + if (!needWriteToStream) + return SZ_OK; + + me->mtc.inProcessed += t->inCodeSize; + + if (t->codeRes == SZ_OK) + if ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK) + if (t->outPreSize != t->outCodeSize + || t->inPreSize != t->inCodeSize) + return SZ_ERROR_FAIL; + + *canRecode = False; + + if (me->outStream) + { + for (;;) + { + size_t cur = size; + size_t written; + if (cur > LZMA2DECMT_STREAM_WRITE_STEP) + cur = LZMA2DECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + me->outProcessed += written; + // me->mtc.writtenTotal += written; + if (written != cur) + return SZ_ERROR_WRITE; + data += cur; + size -= cur; + if (size == 0) + { + *needContinue = needContinue2; + return SZ_OK; + } + RINOK(MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0)); + } + } + + return SZ_ERROR_FAIL; + /* + if (size > me->outBufSize) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBufSize -= size; + me->outBuf += size; + *needContinue = needContinue2; + return SZ_OK; + */ +} + +#endif + + +static SRes Lzma2Dec_Prepare_ST(CLzma2DecMt *p) +{ + if (!p->dec_created) + { + Lzma2Dec_Construct(&p->dec); + p->dec_created = True; + } + + RINOK(Lzma2Dec_Allocate(&p->dec, p->prop, &p->alignOffsetAlloc.vt)); + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + Lzma2Dec_Init(&p->dec); + + return SZ_OK; +} + + +static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p + #ifndef _7ZIP_ST + , BoolInt tMode + #endif + ) +{ + SizeT wrPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CLzma2Dec *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + Lzma2DecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + RINOK(Lzma2Dec_Prepare_ST(p)); + + dec = &p->dec; + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + wrPos = dec->decoder.dicPos; + + for (;;) + { + SizeT dicPos; + SizeT size; + ELzmaFinishMode finishMode; + SizeT inProcessed; + ELzmaStatus status; + SRes res; + + SizeT outProcessed; + BoolInt outFinished; + BoolInt needStop; + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + // p->readProcessed += inLim; + // inLim -= 5; p->readWasFinished = True; // for test + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + dicPos = dec->decoder.dicPos; + { + SizeT next = dec->decoder.dicBufSize; + if (next - wrPos > p->props.outStep_ST) + next = wrPos + p->props.outStep_ST; + size = next - dicPos; + } + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (SizeT)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + inProcessed = inLim - inPos; + + res = Lzma2Dec_DecodeToDic(dec, dicPos + size, inData + inPos, &inProcessed, finishMode, &status); + + inPos += inProcessed; + p->inProcessed += inProcessed; + outProcessed = dec->decoder.dicPos - dicPos; + p->outProcessed += outProcessed; + + outFinished = (p->outSize_Defined && p->outSize <= p->outProcessed); + + needStop = (res != SZ_OK + || (inProcessed == 0 && outProcessed == 0) + || status == LZMA_STATUS_FINISHED_WITH_MARK + || (!p->finishMode && outFinished)); + + if (needStop || outProcessed >= size) + { + SRes res2; + { + size_t writeSize = dec->decoder.dicPos - wrPos; + size_t written = ISeqOutStream_Write(p->outStream, dec->decoder.dic + wrPos, writeSize); + res2 = (written == writeSize) ? SZ_OK : SZ_ERROR_WRITE; + } + + if (dec->decoder.dicPos == dec->decoder.dicBufSize) + dec->decoder.dicPos = 0; + wrPos = dec->decoder.dicPos; + + RINOK(res2); + + if (needStop) + { + if (res != SZ_OK) + return res; + + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + { + if (p->finishMode) + { + if (p->outSize_Defined && p->outSize != p->outProcessed) + return SZ_ERROR_DATA; + } + return SZ_OK; + } + + if (!p->finishMode && outFinished) + return SZ_OK; + + if (status == LZMA_STATUS_NEEDS_MORE_INPUT) + return SZ_ERROR_INPUT_EOF; + + return SZ_ERROR_DATA; + } + } + + if (p->progress) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + RINOK(ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed)); + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + } +} + + + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, const UInt64 *outDataSize, int finishMode, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + UInt64 *inProcessed, + // UInt64 *outProcessed, + int *isMT, + ICompressProgress *progress) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + #ifndef _7ZIP_ST + BoolInt tMode; + #endif + + *inProcessed = 0; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->readWasFinished = False; + + *isMT = False; + + + #ifndef _7ZIP_ST + + tMode = False; + + // p->mtc.parseRes = SZ_OK; + + // p->mtc.numFilledThreads = 0; + // p->mtc.crossStart = 0; + // p->mtc.crossEnd = 0; + // p->mtc.allocError_for_Read_BlockIndex = 0; + // p->mtc.isAllocError = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + Lzma2DecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + + // p->outBuf = NULL; + // p->outBufSize = 0; + /* + if (!outStream) + { + // p->outBuf = outBuf; + // p->outBufSize = *outBufSize; + // *outBufSize = 0; + return SZ_ERROR_PARAM; + } + */ + + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->alignOffsetAlloc.baseAlloc; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.inBufSize = p->props.inBufSize_MT; + + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = Lzma2DecMt_MtCallback_Parse; + vt.PreCode = Lzma2DecMt_MtCallback_PreCode; + vt.Code = Lzma2DecMt_MtCallback_Code; + vt.Write = Lzma2DecMt_MtCallback_Write; + + { + BoolInt needContinue = False; + + SRes res = MtDec_Code(&p->mtc); + + /* + if (!outStream) + *outBufSize = p->outBuf - outBuf; + */ + + *inProcessed = p->mtc.inProcessed; + + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + res = p->mtc.mtProgress.res; + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + if (res == SZ_OK) + return p->mtc.readRes; + return res; + } + + tMode = True; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->inProcessed = p->mtc.inProcessed; + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = Lzma2Dec_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + ); + + *inProcessed = p->inProcessed; + + // res = SZ_OK; // for test + if (res == SZ_OK && p->readRes != SZ_OK) + res = p->readRes; + + /* + #ifndef _7ZIP_ST + if (res == SZ_OK && tMode && p->mtc.parseRes != SZ_OK) + res = p->mtc.parseRes; + #endif + */ + + return res; + } +} + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->inPos = 0; + p->inLim = 0; + + return Lzma2Dec_Prepare_ST(p); +} + + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + ELzmaFinishMode finishMode; + SRes readRes; + size_t size = *outSize; + + *outSize = 0; + *inStreamProcessed = 0; + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (size_t)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + readRes = SZ_OK; + + for (;;) + { + SizeT inCur; + SizeT outCur; + ELzmaStatus status; + SRes res; + + if (p->inPos == p->inLim && readRes == SZ_OK) + { + p->inPos = 0; + p->inLim = p->props.inBufSize_ST; + readRes = ISeqInStream_Read(p->inStream, p->inBuf, &p->inLim); + } + + inCur = p->inLim - p->inPos; + outCur = size; + + res = Lzma2Dec_DecodeToBuf(&p->dec, data, &outCur, + p->inBuf + p->inPos, &inCur, finishMode, &status); + + p->inPos += inCur; + p->inProcessed += inCur; + *inStreamProcessed += inCur; + p->outProcessed += outCur; + *outSize += outCur; + size -= outCur; + data += outCur; + + if (res != 0) + return res; + + /* + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + return readRes; + + if (size == 0 && status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (p->finishMode && p->outSize_Defined && p->outProcessed >= p->outSize) + return SZ_ERROR_DATA; + return readRes; + } + */ + + if (inCur == 0 && outCur == 0) + return readRes; + } +} diff --git a/code/nel/3rdparty/seven_zip/Lzma2DecMt.h b/code/nel/3rdparty/seven_zip/Lzma2DecMt.h new file mode 100644 index 000000000..7791c310b --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Lzma2DecMt.h @@ -0,0 +1,79 @@ +/* Lzma2DecMt.h -- LZMA2 Decoder Multi-thread +2018-02-17 : Igor Pavlov : Public domain */ + +#ifndef __LZMA2_DEC_MT_H +#define __LZMA2_DEC_MT_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t outBlockMax; + size_t inBlockMax; + #endif +} CLzma2DecMtProps; + +/* init to single-thread mode */ +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p); + + +/* ---------- CLzma2DecMtHandle Interface ---------- */ + +/* Lzma2DecMt_ * functions can return the following exit codes: +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater in props + SZ_ERROR_WRITE - ISeqOutStream write callback error + // SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output + SZ_ERROR_PROGRESS - some break from progress callback + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) +*/ + +typedef void * CLzma2DecMtHandle; + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void Lzma2DecMt_Destroy(CLzma2DecMtHandle p); + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - if lzma2 stream must be finished + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + + // out variables: + UInt64 *inProcessed, + int *isMT, /* out: (*isMT == 0), if single thread decoding was used */ + + // UInt64 *outProcessed, + ICompressProgress *progress); + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream); + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed); + + +EXTERN_C_END + +#endif diff --git a/code/ryzom/client/src/seven_zip/Lzma2Enc.cpp b/code/nel/3rdparty/seven_zip/Lzma2Enc.c similarity index 99% rename from code/ryzom/client/src/seven_zip/Lzma2Enc.cpp rename to code/nel/3rdparty/seven_zip/Lzma2Enc.c index 9af0622b0..5c1ad4931 100644 --- a/code/ryzom/client/src/seven_zip/Lzma2Enc.cpp +++ b/code/nel/3rdparty/seven_zip/Lzma2Enc.c @@ -5,7 +5,7 @@ #include -#define _7ZIP_ST +/* #define _7ZIP_ST */ #include "Lzma2Enc.h" diff --git a/code/ryzom/client/src/seven_zip/Lzma2Enc.h b/code/nel/3rdparty/seven_zip/Lzma2Enc.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Lzma2Enc.h rename to code/nel/3rdparty/seven_zip/Lzma2Enc.h diff --git a/code/ryzom/client/src/seven_zip/Lzma86.h b/code/nel/3rdparty/seven_zip/Lzma86.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Lzma86.h rename to code/nel/3rdparty/seven_zip/Lzma86.h diff --git a/code/ryzom/client/src/seven_zip/Lzma86Dec.cpp b/code/nel/3rdparty/seven_zip/Lzma86Dec.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Lzma86Dec.cpp rename to code/nel/3rdparty/seven_zip/Lzma86Dec.c diff --git a/code/ryzom/client/src/seven_zip/Lzma86Enc.cpp b/code/nel/3rdparty/seven_zip/Lzma86Enc.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Lzma86Enc.cpp rename to code/nel/3rdparty/seven_zip/Lzma86Enc.c diff --git a/code/ryzom/client/src/seven_zip/LzmaDec.cpp b/code/nel/3rdparty/seven_zip/LzmaDec.c similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaDec.cpp rename to code/nel/3rdparty/seven_zip/LzmaDec.c diff --git a/code/ryzom/client/src/seven_zip/LzmaDec.h b/code/nel/3rdparty/seven_zip/LzmaDec.h similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaDec.h rename to code/nel/3rdparty/seven_zip/LzmaDec.h diff --git a/code/ryzom/client/src/seven_zip/LzmaEnc.cpp b/code/nel/3rdparty/seven_zip/LzmaEnc.c similarity index 99% rename from code/ryzom/client/src/seven_zip/LzmaEnc.cpp rename to code/nel/3rdparty/seven_zip/LzmaEnc.c index 54d0d0507..46a0db000 100644 --- a/code/ryzom/client/src/seven_zip/LzmaEnc.cpp +++ b/code/nel/3rdparty/seven_zip/LzmaEnc.c @@ -5,8 +5,6 @@ #include -#define _7ZIP_ST - /* #define SHOW_STAT */ /* #define SHOW_STAT2 */ diff --git a/code/ryzom/client/src/seven_zip/LzmaEnc.h b/code/nel/3rdparty/seven_zip/LzmaEnc.h similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaEnc.h rename to code/nel/3rdparty/seven_zip/LzmaEnc.h diff --git a/code/ryzom/client/src/seven_zip/LzmaLib.cpp b/code/nel/3rdparty/seven_zip/LzmaLib.c similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaLib.cpp rename to code/nel/3rdparty/seven_zip/LzmaLib.c diff --git a/code/ryzom/client/src/seven_zip/LzmaLib.h b/code/nel/3rdparty/seven_zip/LzmaLib.h similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaLib.h rename to code/nel/3rdparty/seven_zip/LzmaLib.h diff --git a/code/nel/3rdparty/seven_zip/LzmaUtil.c b/code/nel/3rdparty/seven_zip/LzmaUtil.c new file mode 100644 index 000000000..6bdb2ad6c --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaUtil.c @@ -0,0 +1,258 @@ +/* LzmaUtil.c -- Test application for LZMA compression +2018-07-04 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include +#include +#include + +#include "CpuArch.h" + +#include "Alloc.h" +#include "7zFile.h" +#include "7zVersion.h" +#include "LzmaDec.h" +#include "LzmaEnc.h" + +static const char * const kCantReadMessage = "Can not read input file"; +static const char * const kCantWriteMessage = "Can not write output file"; +static const char * const kCantAllocateMessage = "Can not allocate memory"; +static const char * const kDataErrorMessage = "Data error"; + +static void PrintHelp(char *buffer) +{ + strcat(buffer, + "\nLZMA-C " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n" + "Usage: lzma inputFile outputFile\n" + " e: encode file\n" + " d: decode file\n"); +} + +static int PrintError(char *buffer, const char *message) +{ + strcat(buffer, "\nError: "); + strcat(buffer, message); + strcat(buffer, "\n"); + return 1; +} + +static int PrintErrorNumber(char *buffer, SRes val) +{ + sprintf(buffer + strlen(buffer), "\nError code: %x\n", (unsigned)val); + return 1; +} + +static int PrintUserError(char *buffer) +{ + return PrintError(buffer, "Incorrect command"); +} + + +#define IN_BUF_SIZE (1 << 16) +#define OUT_BUF_SIZE (1 << 16) + + +static SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inStream, + UInt64 unpackSize) +{ + int thereIsSize = (unpackSize != (UInt64)(Int64)-1); + Byte inBuf[IN_BUF_SIZE]; + Byte outBuf[OUT_BUF_SIZE]; + size_t inPos = 0, inSize = 0, outPos = 0; + LzmaDec_Init(state); + for (;;) + { + if (inPos == inSize) + { + inSize = IN_BUF_SIZE; + RINOK(inStream->Read(inStream, inBuf, &inSize)); + inPos = 0; + } + { + SRes res; + SizeT inProcessed = inSize - inPos; + SizeT outProcessed = OUT_BUF_SIZE - outPos; + ELzmaFinishMode finishMode = LZMA_FINISH_ANY; + ELzmaStatus status; + if (thereIsSize && outProcessed > unpackSize) + { + outProcessed = (SizeT)unpackSize; + finishMode = LZMA_FINISH_END; + } + + res = LzmaDec_DecodeToBuf(state, outBuf + outPos, &outProcessed, + inBuf + inPos, &inProcessed, finishMode, &status); + inPos += inProcessed; + outPos += outProcessed; + unpackSize -= outProcessed; + + if (outStream) + if (outStream->Write(outStream, outBuf, outPos) != outPos) + return SZ_ERROR_WRITE; + + outPos = 0; + + if (res != SZ_OK || (thereIsSize && unpackSize == 0)) + return res; + + if (inProcessed == 0 && outProcessed == 0) + { + if (thereIsSize || status != LZMA_STATUS_FINISHED_WITH_MARK) + return SZ_ERROR_DATA; + return res; + } + } + } +} + + +static SRes Decode(ISeqOutStream *outStream, ISeqInStream *inStream) +{ + UInt64 unpackSize; + int i; + SRes res = 0; + + CLzmaDec state; + + /* header: 5 bytes of LZMA properties and 8 bytes of uncompressed size */ + unsigned char header[LZMA_PROPS_SIZE + 8]; + + /* Read and parse header */ + + RINOK(SeqInStream_Read(inStream, header, sizeof(header))); + + unpackSize = 0; + for (i = 0; i < 8; i++) + unpackSize += (UInt64)header[LZMA_PROPS_SIZE + i] << (i * 8); + + LzmaDec_Construct(&state); + RINOK(LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &g_Alloc)); + res = Decode2(&state, outStream, inStream, unpackSize); + LzmaDec_Free(&state, &g_Alloc); + return res; +} + +static SRes Encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize, char *rs) +{ + CLzmaEncHandle enc; + SRes res; + CLzmaEncProps props; + + UNUSED_VAR(rs); + + enc = LzmaEnc_Create(&g_Alloc); + if (enc == 0) + return SZ_ERROR_MEM; + + LzmaEncProps_Init(&props); + res = LzmaEnc_SetProps(enc, &props); + + if (res == SZ_OK) + { + Byte header[LZMA_PROPS_SIZE + 8]; + size_t headerSize = LZMA_PROPS_SIZE; + int i; + + res = LzmaEnc_WriteProperties(enc, header, &headerSize); + for (i = 0; i < 8; i++) + header[headerSize++] = (Byte)(fileSize >> (8 * i)); + if (outStream->Write(outStream, header, headerSize) != headerSize) + res = SZ_ERROR_WRITE; + else + { + if (res == SZ_OK) + res = LzmaEnc_Encode(enc, outStream, inStream, NULL, &g_Alloc, &g_Alloc); + } + } + LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc); + return res; +} + + +static int main2(int numArgs, const char *args[], char *rs) +{ + CFileSeqInStream inStream; + CFileOutStream outStream; + char c; + int res; + int encodeMode; + BoolInt useOutFile = False; + + FileSeqInStream_CreateVTable(&inStream); + File_Construct(&inStream.file); + + FileOutStream_CreateVTable(&outStream); + File_Construct(&outStream.file); + + if (numArgs == 1) + { + PrintHelp(rs); + return 0; + } + + if (numArgs < 3 || numArgs > 4 || strlen(args[1]) != 1) + return PrintUserError(rs); + + c = args[1][0]; + encodeMode = (c == 'e' || c == 'E'); + if (!encodeMode && c != 'd' && c != 'D') + return PrintUserError(rs); + + { + size_t t4 = sizeof(UInt32); + size_t t8 = sizeof(UInt64); + if (t4 != 4 || t8 != 8) + return PrintError(rs, "Incorrect UInt32 or UInt64"); + } + + if (InFile_Open(&inStream.file, args[2]) != 0) + return PrintError(rs, "Can not open input file"); + + if (numArgs > 3) + { + useOutFile = True; + if (OutFile_Open(&outStream.file, args[3]) != 0) + return PrintError(rs, "Can not open output file"); + } + else if (encodeMode) + PrintUserError(rs); + + if (encodeMode) + { + UInt64 fileSize; + File_GetLength(&inStream.file, &fileSize); + res = Encode(&outStream.vt, &inStream.vt, fileSize, rs); + } + else + { + res = Decode(&outStream.vt, useOutFile ? &inStream.vt : NULL); + } + + if (useOutFile) + File_Close(&outStream.file); + File_Close(&inStream.file); + + if (res != SZ_OK) + { + if (res == SZ_ERROR_MEM) + return PrintError(rs, kCantAllocateMessage); + else if (res == SZ_ERROR_DATA) + return PrintError(rs, kDataErrorMessage); + else if (res == SZ_ERROR_WRITE) + return PrintError(rs, kCantWriteMessage); + else if (res == SZ_ERROR_READ) + return PrintError(rs, kCantReadMessage); + return PrintErrorNumber(rs, res); + } + return 0; +} + + +int MY_CDECL main(int numArgs, const char *args[]) +{ + char rs[800] = { 0 }; + int res = main2(numArgs, args, rs); + fputs(rs, stdout); + return res; +} diff --git a/code/ryzom/client/src/seven_zip/Ppmd.h b/code/nel/3rdparty/seven_zip/Ppmd.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Ppmd.h rename to code/nel/3rdparty/seven_zip/Ppmd.h diff --git a/code/ryzom/client/src/seven_zip/Ppmd7.cpp b/code/nel/3rdparty/seven_zip/Ppmd7.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Ppmd7.cpp rename to code/nel/3rdparty/seven_zip/Ppmd7.c diff --git a/code/ryzom/client/src/seven_zip/Ppmd7.h b/code/nel/3rdparty/seven_zip/Ppmd7.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Ppmd7.h rename to code/nel/3rdparty/seven_zip/Ppmd7.h diff --git a/code/ryzom/client/src/seven_zip/Ppmd7Dec.cpp b/code/nel/3rdparty/seven_zip/Ppmd7Dec.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Ppmd7Dec.cpp rename to code/nel/3rdparty/seven_zip/Ppmd7Dec.c diff --git a/code/ryzom/client/src/seven_zip/Ppmd7Enc.cpp b/code/nel/3rdparty/seven_zip/Ppmd7Enc.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Ppmd7Enc.cpp rename to code/nel/3rdparty/seven_zip/Ppmd7Enc.c diff --git a/code/nel/3rdparty/seven_zip/Precomp.h b/code/nel/3rdparty/seven_zip/Precomp.h new file mode 100644 index 000000000..e8ff8b40e --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Precomp.h @@ -0,0 +1,10 @@ +/* Precomp.h -- StdAfx +2013-11-12 : Igor Pavlov : Public domain */ + +#ifndef __7Z_PRECOMP_H +#define __7Z_PRECOMP_H + +#include "Compiler.h" +/* #include "7zTypes.h" */ + +#endif diff --git a/code/ryzom/client/src/seven_zip/RotateDefs.h b/code/nel/3rdparty/seven_zip/RotateDefs.h similarity index 100% rename from code/ryzom/client/src/seven_zip/RotateDefs.h rename to code/nel/3rdparty/seven_zip/RotateDefs.h diff --git a/code/ryzom/client/src/seven_zip/Sha256.cpp b/code/nel/3rdparty/seven_zip/Sha256.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Sha256.cpp rename to code/nel/3rdparty/seven_zip/Sha256.c diff --git a/code/ryzom/client/src/seven_zip/Sha256.h b/code/nel/3rdparty/seven_zip/Sha256.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Sha256.h rename to code/nel/3rdparty/seven_zip/Sha256.h diff --git a/code/ryzom/client/src/seven_zip/Sort.cpp b/code/nel/3rdparty/seven_zip/Sort.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Sort.cpp rename to code/nel/3rdparty/seven_zip/Sort.c diff --git a/code/ryzom/client/src/seven_zip/Sort.h b/code/nel/3rdparty/seven_zip/Sort.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Sort.h rename to code/nel/3rdparty/seven_zip/Sort.h diff --git a/code/ryzom/client/src/seven_zip/Xz.cpp b/code/nel/3rdparty/seven_zip/Xz.c similarity index 100% rename from code/ryzom/client/src/seven_zip/Xz.cpp rename to code/nel/3rdparty/seven_zip/Xz.c diff --git a/code/ryzom/client/src/seven_zip/Xz.h b/code/nel/3rdparty/seven_zip/Xz.h similarity index 100% rename from code/ryzom/client/src/seven_zip/Xz.h rename to code/nel/3rdparty/seven_zip/Xz.h diff --git a/code/ryzom/client/src/seven_zip/XzCrc64.cpp b/code/nel/3rdparty/seven_zip/XzCrc64.c similarity index 100% rename from code/ryzom/client/src/seven_zip/XzCrc64.cpp rename to code/nel/3rdparty/seven_zip/XzCrc64.c diff --git a/code/ryzom/client/src/seven_zip/XzCrc64.h b/code/nel/3rdparty/seven_zip/XzCrc64.h similarity index 100% rename from code/ryzom/client/src/seven_zip/XzCrc64.h rename to code/nel/3rdparty/seven_zip/XzCrc64.h diff --git a/code/ryzom/client/src/seven_zip/XzCrc64Opt.cpp b/code/nel/3rdparty/seven_zip/XzCrc64Opt.c similarity index 100% rename from code/ryzom/client/src/seven_zip/XzCrc64Opt.cpp rename to code/nel/3rdparty/seven_zip/XzCrc64Opt.c diff --git a/code/ryzom/client/src/seven_zip/XzDec.cpp b/code/nel/3rdparty/seven_zip/XzDec.c similarity index 100% rename from code/ryzom/client/src/seven_zip/XzDec.cpp rename to code/nel/3rdparty/seven_zip/XzDec.c diff --git a/code/ryzom/client/src/seven_zip/XzEnc.cpp b/code/nel/3rdparty/seven_zip/XzEnc.c similarity index 100% rename from code/ryzom/client/src/seven_zip/XzEnc.cpp rename to code/nel/3rdparty/seven_zip/XzEnc.c diff --git a/code/ryzom/client/src/seven_zip/XzEnc.h b/code/nel/3rdparty/seven_zip/XzEnc.h similarity index 100% rename from code/ryzom/client/src/seven_zip/XzEnc.h rename to code/nel/3rdparty/seven_zip/XzEnc.h diff --git a/code/ryzom/client/src/seven_zip/XzIn.cpp b/code/nel/3rdparty/seven_zip/XzIn.c similarity index 100% rename from code/ryzom/client/src/seven_zip/XzIn.cpp rename to code/nel/3rdparty/seven_zip/XzIn.c diff --git a/code/nel/CMakeLists.txt b/code/nel/CMakeLists.txt index 3470fcebb..1530e75e3 100644 --- a/code/nel/CMakeLists.txt +++ b/code/nel/CMakeLists.txt @@ -56,6 +56,7 @@ IF(WITH_INSTALL_LIBRARIES) ADD_SUBDIRECTORY(include) ENDIF() +ADD_SUBDIRECTORY(3rdparty) ADD_SUBDIRECTORY(src) IF(WITH_NEL_SAMPLES) diff --git a/code/nel/include/nel/misc/app_context.h b/code/nel/include/nel/misc/app_context.h index bf73ddb83..6afe62c79 100644 --- a/code/nel/include/nel/misc/app_context.h +++ b/code/nel/include/nel/misc/app_context.h @@ -237,6 +237,22 @@ namespace NLMISC } \ private: +#define NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(className) \ + NLMISC_SAFE_SINGLETON_DECL(className); \ + \ + public: \ + static void releaseInstance() \ + { \ + if (_Instance) \ + { \ + NLMISC::INelContext::getInstance().releaseSingletonPointer(#className, _Instance); \ + delete _Instance; \ + _Instance = NULL; \ + } \ + } \ + private: + + /** The same as above, but generate a getInstance method that * return a pointer instead of a reference */ @@ -304,5 +320,6 @@ void initNelLibrary(CLibrary &lib); } // namespace NLMISC +#include #endif //APP_CONTEXT_H diff --git a/code/nel/include/nel/misc/http_package_provider.h b/code/nel/include/nel/misc/http_package_provider.h new file mode 100644 index 000000000..b4b82d70a --- /dev/null +++ b/code/nel/include/nel/misc/http_package_provider.h @@ -0,0 +1,50 @@ +// NeL - MMORPG Framework +// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_HTTP_PACKAGE_PROVIDER_H +#define NLMISC_HTTP_PACKAGE_PROVIDER_H + +#include +#include + +namespace NLMISC { + +class CHttpPackageProvider : public IStreamedPackageProvider +{ +public: + CHttpPackageProvider(); + virtual ~CHttpPackageProvider(); + + /// Download a file. This call is blocking + /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. + /// hash: [in] + virtual bool getFile(std::string &filePath, const CHashKey &hash, const std::string &name) NL_OVERRIDE; + +public: + /// Set storage path (ex. stream/) + std::string Path; + + /// Loads a package into the package manager (ex. http://cdn.ryzom.dev/open/stream/) + typedef std::vector THosts; + THosts Hosts; + +}; /* class CHttpPackageProvider */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */ + +/* end of file */ diff --git a/code/nel/include/nel/misc/i_streamed_package_provider.h b/code/nel/include/nel/misc/i_streamed_package_provider.h new file mode 100644 index 000000000..320e3cdfd --- /dev/null +++ b/code/nel/include/nel/misc/i_streamed_package_provider.h @@ -0,0 +1,44 @@ +// NeL - MMORPG Framework +// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_PROVIDER_H +#define NLMISC_STREAMED_PACKAGE_PROVIDER_H + +#include +#include +#include + +namespace NLMISC { + +class IStreamedPackageProvider +{ +public: + IStreamedPackageProvider(); + virtual ~IStreamedPackageProvider(); + + /// Download a file. This call is blocking + /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. + /// hash: [in] + /// name: [in] name for debugging + virtual bool getFile(std::string &filePath, const CHashKey &hash, const std::string &name = "") = 0; + +}; /* class IStreamedPackageProvider */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_PROVIDER_H */ + +/* end of file */ diff --git a/code/nel/include/nel/misc/path.h b/code/nel/include/nel/misc/path.h index f42acc91b..de46532b7 100644 --- a/code/nel/include/nel/misc/path.h +++ b/code/nel/include/nel/misc/path.h @@ -74,6 +74,9 @@ public: /** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included (the extention (Nel Big File) is used to know that it's a big file) */ void addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); + + /** Sale but for .snp (Streamed NeL Package) */ + void addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); /** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included */ void addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); @@ -276,8 +279,8 @@ private: struct CMCFileEntry { char *Name; // Normal case (the search is done by using nlstricmp) - uint32 idPath : 16; // Path (not with file at the end) - look in the SSMpath (65536 different path allowed) - uint32 idExt : 15; // real extension of the file if remapped - look in the SSMext (32768 different extension allowed) + uint32 idPath : 20; // Path (not with file at the end) - look in the SSMpath (1048576 different path allowed) + uint32 idExt : 11; // real extension of the file if remapped - look in the SSMext (2048 different extension allowed) uint32 Remapped : 1; // true if the file is remapped }; @@ -371,6 +374,9 @@ public: /** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included (the extention (Nel Big File) is used to know that it's a big file) */ static void addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); + /** Same but Streamed Package */ + static void addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); + /** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included */ static void addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); diff --git a/code/ryzom/client/src/seven_zip/seven_zip.h b/code/nel/include/nel/misc/seven_zip.h similarity index 86% rename from code/ryzom/client/src/seven_zip/seven_zip.h rename to code/nel/include/nel/misc/seven_zip.h index e471a9ac0..469b24bb9 100644 --- a/code/ryzom/client/src/seven_zip/seven_zip.h +++ b/code/nel/include/nel/misc/seven_zip.h @@ -18,6 +18,9 @@ #define SEVEN_ZIP_H #include +#include + +namespace NLMISC { // utility func to decompress a monofile 7zip archive bool unpack7Zip(const std::string &sevenZipFileName, const std::string &destFileName); @@ -25,7 +28,12 @@ bool unpack7Zip(const std::string &sevenZipFileName, const std::string &destFile // utility func to decompress a single LZMA packed file bool unpackLZMA(const std::string &lzmaFileName, const std::string &destFileName); +// utility func to decompress a single LZMA packed file +bool unpackLZMA(const std::string &lzmaFileName, const std::string &destFileName, CHashKey &sha1); + // utility func to compress a single file to LZMA packed file bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName); +} + #endif diff --git a/code/nel/include/nel/misc/sha1.h b/code/nel/include/nel/misc/sha1.h index ae42c3c1e..33db14f75 100644 --- a/code/nel/include/nel/misc/sha1.h +++ b/code/nel/include/nel/misc/sha1.h @@ -23,6 +23,8 @@ #include +namespace NLMISC { + struct CHashKey { CHashKey () { HashKeyString.resize(20); } @@ -109,4 +111,6 @@ CHashKey getSHA1(const uint8 *buffer, uint32 size); // This function get a buffer and key with size and returns his HMAC-SHA1 hash key CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint32 key_len); +} + #endif // NL_SHA1_H diff --git a/code/nel/include/nel/misc/streamed_package.h b/code/nel/include/nel/misc/streamed_package.h new file mode 100644 index 000000000..9040598d6 --- /dev/null +++ b/code/nel/include/nel/misc/streamed_package.h @@ -0,0 +1,59 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_H +#define NLMISC_STREAMED_PACKAGE_H + +#include +#include + +namespace NLMISC { + +class CStreamedPackage +{ +public: + struct CEntry + { + std::string Name; + CHashKey Hash; + uint32 Size; + uint32 LastModified; + + void serial(NLMISC::IStream &f) throw(NLMISC::EStream); + + }; + +public: + CStreamedPackage(); + ~CStreamedPackage(); + + void serial(NLMISC::IStream &f) throw(NLMISC::EStream); + + /// result: [out] ex. /00/00/000000000.. + /// hash: [in] + static void makePath(std::string &result, const CHashKey &hash); + +public: + typedef std::vector TEntries; + TEntries Entries; + +}; /* class CStreamedPackage */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */ + +/* end of file */ diff --git a/code/nel/include/nel/misc/streamed_package_manager.h b/code/nel/include/nel/misc/streamed_package_manager.h new file mode 100644 index 000000000..85d13db59 --- /dev/null +++ b/code/nel/include/nel/misc/streamed_package_manager.h @@ -0,0 +1,71 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H +#define NLMISC_STREAMED_PACKAGE_MANAGER_H + +#include +#include +#include +#include + +namespace NLMISC { + +class CStreamedPackageManager +{ + NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(CStreamedPackageManager); + +public: + CStreamedPackageManager(); + ~CStreamedPackageManager(); + + /// package: ex. /games/nel/data/characters_maps_hr.snp + bool loadPackage(const std::string &package); + + /// Get a file list + void list(std::vector &fileNames, const std::string &package); + + /// Unload all packages + void unloadAll(); + + /// Get an existing file or download if necessary. This call is blocking + /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. + /// fileName: [in] ex. fy_hof_underwear.dds + bool getFile(std::string &filePath, const std::string &fileName); + + /// Get file size + /// fileSize: [out] + /// fileName: [in] ex. fy_hof_underwear.dds + bool getFileSize(uint32 &fileSize, const std::string &fileName); + +public: + /// Streamed package provider. This downloads the files + IStreamedPackageProvider *Provider; + +private: + typedef std::map TPackages; + TPackages m_Packages; + + typedef std::map TEntries; + TEntries m_Entries; + +}; /* class CStreamedPackageManager */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */ + +/* end of file */ diff --git a/code/nel/include/nel/misc/types_nl.h b/code/nel/include/nel/misc/types_nl.h index 353f50490..f42f410e6 100644 --- a/code/nel/include/nel/misc/types_nl.h +++ b/code/nel/include/nel/misc/types_nl.h @@ -479,6 +479,7 @@ extern void operator delete[](void *p) throw(); # define CHashMultiMap NL_ISO_STDTR1_NAMESPACE::unordered_multimap # define CUniquePtr ::std::auto_ptr # define CUniquePtrMove +# define NL_OVERRIDE #elif defined(NL_COMP_VC) && (NL_COMP_VC_VERSION >= 70 && NL_COMP_VC_VERSION <= 90) // VC7 through 9 # include # include @@ -487,6 +488,7 @@ extern void operator delete[](void *p) throw(); # define CHashMultiMap stdext::hash_multimap # define CUniquePtr ::std::auto_ptr # define CUniquePtrMove +# define NL_OVERRIDE #elif defined(NL_COMP_GCC) // GCC4 # include # include @@ -526,6 +528,10 @@ template<> struct hash */ typedef uint16 ucchar; +#ifndef NL_OVERRIDE +#define NL_OVERRIDE override +#endif + #if defined(NL_OS_WINDOWS) && (defined(UNICODE) || defined(_UNICODE)) #define nltmain wmain #define nltWinMain wWinMain diff --git a/code/nel/src/misc/CMakeLists.txt b/code/nel/src/misc/CMakeLists.txt index e8470983c..091e181cc 100644 --- a/code/nel/src/misc/CMakeLists.txt +++ b/code/nel/src/misc/CMakeLists.txt @@ -25,6 +25,8 @@ FILE(GLOB NLMISC_FILESYSTEM big_file.cpp ../../include/nel/misc/big_file.h *_xml.cpp ../../include/nel/misc/*_xml.h xml_*.cpp ../../include/nel/misc/xml_*.h + streamed_*.cpp ../../include/nel/misc/streamed_*.h + seven_zip.cpp ../../include/nel/misc/seven_zip.h ) FILE(GLOB NLMISC_STREAM @@ -209,15 +211,17 @@ IF(UNIX) ENDIF() ENDIF() +INCLUDE_DIRECTORIES(../../3rdparty) INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR} ${PNG_INCLUDE_DIR} ${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR} config_file) -TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES}) +TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES} nel_sevenzip) + NL_DEFAULT_PROPS(nelmisc "NeL, Library: NeL Misc") NL_ADD_RUNTIME_FLAGS(nelmisc) NL_ADD_LIB_SUFFIX(nelmisc) -ADD_DEFINITIONS(${LIBXML2_DEFINITIONS} ${CURL_DEFINITIONS} ) +ADD_DEFINITIONS(${LIBXML2_DEFINITIONS} ${CURL_DEFINITIONS}) IF(WITH_PCH) ADD_NATIVE_PRECOMPILED_HEADER(nelmisc ${CMAKE_CURRENT_SOURCE_DIR}/stdmisc.h ${CMAKE_CURRENT_SOURCE_DIR}/stdmisc.cpp) diff --git a/code/nel/src/misc/file.cpp b/code/nel/src/misc/file.cpp index 834b11a8d..5bb82c454 100644 --- a/code/nel/src/misc/file.cpp +++ b/code/nel/src/misc/file.cpp @@ -23,6 +23,7 @@ #include "nel/misc/command.h" #include "nel/misc/sstring.h" #include "nel/misc/xml_pack.h" +#include "nel/misc/streamed_package_manager.h" #ifndef NL_OS_WINDOWS #include @@ -211,6 +212,40 @@ bool CIFile::open(const std::string &path, bool text) _F = CXMLPack::getInstance().getFile (path, _FileSize, _BigFileOffset, dummy, _AlwaysOpened); } } + else if (pos > 3 && path[pos-3] == 's' && path[pos-2] == 'n' && path[pos-1] == 'p') + { + // nldebug("Opening a streamed package file"); + + _IsInXMLPackFile = false; + _IsInBigFile = false; + _BigFileOffset = 0; + _AlwaysOpened = false; + std::string filePath; + if (CStreamedPackageManager::getInstance().getFile (filePath, path.substr(pos+1))) + { + _F = fopen (filePath.c_str(), mode); + if (_F != NULL) + { + _FileSize=CFile::getFileSize(_F); + if (_FileSize == 0) + { + nlwarning("FILE: Size of file '%s' is 0", path.c_str()); + fclose(_F); + _F = NULL; + } + } + else + { + nlwarning("Failed to open file '%s', error %u : %s", path.c_str(), errno, strerror(errno)); + _FileSize = 0; + } + } + else + { + // TEMPORARY ERROR + // nlerror("File '%s' not in streamed package", path.c_str()); + } + } else { // bnp file diff --git a/code/nel/src/misc/http_package_provider.cpp b/code/nel/src/misc/http_package_provider.cpp new file mode 100644 index 000000000..035fa2a5d --- /dev/null +++ b/code/nel/src/misc/http_package_provider.cpp @@ -0,0 +1,145 @@ +// NeL - MMORPG Framework +// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// 3rd Party includes +#include + +// Project includes +#include +#include +#include +#include +#include +#include + +namespace NLMISC +{ + +CHttpPackageProvider::CHttpPackageProvider() +{ + // init +} + +CHttpPackageProvider::~CHttpPackageProvider() +{ + // release +} + +bool CHttpPackageProvider::getFile(std::string &filePath, const CHashKey &hash, const std::string &name) +{ + CStreamedPackage::makePath(filePath, hash); + std::string downloadUrlFile = filePath + ".lzma"; + filePath = Path + filePath; + std::string downloadPath = filePath + ".download." + toString(rand() * rand()); + + std::string storageDirectory = CFile::getPath(downloadPath); + CFile::createDirectoryTree(storageDirectory); + /*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory)) + { + nldebug("Unable to create directory '%s'", storageDirectory.c_str()); + return false; + }*/ + + // download + for (;;) + { + if (CFile::fileExists(filePath)) + return true; + + std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile; + nldebug("Download streamed package '%s' from '%s'", name.c_str(), downloadUrl.c_str()); + + FILE *fp = fopen(downloadPath.c_str(), "wb"); + if (fp == NULL) + { + nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), name.c_str()); + return false; + } + + CURL *curl; + CURLcode res; + curl = curl_easy_init(); + if (curl) + { + curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_FILE, fp); + res = curl_easy_perform(curl); + long r; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r); + curl_easy_cleanup(curl); + + bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/; + fclose(fp); + + if (diskFull) + { + CFile::deleteFile(downloadPath); + throw EDiskFullError(downloadPath); + } + + if (res != CURLE_OK || r < 200 || r >= 300) + { + CFile::deleteFile(downloadPath); + nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str()); + nlSleep(1000); + continue; + } + } + else + { + nldebug("Curl initialize failed"); + fclose(fp); + CFile::deleteFile(downloadPath); + return false; + } + + // ok! + break; + } + + // extract into file + std::string unpackPath = filePath + ".extract." + toString(rand() * rand()); + + CHashKey outHash; + if (!unpackLZMA(downloadPath, unpackPath, outHash)) + { + return false; + } + + if (!(outHash == hash)) + { + std::string wantHashS = hash.toString(); + std::string outHashS = outHash.toString(); + nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str()); + return false; + } + + if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str())) + { + nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str()); + // in case downloaded from another thread + return CFile::fileExists(filePath); + } + + return true; +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/src/misc/i_streamed_package_provider.cpp b/code/nel/src/misc/i_streamed_package_provider.cpp new file mode 100644 index 000000000..12c873760 --- /dev/null +++ b/code/nel/src/misc/i_streamed_package_provider.cpp @@ -0,0 +1,37 @@ +// NeL - MMORPG Framework +// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// Project includes +#include + +namespace NLMISC +{ + +IStreamedPackageProvider::IStreamedPackageProvider() +{ + // init +} + +IStreamedPackageProvider::~IStreamedPackageProvider() +{ + // release +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index 961a907b4..e563ad810 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -23,6 +23,7 @@ #include "nel/misc/progress_callback.h" #include "nel/misc/file.h" #include "nel/misc/xml_pack.h" +#include "nel/misc/streamed_package_manager.h" #ifdef NL_OS_WINDOWS # include @@ -288,6 +289,7 @@ void CFileContainer::clearMap () nlassert(!_MemoryCompressed); _Files.clear (); CBigFile::getInstance().removeAll (); + CStreamedPackageManager::getInstance().unloadAll (); NL_DISPLAY_PATH("PATH: CPath::clearMap(): map directory cleared"); } @@ -336,7 +338,7 @@ void CFileContainer::remapExtension (const string &ext1, const string &ext2, boo nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): can't remap empty extension", ext1lwr.c_str(), ext2lwr.c_str(), substitute); } - if (ext1lwr == "bnp" || ext2lwr == "bnp") + if (ext1lwr == "bnp" || ext2lwr == "bnp" || ext1lwr == "snp" || ext2lwr == "snp") { nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): you can't remap a big file", ext1lwr.c_str(), ext2lwr.c_str(), substitute); } @@ -1162,16 +1164,26 @@ void CFileContainer::addSearchFile (const string &file, bool remap, const string return; } + std::string fileExtension = CFile::getExtension(newFile); + // check if it s a big file - if (CFile::getExtension(newFile) == "bnp") + if (fileExtension == "bnp") { NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is a big file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str()); addSearchBigFile(file, false, false, progressCallBack); return; } + // check if it s a streamed package + if (fileExtension == "snp") + { + NL_DISPLAY_PATH ("PATH: CPath::addSearchStreamedPackage(%s, %d, '%s'): '%s' is a streamed package, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str()); + addSearchStreamedPackage(file, false, false, progressCallBack); + return; + } + // check if it s an xml pack file - if (CFile::getExtension(newFile) == "xml_pack") + if (fileExtension == "xml_pack") { NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is an xml pack file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str()); addSearchXmlpackFile(file, false, false, progressCallBack); @@ -1392,6 +1404,64 @@ void CFileContainer::addSearchBigFile (const string &sBigFilename, bool recurse, fclose (Handle); } +void CPath::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack) +{ + getInstance()->_FileContainer.addSearchBigFile(filename, recurse, alternative, progressCallBack); +} + +void CFileContainer::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack) +{ + // Check if filename is not empty + if (filename.empty()) + { + nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): can't add empty file, skip it", filename.c_str(), recurse, alternative); + return; + } + // Check if the file exists + if (!CFile::isExists(filename)) + { + nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not found, skip it", filename.c_str(), recurse, alternative, filename.c_str()); + return; + } + // Check if it s a file + if (CFile::isDirectory(filename)) + { + nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not a file, skip it", filename.c_str(), recurse, alternative, filename.c_str()); + return; + } + + // Add the file itself + std::string packname = NLMISC::toLower(CFile::getFilename(filename)); + insertFileInMap(packname, filename, false, CFile::getExtension(filename)); + + // Add the package to the package manager + if (CStreamedPackageManager::getInstance().loadPackage(filename)) + { + std::vector fileNames; + CStreamedPackageManager::getInstance().list(fileNames, packname); + + for (std::vector::iterator it(fileNames.begin()), end(fileNames.end()); it != end; ++it) + { + // Add the file to the lookup + std::string filePackageName = packname + "@" + (*it); + // nldebug("Insert '%s'", filePackageName.c_str()); + insertFileInMap((*it), filePackageName, false, CFile::getExtension(*it)); + + // Remapped extensions + std::string ext = CFile::getExtension(*it); + for (uint j = 0; j < _Extensions.size (); j++) + { + if (_Extensions[j].first == ext) + { + // Need to remap + insertFileInMap(CFile::getFilenameWithoutExtension(*it) + "." + _Extensions[j].second, + filePackageName, true, _Extensions[j].first); + } + } + } + } +} + // WARNING : recurse is not used void CPath::addSearchXmlpackFile (const string &sXmlpackFilename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack) { @@ -1679,7 +1749,7 @@ void CFileContainer::memoryCompress() while (it != _Files.end()) { string sTmp = SSMpath.get(it->second.idPath); - if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && !it->second.Remapped) + if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && (sTmp.find("snp@") == string::npos) && !it->second.Remapped) { // This is a file included in a bigfile (so the name is in the bigfile manager) } @@ -1702,7 +1772,7 @@ void CFileContainer::memoryCompress() { CFileEntry &rFE = it->second; string sTmp = SSMpath.get(rFE.idPath); - if (sTmp.find("@") == string::npos || sTmp.find("@@") != string::npos || rFE.Remapped) + if ((sTmp.find("@") == string::npos) || (sTmp.find("@@") != string::npos) || (sTmp.find("snp@") != string::npos) || rFE.Remapped) { strcpy(_AllFileNames+nSize, rFE.Name.c_str()); _MCFiles[nNb].Name = _AllFileNames+nSize; @@ -1999,6 +2069,7 @@ string CFile::findNewFile (const string &filename) // \warning doesn't work with big file uint32 CFile::getFileSize (const std::string &filename) { + std::string::size_type pos; if (filename.find("@@") != string::npos) { uint32 fs = 0, bfo; @@ -2006,12 +2077,21 @@ uint32 CFile::getFileSize (const std::string &filename) CXMLPack::getInstance().getFile (filename, fs, bfo, c, d); return fs; } - else if (filename.find('@') != string::npos) + else if ((pos = filename.find('@')) != string::npos) { - uint32 fs = 0, bfo; - bool c, d; - CBigFile::getInstance().getFile (filename, fs, bfo, c, d); - return fs; + if (pos > 3 && filename[pos-3] == 's' && filename[pos-2] == 'n' && filename[pos-1] == 'p') + { + uint32 fs = 0; + CStreamedPackageManager::getInstance().getFileSize (fs, filename.substr(pos+1)); + return fs; + } + else + { + uint32 fs = 0, bfo; + bool c, d; + CBigFile::getInstance().getFile (filename, fs, bfo, c, d); + return fs; + } } else { diff --git a/code/ryzom/client/src/seven_zip/seven_zip.cpp b/code/nel/src/misc/seven_zip.cpp similarity index 80% rename from code/ryzom/client/src/seven_zip/seven_zip.cpp rename to code/nel/src/misc/seven_zip.cpp index c52c790e9..6f7f2f496 100644 --- a/code/ryzom/client/src/seven_zip/seven_zip.cpp +++ b/code/nel/src/misc/seven_zip.cpp @@ -14,17 +14,18 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -#include "seven_zip.h" +#include -#include "nel/misc/types_nl.h" -#include "nel/misc/file.h" -#include "nel/misc/path.h" +#include +#include +#include -#include "7z.h" -#include "7zAlloc.h" -#include "7zCrc.h" -#include "7zVersion.h" -#include "LzmaLib.h" +#include +#include +#include +#include +#include +#include #include @@ -39,6 +40,7 @@ using namespace NLMISC; #define new DEBUG_NEW #endif +namespace NLMISC { /// Input stream class for 7zip archive class CNel7ZipInStream : public ISeekInStream @@ -273,6 +275,85 @@ bool unpackLZMA(const std::string &lzmaFile, const std::string &destFileName) return true; } +bool unpackLZMA(const std::string &lzmaFile, const std::string &destFileName, CHashKey &sha1) +{ + nldebug("unpackLZMA: decompress LZMA file '%s' to '%s", lzmaFile.c_str(), destFileName.c_str()); + + // open input file + CIFile inStream(lzmaFile); + uint32 inSize = inStream.getFileSize(); + + if (inSize < LZMA_PROPS_SIZE + 8) + { + nlwarning("unpackLZMA: Invalid file size, too small file '%s'", lzmaFile.c_str()); + return false; + } + + // allocate input buffer for props + std::vector propsBuffer(LZMA_PROPS_SIZE); + + // size of LZMA content + inSize -= LZMA_PROPS_SIZE + 8; + + // allocate input buffer for lzma data + std::vector inBuffer(inSize); + + uint64 fileSize = 0; + + try + { + // read props + inStream.serialBuffer(&propsBuffer[0], LZMA_PROPS_SIZE); + + // read uncompressed size + inStream.serial(fileSize); + + // read lzma content + inStream.serialBuffer(&inBuffer[0], inSize); + } + catch(const EReadError &e) + { + nlwarning("unpackLZMA: Error while reading '%s': %s", lzmaFile.c_str(), e.what()); + return false; + } + + // allocate the output buffer + std::vector outBuffer(fileSize); + + // in and out file sizes + SizeT outProcessed = (SizeT)fileSize; + SizeT inProcessed = (SizeT)inSize; + + // decompress the file in memory + sint res = LzmaUncompress(&outBuffer[0], &outProcessed, &inBuffer[0], &inProcessed, &propsBuffer[0], LZMA_PROPS_SIZE); + + if (res != 0 || outProcessed != fileSize) + { + nlwarning("unpackLZMA: Failed to decode lzma file '%s' with status %d", lzmaFile.c_str(), res); + return false; + } + + // get hash + sha1 = getSHA1(&outBuffer[0], outBuffer.size()); + + // store on output buffer + COFile outStream(destFileName); + + try + { + // write content + outStream.serialBuffer(&outBuffer[0], (uint)fileSize); + } + catch(const EFile &e) + { + nlwarning("unpackLZMA: Error while writing '%s': %s", destFileName.c_str(), e.what()); + CFile::deleteFile(destFileName); + return false; + } + + return true; +} + bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName) { nldebug("packLZMA: compress '%s' to LZMA file '%s", srcFileName.c_str(), lzmaFileName.c_str()); @@ -371,3 +452,5 @@ bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName) return false; } + +} diff --git a/code/nel/src/misc/sha1.cpp b/code/nel/src/misc/sha1.cpp index 8afa0c039..c6e5701ec 100644 --- a/code/nel/src/misc/sha1.cpp +++ b/code/nel/src/misc/sha1.cpp @@ -35,67 +35,55 @@ using namespace NLMISC; #define new DEBUG_NEW #endif -// -// Types -// - -/* - * If you do not have the ISO standard stdint.h header file, then you - * must typedef the following: - * name meaning - * uint32_t unsigned 32 bit integer - * uint8_t unsigned 8 bit integer (i.e., unsigned char) - * int_least16_t integer of >= 16 bits - * - */ - -typedef unsigned int uint32_t; -typedef unsigned char uint8_t; -typedef short int_least16_t; +namespace NLMISC { // // Constantes // static const int bufferSize = 100000; -static uint8_t buffer[bufferSize]; - -enum -{ - shaSuccess = 0, - shaNull, /* Null pointer parameter */ - shaInputTooLong, /* input data too long */ - shaStateError /* called Input after Result */ -}; +static uint8 buffer[bufferSize]; #define SHA1HashSize 20 +// +// Types +// + /* - * This structure will hold context information for the SHA-1 - * hashing operation - */ +* This structure will hold context information for the SHA-1 +* hashing operation +*/ typedef struct SHA1Context { - uint32_t Intermediate_Hash[SHA1HashSize/4]; /* Message Digest */ + uint32 Intermediate_Hash[SHA1HashSize/4]; /* Message Digest */ - uint32_t Length_Low; /* Message length in bits */ - uint32_t Length_High; /* Message length in bits */ + uint32 Length_Low; /* Message length in bits */ + uint32 Length_High; /* Message length in bits */ - /* Index into message block array */ - int_least16_t Message_Block_Index; - uint8_t Message_Block[64]; /* 512-bit message blocks */ + /* Index into message block array */ + sint16 Message_Block_Index; + uint8 Message_Block[64]; /* 512-bit message blocks */ - int Computed; /* Is the digest computed? */ - int Corrupted; /* Is the message digest corrupted? */ + int Computed; /* Is the digest computed? */ + int Corrupted; /* Is the message digest corrupted? */ } SHA1Context; +enum +{ + shaSuccess = 0, + shaNull, /* Null pointer parameter */ + shaInputTooLong, /* input data too long */ + shaStateError /* called Input after Result */ +}; + // // Function Prototypes // int SHA1Reset (SHA1Context *); -int SHA1Input (SHA1Context *, const uint8_t *, unsigned int); -int SHA1Result (SHA1Context *, uint8_t Message_Digest[SHA1HashSize]); +int SHA1Input (SHA1Context *, const uint8 *, unsigned int); +int SHA1Result (SHA1Context *, uint8 Message_Digest[SHA1HashSize]); // // Functions @@ -105,7 +93,7 @@ CHashKey getSHA1(const uint8 *buffer, uint32 size) { SHA1Context sha; int err; - uint8_t Message_Digest[20]; + uint8 Message_Digest[20]; err = SHA1Reset(&sha); if (err) @@ -114,7 +102,7 @@ CHashKey getSHA1(const uint8 *buffer, uint32 size) return CHashKey(); } - err = SHA1Input(&sha, (const uint8_t*)buffer, size); + err = SHA1Input(&sha, (const uint8*)buffer, size); if (err) { nlwarning ("SHA: SHA1Input Error %d.\n", err); @@ -136,7 +124,7 @@ CHashKey getSHA1(const string &filename, bool forcePath) { SHA1Context sha; int err; - uint8_t Message_Digest[20]; + uint8 Message_Digest[20]; //printf("reading '%s'\n", findData.cFileName); CIFile ifile; @@ -201,9 +189,9 @@ CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint3 { SHA1Context sha; - uint8_t SHA1_Key[64]; - uint8_t SHA1_Key1[20]; - uint8_t SHA1_Key2[20]; + uint8 SHA1_Key[64]; + uint8 SHA1_Key1[20]; + uint8 SHA1_Key2[20]; string buffer1; string buffer2; @@ -214,9 +202,9 @@ CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint3 // If lenght of key > 64 use sha1 hash if (key_len > 64) { - uint8_t SHA1_Key0[20]; + uint8 SHA1_Key0[20]; SHA1Reset(&sha); - SHA1Input(&sha, (const uint8_t*)key, key_len); + SHA1Input(&sha, (const uint8*)key, key_len); SHA1Result(&sha, SHA1_Key0); CHashKey hk0 (SHA1_Key0); for (uint i = 0; i < 20; i++) @@ -236,7 +224,7 @@ CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint3 // Get hash SHA1Reset(&sha); - SHA1Input(&sha, (const uint8_t*)buffer1.c_str(), (uint)buffer1.size()); + SHA1Input(&sha, (const uint8*)buffer1.c_str(), (uint)buffer1.size()); SHA1Result(&sha, SHA1_Key1); CHashKey hk1 (SHA1_Key1); @@ -250,7 +238,7 @@ CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint3 // Get new hash SHA1Reset(&sha); - SHA1Input(&sha, (const uint8_t*)buffer2.c_str(), (uint)buffer2.size()); + SHA1Input(&sha, (const uint8*)buffer2.c_str(), (uint)buffer2.size()); SHA1Result(&sha, SHA1_Key2); CHashKey hk (SHA1_Key2); @@ -330,7 +318,7 @@ int SHA1Reset(SHA1Context *context) * */ int SHA1Result( SHA1Context *context, - uint8_t Message_Digest[SHA1HashSize]) + uint8 Message_Digest[SHA1HashSize]) { int i; @@ -360,7 +348,7 @@ int SHA1Result( SHA1Context *context, for(i = 0; i < SHA1HashSize; ++i) { - Message_Digest[i] = uint8_t((context->Intermediate_Hash[i>>2] + Message_Digest[i] = uint8((context->Intermediate_Hash[i>>2] >> 8 * ( 3 - ( i & 0x03 ) ) ) & 0xff ); } @@ -388,7 +376,7 @@ int SHA1Result( SHA1Context *context, * */ int SHA1Input( SHA1Context *context, - const uint8_t *message_array, + const uint8 *message_array, unsigned length) { if (!length) @@ -462,16 +450,16 @@ int SHA1Input( SHA1Context *context, */ void SHA1ProcessMessageBlock(SHA1Context *context) { - const uint32_t K[] = { /* Constants defined in SHA-1 */ + const uint32 K[] = { /* Constants defined in SHA-1 */ 0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC, 0xCA62C1D6 }; int t; /* Loop counter */ - uint32_t temp; /* Temporary word value */ - uint32_t W[80]; /* Word sequence */ - uint32_t A, B, C, D, E; /* Word buffers */ + uint32 temp; /* Temporary word value */ + uint32 W[80]; /* Word sequence */ + uint32 A, B, C, D, E; /* Word buffers */ /* * Initialize the first 16 words in the array W @@ -607,14 +595,17 @@ void SHA1PadMessage(SHA1Context *context) /* * Store the message length as the last 8 octets */ - context->Message_Block[56] = uint8_t((context->Length_High >> 24)&0xff); - context->Message_Block[57] = uint8_t((context->Length_High >> 16)&0xff); - context->Message_Block[58] = uint8_t((context->Length_High >> 8)&0xff); - context->Message_Block[59] = uint8_t((context->Length_High)&0xff); - context->Message_Block[60] = uint8_t((context->Length_Low >> 24)&0xff); - context->Message_Block[61] = uint8_t((context->Length_Low >> 16)&0xff); - context->Message_Block[62] = uint8_t((context->Length_Low >> 8)&0xff); - context->Message_Block[63] = uint8_t((context->Length_Low)&0xff); + context->Message_Block[56] = uint8((context->Length_High >> 24)&0xff); + context->Message_Block[57] = uint8((context->Length_High >> 16)&0xff); + context->Message_Block[58] = uint8((context->Length_High >> 8)&0xff); + context->Message_Block[59] = uint8((context->Length_High)&0xff); + context->Message_Block[60] = uint8((context->Length_Low >> 24)&0xff); + context->Message_Block[61] = uint8((context->Length_Low >> 16)&0xff); + context->Message_Block[62] = uint8((context->Length_Low >> 8)&0xff); + context->Message_Block[63] = uint8((context->Length_Low)&0xff); SHA1ProcessMessageBlock(context); } + +} + diff --git a/code/nel/src/misc/streamed_package.cpp b/code/nel/src/misc/streamed_package.cpp new file mode 100644 index 000000000..fed7d1dc3 --- /dev/null +++ b/code/nel/src/misc/streamed_package.cpp @@ -0,0 +1,66 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// Project includes +#include +#include + +namespace NLMISC { + +CStreamedPackage::CStreamedPackage() +{ + // init +} + +CStreamedPackage::~CStreamedPackage() +{ + // release +} + +void CStreamedPackage::serial(NLMISC::IStream &f) throw(NLMISC::EStream) +{ + f.serialCheck(NELID("SNPK")); + + uint version = 1; + f.serialVersion(version); + + f.serialCont(Entries); +} + +void CStreamedPackage::CEntry::serial(NLMISC::IStream &f) throw(NLMISC::EStream) +{ + uint version = 1; + f.serialVersion(version); + + f.serial(Name); + f.serial(Hash); + f.serial(Size); + f.serial(LastModified); +} + +void CStreamedPackage::makePath(std::string &result, const CHashKey &hash) +{ + std::string lowerHash = NLMISC::toLower(hash.toString()); + result = std::string("/") + lowerHash.substr(0, 2) + + "/" + lowerHash.substr(2, 2) + + "/" + lowerHash.substr(4); +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp new file mode 100644 index 000000000..ac605b049 --- /dev/null +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -0,0 +1,126 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// Project includes +#include +#include +#include +#include + +namespace NLMISC +{ + +NLMISC_SAFE_SINGLETON_IMPL(CStreamedPackageManager); + +CStreamedPackageManager::CStreamedPackageManager() +{ + // init +} + +CStreamedPackageManager::~CStreamedPackageManager() +{ + // release +} + +bool CStreamedPackageManager::loadPackage(const std::string &package) +{ + nldebug("Load package '%s'", package.c_str()); + + std::string packname = NLMISC::toLower(CFile::getFilename(package)); + m_Packages[packname] = CStreamedPackage(); + std::map::iterator it = m_Packages.find(packname); + CStreamedPackage &p = it->second; + + try + { + CIFile fi; + fi.open(package); + fi.serial(p); + } + catch (Exception &e) + { + nlerror("Package serial exception: '%s'", e.what()); + m_Packages.erase(it); + return false; + } + + for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + m_Entries[entry.Name] = &entry; + } + + return true; +} + +void CStreamedPackageManager::list(std::vector &fileNames, const std::string &package) +{ + // nldebug("List package '%s'", package.c_str()); + + std::map::iterator it = m_Packages.find(package); + CStreamedPackage &p = it->second; + + for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + fileNames.push_back(entry.Name); + } +} + +void CStreamedPackageManager::unloadAll() +{ + m_Packages.clear(); + m_Entries.clear(); +} + +bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName) +{ + // nldebug("Get file path for streamed file '%s'", fileName.c_str()); + + IStreamedPackageProvider *provider = Provider; + if (!provider) + { + nlerrornoex("No streamed package provider was set"); + return false; + } + + TEntries::iterator it = m_Entries.find(fileName); + if (it == m_Entries.end()) + { + return false; + } + const CStreamedPackage::CEntry *entry = it->second; + + return provider->getFile(filePath, entry->Hash, fileName); + + return true; +} + +bool CStreamedPackageManager::getFileSize(uint32 &fileSize, const std::string &fileName) +{ + // nldebug("Get file size for streamed file '%s'", fileName.c_str()); + TEntries::iterator it = m_Entries.find(fileName); + if (it == m_Entries.end()) + return false; + fileSize = it->second->Size; + return true; +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/tools/build_gamedata/configuration/tools.py b/code/nel/tools/build_gamedata/configuration/tools.py index 19cf5c9a5..ebb1d07da 100755 --- a/code/nel/tools/build_gamedata/configuration/tools.py +++ b/code/nel/tools/build_gamedata/configuration/tools.py @@ -91,6 +91,7 @@ BuildClodBankTool = "build_clod_bank" SheetsPackerTool = "sheets_packer" SheetsPackerShardTool = "sheets_packer_shard" BnpMakeTool = "bnp_make" +SnpMakeTool = "snp_make" AiBuildWmapTool = "ai_build_wmap" TgaCutTool = "tga_cut" PatchGenTool = "patch_gen" diff --git a/code/nel/tools/build_gamedata/d1_client_patch.py b/code/nel/tools/build_gamedata/d1_client_patch.py index 449d6e0b7..ea4cc742c 100755 --- a/code/nel/tools/build_gamedata/d1_client_patch.py +++ b/code/nel/tools/build_gamedata/d1_client_patch.py @@ -51,6 +51,7 @@ printLog(log, "") # Find tools BnpMake = findTool(log, ToolDirectories, BnpMakeTool, ToolSuffix) +SnpMake = findTool(log, ToolDirectories, SnpMakeTool, ToolSuffix); PatchGen = findTool(log, ToolDirectories, PatchGenTool, ToolSuffix) printLog(log, "") @@ -90,20 +91,23 @@ else: inCategories = 1 cfg.write("\t<_Categories>\n") for category in InstallClientData: + packExt = ".bnp" + if (category["StreamedPackages"]): + packExt = ".snp" cfg.write("\t\t<_Category>\n") cfg.write("\t\t\t<_Name type=\"STRING\" value=\"" + category["Name"] + "\"/>\n") if category["UnpackTo"] != None: if category["UnpackTo"] != "": cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./" + category["UnpackTo"] + "/\"/>\n") else: - cfg.write("\t\t\t<_UnpackTo type=\"SINT32\" value=\"./\"/>\n") + cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./\"/>\n") cfg.write("\t\t\t<_IsOptional type=\"SINT32\" value=\"" + str(category["IsOptional"]) + "\"/>\n") cfg.write("\t\t\t<_IsIncremental type=\"SINT32\" value=\"" + str(category["IsIncremental"]) + "\"/>\n") for package in category["Packages"]: if (len(package[1]) > 0): cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[1][0] + "\"/>\n") else: - cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[0] + ".bnp\"/>\n") + cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[0] + packExt + "\"/>\n") for ref in category["Refs"]: cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + ref + "_.ref\"/>\n") cfg.write("\t\t\n") @@ -121,11 +125,14 @@ else: targetPath = ClientPatchDirectory + "/bnp" mkPath(log, targetPath) for category in InstallClientData: + packExt = ".bnp" + if (category["StreamedPackages"]): + packExt = ".snp" for package in category["Packages"]: printLog(log, "PACKAGE " + package[0]) sourcePath = InstallDirectory + "/" + package[0] mkPath(log, sourcePath) - targetBnp = targetPath + "/" + package[0] + ".bnp" + targetBnp = targetPath + "/" + package[0] + packExt if (len(package[1]) > 0): targetBnp = targetPath + "/" + package[1][0] printLog(log, "TARGET " + package[1][0]) @@ -135,8 +142,16 @@ else: else: needUpdateBnp = needUpdateDirNoSubdirFile(log, sourcePath, targetBnp) if (needUpdateBnp): - printLog(log, "BNP " + targetBnp) - subprocess.call([ BnpMake, "-p", sourcePath, "-o", targetBnp ] + package[1][1:]) + if (category["StreamedPackages"]): + printLog(log, "SNP " + targetBnp) + # cwDir = os.getcwd().replace("\\", "/") + # toolDir = os.path.dirname(Lzma).replace("\\", "/") + # os.chdir(toolDir) + subprocess.call([ SnpMake, "-p", sourcePath, targetBnp, ClientPatchDirectory + "/stream" ] + package[1][1:]) + # os.chdir(cwDir) + else: + printLog(log, "BNP " + targetBnp) + subprocess.call([ BnpMake, "-p", sourcePath, "-o", targetBnp ] + package[1][1:]) else: printLog(log, "SKIP " + targetBnp) printLog(log, "") diff --git a/code/nel/tools/build_gamedata/d2_client_install.py b/code/nel/tools/build_gamedata/d2_client_install.py index febaef656..bb399447f 100755 --- a/code/nel/tools/build_gamedata/d2_client_install.py +++ b/code/nel/tools/build_gamedata/d2_client_install.py @@ -47,6 +47,9 @@ printLog(log, "") for category in InstallClientData: printLog(log, "CATEGORY " + category["Name"]) + packExt = ".bnp" + if (category["StreamedPackages"]): + packExt = ".snp" if (category["UnpackTo"] != None): targetPath = ClientInstallDirectory if (category["UnpackTo"] != ""): @@ -62,8 +65,8 @@ for category in InstallClientData: mkPath(log, targetPath) for package in category["Packages"]: printLog(log, "PACKAGE " + package[0]) - sourceBnp = sourcePath + "/" + package[0] + ".bnp" - targetBnp = targetPath + "/" + package[0] + ".bnp" + sourceBnp = sourcePath + "/" + package[0] + packExt + targetBnp = targetPath + "/" + package[0] + packExt if (len(package[1]) > 0): sourceBnp = sourcePath + "/" + package[1][0] targetBnp = targetPath + "/" + package[1][0] diff --git a/code/nel/tools/misc/CMakeLists.txt b/code/nel/tools/misc/CMakeLists.txt index 8f962d1f2..f9112a823 100644 --- a/code/nel/tools/misc/CMakeLists.txt +++ b/code/nel/tools/misc/CMakeLists.txt @@ -4,7 +4,15 @@ IF(WITH_QT OR WITH_QT5) ENDIF() IF(WITH_NEL_TOOLS) - SUBDIRS(bnp_make disp_sheet_id extract_filename lock make_sheet_id xml_packer) + SUBDIRS( + bnp_make + snp_make + disp_sheet_id + extract_filename + lock + make_sheet_id + xml_packer + ) IF(WITH_QT OR WITH_QT5) ADD_SUBDIRECTORY(words_dic_qt) diff --git a/code/nel/tools/misc/snp_make/CMakeLists.txt b/code/nel/tools/misc/snp_make/CMakeLists.txt new file mode 100644 index 000000000..a1b5f388d --- /dev/null +++ b/code/nel/tools/misc/snp_make/CMakeLists.txt @@ -0,0 +1,9 @@ +FILE(GLOB SRC *.cpp *.h) + +ADD_EXECUTABLE(snp_make ${SRC}) + +TARGET_LINK_LIBRARIES(snp_make nelmisc) +NL_DEFAULT_PROPS(snp_make "NeL, Tools, Misc: snp_make") +NL_ADD_RUNTIME_FLAGS(snp_make) + +INSTALL(TARGETS snp_make RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT toolsmisc) diff --git a/code/nel/tools/misc/snp_make/main.cpp b/code/nel/tools/misc/snp_make/main.cpp new file mode 100644 index 000000000..a86d497d4 --- /dev/null +++ b/code/nel/tools/misc/snp_make/main.cpp @@ -0,0 +1,360 @@ +// NeL - MMORPG Framework +// Copyright (C) 2010 Winch Gate Property Limited +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "nel/misc/types_nl.h" + +#include +#include + +#ifdef NL_OS_WINDOWS +# include +# include +#endif + +#include +#include + +#include "nel/misc/debug.h" +#include "nel/misc/file.h" +#include "nel/misc/path.h" +#include "nel/misc/algo.h" +#include "nel/misc/common.h" +#include "nel/misc/streamed_package.h" +#include "nel/misc/seven_zip.h" + +using namespace std; +using namespace NLMISC; + +// --------------------------------------------------------------------------- + +class CWildCard +{ +public: + string Expression; + bool Not; +}; +std::vector WildCards; + +std::string SourceDirectory; +std::string PackageFileName; +std::string StreamDirectory; + +CStreamedPackage Package; + +// --------------------------------------------------------------------------- + +bool keepFile (const char *fileName) +{ + uint i; + bool ifPresent = false; + bool ifTrue = false; + string file = toLower(CFile::getFilename (fileName)); + for (i=0; i [option] ... [option]\n"); + printf (" option : \n"); + printf (" -if wildcard : add the file if it matches the wilcard (at least one 'if' conditions must be met for a file to be adding)\n"); + printf (" -ifnot wildcard : add the file if it doesn't match the wilcard (all the 'ifnot' conditions must be met for a file to be adding)\n"); + printf (" Pack the directory to a snp file\n"); + printf (" snp_make -l \n"); + printf (" List the files contained in the snp file\n"); +} + +// --------------------------------------------------------------------------- + +void generateLZMA(const std::string &sourceFile, const std::string &outputFile) +{ + NLMISC::packLZMA(sourceFile, outputFile); + + /* + std::string cmd="lzma e "; + cmd+=" "+sourceFile+" "+outputFile; + nlinfo("executing system command: %s",cmd.c_str()); +#ifdef NL_OS_WINDOWS + _spawnlp(_P_WAIT, "lzma.exe","lzma.exe", "e", sourceFile.c_str(), outputFile.c_str(), NULL); +#else // NL_OS_WINDOWS + sint error = system (cmd.c_str()); + if (error) + nlwarning("'%s' failed with error code %d", cmd.c_str(), error); +#endif // NL_OS_WINDOWS + */ +} + +// --------------------------------------------------------------------------- + +uint readOptions (int nNbArg, char **ppArgs) +{ + uint i; + uint optionCount = 0; + for (i=0; i<(uint)nNbArg; i++) + { + // If ? + if ((strcmp (ppArgs[i], "-if") == 0) && ((i+1)<(uint)nNbArg)) + { + CWildCard card; + card.Expression = toLower(string(ppArgs[i+1])); + card.Not = false; + WildCards.push_back (card); + optionCount += 2; + } + // If not ? + if ((strcmp (ppArgs[i], "-ifnot") == 0) && ((i+1)<(uint)nNbArg)) + { + CWildCard card; + card.Expression = toLower(string(ppArgs[i+1])); + card.Not = true; + WildCards.push_back (card); + optionCount += 2; + } + } + return optionCount; +} + +// --------------------------------------------------------------------------- +int main (int nNbArg, char **ppArgs) +{ + NLMISC::CApplicationContext myApplicationContext; + + if (nNbArg < 3) + { + usage(); + return -1; + } + + if ((strcmp(ppArgs[1], "/p") == 0) || (strcmp(ppArgs[1], "/P") == 0) || + (strcmp(ppArgs[1], "-p") == 0) || (strcmp(ppArgs[1], "-P") == 0)) + { + if (nNbArg < 5) + { + usage(); + return -1; + } + + SourceDirectory = ppArgs[2]; + PackageFileName = ppArgs[3]; + StreamDirectory = ppArgs[4]; + readOptions(nNbArg, ppArgs); + + nldebug("Make streamed package: '%s'", PackageFileName.c_str()); + + if (CFile::fileExists(PackageFileName)) + { + nldebug("Update existing package"); + try + { + CIFile fi; + fi.open(PackageFileName); + fi.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + } + else + { + nldebug("New package"); + } + + std::vector pathContent; // contains full pathnames + std::vector nameContent; // only filename + CPath::getPathContent(SourceDirectory, true, false, true, pathContent); + nameContent.reserve(pathContent.size()); + for (std::vector::size_type i = 0; i < pathContent.size(); ++i) + { + const std::string &file = pathContent[i]; + if (keepFile(file.c_str())) + { + std::string fileName = NLMISC::toLower(CFile::getFilename(file)); + // nldebug("File: '%s' ('%s')", file.c_str(), fileName.c_str()); + nameContent.push_back(fileName); + nlassert(nameContent.size() == (i + 1)); + } + else + { + // Not included in this package + pathContent.erase(pathContent.begin() + i); + --i; + } + } + + std::vector packageIndex; // index of file in package + packageIndex.resize(pathContent.size(), -1); + + for (CStreamedPackage::TEntries::size_type i = 0; i < Package.Entries.size(); ++i) + { + const CStreamedPackage::CEntry &entry = Package.Entries[i]; + + sint foundIndex = -1; // find index in found file list + for (std::vector::size_type j = 0; j < pathContent.size(); ++j) + { + if (nameContent[j] == entry.Name) + { + foundIndex = j; + break; + } + } + + if (foundIndex < 0) + { + nlinfo("File no longer exists: '%s'", entry.Name.c_str()); + Package.Entries.erase(Package.Entries.begin() + i); + --i; + } + else + { + // File still exists, map it + packageIndex[foundIndex] = i; + } + } + + for (std::vector::size_type i = 0; i < pathContent.size(); ++i) + { + sint pidx = packageIndex[i]; + const std::string &name = nameContent[i]; + const std::string &path = pathContent[i]; + + if (pidx < 0) + { + nlinfo("File added: '%s'", name.c_str()); + pidx = Package.Entries.size(); + Package.Entries.push_back(CStreamedPackage::CEntry()); + Package.Entries[pidx].Name = name; + Package.Entries[pidx].LastModified = 0; + Package.Entries[pidx].Size = 0; + } + else + { + nlinfo("File check for changes: '%s'", name.c_str()); + } + + CStreamedPackage::CEntry &entry = Package.Entries[pidx]; + + std::string targetLzmaOld; // in case lzma wasn't made make sure it exists a second run + CStreamedPackage::makePath(targetLzmaOld, entry.Hash); + targetLzmaOld = StreamDirectory + targetLzmaOld + ".lzma"; + + uint32 lastModified = CFile::getFileModificationDate(path); + uint32 fileSize = CFile::getFileSize(path); + if (lastModified > entry.LastModified || fileSize != entry.Size || !CFile::fileExists(targetLzmaOld)) + { + entry.LastModified = lastModified; + + nlinfo("Calculate file hash"); + CHashKey hash = getSHA1(path, true); + /*nldebug("%s", hash.toString().c_str()); + std::string hashPath; + CStreamedPackage::makePath(hashPath, hash); + nldebug("%s", hashPath.c_str());*/ + + if (hash == entry.Hash && fileSize == entry.Size) + { + // File has not changed + } + else + { + nlinfo("File changed"); + entry.Hash = hash; + entry.Size = fileSize; + } + + std::string targetLzma; // in case lzma wasn't made make sure it exists a second run + CStreamedPackage::makePath(targetLzma, entry.Hash); + targetLzma = StreamDirectory + targetLzma + ".lzma"; + + if (!CFile::fileExists(targetLzma)) + { + // make the compressed file + nlinfo("%s -> %s", path.c_str(), targetLzma.c_str()); + CFile::createDirectoryTree(CFile::getPath(targetLzma)); + generateLZMA(path, targetLzma); + } + } + } + + try + { + nldebug("Store package '%s'", PackageFileName.c_str()); + COFile fo; + fo.open(PackageFileName); + fo.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + + return 0; + } + + if ((strcmp(ppArgs[1], "/l") == 0) || (strcmp(ppArgs[1], "/L") == 0) || + (strcmp(ppArgs[1], "-l") == 0) || (strcmp(ppArgs[1], "-L") == 0)) + { + PackageFileName = ppArgs[2]; + if (!CFile::fileExists(PackageFileName)) + { + nlwarning("ERROR (snp_make) : package doesn't exist: '%s'", PackageFileName.c_str()); + return -1; + } + + try + { + CIFile fi; + fi.open(PackageFileName); + fi.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + + for (CStreamedPackage::TEntries::const_iterator it(Package.Entries.begin()), end(Package.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + + printf("List files in '%s'", PackageFileName.c_str()); + printf("%s { Hash: '%s', Size: '%u', LastModified: '%u' }", entry.Name.c_str(), entry.Hash.toString().c_str(), entry.Size, entry.LastModified); + } + + return 0; + } + + usage (); + return -1; +} diff --git a/code/ryzom/client/CMakeLists.txt b/code/ryzom/client/CMakeLists.txt index 43897c905..7a6cf7424 100644 --- a/code/ryzom/client/CMakeLists.txt +++ b/code/ryzom/client/CMakeLists.txt @@ -1,5 +1,4 @@ # Need clientsheets lib for sheets packer tool -# Need seven_zip for patch_gen and ryzom_installer ADD_SUBDIRECTORY(src) IF(WITH_RYZOM_CLIENT) diff --git a/code/ryzom/client/src/CMakeLists.txt b/code/ryzom/client/src/CMakeLists.txt index 4116fc75b..99b9a6773 100644 --- a/code/ryzom/client/src/CMakeLists.txt +++ b/code/ryzom/client/src/CMakeLists.txt @@ -3,11 +3,6 @@ IF(WITH_RYZOM_TOOLS OR WITH_RYZOM_CLIENT) ADD_SUBDIRECTORY(client_sheets) ENDIF() -IF(WITH_RYZOM_TOOLS OR WITH_RYZOM_CLIENT OR WITH_RYZOM_INSTALLER) - # Need seven_zip lib for patch_gen tool - ADD_SUBDIRECTORY(seven_zip) -ENDIF() - IF(WITH_RYZOM_CLIENT) IF(NOT WITH_GUI) MESSAGE(FATAL_ERROR "The client cannot be built without the NeL GUI Library (WITH_GUI)") @@ -158,6 +153,8 @@ IF(WITH_RYZOM_CLIENT) ${OPENSSL_INCLUDE_DIR} ) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/nel/3rdparty) + IF(STEAM_FOUND) INCLUDE_DIRECTORIES(${STEAM_INCLUDE_DIRS}) ENDIF() @@ -170,7 +167,7 @@ IF(WITH_RYZOM_CLIENT) nel3d nelgui nelsound - ryzom_sevenzip + nel_sevenzip ryzom_clientsheets ryzom_gameshare nelpacs diff --git a/code/ryzom/client/src/client_cfg.cpp b/code/ryzom/client/src/client_cfg.cpp index aac38c885..23f5cc961 100644 --- a/code/ryzom/client/src/client_cfg.cpp +++ b/code/ryzom/client/src/client_cfg.cpp @@ -472,6 +472,8 @@ CClientConfig::CClientConfig() ColorShout = CRGBA(150,0,0,255); // Default Shout color. ColorTalk = CRGBA(255,255,255,255); // Default Talk color. + StreamedPackagePath = "stream"; + // MP3 player MediaPlayerDirectory = "music"; MediaPlayerAutoPlay = false; @@ -1288,18 +1290,25 @@ void CClientConfig::setValues() ////////// // MISC // + // Pre Data Path. READ_STRINGVECTOR_FV(PreDataPath); // Data Path. READ_STRINGVECTOR_FV(DataPath); - // List of files that trigger R2ED reload when touched - READ_STRINGVECTOR_FV(R2EDReloadFiles); - // Data Path no recurse. READ_STRINGVECTOR_FV(DataPathNoRecurse); + // Streamed package path + READ_STRING_FV(StreamedPackagePath); + + // Streamed package hosts + READ_STRINGVECTOR_FV(StreamedPackageHosts); + + // List of files that trigger R2ED reload when touched + READ_STRINGVECTOR_FV(R2EDReloadFiles); + // Update packed sheet Path READ_STRINGVECTOR_FV(UpdatePackedSheetPath); diff --git a/code/ryzom/client/src/client_cfg.h b/code/ryzom/client/src/client_cfg.h index 7c1fa2ad6..7d2dbac37 100644 --- a/code/ryzom/client/src/client_cfg.h +++ b/code/ryzom/client/src/client_cfg.h @@ -376,6 +376,10 @@ struct CClientConfig std::vector DataPath; /// Data Path no recurse. std::vector DataPathNoRecurse; + /// Streamed package path + std::string StreamedPackagePath; + /// Streamed package hosts + std::vector StreamedPackageHosts; // TODO: From 'domain' SQL table /// Update packed sheet Path. std::vector UpdatePackedSheetPath; /// True if we want the packed sheet to be updated if needed diff --git a/code/ryzom/client/src/global.cpp b/code/ryzom/client/src/global.cpp index 4b8e1aad7..41d41160c 100644 --- a/code/ryzom/client/src/global.cpp +++ b/code/ryzom/client/src/global.cpp @@ -24,6 +24,9 @@ using namespace NLMISC; // *************************************************************************** +// Data +NLMISC::CHttpPackageProvider *HttpPackageProvider = NULL; + // Main System NL3D::UDriver *Driver = NULL; // The main 3D Driver NL3D::IStereoDisplay *StereoDisplay = NULL; // Stereo display diff --git a/code/ryzom/client/src/global.h b/code/ryzom/client/src/global.h index 78cebcec9..75ba2381d 100644 --- a/code/ryzom/client/src/global.h +++ b/code/ryzom/client/src/global.h @@ -28,6 +28,11 @@ // *************************************************************************** +namespace NLMISC +{ + class CHttpPackageProvider; +} + namespace NL3D { class UDriver; @@ -78,6 +83,9 @@ const float ExtraZoneLoadingVision = 100.f; // *************************************************************************** +// Data +extern NLMISC::CHttpPackageProvider *HttpPackageProvider; // Http provider from on-the-fly downloaded game data + // Main System extern NL3D::UDriver *Driver; // The main 3D Driver extern NL3D::IStereoDisplay *StereoDisplay; // Stereo display diff --git a/code/ryzom/client/src/init.cpp b/code/ryzom/client/src/init.cpp index abbf8b6b4..85eb50b51 100644 --- a/code/ryzom/client/src/init.cpp +++ b/code/ryzom/client/src/init.cpp @@ -34,6 +34,8 @@ #include "nel/misc/system_info.h" #include "nel/misc/block_memory.h" #include "nel/misc/system_utils.h" +#include "nel/misc/streamed_package_manager.h" +#include "nel/misc/http_package_provider.h" #include "nel/misc/cmd_args.h" // 3D Interface. #include "nel/3d/bloom_effect.h" @@ -157,6 +159,8 @@ bool LastScreenSaverEnabled = false; extern void registerInterfaceElements(); extern CContinentManager ContinentMngr; +extern NLMISC::CCmdArgs Args; + // Tips of the day count #define RZ_NUM_TIPS 17 ucstring TipsOfTheDay; @@ -660,8 +664,6 @@ void initStereoDisplayDevice() } // we want to get executable directory -extern NLMISC::CCmdArgs Args; - static void addPaths(IProgressCallback &progress, const std::vector &paths, bool recurse) { // all prefixes for paths @@ -736,6 +738,19 @@ static void addPaths(IProgressCallback &progress, const std::vector } } +void initStreamedPackageManager(NLMISC::IProgressCallback &progress) +{ + CStreamedPackageManager &spm = CStreamedPackageManager::getInstance(); + nlassert(!spm.Provider); // If this asserts, init was called twice without release + nlassert(!HttpPackageProvider); // Idem + CHttpPackageProvider *hpp = new CHttpPackageProvider(); + hpp->Path = ClientCfg.StreamedPackagePath; + for (uint i = 0; i < ClientCfg.StreamedPackageHosts.size(); i++) + hpp->Hosts.push_back(ClientCfg.StreamedPackageHosts[i]); + spm.Provider = hpp; + HttpPackageProvider = hpp; +} + void addSearchPaths(IProgressCallback &progress) { // Add search path of UI addon. Allow only a subset of files. @@ -976,6 +991,7 @@ void prelogInit() CPath::remapExtension ("png", "tga", true); FPU_CHECKER_ONCE + initStreamedPackageManager(ProgressBar); addPreDataPaths(ProgressBar); FPU_CHECKER_ONCE diff --git a/code/ryzom/client/src/init.h b/code/ryzom/client/src/init.h index f179bf13d..4ef738828 100644 --- a/code/ryzom/client/src/init.h +++ b/code/ryzom/client/src/init.h @@ -36,6 +36,7 @@ void prelogInit(); // Initialize the application after login step void postlogInit(); +void initStreamedPackageManager(NLMISC::IProgressCallback &progress); void addSearchPaths(NLMISC::IProgressCallback &progress); void addPreDataPaths(NLMISC::IProgressCallback &progress); diff --git a/code/ryzom/client/src/login.cpp b/code/ryzom/client/src/login.cpp index eefe56188..4bc8c10be 100644 --- a/code/ryzom/client/src/login.cpp +++ b/code/ryzom/client/src/login.cpp @@ -28,6 +28,7 @@ #include "nel/misc/thread.h" #include "nel/misc/big_file.h" #include "nel/misc/system_utils.h" +#include "nel/misc/streamed_package_manager.h" #include "nel/net/tcp_sock.h" #include "nel/3d/u_driver.h" @@ -1686,6 +1687,7 @@ void initPatch() CBGDownloaderAccess::getInstance().startTask(taskDesc, string(), true /* showDownloader */); // no command line since bg downloader should already be started // release lock on bnp, so that they can be written NLMISC::CBigFile::getInstance().removeAll(); + NLMISC::CStreamedPackageManager::getInstance().unloadAll(); } NLGUI::CDBManager::getInstance()->getDbProp("UI:VARIABLES:SCREEN")->setValue32(UI_VARIABLES_SCREEN_PATCHING); diff --git a/code/ryzom/client/src/login_patch.cpp b/code/ryzom/client/src/login_patch.cpp index bde537f21..ca91fcbfa 100644 --- a/code/ryzom/client/src/login_patch.cpp +++ b/code/ryzom/client/src/login_patch.cpp @@ -48,6 +48,7 @@ #include "nel/misc/big_file.h" #include "nel/misc/i18n.h" #include "nel/misc/cmd_args.h" +#include "nel/misc/seven_zip.h" #include "nel/misc/curl_certificates.h" #include "game_share/bg_downloader_msg.h" @@ -56,7 +57,6 @@ #include "login.h" #include "user_agent.h" -#include "seven_zip/seven_zip.h" #ifndef RY_BG_DOWNLOADER #include "client_cfg.h" @@ -2506,7 +2506,7 @@ void CPatchThread::run() CPatchManager::SFileToPatch &rFTP = AllFilesToPatch[i]; string ext = NLMISC::CFile::getExtension(rFTP.FileName); - if (ext == "bnp") + if (ext == "bnp" || ext == "snp") { float oldCurrentFilePatched = CurrentFilePatched; processFile (rFTP); diff --git a/code/ryzom/client/src/release.cpp b/code/ryzom/client/src/release.cpp index 97265dcaa..6686fea5d 100644 --- a/code/ryzom/client/src/release.cpp +++ b/code/ryzom/client/src/release.cpp @@ -25,6 +25,7 @@ #include "nel/misc/debug.h" #include "nel/misc/async_file_manager.h" #include "nel/misc/system_utils.h" +#include "nel/misc/streamed_package_manager.h" // 3D Interface. #include "nel/3d/bloom_effect.h" #include "nel/3d/fxaa.h" @@ -666,6 +667,9 @@ void release() R2::CObjectSerializer::releaseInstance(); NLMISC::CBigFile::getInstance().removeAll(); NLMISC::CBigFile::releaseInstance(); + NLMISC::CStreamedPackageManager::releaseInstance(); + delete HttpPackageProvider; + HttpPackageProvider = NULL; NL3D::CFastHLSModifier::releaseInstance(); CLandscapePolyDrawer::releaseInstance(); NL3D::CParticleSystemShape::releaseInstance(); diff --git a/code/ryzom/client/src/seven_zip/7zMain.cpp b/code/ryzom/client/src/seven_zip/7zMain.cpp deleted file mode 100644 index 59ecd2885..000000000 --- a/code/ryzom/client/src/seven_zip/7zMain.cpp +++ /dev/null @@ -1,686 +0,0 @@ -/* 7zMain.c - Test application for 7z Decoder -2019-02-02 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include -#include - -#include "CpuArch.h" - -#include "7z.h" -#include "7zAlloc.h" -#include "7zBuf.h" -#include "7zCrc.h" -#include "7zFile.h" -#include "7zVersion.h" - -#ifndef USE_WINDOWS_FILE -/* for mkdir */ -#ifdef _WIN32 -#include -#else -#include -#include -#endif -#endif - - -#define kInputBufSize ((size_t)1 << 18) - -static const ISzAlloc g_Alloc = { SzAlloc, SzFree }; - - -static void Print(const char *s) -{ - fputs(s, stdout); -} - - -static int Buf_EnsureSize(CBuf *dest, size_t size) -{ - if (dest->size >= size) - return 1; - Buf_Free(dest, &g_Alloc); - return Buf_Create(dest, size, &g_Alloc); -} - -#ifndef _WIN32 -#define _USE_UTF8 -#endif - -/* #define _USE_UTF8 */ - -#ifdef _USE_UTF8 - -#define _UTF8_START(n) (0x100 - (1 << (7 - (n)))) - -#define _UTF8_RANGE(n) (((UInt32)1) << ((n) * 5 + 6)) - -#define _UTF8_HEAD(n, val) ((Byte)(_UTF8_START(n) + (val >> (6 * (n))))) -#define _UTF8_CHAR(n, val) ((Byte)(0x80 + (((val) >> (6 * (n))) & 0x3F))) - -static size_t Utf16_To_Utf8_Calc(const UInt16 *src, const UInt16 *srcLim) -{ - size_t size = 0; - for (;;) - { - UInt32 val; - if (src == srcLim) - return size; - - size++; - val = *src++; - - if (val < 0x80) - continue; - - if (val < _UTF8_RANGE(1)) - { - size++; - continue; - } - - if (val >= 0xD800 && val < 0xDC00 && src != srcLim) - { - UInt32 c2 = *src; - if (c2 >= 0xDC00 && c2 < 0xE000) - { - src++; - size += 3; - continue; - } - } - - size += 2; - } -} - -static Byte *Utf16_To_Utf8(Byte *dest, const UInt16 *src, const UInt16 *srcLim) -{ - for (;;) - { - UInt32 val; - if (src == srcLim) - return dest; - - val = *src++; - - if (val < 0x80) - { - *dest++ = (char)val; - continue; - } - - if (val < _UTF8_RANGE(1)) - { - dest[0] = _UTF8_HEAD(1, val); - dest[1] = _UTF8_CHAR(0, val); - dest += 2; - continue; - } - - if (val >= 0xD800 && val < 0xDC00 && src != srcLim) - { - UInt32 c2 = *src; - if (c2 >= 0xDC00 && c2 < 0xE000) - { - src++; - val = (((val - 0xD800) << 10) | (c2 - 0xDC00)) + 0x10000; - dest[0] = _UTF8_HEAD(3, val); - dest[1] = _UTF8_CHAR(2, val); - dest[2] = _UTF8_CHAR(1, val); - dest[3] = _UTF8_CHAR(0, val); - dest += 4; - continue; - } - } - - dest[0] = _UTF8_HEAD(2, val); - dest[1] = _UTF8_CHAR(1, val); - dest[2] = _UTF8_CHAR(0, val); - dest += 3; - } -} - -static SRes Utf16_To_Utf8Buf(CBuf *dest, const UInt16 *src, size_t srcLen) -{ - size_t destLen = Utf16_To_Utf8_Calc(src, src + srcLen); - destLen += 1; - if (!Buf_EnsureSize(dest, destLen)) - return SZ_ERROR_MEM; - *Utf16_To_Utf8(dest->data, src, src + srcLen) = 0; - return SZ_OK; -} - -#endif - -static SRes Utf16_To_Char(CBuf *buf, const UInt16 *s - #ifndef _USE_UTF8 - , UINT codePage - #endif - ) -{ - unsigned len = 0; - for (len = 0; s[len] != 0; len++); - - #ifndef _USE_UTF8 - { - unsigned size = len * 3 + 100; - if (!Buf_EnsureSize(buf, size)) - return SZ_ERROR_MEM; - { - buf->data[0] = 0; - if (len != 0) - { - char defaultChar = '_'; - BOOL defUsed; - unsigned numChars = 0; - numChars = WideCharToMultiByte(codePage, 0, (LPCWSTR)s, len, (char *)buf->data, size, &defaultChar, &defUsed); - if (numChars == 0 || numChars >= size) - return SZ_ERROR_FAIL; - buf->data[numChars] = 0; - } - return SZ_OK; - } - } - #else - return Utf16_To_Utf8Buf(buf, s, len); - #endif -} - -#ifdef _WIN32 - #ifndef USE_WINDOWS_FILE - static UINT g_FileCodePage = CP_ACP; - #endif - #define MY_FILE_CODE_PAGE_PARAM ,g_FileCodePage -#else - #define MY_FILE_CODE_PAGE_PARAM -#endif - -static WRes MyCreateDir(const UInt16 *name) -{ - #ifdef USE_WINDOWS_FILE - - return CreateDirectoryW((LPCWSTR)name, NULL) ? 0 : GetLastError(); - - #else - - CBuf buf; - WRes res; - Buf_Init(&buf); - RINOK(Utf16_To_Char(&buf, name MY_FILE_CODE_PAGE_PARAM)); - - res = - #ifdef _WIN32 - _mkdir((const char *)buf.data) - #else - mkdir((const char *)buf.data, 0777) - #endif - == 0 ? 0 : errno; - Buf_Free(&buf, &g_Alloc); - return res; - - #endif -} - -static WRes OutFile_OpenUtf16(CSzFile *p, const UInt16 *name) -{ - #ifdef USE_WINDOWS_FILE - return OutFile_OpenW(p, (LPCWSTR)name); - #else - CBuf buf; - WRes res; - Buf_Init(&buf); - RINOK(Utf16_To_Char(&buf, name MY_FILE_CODE_PAGE_PARAM)); - res = OutFile_Open(p, (const char *)buf.data); - Buf_Free(&buf, &g_Alloc); - return res; - #endif -} - - -static SRes PrintString(const UInt16 *s) -{ - CBuf buf; - SRes res; - Buf_Init(&buf); - res = Utf16_To_Char(&buf, s - #ifndef _USE_UTF8 - , CP_OEMCP - #endif - ); - if (res == SZ_OK) - Print((const char *)buf.data); - Buf_Free(&buf, &g_Alloc); - return res; -} - -static void UInt64ToStr(UInt64 value, char *s, int numDigits) -{ - char temp[32]; - int pos = 0; - do - { - temp[pos++] = (char)('0' + (unsigned)(value % 10)); - value /= 10; - } - while (value != 0); - - for (numDigits -= pos; numDigits > 0; numDigits--) - *s++ = ' '; - - do - *s++ = temp[--pos]; - while (pos); - *s = '\0'; -} - -static char *UIntToStr(char *s, unsigned value, int numDigits) -{ - char temp[16]; - int pos = 0; - do - temp[pos++] = (char)('0' + (value % 10)); - while (value /= 10); - - for (numDigits -= pos; numDigits > 0; numDigits--) - *s++ = '0'; - - do - *s++ = temp[--pos]; - while (pos); - *s = '\0'; - return s; -} - -static void UIntToStr_2(char *s, unsigned value) -{ - s[0] = (char)('0' + (value / 10)); - s[1] = (char)('0' + (value % 10)); -} - -#define PERIOD_4 (4 * 365 + 1) -#define PERIOD_100 (PERIOD_4 * 25 - 1) -#define PERIOD_400 (PERIOD_100 * 4 + 1) - -static void ConvertFileTimeToString(const CNtfsFileTime *nt, char *s) -{ - unsigned year, mon, hour, min, sec; - Byte ms[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; - unsigned t; - UInt32 v; - UInt64 v64 = nt->Low | ((UInt64)nt->High << 32); - v64 /= 10000000; - sec = (unsigned)(v64 % 60); v64 /= 60; - min = (unsigned)(v64 % 60); v64 /= 60; - hour = (unsigned)(v64 % 24); v64 /= 24; - - v = (UInt32)v64; - - year = (unsigned)(1601 + v / PERIOD_400 * 400); - v %= PERIOD_400; - - t = v / PERIOD_100; if (t == 4) t = 3; year += t * 100; v -= t * PERIOD_100; - t = v / PERIOD_4; if (t == 25) t = 24; year += t * 4; v -= t * PERIOD_4; - t = v / 365; if (t == 4) t = 3; year += t; v -= t * 365; - - if (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) - ms[1] = 29; - for (mon = 0;; mon++) - { - unsigned d = ms[mon]; - if (v < d) - break; - v -= d; - } - s = UIntToStr(s, year, 4); *s++ = '-'; - UIntToStr_2(s, mon + 1); s[2] = '-'; s += 3; - UIntToStr_2(s, (unsigned)v + 1); s[2] = ' '; s += 3; - UIntToStr_2(s, hour); s[2] = ':'; s += 3; - UIntToStr_2(s, min); s[2] = ':'; s += 3; - UIntToStr_2(s, sec); s[2] = 0; -} - -static void PrintLF() -{ - Print("\n"); -} - -static void PrintError(char *s) -{ - Print("\nERROR: "); - Print(s); - PrintLF(); -} - -static void GetAttribString(UInt32 wa, BoolInt isDir, char *s) -{ - #ifdef USE_WINDOWS_FILE - s[0] = (char)(((wa & FILE_ATTRIBUTE_DIRECTORY) != 0 || isDir) ? 'D' : '.'); - s[1] = (char)(((wa & FILE_ATTRIBUTE_READONLY ) != 0) ? 'R': '.'); - s[2] = (char)(((wa & FILE_ATTRIBUTE_HIDDEN ) != 0) ? 'H': '.'); - s[3] = (char)(((wa & FILE_ATTRIBUTE_SYSTEM ) != 0) ? 'S': '.'); - s[4] = (char)(((wa & FILE_ATTRIBUTE_ARCHIVE ) != 0) ? 'A': '.'); - s[5] = 0; - #else - s[0] = (char)(((wa & (1 << 4)) != 0 || isDir) ? 'D' : '.'); - s[1] = 0; - #endif -} - - -// #define NUM_PARENTS_MAX 128 - -int MY_CDECL main(int numargs, char *args[]) -{ - ISzAlloc allocImp; - ISzAlloc allocTempImp; - - CFileInStream archiveStream; - CLookToRead2 lookStream; - CSzArEx db; - SRes res; - UInt16 *temp = NULL; - size_t tempSize = 0; - // UInt32 parents[NUM_PARENTS_MAX]; - - Print("\n7z Decoder " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n"); - - if (numargs == 1) - { - Print( - "Usage: 7zDec \n\n" - "\n" - " e: Extract files from archive (without using directory names)\n" - " l: List contents of archive\n" - " t: Test integrity of archive\n" - " x: eXtract files with full paths\n"); - return 0; - } - - if (numargs < 3) - { - PrintError("incorrect command"); - return 1; - } - - #if defined(_WIN32) && !defined(USE_WINDOWS_FILE) && !defined(UNDER_CE) - g_FileCodePage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; - #endif - - - allocImp = g_Alloc; - allocTempImp = g_Alloc; - - #ifdef UNDER_CE - if (InFile_OpenW(&archiveStream.file, L"\test.7z")) - #else - if (InFile_Open(&archiveStream.file, args[2])) - #endif - { - PrintError("can not open input file"); - return 1; - } - - FileInStream_CreateVTable(&archiveStream); - LookToRead2_CreateVTable(&lookStream, False); - lookStream.buf = NULL; - - res = SZ_OK; - - { - lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); - if (!lookStream.buf) - res = SZ_ERROR_MEM; - else - { - lookStream.bufSize = kInputBufSize; - lookStream.realStream = &archiveStream.vt; - LookToRead2_Init(&lookStream); - } - } - - CrcGenerateTable(); - - SzArEx_Init(&db); - - if (res == SZ_OK) - { - res = SzArEx_Open(&db, &lookStream.vt, &allocImp, &allocTempImp); - } - - if (res == SZ_OK) - { - char *command = args[1]; - int listCommand = 0, testCommand = 0, fullPaths = 0; - - if (strcmp(command, "l") == 0) listCommand = 1; - else if (strcmp(command, "t") == 0) testCommand = 1; - else if (strcmp(command, "e") == 0) { } - else if (strcmp(command, "x") == 0) { fullPaths = 1; } - else - { - PrintError("incorrect command"); - res = SZ_ERROR_FAIL; - } - - if (res == SZ_OK) - { - UInt32 i; - - /* - if you need cache, use these 3 variables. - if you use external function, you can make these variable as static. - */ - UInt32 blockIndex = 0xFFFFFFFF; /* it can have any value before first call (if outBuffer = 0) */ - Byte *outBuffer = 0; /* it must be 0 before first call for each new archive. */ - size_t outBufferSize = 0; /* it can have any value before first call (if outBuffer = 0) */ - - for (i = 0; i < db.NumFiles; i++) - { - size_t offset = 0; - size_t outSizeProcessed = 0; - // const CSzFileItem *f = db.Files + i; - size_t len; - unsigned isDir = SzArEx_IsDir(&db, i); - if (listCommand == 0 && isDir && !fullPaths) - continue; - len = SzArEx_GetFileNameUtf16(&db, i, NULL); - // len = SzArEx_GetFullNameLen(&db, i); - - if (len > tempSize) - { - SzFree(NULL, temp); - tempSize = len; - temp = (UInt16 *)SzAlloc(NULL, tempSize * sizeof(temp[0])); - if (!temp) - { - res = SZ_ERROR_MEM; - break; - } - } - - SzArEx_GetFileNameUtf16(&db, i, temp); - /* - if (SzArEx_GetFullNameUtf16_Back(&db, i, temp + len) != temp) - { - res = SZ_ERROR_FAIL; - break; - } - */ - - if (listCommand) - { - char attr[8], s[32], t[32]; - UInt64 fileSize; - - GetAttribString(SzBitWithVals_Check(&db.Attribs, i) ? db.Attribs.Vals[i] : 0, isDir, attr); - - fileSize = SzArEx_GetFileSize(&db, i); - UInt64ToStr(fileSize, s, 10); - - if (SzBitWithVals_Check(&db.MTime, i)) - ConvertFileTimeToString(&db.MTime.Vals[i], t); - else - { - size_t j; - for (j = 0; j < 19; j++) - t[j] = ' '; - t[j] = '\0'; - } - - Print(t); - Print(" "); - Print(attr); - Print(" "); - Print(s); - Print(" "); - res = PrintString(temp); - if (res != SZ_OK) - break; - if (isDir) - Print("/"); - PrintLF(); - continue; - } - - Print(testCommand ? - "Testing ": - "Extracting "); - res = PrintString(temp); - if (res != SZ_OK) - break; - - if (isDir) - Print("/"); - else - { - res = SzArEx_Extract(&db, &lookStream.vt, i, - &blockIndex, &outBuffer, &outBufferSize, - &offset, &outSizeProcessed, - &allocImp, &allocTempImp); - if (res != SZ_OK) - break; - } - - if (!testCommand) - { - CSzFile outFile; - size_t processedSize; - size_t j; - UInt16 *name = (UInt16 *)temp; - const UInt16 *destPath = (const UInt16 *)name; - - for (j = 0; name[j] != 0; j++) - if (name[j] == '/') - { - if (fullPaths) - { - name[j] = 0; - MyCreateDir(name); - name[j] = CHAR_PATH_SEPARATOR; - } - else - destPath = name + j + 1; - } - - if (isDir) - { - MyCreateDir(destPath); - PrintLF(); - continue; - } - else if (OutFile_OpenUtf16(&outFile, destPath)) - { - PrintError("can not open output file"); - res = SZ_ERROR_FAIL; - break; - } - - processedSize = outSizeProcessed; - - if (File_Write(&outFile, outBuffer + offset, &processedSize) != 0 || processedSize != outSizeProcessed) - { - PrintError("can not write output file"); - res = SZ_ERROR_FAIL; - break; - } - - #ifdef USE_WINDOWS_FILE - { - FILETIME mtime, ctime; - FILETIME *mtimePtr = NULL; - FILETIME *ctimePtr = NULL; - - if (SzBitWithVals_Check(&db.MTime, i)) - { - const CNtfsFileTime *t = &db.MTime.Vals[i]; - mtime.dwLowDateTime = (DWORD)(t->Low); - mtime.dwHighDateTime = (DWORD)(t->High); - mtimePtr = &mtime; - } - if (SzBitWithVals_Check(&db.CTime, i)) - { - const CNtfsFileTime *t = &db.CTime.Vals[i]; - ctime.dwLowDateTime = (DWORD)(t->Low); - ctime.dwHighDateTime = (DWORD)(t->High); - ctimePtr = &ctime; - } - if (mtimePtr || ctimePtr) - SetFileTime(outFile.handle, ctimePtr, NULL, mtimePtr); - } - #endif - - if (File_Close(&outFile)) - { - PrintError("can not close output file"); - res = SZ_ERROR_FAIL; - break; - } - - #ifdef USE_WINDOWS_FILE - if (SzBitWithVals_Check(&db.Attribs, i)) - { - UInt32 attrib = db.Attribs.Vals[i]; - /* p7zip stores posix attributes in high 16 bits and adds 0x8000 as marker. - We remove posix bits, if we detect posix mode field */ - if ((attrib & 0xF0000000) != 0) - attrib &= 0x7FFF; - SetFileAttributesW((LPCWSTR)destPath, attrib); - } - #endif - } - PrintLF(); - } - ISzAlloc_Free(&allocImp, outBuffer); - } - } - - SzFree(NULL, temp); - SzArEx_Free(&db, &allocImp); - ISzAlloc_Free(&allocImp, lookStream.buf); - - File_Close(&archiveStream.file); - - if (res == SZ_OK) - { - Print("\nEverything is Ok\n"); - return 0; - } - - if (res == SZ_ERROR_UNSUPPORTED) - PrintError("decoder doesn't support this archive"); - else if (res == SZ_ERROR_MEM) - PrintError("can not allocate memory"); - else if (res == SZ_ERROR_CRC) - PrintError("CRC error"); - else - { - char s[32]; - UInt64ToStr(res, s, 0); - PrintError(s); - } - - return 1; -} diff --git a/code/ryzom/client/src/seven_zip/CMakeLists.txt b/code/ryzom/client/src/seven_zip/CMakeLists.txt deleted file mode 100644 index d62750e17..000000000 --- a/code/ryzom/client/src/seven_zip/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) - -FILE(GLOB LIB_SRC *.cpp *.h) - -LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) - -NL_TARGET_LIB(ryzom_sevenzip ${LIB_SRC}) -TARGET_LINK_LIBRARIES(ryzom_sevenzip ${NELMISC_LIBRARIES}) -NL_DEFAULT_PROPS(ryzom_sevenzip "Ryzom, Library: Seven Zip") -NL_ADD_RUNTIME_FLAGS(ryzom_sevenzip) -NL_ADD_LIB_SUFFIX(ryzom_sevenzip) - -ADD_DEFINITIONS(-DUNICODE -D_UNICODE -D_7ZIP_PPMD_SUPPPORT) - -IF(WITH_PCH) - ADD_NATIVE_PRECOMPILED_HEADER(ryzom_sevenzip ${CMAKE_CURRENT_SOURCE_DIR}/Precomp.h ${CMAKE_CURRENT_SOURCE_DIR}/Precomp.cpp) -ENDIF() - -IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) - INSTALL(TARGETS ryzom_sevenzip LIBRARY DESTINATION ${RYZOM_LIB_PREFIX} ARCHIVE DESTINATION ${RYZOM_LIB_PREFIX} COMPONENT libraries) -ENDIF() - -IF(WITH_RYZOM_TOOLS) - ADD_EXECUTABLE(7zDec ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) - - TARGET_LINK_LIBRARIES(7zDec ryzom_sevenzip) - NL_DEFAULT_PROPS(7zDec "Ryzom, Tools, Misc: Seven Zip Decoder") - NL_ADD_RUNTIME_FLAGS(7zDec) - - INSTALL(TARGETS 7zDec RUNTIME DESTINATION ${RYZOM_BIN_PREFIX} COMPONENT tools) -ENDIF() diff --git a/code/ryzom/client/src/seven_zip/Precomp.cpp b/code/ryzom/client/src/seven_zip/Precomp.cpp deleted file mode 100644 index 01605e3c2..000000000 --- a/code/ryzom/client/src/seven_zip/Precomp.cpp +++ /dev/null @@ -1,4 +0,0 @@ -/* Precomp.c -- StdAfx -2013-01-21 : Igor Pavlov : Public domain */ - -#include "Precomp.h" diff --git a/code/ryzom/client/src/seven_zip/Precomp.h b/code/ryzom/client/src/seven_zip/Precomp.h deleted file mode 100644 index dcaf49475..000000000 --- a/code/ryzom/client/src/seven_zip/Precomp.h +++ /dev/null @@ -1,19 +0,0 @@ -/* Precomp.h -- StdAfx -2013-11-12 : Igor Pavlov : Public domain */ - -#ifndef __7Z_PRECOMP_H -#define __7Z_PRECOMP_H - -#if defined(_MSC_VER) && defined(_DEBUG) - #define _CRTDBG_MAP_ALLOC - #include - #include - #define DEBUG_NEW new(_NORMAL_BLOCK, __FILE__, __LINE__) -#endif - -#define _7ZIP_ST - -#include "Compiler.h" -/* #include "7zTypes.h" */ - -#endif diff --git a/code/ryzom/client/src/seven_zip/readme.txt b/code/ryzom/client/src/seven_zip/readme.txt deleted file mode 100644 index e9798c1df..000000000 --- a/code/ryzom/client/src/seven_zip/readme.txt +++ /dev/null @@ -1,22 +0,0 @@ - - -This is an extract of some files from the 7zip SDK. - -At time of writing (2007-01-12) there is no dll or library project in the 7zip -distribution (lzma443), so I build a custum project and copied in the 7zip -files needed to do 7zip/lzma extraction. - -To update this library, download the latest 7zip SDK and copy all file from - /C/7zip/Archive/7z_C -for the 7zip archive file format support - -and the 3 files needed from - /C/7zip/Compress/LZMA_C - (lzmaDecode.h lzmaDecode.cpp lzmaTypes.h) -for the lzma decrompression lib. - - -NB : If callback support must be enabled, add "#define _LZMA_IN_CB" in - 7zTypes.h - - diff --git a/code/ryzom/client/src/streamable_ig.cpp b/code/ryzom/client/src/streamable_ig.cpp index eaaea24f5..3216b2ee1 100644 --- a/code/ryzom/client/src/streamable_ig.cpp +++ b/code/ryzom/client/src/streamable_ig.cpp @@ -93,8 +93,10 @@ CStreamableIG::~CStreamableIG() H_AUTO_USE(RZ_StremableIG) if (!_Linked) { + #ifdef NL_DEBUG if(!ClientCfg.Light) nlwarning("Loading async %p", this); + #endif #ifdef NL_DEBUG //nlinfo("Loading async : %s", Name.c_str()); #endif diff --git a/code/ryzom/common/src/game_share/bnp_patch.cpp b/code/ryzom/common/src/game_share/bnp_patch.cpp index baac733e8..72c484c84 100644 --- a/code/ryzom/common/src/game_share/bnp_patch.cpp +++ b/code/ryzom/common/src/game_share/bnp_patch.cpp @@ -74,7 +74,7 @@ bool CBNPFileVersion::setup(const std::string &fileName, uint32 versionNumber) BOMB_IF(!NLMISC::CFile::fileExists(fileName),("File not found: "+fileName).c_str(),return false); // generate a hash key for the file and store it in a vector of uint32 - CHashKey hashKey= getSHA1(fileName); + NLMISC::CHashKey hashKey= NLMISC::getSHA1(fileName); nlassert(hashKey.HashKeyString.size()==20); _HashKey.clear(); for (uint32 i=0;i<5;++i) @@ -137,10 +137,10 @@ uint32 CBNPFileVersion::getPatchSize() const return _PatchSize; } -CHashKey CBNPFileVersion::getHashKey() const +NLMISC::CHashKey CBNPFileVersion::getHashKey() const { nlassert(_HashKey.size()==5); - CHashKey hashKey; + NLMISC::CHashKey hashKey; for (uint32 i=0;i<5;++i) { *(uint32*)&hashKey.HashKeyString[4*i]=_HashKey[i]; diff --git a/code/ryzom/common/src/game_share/bnp_patch.h b/code/ryzom/common/src/game_share/bnp_patch.h index 395727bf0..b2d602194 100644 --- a/code/ryzom/common/src/game_share/bnp_patch.h +++ b/code/ryzom/common/src/game_share/bnp_patch.h @@ -72,7 +72,7 @@ public: uint32 get7ZFileSize() const; uint32 getFileSize() const; uint32 getPatchSize() const; - CHashKey getHashKey() const; + NLMISC::CHashKey getHashKey() const; // == operator bool operator==(const CBNPFileVersion& other) const; diff --git a/code/ryzom/tools/client/client_patcher/CMakeLists.txt b/code/ryzom/tools/client/client_patcher/CMakeLists.txt index ad2eeb629..810f4ccfd 100644 --- a/code/ryzom/tools/client/client_patcher/CMakeLists.txt +++ b/code/ryzom/tools/client/client_patcher/CMakeLists.txt @@ -23,7 +23,7 @@ TARGET_LINK_LIBRARIES(ryzom_client_patcher nelmisc nelnet ryzom_gameshare - ryzom_sevenzip + nel_sevenzip ${CURL_LIBRARIES}) IF(APPLE) diff --git a/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp b/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp index db8481453..b176b496c 100644 --- a/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp +++ b/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp @@ -296,7 +296,7 @@ void CScreenshotIslands::searchIslandsBorders() zonelFiles.clear(); string bnpFileName = itCont->first + ".bnp"; - CBigFile::getInstance().list(bnpFileName.c_str(), filenames); + CBigFile::getInstance().list(bnpFileName.c_str(), filenames); // FIXME FIXME NOT READING FROM BNP! for(uint i=0; i fileList; NLMISC::CPath::getPathContent(_BnpDirectory,false,false,true,fileList); for (uint32 i=0;i