Merge branch 'feature/streamed-package' into develop
kaetemi
5 years ago
commit
dbef81c3c2
@ -0,0 +1,2 @@
|
||||
SET(SEVENZIP_LIBRARY "nel_sevenzip")
|
||||
ADD_SUBDIRECTORY(seven_zip)
|
||||
@ -0,0 +1,27 @@
|
||||
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR})
|
||||
|
||||
FILE(GLOB LIB_SRC *.cpp *.c *.h)
|
||||
|
||||
LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/LzmaUtil.c)
|
||||
|
||||
NL_TARGET_LIB(nel_sevenzip ${LIB_SRC})
|
||||
# TARGET_LINK_LIBRARIES(nel_sevenzip ${PLATFORM_LINKFLAGS})
|
||||
NL_DEFAULT_PROPS(nel_sevenzip "NeL, 3rd Party: Seven Zip")
|
||||
NL_ADD_RUNTIME_FLAGS(nel_sevenzip)
|
||||
NL_ADD_LIB_SUFFIX(nel_sevenzip)
|
||||
|
||||
ADD_DEFINITIONS(-D_7ZIP_ST)
|
||||
|
||||
IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC)
|
||||
INSTALL(TARGETS nel_sevenzip LIBRARY DESTINATION ${NL_LIB_PREFIX} ARCHIVE DESTINATION ${NL_LIB_PREFIX} COMPONENT libraries)
|
||||
ENDIF()
|
||||
|
||||
IF(WITH_NEL_TOOLS)
|
||||
ADD_EXECUTABLE(lzma ${CMAKE_CURRENT_SOURCE_DIR}/LzmaUtil.c)
|
||||
|
||||
TARGET_LINK_LIBRARIES(lzma nel_sevenzip)
|
||||
NL_DEFAULT_PROPS(lzma "NeL, 3rd Party: LZMA")
|
||||
NL_ADD_RUNTIME_FLAGS(lzma)
|
||||
|
||||
INSTALL(TARGETS lzma RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT tools)
|
||||
ENDIF()
|
||||
@ -0,0 +1,108 @@
|
||||
/* DllSecur.c -- DLL loading security
|
||||
2018-02-21 : Igor Pavlov : Public domain */
|
||||
|
||||
#include "Precomp.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
#include <windows.h>
|
||||
|
||||
#include "DllSecur.h"
|
||||
|
||||
#ifndef UNDER_CE
|
||||
|
||||
typedef BOOL (WINAPI *Func_SetDefaultDllDirectories)(DWORD DirectoryFlags);
|
||||
|
||||
#define MY_LOAD_LIBRARY_SEARCH_USER_DIRS 0x400
|
||||
#define MY_LOAD_LIBRARY_SEARCH_SYSTEM32 0x800
|
||||
|
||||
static const char * const g_Dlls =
|
||||
#ifndef _CONSOLE
|
||||
"UXTHEME\0"
|
||||
#endif
|
||||
"USERENV\0"
|
||||
"SETUPAPI\0"
|
||||
"APPHELP\0"
|
||||
"PROPSYS\0"
|
||||
"DWMAPI\0"
|
||||
"CRYPTBASE\0"
|
||||
"OLEACC\0"
|
||||
"CLBCATQ\0"
|
||||
"VERSION\0"
|
||||
;
|
||||
|
||||
#endif
|
||||
|
||||
void My_SetDefaultDllDirectories()
|
||||
{
|
||||
#ifndef UNDER_CE
|
||||
|
||||
OSVERSIONINFO vi;
|
||||
vi.dwOSVersionInfoSize = sizeof(vi);
|
||||
GetVersionEx(&vi);
|
||||
if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0)
|
||||
{
|
||||
Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories)
|
||||
GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories");
|
||||
if (setDllDirs)
|
||||
if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS))
|
||||
return;
|
||||
}
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
void LoadSecurityDlls()
|
||||
{
|
||||
#ifndef UNDER_CE
|
||||
|
||||
wchar_t buf[MAX_PATH + 100];
|
||||
|
||||
{
|
||||
// at Vista (ver 6.0) : CoCreateInstance(CLSID_ShellLink, ...) doesn't work after SetDefaultDllDirectories() : Check it ???
|
||||
OSVERSIONINFO vi;
|
||||
vi.dwOSVersionInfoSize = sizeof(vi);
|
||||
if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0)
|
||||
{
|
||||
Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories)
|
||||
GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories");
|
||||
if (setDllDirs)
|
||||
if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS))
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
unsigned len = GetSystemDirectoryW(buf, MAX_PATH + 2);
|
||||
if (len == 0 || len > MAX_PATH)
|
||||
return;
|
||||
}
|
||||
{
|
||||
const char *dll;
|
||||
unsigned pos = (unsigned)lstrlenW(buf);
|
||||
|
||||
if (buf[pos - 1] != '\\')
|
||||
buf[pos++] = '\\';
|
||||
|
||||
for (dll = g_Dlls; dll[0] != 0;)
|
||||
{
|
||||
unsigned k = 0;
|
||||
for (;;)
|
||||
{
|
||||
char c = *dll++;
|
||||
buf[pos + k] = (Byte)c;
|
||||
k++;
|
||||
if (c == 0)
|
||||
break;
|
||||
}
|
||||
|
||||
lstrcatW(buf, L".dll");
|
||||
LoadLibraryExW(buf, NULL, LOAD_WITH_ALTERED_SEARCH_PATH);
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
@ -0,0 +1,20 @@
|
||||
/* DllSecur.h -- DLL loading for security
|
||||
2018-02-19 : Igor Pavlov : Public domain */
|
||||
|
||||
#ifndef __DLL_SECUR_H
|
||||
#define __DLL_SECUR_H
|
||||
|
||||
#include "7zTypes.h"
|
||||
|
||||
EXTERN_C_BEGIN
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
void My_SetDefaultDllDirectories();
|
||||
void LoadSecurityDlls();
|
||||
|
||||
#endif
|
||||
|
||||
EXTERN_C_END
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,79 @@
|
||||
/* Lzma2DecMt.h -- LZMA2 Decoder Multi-thread
|
||||
2018-02-17 : Igor Pavlov : Public domain */
|
||||
|
||||
#ifndef __LZMA2_DEC_MT_H
|
||||
#define __LZMA2_DEC_MT_H
|
||||
|
||||
#include "7zTypes.h"
|
||||
|
||||
EXTERN_C_BEGIN
|
||||
|
||||
typedef struct
|
||||
{
|
||||
size_t inBufSize_ST;
|
||||
size_t outStep_ST;
|
||||
|
||||
#ifndef _7ZIP_ST
|
||||
unsigned numThreads;
|
||||
size_t inBufSize_MT;
|
||||
size_t outBlockMax;
|
||||
size_t inBlockMax;
|
||||
#endif
|
||||
} CLzma2DecMtProps;
|
||||
|
||||
/* init to single-thread mode */
|
||||
void Lzma2DecMtProps_Init(CLzma2DecMtProps *p);
|
||||
|
||||
|
||||
/* ---------- CLzma2DecMtHandle Interface ---------- */
|
||||
|
||||
/* Lzma2DecMt_ * functions can return the following exit codes:
|
||||
SRes:
|
||||
SZ_OK - OK
|
||||
SZ_ERROR_MEM - Memory allocation error
|
||||
SZ_ERROR_PARAM - Incorrect paramater in props
|
||||
SZ_ERROR_WRITE - ISeqOutStream write callback error
|
||||
// SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output
|
||||
SZ_ERROR_PROGRESS - some break from progress callback
|
||||
SZ_ERROR_THREAD - error in multithreading functions (only for Mt version)
|
||||
*/
|
||||
|
||||
typedef void * CLzma2DecMtHandle;
|
||||
|
||||
CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid);
|
||||
void Lzma2DecMt_Destroy(CLzma2DecMtHandle p);
|
||||
|
||||
SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p,
|
||||
Byte prop,
|
||||
const CLzma2DecMtProps *props,
|
||||
ISeqOutStream *outStream,
|
||||
const UInt64 *outDataSize, // NULL means undefined
|
||||
int finishMode, // 0 - partial unpacking is allowed, 1 - if lzma2 stream must be finished
|
||||
// Byte *outBuf, size_t *outBufSize,
|
||||
ISeqInStream *inStream,
|
||||
// const Byte *inData, size_t inDataSize,
|
||||
|
||||
// out variables:
|
||||
UInt64 *inProcessed,
|
||||
int *isMT, /* out: (*isMT == 0), if single thread decoding was used */
|
||||
|
||||
// UInt64 *outProcessed,
|
||||
ICompressProgress *progress);
|
||||
|
||||
|
||||
/* ---------- Read from CLzma2DecMtHandle Interface ---------- */
|
||||
|
||||
SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp,
|
||||
Byte prop,
|
||||
const CLzma2DecMtProps *props,
|
||||
const UInt64 *outDataSize, int finishMode,
|
||||
ISeqInStream *inStream);
|
||||
|
||||
SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp,
|
||||
Byte *data, size_t *outSize,
|
||||
UInt64 *inStreamProcessed);
|
||||
|
||||
|
||||
EXTERN_C_END
|
||||
|
||||
#endif
|
||||
@ -0,0 +1,258 @@
|
||||
/* LzmaUtil.c -- Test application for LZMA compression
|
||||
2018-07-04 : Igor Pavlov : Public domain */
|
||||
|
||||
#include "Precomp.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "CpuArch.h"
|
||||
|
||||
#include "Alloc.h"
|
||||
#include "7zFile.h"
|
||||
#include "7zVersion.h"
|
||||
#include "LzmaDec.h"
|
||||
#include "LzmaEnc.h"
|
||||
|
||||
static const char * const kCantReadMessage = "Can not read input file";
|
||||
static const char * const kCantWriteMessage = "Can not write output file";
|
||||
static const char * const kCantAllocateMessage = "Can not allocate memory";
|
||||
static const char * const kDataErrorMessage = "Data error";
|
||||
|
||||
static void PrintHelp(char *buffer)
|
||||
{
|
||||
strcat(buffer,
|
||||
"\nLZMA-C " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n"
|
||||
"Usage: lzma <e|d> inputFile outputFile\n"
|
||||
" e: encode file\n"
|
||||
" d: decode file\n");
|
||||
}
|
||||
|
||||
static int PrintError(char *buffer, const char *message)
|
||||
{
|
||||
strcat(buffer, "\nError: ");
|
||||
strcat(buffer, message);
|
||||
strcat(buffer, "\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int PrintErrorNumber(char *buffer, SRes val)
|
||||
{
|
||||
sprintf(buffer + strlen(buffer), "\nError code: %x\n", (unsigned)val);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int PrintUserError(char *buffer)
|
||||
{
|
||||
return PrintError(buffer, "Incorrect command");
|
||||
}
|
||||
|
||||
|
||||
#define IN_BUF_SIZE (1 << 16)
|
||||
#define OUT_BUF_SIZE (1 << 16)
|
||||
|
||||
|
||||
static SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inStream,
|
||||
UInt64 unpackSize)
|
||||
{
|
||||
int thereIsSize = (unpackSize != (UInt64)(Int64)-1);
|
||||
Byte inBuf[IN_BUF_SIZE];
|
||||
Byte outBuf[OUT_BUF_SIZE];
|
||||
size_t inPos = 0, inSize = 0, outPos = 0;
|
||||
LzmaDec_Init(state);
|
||||
for (;;)
|
||||
{
|
||||
if (inPos == inSize)
|
||||
{
|
||||
inSize = IN_BUF_SIZE;
|
||||
RINOK(inStream->Read(inStream, inBuf, &inSize));
|
||||
inPos = 0;
|
||||
}
|
||||
{
|
||||
SRes res;
|
||||
SizeT inProcessed = inSize - inPos;
|
||||
SizeT outProcessed = OUT_BUF_SIZE - outPos;
|
||||
ELzmaFinishMode finishMode = LZMA_FINISH_ANY;
|
||||
ELzmaStatus status;
|
||||
if (thereIsSize && outProcessed > unpackSize)
|
||||
{
|
||||
outProcessed = (SizeT)unpackSize;
|
||||
finishMode = LZMA_FINISH_END;
|
||||
}
|
||||
|
||||
res = LzmaDec_DecodeToBuf(state, outBuf + outPos, &outProcessed,
|
||||
inBuf + inPos, &inProcessed, finishMode, &status);
|
||||
inPos += inProcessed;
|
||||
outPos += outProcessed;
|
||||
unpackSize -= outProcessed;
|
||||
|
||||
if (outStream)
|
||||
if (outStream->Write(outStream, outBuf, outPos) != outPos)
|
||||
return SZ_ERROR_WRITE;
|
||||
|
||||
outPos = 0;
|
||||
|
||||
if (res != SZ_OK || (thereIsSize && unpackSize == 0))
|
||||
return res;
|
||||
|
||||
if (inProcessed == 0 && outProcessed == 0)
|
||||
{
|
||||
if (thereIsSize || status != LZMA_STATUS_FINISHED_WITH_MARK)
|
||||
return SZ_ERROR_DATA;
|
||||
return res;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static SRes Decode(ISeqOutStream *outStream, ISeqInStream *inStream)
|
||||
{
|
||||
UInt64 unpackSize;
|
||||
int i;
|
||||
SRes res = 0;
|
||||
|
||||
CLzmaDec state;
|
||||
|
||||
/* header: 5 bytes of LZMA properties and 8 bytes of uncompressed size */
|
||||
unsigned char header[LZMA_PROPS_SIZE + 8];
|
||||
|
||||
/* Read and parse header */
|
||||
|
||||
RINOK(SeqInStream_Read(inStream, header, sizeof(header)));
|
||||
|
||||
unpackSize = 0;
|
||||
for (i = 0; i < 8; i++)
|
||||
unpackSize += (UInt64)header[LZMA_PROPS_SIZE + i] << (i * 8);
|
||||
|
||||
LzmaDec_Construct(&state);
|
||||
RINOK(LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &g_Alloc));
|
||||
res = Decode2(&state, outStream, inStream, unpackSize);
|
||||
LzmaDec_Free(&state, &g_Alloc);
|
||||
return res;
|
||||
}
|
||||
|
||||
static SRes Encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize, char *rs)
|
||||
{
|
||||
CLzmaEncHandle enc;
|
||||
SRes res;
|
||||
CLzmaEncProps props;
|
||||
|
||||
UNUSED_VAR(rs);
|
||||
|
||||
enc = LzmaEnc_Create(&g_Alloc);
|
||||
if (enc == 0)
|
||||
return SZ_ERROR_MEM;
|
||||
|
||||
LzmaEncProps_Init(&props);
|
||||
res = LzmaEnc_SetProps(enc, &props);
|
||||
|
||||
if (res == SZ_OK)
|
||||
{
|
||||
Byte header[LZMA_PROPS_SIZE + 8];
|
||||
size_t headerSize = LZMA_PROPS_SIZE;
|
||||
int i;
|
||||
|
||||
res = LzmaEnc_WriteProperties(enc, header, &headerSize);
|
||||
for (i = 0; i < 8; i++)
|
||||
header[headerSize++] = (Byte)(fileSize >> (8 * i));
|
||||
if (outStream->Write(outStream, header, headerSize) != headerSize)
|
||||
res = SZ_ERROR_WRITE;
|
||||
else
|
||||
{
|
||||
if (res == SZ_OK)
|
||||
res = LzmaEnc_Encode(enc, outStream, inStream, NULL, &g_Alloc, &g_Alloc);
|
||||
}
|
||||
}
|
||||
LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
static int main2(int numArgs, const char *args[], char *rs)
|
||||
{
|
||||
CFileSeqInStream inStream;
|
||||
CFileOutStream outStream;
|
||||
char c;
|
||||
int res;
|
||||
int encodeMode;
|
||||
BoolInt useOutFile = False;
|
||||
|
||||
FileSeqInStream_CreateVTable(&inStream);
|
||||
File_Construct(&inStream.file);
|
||||
|
||||
FileOutStream_CreateVTable(&outStream);
|
||||
File_Construct(&outStream.file);
|
||||
|
||||
if (numArgs == 1)
|
||||
{
|
||||
PrintHelp(rs);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (numArgs < 3 || numArgs > 4 || strlen(args[1]) != 1)
|
||||
return PrintUserError(rs);
|
||||
|
||||
c = args[1][0];
|
||||
encodeMode = (c == 'e' || c == 'E');
|
||||
if (!encodeMode && c != 'd' && c != 'D')
|
||||
return PrintUserError(rs);
|
||||
|
||||
{
|
||||
size_t t4 = sizeof(UInt32);
|
||||
size_t t8 = sizeof(UInt64);
|
||||
if (t4 != 4 || t8 != 8)
|
||||
return PrintError(rs, "Incorrect UInt32 or UInt64");
|
||||
}
|
||||
|
||||
if (InFile_Open(&inStream.file, args[2]) != 0)
|
||||
return PrintError(rs, "Can not open input file");
|
||||
|
||||
if (numArgs > 3)
|
||||
{
|
||||
useOutFile = True;
|
||||
if (OutFile_Open(&outStream.file, args[3]) != 0)
|
||||
return PrintError(rs, "Can not open output file");
|
||||
}
|
||||
else if (encodeMode)
|
||||
PrintUserError(rs);
|
||||
|
||||
if (encodeMode)
|
||||
{
|
||||
UInt64 fileSize;
|
||||
File_GetLength(&inStream.file, &fileSize);
|
||||
res = Encode(&outStream.vt, &inStream.vt, fileSize, rs);
|
||||
}
|
||||
else
|
||||
{
|
||||
res = Decode(&outStream.vt, useOutFile ? &inStream.vt : NULL);
|
||||
}
|
||||
|
||||
if (useOutFile)
|
||||
File_Close(&outStream.file);
|
||||
File_Close(&inStream.file);
|
||||
|
||||
if (res != SZ_OK)
|
||||
{
|
||||
if (res == SZ_ERROR_MEM)
|
||||
return PrintError(rs, kCantAllocateMessage);
|
||||
else if (res == SZ_ERROR_DATA)
|
||||
return PrintError(rs, kDataErrorMessage);
|
||||
else if (res == SZ_ERROR_WRITE)
|
||||
return PrintError(rs, kCantWriteMessage);
|
||||
else if (res == SZ_ERROR_READ)
|
||||
return PrintError(rs, kCantReadMessage);
|
||||
return PrintErrorNumber(rs, res);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int MY_CDECL main(int numArgs, const char *args[])
|
||||
{
|
||||
char rs[800] = { 0 };
|
||||
int res = main2(numArgs, args, rs);
|
||||
fputs(rs, stdout);
|
||||
return res;
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
/* Precomp.h -- StdAfx
|
||||
2013-11-12 : Igor Pavlov : Public domain */
|
||||
|
||||
#ifndef __7Z_PRECOMP_H
|
||||
#define __7Z_PRECOMP_H
|
||||
|
||||
#include "Compiler.h"
|
||||
/* #include "7zTypes.h" */
|
||||
|
||||
#endif
|
||||
@ -0,0 +1,50 @@
|
||||
// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
|
||||
// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be)
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#ifndef NLMISC_HTTP_PACKAGE_PROVIDER_H
|
||||
#define NLMISC_HTTP_PACKAGE_PROVIDER_H
|
||||
|
||||
#include <nel/misc/types_nl.h>
|
||||
#include <nel/misc/i_streamed_package_provider.h>
|
||||
|
||||
namespace NLMISC {
|
||||
|
||||
class CHttpPackageProvider : public IStreamedPackageProvider
|
||||
{
|
||||
public:
|
||||
CHttpPackageProvider();
|
||||
virtual ~CHttpPackageProvider();
|
||||
|
||||
/// Download a file. This call is blocking
|
||||
/// filePath: [out] ex. /games/nel/stream/00/00/000000000..
|
||||
/// hash: [in]
|
||||
virtual bool getFile(std::string &filePath, const CHashKey &hash, const std::string &name) NL_OVERRIDE;
|
||||
|
||||
public:
|
||||
/// Set storage path (ex. stream/)
|
||||
std::string Path;
|
||||
|
||||
/// Loads a package into the package manager (ex. http://cdn.ryzom.dev/open/stream/)
|
||||
typedef std::vector<std::string> THosts;
|
||||
THosts Hosts;
|
||||
|
||||
}; /* class CHttpPackageProvider */
|
||||
|
||||
} /* namespace NLMISC */
|
||||
|
||||
#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */
|
||||
|
||||
/* end of file */
|
||||
@ -0,0 +1,44 @@
|
||||
// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
|
||||
// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be)
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#ifndef NLMISC_STREAMED_PACKAGE_PROVIDER_H
|
||||
#define NLMISC_STREAMED_PACKAGE_PROVIDER_H
|
||||
|
||||
#include <nel/misc/types_nl.h>
|
||||
#include <nel/misc/sha1.h>
|
||||
#include <string>
|
||||
|
||||
namespace NLMISC {
|
||||
|
||||
class IStreamedPackageProvider
|
||||
{
|
||||
public:
|
||||
IStreamedPackageProvider();
|
||||
virtual ~IStreamedPackageProvider();
|
||||
|
||||
/// Download a file. This call is blocking
|
||||
/// filePath: [out] ex. /games/nel/stream/00/00/000000000..
|
||||
/// hash: [in]
|
||||
/// name: [in] name for debugging
|
||||
virtual bool getFile(std::string &filePath, const CHashKey &hash, const std::string &name = "") = 0;
|
||||
|
||||
}; /* class IStreamedPackageProvider */
|
||||
|
||||
} /* namespace NLMISC */
|
||||
|
||||
#endif /* #ifndef NLMISC_STREAMED_PACKAGE_PROVIDER_H */
|
||||
|
||||
/* end of file */
|
||||
@ -0,0 +1,59 @@
|
||||
// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
|
||||
// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be)
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#ifndef NLMISC_STREAMED_PACKAGE_H
|
||||
#define NLMISC_STREAMED_PACKAGE_H
|
||||
|
||||
#include <nel/misc/types_nl.h>
|
||||
#include <nel/misc/sha1.h>
|
||||
|
||||
namespace NLMISC {
|
||||
|
||||
class CStreamedPackage
|
||||
{
|
||||
public:
|
||||
struct CEntry
|
||||
{
|
||||
std::string Name;
|
||||
CHashKey Hash;
|
||||
uint32 Size;
|
||||
uint32 LastModified;
|
||||
|
||||
void serial(NLMISC::IStream &f) throw(NLMISC::EStream);
|
||||
|
||||
};
|
||||
|
||||
public:
|
||||
CStreamedPackage();
|
||||
~CStreamedPackage();
|
||||
|
||||
void serial(NLMISC::IStream &f) throw(NLMISC::EStream);
|
||||
|
||||
/// result: [out] ex. /00/00/000000000..
|
||||
/// hash: [in]
|
||||
static void makePath(std::string &result, const CHashKey &hash);
|
||||
|
||||
public:
|
||||
typedef std::vector<CEntry> TEntries;
|
||||
TEntries Entries;
|
||||
|
||||
}; /* class CStreamedPackage */
|
||||
|
||||
} /* namespace NLMISC */
|
||||
|
||||
#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */
|
||||
|
||||
/* end of file */
|
||||
@ -0,0 +1,71 @@
|
||||
// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
|
||||
// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be)
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H
|
||||
#define NLMISC_STREAMED_PACKAGE_MANAGER_H
|
||||
|
||||
#include <nel/misc/types_nl.h>
|
||||
#include <nel/misc/app_context.h>
|
||||
#include <nel/misc/streamed_package.h>
|
||||
#include <nel/misc/i_streamed_package_provider.h>
|
||||
|
||||
namespace NLMISC {
|
||||
|
||||
class CStreamedPackageManager
|
||||
{
|
||||
NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(CStreamedPackageManager);
|
||||
|
||||
public:
|
||||
CStreamedPackageManager();
|
||||
~CStreamedPackageManager();
|
||||
|
||||
/// package: ex. /games/nel/data/characters_maps_hr.snp
|
||||
bool loadPackage(const std::string &package);
|
||||
|
||||
/// Get a file list
|
||||
void list(std::vector<std::string> &fileNames, const std::string &package);
|
||||
|
||||
/// Unload all packages
|
||||
void unloadAll();
|
||||
|
||||
/// Get an existing file or download if necessary. This call is blocking
|
||||
/// filePath: [out] ex. /games/nel/stream/00/00/000000000..
|
||||
/// fileName: [in] ex. fy_hof_underwear.dds
|
||||
bool getFile(std::string &filePath, const std::string &fileName);
|
||||
|
||||
/// Get file size
|
||||
/// fileSize: [out]
|
||||
/// fileName: [in] ex. fy_hof_underwear.dds
|
||||
bool getFileSize(uint32 &fileSize, const std::string &fileName);
|
||||
|
||||
public:
|
||||
/// Streamed package provider. This downloads the files
|
||||
IStreamedPackageProvider *Provider;
|
||||
|
||||
private:
|
||||
typedef std::map<std::string, CStreamedPackage> TPackages;
|
||||
TPackages m_Packages;
|
||||
|
||||
typedef std::map<std::string, const CStreamedPackage::CEntry *> TEntries;
|
||||
TEntries m_Entries;
|
||||
|
||||
}; /* class CStreamedPackageManager */
|
||||
|
||||
} /* namespace NLMISC */
|
||||
|
||||
#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */
|
||||
|
||||
/* end of file */
|
||||
@ -0,0 +1,145 @@
|
||||
// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
|
||||
// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be)
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#include "stdmisc.h"
|
||||
|
||||
// 3rd Party includes
|
||||
#include <curl/curl.h>
|
||||
|
||||
// Project includes
|
||||
#include <nel/misc/http_package_provider.h>
|
||||
#include <nel/misc/streamed_package.h>
|
||||
#include <nel/misc/file.h>
|
||||
#include <nel/misc/path.h>
|
||||
#include <nel/misc/seven_zip.h>
|
||||
#include <nel/misc/sha1.h>
|
||||
|
||||
namespace NLMISC
|
||||
{
|
||||
|
||||
CHttpPackageProvider::CHttpPackageProvider()
|
||||
{
|
||||
// init
|
||||
}
|
||||
|
||||
CHttpPackageProvider::~CHttpPackageProvider()
|
||||
{
|
||||
// release
|
||||
}
|
||||
|
||||
bool CHttpPackageProvider::getFile(std::string &filePath, const CHashKey &hash, const std::string &name)
|
||||
{
|
||||
CStreamedPackage::makePath(filePath, hash);
|
||||
std::string downloadUrlFile = filePath + ".lzma";
|
||||
filePath = Path + filePath;
|
||||
std::string downloadPath = filePath + ".download." + toString(rand() * rand());
|
||||
|
||||
std::string storageDirectory = CFile::getPath(downloadPath);
|
||||
CFile::createDirectoryTree(storageDirectory);
|
||||
/*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory))
|
||||
{
|
||||
nldebug("Unable to create directory '%s'", storageDirectory.c_str());
|
||||
return false;
|
||||
}*/
|
||||
|
||||
// download
|
||||
for (;;)
|
||||
{
|
||||
if (CFile::fileExists(filePath))
|
||||
return true;
|
||||
|
||||
std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile;
|
||||
nldebug("Download streamed package '%s' from '%s'", name.c_str(), downloadUrl.c_str());
|
||||
|
||||
FILE *fp = fopen(downloadPath.c_str(), "wb");
|
||||
if (fp == NULL)
|
||||
{
|
||||
nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), name.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
CURL *curl;
|
||||
CURLcode res;
|
||||
curl = curl_easy_init();
|
||||
if (curl)
|
||||
{
|
||||
curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_FILE, fp);
|
||||
res = curl_easy_perform(curl);
|
||||
long r;
|
||||
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r);
|
||||
curl_easy_cleanup(curl);
|
||||
|
||||
bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/;
|
||||
fclose(fp);
|
||||
|
||||
if (diskFull)
|
||||
{
|
||||
CFile::deleteFile(downloadPath);
|
||||
throw EDiskFullError(downloadPath);
|
||||
}
|
||||
|
||||
if (res != CURLE_OK || r < 200 || r >= 300)
|
||||
{
|
||||
CFile::deleteFile(downloadPath);
|
||||
nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str());
|
||||
nlSleep(1000);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
nldebug("Curl initialize failed");
|
||||
fclose(fp);
|
||||
CFile::deleteFile(downloadPath);
|
||||
return false;
|
||||
}
|
||||
|
||||
// ok!
|
||||
break;
|
||||
}
|
||||
|
||||
// extract into file
|
||||
std::string unpackPath = filePath + ".extract." + toString(rand() * rand());
|
||||
|
||||
CHashKey outHash;
|
||||
if (!unpackLZMA(downloadPath, unpackPath, outHash))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!(outHash == hash))
|
||||
{
|
||||
std::string wantHashS = hash.toString();
|
||||
std::string outHashS = outHash.toString();
|
||||
nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str()))
|
||||
{
|
||||
nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str());
|
||||
// in case downloaded from another thread
|
||||
return CFile::fileExists(filePath);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} /* namespace NLMISC */
|
||||
|
||||
/* end of file */
|
||||
@ -0,0 +1,37 @@
|
||||
// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
|
||||
// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be)
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#include "stdmisc.h"
|
||||
|
||||
// Project includes
|
||||
#include <nel/misc/i_streamed_package_provider.h>
|
||||
|
||||
namespace NLMISC
|
||||
{
|
||||
|
||||
IStreamedPackageProvider::IStreamedPackageProvider()
|
||||
{
|
||||
// init
|
||||
}
|
||||
|
||||
IStreamedPackageProvider::~IStreamedPackageProvider()
|
||||
{
|
||||
// release
|
||||
}
|
||||
|
||||
} /* namespace NLMISC */
|
||||
|
||||
/* end of file */
|
||||
@ -0,0 +1,66 @@
|
||||
// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
|
||||
// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be)
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#include "stdmisc.h"
|
||||
|
||||
// Project includes
|
||||
#include <nel/misc/streamed_package.h>
|
||||
#include <nel/misc/stream.h>
|
||||
|
||||
namespace NLMISC {
|
||||
|
||||
CStreamedPackage::CStreamedPackage()
|
||||
{
|
||||
// init
|
||||
}
|
||||
|
||||
CStreamedPackage::~CStreamedPackage()
|
||||
{
|
||||
// release
|
||||
}
|
||||
|
||||
void CStreamedPackage::serial(NLMISC::IStream &f) throw(NLMISC::EStream)
|
||||
{
|
||||
f.serialCheck(NELID("SNPK"));
|
||||
|
||||
uint version = 1;
|
||||
f.serialVersion(version);
|
||||
|
||||
f.serialCont(Entries);
|
||||
}
|
||||
|
||||
void CStreamedPackage::CEntry::serial(NLMISC::IStream &f) throw(NLMISC::EStream)
|
||||
{
|
||||
uint version = 1;
|
||||
f.serialVersion(version);
|
||||
|
||||
f.serial(Name);
|
||||
f.serial(Hash);
|
||||
f.serial(Size);
|
||||
f.serial(LastModified);
|
||||
}
|
||||
|
||||
void CStreamedPackage::makePath(std::string &result, const CHashKey &hash)
|
||||
{
|
||||
std::string lowerHash = NLMISC::toLower(hash.toString());
|
||||
result = std::string("/") + lowerHash.substr(0, 2)
|
||||
+ "/" + lowerHash.substr(2, 2)
|
||||
+ "/" + lowerHash.substr(4);
|
||||
}
|
||||
|
||||
} /* namespace NLMISC */
|
||||
|
||||
/* end of file */
|
||||
@ -0,0 +1,126 @@
|
||||
// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
|
||||
// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be)
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
#include "stdmisc.h"
|
||||
|
||||
// Project includes
|
||||
#include <nel/misc/streamed_package_manager.h>
|
||||
#include <nel/misc/file.h>
|
||||
#include <nel/misc/path.h>
|
||||
#include <nel/misc/sha1.h>
|
||||
|
||||
namespace NLMISC
|
||||
{
|
||||
|
||||
NLMISC_SAFE_SINGLETON_IMPL(CStreamedPackageManager);
|
||||
|
||||
CStreamedPackageManager::CStreamedPackageManager()
|
||||
{
|
||||
// init
|
||||
}
|
||||
|
||||
CStreamedPackageManager::~CStreamedPackageManager()
|
||||
{
|
||||
// release
|
||||
}
|
||||
|
||||
bool CStreamedPackageManager::loadPackage(const std::string &package)
|
||||
{
|
||||
nldebug("Load package '%s'", package.c_str());
|
||||
|
||||
std::string packname = NLMISC::toLower(CFile::getFilename(package));
|
||||
m_Packages[packname] = CStreamedPackage();
|
||||
std::map<std::string, CStreamedPackage>::iterator it = m_Packages.find(packname);
|
||||
CStreamedPackage &p = it->second;
|
||||
|
||||
try
|
||||
{
|
||||
CIFile fi;
|
||||
fi.open(package);
|
||||
fi.serial(p);
|
||||
}
|
||||
catch (Exception &e)
|
||||
{
|
||||
nlerror("Package serial exception: '%s'", e.what());
|
||||
m_Packages.erase(it);
|
||||
return false;
|
||||
}
|
||||
|
||||
for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it)
|
||||
{
|
||||
const CStreamedPackage::CEntry &entry = (*it);
|
||||
m_Entries[entry.Name] = &entry;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void CStreamedPackageManager::list(std::vector<std::string> &fileNames, const std::string &package)
|
||||
{
|
||||
// nldebug("List package '%s'", package.c_str());
|
||||
|
||||
std::map<std::string, CStreamedPackage>::iterator it = m_Packages.find(package);
|
||||
CStreamedPackage &p = it->second;
|
||||
|
||||
for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it)
|
||||
{
|
||||
const CStreamedPackage::CEntry &entry = (*it);
|
||||
fileNames.push_back(entry.Name);
|
||||
}
|
||||
}
|
||||
|
||||
void CStreamedPackageManager::unloadAll()
|
||||
{
|
||||
m_Packages.clear();
|
||||
m_Entries.clear();
|
||||
}
|
||||
|
||||
bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName)
|
||||
{
|
||||
// nldebug("Get file path for streamed file '%s'", fileName.c_str());
|
||||
|
||||
IStreamedPackageProvider *provider = Provider;
|
||||
if (!provider)
|
||||
{
|
||||
nlerrornoex("No streamed package provider was set");
|
||||
return false;
|
||||
}
|
||||
|
||||
TEntries::iterator it = m_Entries.find(fileName);
|
||||
if (it == m_Entries.end())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
const CStreamedPackage::CEntry *entry = it->second;
|
||||
|
||||
return provider->getFile(filePath, entry->Hash, fileName);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool CStreamedPackageManager::getFileSize(uint32 &fileSize, const std::string &fileName)
|
||||
{
|
||||
// nldebug("Get file size for streamed file '%s'", fileName.c_str());
|
||||
TEntries::iterator it = m_Entries.find(fileName);
|
||||
if (it == m_Entries.end())
|
||||
return false;
|
||||
fileSize = it->second->Size;
|
||||
return true;
|
||||
}
|
||||
|
||||
} /* namespace NLMISC */
|
||||
|
||||
/* end of file */
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue