From 9c3e8674ae6113d0f99005a49881b0634c26bb3c Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 24 Sep 2014 16:08:16 +0200 Subject: [PATCH 01/20] Add tool to build streamed package, ref #179 --HG-- branch : feature-streamed-package --- code/nel/include/nel/misc/streamed_package.h | 56 +++ .../nel/misc/streamed_package_manager.h | 34 ++ code/nel/src/misc/streamed_package.cpp | 66 ++++ .../nel/src/misc/streamed_package_manager.cpp | 36 ++ code/nel/tools/misc/CMakeLists.txt | 2 +- code/nel/tools/misc/snp_make/CMakeLists.txt | 9 + code/nel/tools/misc/snp_make/main.cpp | 355 ++++++++++++++++++ .../tools/patch_gen/patch_gen_common.cpp | 3 +- 8 files changed, 559 insertions(+), 2 deletions(-) create mode 100644 code/nel/include/nel/misc/streamed_package.h create mode 100644 code/nel/include/nel/misc/streamed_package_manager.h create mode 100644 code/nel/src/misc/streamed_package.cpp create mode 100644 code/nel/src/misc/streamed_package_manager.cpp create mode 100644 code/nel/tools/misc/snp_make/CMakeLists.txt create mode 100644 code/nel/tools/misc/snp_make/main.cpp diff --git a/code/nel/include/nel/misc/streamed_package.h b/code/nel/include/nel/misc/streamed_package.h new file mode 100644 index 000000000..b27e6658f --- /dev/null +++ b/code/nel/include/nel/misc/streamed_package.h @@ -0,0 +1,56 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_H +#define NLMISC_STREAMED_PACKAGE_H + +#include + +namespace NLMISC { + +class CStreamedPackage +{ +public: + struct CEntry + { + std::string Name; + CHashKey Hash; + uint32 Size; + uint32 LastModified; + + void serial(NLMISC::IStream &f) throw(NLMISC::EStream); + + }; + +public: + CStreamedPackage(); + ~CStreamedPackage(); + + void serial(NLMISC::IStream &f) throw(NLMISC::EStream); + + static void makePath(std::string &result, const CHashKey &hash); + +public: + typedef std::vector TEntries; + TEntries Entries; + +}; /* class CStreamedPackage */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */ + +/* end of file */ diff --git a/code/nel/include/nel/misc/streamed_package_manager.h b/code/nel/include/nel/misc/streamed_package_manager.h new file mode 100644 index 000000000..0bec38584 --- /dev/null +++ b/code/nel/include/nel/misc/streamed_package_manager.h @@ -0,0 +1,34 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_H +#define NLMISC_STREAMED_PACKAGE_H + +namespace NLMISC { + +class CStreamedPackageManager +{ +public: + CStreamedPackageManager(); + ~CStreamedPackageManager(); + +}; /* class CStreamedPackageManager */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */ + +/* end of file */ diff --git a/code/nel/src/misc/streamed_package.cpp b/code/nel/src/misc/streamed_package.cpp new file mode 100644 index 000000000..fed7d1dc3 --- /dev/null +++ b/code/nel/src/misc/streamed_package.cpp @@ -0,0 +1,66 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// Project includes +#include +#include + +namespace NLMISC { + +CStreamedPackage::CStreamedPackage() +{ + // init +} + +CStreamedPackage::~CStreamedPackage() +{ + // release +} + +void CStreamedPackage::serial(NLMISC::IStream &f) throw(NLMISC::EStream) +{ + f.serialCheck(NELID("SNPK")); + + uint version = 1; + f.serialVersion(version); + + f.serialCont(Entries); +} + +void CStreamedPackage::CEntry::serial(NLMISC::IStream &f) throw(NLMISC::EStream) +{ + uint version = 1; + f.serialVersion(version); + + f.serial(Name); + f.serial(Hash); + f.serial(Size); + f.serial(LastModified); +} + +void CStreamedPackage::makePath(std::string &result, const CHashKey &hash) +{ + std::string lowerHash = NLMISC::toLower(hash.toString()); + result = std::string("/") + lowerHash.substr(0, 2) + + "/" + lowerHash.substr(2, 2) + + "/" + lowerHash.substr(4); +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp new file mode 100644 index 000000000..2d04cb95a --- /dev/null +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -0,0 +1,36 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// Project includes +#include + +namespace NLMISC { + +CStreamedPackageManager::CStreamedPackageManager() +{ + // init +} + +CStreamedPackageManager::~CStreamedPackageManager() +{ + // release +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/tools/misc/CMakeLists.txt b/code/nel/tools/misc/CMakeLists.txt index 5386cbbc6..76399f6a3 100644 --- a/code/nel/tools/misc/CMakeLists.txt +++ b/code/nel/tools/misc/CMakeLists.txt @@ -1,4 +1,4 @@ -SUBDIRS(bnp_make disp_sheet_id extract_filename lock make_sheet_id xml_packer) +SUBDIRS(bnp_make snp_make disp_sheet_id extract_filename lock make_sheet_id xml_packer) IF(WITH_QT) ADD_SUBDIRECTORY(words_dic_qt) diff --git a/code/nel/tools/misc/snp_make/CMakeLists.txt b/code/nel/tools/misc/snp_make/CMakeLists.txt new file mode 100644 index 000000000..a1b5f388d --- /dev/null +++ b/code/nel/tools/misc/snp_make/CMakeLists.txt @@ -0,0 +1,9 @@ +FILE(GLOB SRC *.cpp *.h) + +ADD_EXECUTABLE(snp_make ${SRC}) + +TARGET_LINK_LIBRARIES(snp_make nelmisc) +NL_DEFAULT_PROPS(snp_make "NeL, Tools, Misc: snp_make") +NL_ADD_RUNTIME_FLAGS(snp_make) + +INSTALL(TARGETS snp_make RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT toolsmisc) diff --git a/code/nel/tools/misc/snp_make/main.cpp b/code/nel/tools/misc/snp_make/main.cpp new file mode 100644 index 000000000..61fc11810 --- /dev/null +++ b/code/nel/tools/misc/snp_make/main.cpp @@ -0,0 +1,355 @@ +// NeL - MMORPG Framework +// Copyright (C) 2010 Winch Gate Property Limited +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "nel/misc/types_nl.h" + +#include +#include + +#ifdef NL_OS_WINDOWS +# include +# include +#endif + +#include +#include + +#include "nel/misc/debug.h" +#include "nel/misc/file.h" +#include "nel/misc/path.h" +#include "nel/misc/algo.h" +#include "nel/misc/common.h" +#include "nel/misc/streamed_package.h" + +using namespace std; +using namespace NLMISC; + +// --------------------------------------------------------------------------- + +class CWildCard +{ +public: + string Expression; + bool Not; +}; +std::vector WildCards; + +std::string SourceDirectory; +std::string PackageFileName; +std::string StreamDirectory; + +CStreamedPackage Package; + +// --------------------------------------------------------------------------- + +bool keepFile (const char *fileName) +{ + uint i; + bool ifPresent = false; + bool ifTrue = false; + string file = toLower(CFile::getFilename (fileName)); + for (i=0; i [option] ... [option]\n"); + printf (" option : \n"); + printf (" -if wildcard : add the file if it matches the wilcard (at least one 'if' conditions must be met for a file to be adding)\n"); + printf (" -ifnot wildcard : add the file if it doesn't match the wilcard (all the 'ifnot' conditions must be met for a file to be adding)\n"); + printf (" Pack the directory to a snp file\n"); + printf (" snp_make /l \n"); + printf (" List the files contained in the snp file\n"); +} + +// --------------------------------------------------------------------------- + +void generateLZMA(const std::string sourceFile, const std::string &outputFile) +{ + std::string cmd="lzma e "; + cmd+=" "+sourceFile+" "+outputFile; + nlinfo("executing system command: %s",cmd.c_str()); +#ifdef NL_OS_WINDOWS + _spawnlp(_P_WAIT, "lzma.exe","lzma.exe", "e", sourceFile.c_str(), outputFile.c_str(), NULL); +#else // NL_OS_WINDOWS + sint error = system (cmd.c_str()); + if (error) + nlwarning("'%s' failed with error code %d", cmd.c_str(), error); +#endif // NL_OS_WINDOWS +} + +// --------------------------------------------------------------------------- + +uint readOptions (int nNbArg, char **ppArgs) +{ + uint i; + uint optionCount = 0; + for (i=0; i<(uint)nNbArg; i++) + { + // If ? + if ((strcmp (ppArgs[i], "-if") == 0) && ((i+1)<(uint)nNbArg)) + { + CWildCard card; + card.Expression = toLower(string(ppArgs[i+1])); + card.Not = false; + WildCards.push_back (card); + optionCount += 2; + } + // If not ? + if ((strcmp (ppArgs[i], "-ifnot") == 0) && ((i+1)<(uint)nNbArg)) + { + CWildCard card; + card.Expression = toLower(string(ppArgs[i+1])); + card.Not = true; + WildCards.push_back (card); + optionCount += 2; + } + } + return optionCount; +} + +// --------------------------------------------------------------------------- +int main (int nNbArg, char **ppArgs) +{ + NLMISC::CApplicationContext myApplicationContext; + + if (nNbArg < 3) + { + usage(); + return -1; + } + + if ((strcmp(ppArgs[1], "/p") == 0) || (strcmp(ppArgs[1], "/P") == 0) || + (strcmp(ppArgs[1], "-p") == 0) || (strcmp(ppArgs[1], "-P") == 0)) + { + if (nNbArg < 5) + { + usage(); + return -1; + } + + SourceDirectory = ppArgs[2]; + PackageFileName = ppArgs[3]; + StreamDirectory = ppArgs[4]; + readOptions(nNbArg, ppArgs); + + nldebug("Make streamed package: '%s'", PackageFileName.c_str()); + + if (CFile::fileExists(PackageFileName)) + { + nldebug("Update existing package"); + try + { + CIFile fi; + fi.open(PackageFileName); + fi.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + } + else + { + nldebug("New package"); + } + + std::vector pathContent; // contains full pathnames + std::vector nameContent; // only filename + CPath::getPathContent(SourceDirectory, true, false, true, pathContent); + nameContent.reserve(pathContent.size()); + for (std::vector::size_type i = 0; i < pathContent.size(); ++i) + { + const std::string &file = pathContent[i]; + if (keepFile(file.c_str())) + { + std::string fileName = NLMISC::toLower(CFile::getFilename(file)); + // nldebug("File: '%s' ('%s')", file.c_str(), fileName.c_str()); + nameContent.push_back(fileName); + nlassert(nameContent.size() == (i + 1)); + } + else + { + // Not included in this package + pathContent.erase(pathContent.begin() + i); + --i; + } + } + + std::vector packageIndex; // index of file in package + packageIndex.resize(pathContent.size(), -1); + + for (CStreamedPackage::TEntries::size_type i = 0; i < Package.Entries.size(); ++i) + { + const CStreamedPackage::CEntry &entry = Package.Entries[i]; + + sint foundIndex = -1; // find index in found file list + for (std::vector::size_type j = 0; j < pathContent.size(); ++j) + { + if (nameContent[j] == entry.Name) + { + foundIndex = j; + break; + } + } + + if (foundIndex < 0) + { + nlinfo("File no longer exists: '%s'", entry.Name.c_str()); + Package.Entries.erase(Package.Entries.begin() + i); + --i; + } + else + { + // File still exists, map it + packageIndex[foundIndex] = i; + } + } + + for (std::vector::size_type i = 0; i < pathContent.size(); ++i) + { + sint pidx = packageIndex[i]; + const std::string &name = nameContent[i]; + const std::string &path = pathContent[i]; + + if (pidx < 0) + { + nlinfo("File added: '%s'", name.c_str()); + pidx = Package.Entries.size(); + Package.Entries.push_back(CStreamedPackage::CEntry()); + Package.Entries[pidx].Name = name; + Package.Entries[pidx].LastModified = 0; + Package.Entries[pidx].Size = 0; + } + else + { + nlinfo("File check for changes: '%s'", name.c_str()); + } + + CStreamedPackage::CEntry &entry = Package.Entries[pidx]; + + std::string targetLzmaOld; // in case lzma wasn't made make sure it exists a second run + CStreamedPackage::makePath(targetLzmaOld, entry.Hash); + targetLzmaOld = StreamDirectory + targetLzmaOld + ".lzma"; + + uint32 lastModified = CFile::getFileModificationDate(path); + uint32 fileSize = CFile::getFileSize(path); + if (lastModified > entry.LastModified || fileSize != entry.Size || !CFile::fileExists(targetLzmaOld)) + { + entry.LastModified = lastModified; + + nlinfo("Calculate file hash"); + CHashKey hash = getSHA1(path, true); + /*nldebug("%s", hash.toString().c_str()); + std::string hashPath; + CStreamedPackage::makePath(hashPath, hash); + nldebug("%s", hashPath.c_str());*/ + + if (hash == entry.Hash && fileSize == entry.Size) + { + // File has not changed + } + else + { + nlinfo("File changed"); + entry.Hash = hash; + entry.Size = fileSize; + } + + std::string targetLzma; // in case lzma wasn't made make sure it exists a second run + CStreamedPackage::makePath(targetLzma, entry.Hash); + targetLzma = StreamDirectory + targetLzma + ".lzma"; + + if (!CFile::fileExists(targetLzma)) + { + // make the compressed file + nlinfo("%s -> %s", path.c_str(), targetLzma.c_str()); + CFile::createDirectoryTree(CFile::getPath(targetLzma)); + generateLZMA(path, targetLzma); + } + } + } + + try + { + nldebug("Store package '%s'", PackageFileName.c_str()); + COFile fo; + fo.open(PackageFileName); + fo.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + + return 0; + } + + if ((strcmp(ppArgs[1], "/l") == 0) || (strcmp(ppArgs[1], "/L") == 0) || + (strcmp(ppArgs[1], "-l") == 0) || (strcmp(ppArgs[1], "-L") == 0)) + { + PackageFileName = ppArgs[2]; + if (!CFile::fileExists(PackageFileName)) + { + nlwarning("ERROR (snp_make) : package doesn't exist: '%s'", PackageFileName.c_str()); + return -1; + } + + try + { + CIFile fi; + fi.open(PackageFileName); + fi.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + + for (CStreamedPackage::TEntries::const_iterator it(Package.Entries.begin()), end(Package.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + + printf("List files in '%s'", PackageFileName.c_str()); + printf("%s { Hash: '%s', Size: '%u', LastModified: '%u' }", entry.Name.c_str(), entry.Hash.toString().c_str(), entry.Size, entry.LastModified); + } + + return 0; + } + + usage (); + return -1; +} diff --git a/code/ryzom/tools/patch_gen/patch_gen_common.cpp b/code/ryzom/tools/patch_gen/patch_gen_common.cpp index 3d28a2439..127be5053 100644 --- a/code/ryzom/tools/patch_gen/patch_gen_common.cpp +++ b/code/ryzom/tools/patch_gen/patch_gen_common.cpp @@ -442,7 +442,8 @@ void CPackageDescription::buildDefaultFileList() std::vector fileList; NLMISC::CPath::getPathContent(_BnpDirectory,false,false,true,fileList); for (uint32 i=0;i Date: Wed, 24 Sep 2014 17:51:05 +0200 Subject: [PATCH 02/20] Move 7zip library --HG-- branch : feature-streamed-package --- code/CMakeLists.txt | 1 + code/nel/3rdparty/CMakeLists.txt | 2 ++ .../3rdparty}/seven_zip/7zAlloc.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zAlloc.h | 0 .../3rdparty}/seven_zip/7zBuffer.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zBuffer.h | 0 .../src => nel/3rdparty}/seven_zip/7zCrc.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zCrc.h | 0 .../3rdparty}/seven_zip/7zDecode.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zDecode.h | 0 .../3rdparty}/seven_zip/7zExtract.cpp | 0 .../3rdparty}/seven_zip/7zExtract.h | 0 .../3rdparty}/seven_zip/7zHeader.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zHeader.h | 0 .../src => nel/3rdparty}/seven_zip/7zIn.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zIn.h | 0 .../src => nel/3rdparty}/seven_zip/7zItem.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zItem.h | 0 .../src => nel/3rdparty}/seven_zip/7zMain.cpp | 0 .../3rdparty}/seven_zip/7zMethodID.cpp | 0 .../3rdparty}/seven_zip/7zMethodID.h | 0 .../src => nel/3rdparty}/seven_zip/7zTypes.h | 0 .../3rdparty}/seven_zip/BranchTypes.h | 0 .../3rdparty}/seven_zip/BranchX86.cpp | 0 .../3rdparty}/seven_zip/BranchX86.h | 0 code/nel/3rdparty/seven_zip/CMakeLists.txt | 27 +++++++++++++++++++ .../3rdparty}/seven_zip/LzmaDecode.cpp | 0 .../3rdparty}/seven_zip/LzmaDecode.h | 0 .../3rdparty}/seven_zip/LzmaRamDecode.cpp | 0 .../3rdparty}/seven_zip/LzmaRamDecode.h | 0 .../3rdparty}/seven_zip/LzmaTypes.h | 0 .../src => nel/3rdparty}/seven_zip/readme.txt | 0 code/nel/CMakeLists.txt | 1 + code/nel/src/misc/CMakeLists.txt | 2 +- code/ryzom/client/src/CMakeLists.txt | 2 -- .../ryzom/client/src/seven_zip/CMakeLists.txt | 27 ------------------- .../client/client_patcher/CMakeLists.txt | 2 +- 37 files changed, 33 insertions(+), 31 deletions(-) create mode 100644 code/nel/3rdparty/CMakeLists.txt rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zAlloc.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zAlloc.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zBuffer.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zBuffer.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zCrc.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zCrc.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zDecode.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zDecode.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zExtract.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zExtract.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zHeader.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zHeader.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zIn.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zIn.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zItem.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zItem.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zMain.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zMethodID.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zMethodID.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zTypes.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/BranchTypes.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/BranchX86.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/BranchX86.h (100%) create mode 100644 code/nel/3rdparty/seven_zip/CMakeLists.txt rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/LzmaDecode.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/LzmaDecode.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/LzmaRamDecode.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/LzmaRamDecode.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/LzmaTypes.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/readme.txt (100%) delete mode 100644 code/ryzom/client/src/seven_zip/CMakeLists.txt diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index 071554e06..80bc52421 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -177,6 +177,7 @@ IF(WITH_NEL) ENDIF(WITH_GUI) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/nel/include ${LIBXML2_INCLUDE_DIR}) + INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/nel/3rdparty) ADD_DEFINITIONS(${LIBXML2_DEFINITIONS}) ADD_SUBDIRECTORY(nel) ENDIF(WITH_NEL) diff --git a/code/nel/3rdparty/CMakeLists.txt b/code/nel/3rdparty/CMakeLists.txt new file mode 100644 index 000000000..bdd3e5fcd --- /dev/null +++ b/code/nel/3rdparty/CMakeLists.txt @@ -0,0 +1,2 @@ +SET(SEVENZIP_LIBRARY "nel_sevenzip") +ADD_SUBDIRECTORY(seven_zip) diff --git a/code/ryzom/client/src/seven_zip/7zAlloc.cpp b/code/nel/3rdparty/seven_zip/7zAlloc.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zAlloc.cpp rename to code/nel/3rdparty/seven_zip/7zAlloc.cpp diff --git a/code/ryzom/client/src/seven_zip/7zAlloc.h b/code/nel/3rdparty/seven_zip/7zAlloc.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zAlloc.h rename to code/nel/3rdparty/seven_zip/7zAlloc.h diff --git a/code/ryzom/client/src/seven_zip/7zBuffer.cpp b/code/nel/3rdparty/seven_zip/7zBuffer.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zBuffer.cpp rename to code/nel/3rdparty/seven_zip/7zBuffer.cpp diff --git a/code/ryzom/client/src/seven_zip/7zBuffer.h b/code/nel/3rdparty/seven_zip/7zBuffer.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zBuffer.h rename to code/nel/3rdparty/seven_zip/7zBuffer.h diff --git a/code/ryzom/client/src/seven_zip/7zCrc.cpp b/code/nel/3rdparty/seven_zip/7zCrc.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zCrc.cpp rename to code/nel/3rdparty/seven_zip/7zCrc.cpp diff --git a/code/ryzom/client/src/seven_zip/7zCrc.h b/code/nel/3rdparty/seven_zip/7zCrc.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zCrc.h rename to code/nel/3rdparty/seven_zip/7zCrc.h diff --git a/code/ryzom/client/src/seven_zip/7zDecode.cpp b/code/nel/3rdparty/seven_zip/7zDecode.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zDecode.cpp rename to code/nel/3rdparty/seven_zip/7zDecode.cpp diff --git a/code/ryzom/client/src/seven_zip/7zDecode.h b/code/nel/3rdparty/seven_zip/7zDecode.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zDecode.h rename to code/nel/3rdparty/seven_zip/7zDecode.h diff --git a/code/ryzom/client/src/seven_zip/7zExtract.cpp b/code/nel/3rdparty/seven_zip/7zExtract.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zExtract.cpp rename to code/nel/3rdparty/seven_zip/7zExtract.cpp diff --git a/code/ryzom/client/src/seven_zip/7zExtract.h b/code/nel/3rdparty/seven_zip/7zExtract.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zExtract.h rename to code/nel/3rdparty/seven_zip/7zExtract.h diff --git a/code/ryzom/client/src/seven_zip/7zHeader.cpp b/code/nel/3rdparty/seven_zip/7zHeader.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zHeader.cpp rename to code/nel/3rdparty/seven_zip/7zHeader.cpp diff --git a/code/ryzom/client/src/seven_zip/7zHeader.h b/code/nel/3rdparty/seven_zip/7zHeader.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zHeader.h rename to code/nel/3rdparty/seven_zip/7zHeader.h diff --git a/code/ryzom/client/src/seven_zip/7zIn.cpp b/code/nel/3rdparty/seven_zip/7zIn.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zIn.cpp rename to code/nel/3rdparty/seven_zip/7zIn.cpp diff --git a/code/ryzom/client/src/seven_zip/7zIn.h b/code/nel/3rdparty/seven_zip/7zIn.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zIn.h rename to code/nel/3rdparty/seven_zip/7zIn.h diff --git a/code/ryzom/client/src/seven_zip/7zItem.cpp b/code/nel/3rdparty/seven_zip/7zItem.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zItem.cpp rename to code/nel/3rdparty/seven_zip/7zItem.cpp diff --git a/code/ryzom/client/src/seven_zip/7zItem.h b/code/nel/3rdparty/seven_zip/7zItem.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zItem.h rename to code/nel/3rdparty/seven_zip/7zItem.h diff --git a/code/ryzom/client/src/seven_zip/7zMain.cpp b/code/nel/3rdparty/seven_zip/7zMain.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zMain.cpp rename to code/nel/3rdparty/seven_zip/7zMain.cpp diff --git a/code/ryzom/client/src/seven_zip/7zMethodID.cpp b/code/nel/3rdparty/seven_zip/7zMethodID.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zMethodID.cpp rename to code/nel/3rdparty/seven_zip/7zMethodID.cpp diff --git a/code/ryzom/client/src/seven_zip/7zMethodID.h b/code/nel/3rdparty/seven_zip/7zMethodID.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zMethodID.h rename to code/nel/3rdparty/seven_zip/7zMethodID.h diff --git a/code/ryzom/client/src/seven_zip/7zTypes.h b/code/nel/3rdparty/seven_zip/7zTypes.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zTypes.h rename to code/nel/3rdparty/seven_zip/7zTypes.h diff --git a/code/ryzom/client/src/seven_zip/BranchTypes.h b/code/nel/3rdparty/seven_zip/BranchTypes.h similarity index 100% rename from code/ryzom/client/src/seven_zip/BranchTypes.h rename to code/nel/3rdparty/seven_zip/BranchTypes.h diff --git a/code/ryzom/client/src/seven_zip/BranchX86.cpp b/code/nel/3rdparty/seven_zip/BranchX86.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/BranchX86.cpp rename to code/nel/3rdparty/seven_zip/BranchX86.cpp diff --git a/code/ryzom/client/src/seven_zip/BranchX86.h b/code/nel/3rdparty/seven_zip/BranchX86.h similarity index 100% rename from code/ryzom/client/src/seven_zip/BranchX86.h rename to code/nel/3rdparty/seven_zip/BranchX86.h diff --git a/code/nel/3rdparty/seven_zip/CMakeLists.txt b/code/nel/3rdparty/seven_zip/CMakeLists.txt new file mode 100644 index 000000000..d2d2e690d --- /dev/null +++ b/code/nel/3rdparty/seven_zip/CMakeLists.txt @@ -0,0 +1,27 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +FILE(GLOB LIB_SRC *.cpp *.h) + +LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) + +NL_TARGET_LIB(nel_sevenzip ${LIB_SRC}) +# TARGET_LINK_LIBRARIES(nel_sevenzip ${PLATFORM_LINKFLAGS}) +NL_DEFAULT_PROPS(nel_sevenzip "NeL, 3rd Party: Seven Zip") +NL_ADD_RUNTIME_FLAGS(nel_sevenzip) +NL_ADD_LIB_SUFFIX(nel_sevenzip) + +ADD_DEFINITIONS(-D_SZ_ONE_DIRECTORY) + +IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) + INSTALL(TARGETS nel_sevenzip LIBRARY DESTINATION ${NL_LIB_PREFIX} ARCHIVE DESTINATION ${NL_LIB_PREFIX} COMPONENT libraries) +ENDIF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) + +IF(WITH_TOOLS) + ADD_EXECUTABLE(7zDec ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) + + TARGET_LINK_LIBRARIES(7zDec nel_sevenzip) + NL_DEFAULT_PROPS(7zDec "NeL, 3rd Party: Seven Zip Decoder") + NL_ADD_RUNTIME_FLAGS(7zDec) + + INSTALL(TARGETS 7zDec RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT tools) +ENDIF(WITH_TOOLS) diff --git a/code/ryzom/client/src/seven_zip/LzmaDecode.cpp b/code/nel/3rdparty/seven_zip/LzmaDecode.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaDecode.cpp rename to code/nel/3rdparty/seven_zip/LzmaDecode.cpp diff --git a/code/ryzom/client/src/seven_zip/LzmaDecode.h b/code/nel/3rdparty/seven_zip/LzmaDecode.h similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaDecode.h rename to code/nel/3rdparty/seven_zip/LzmaDecode.h diff --git a/code/ryzom/client/src/seven_zip/LzmaRamDecode.cpp b/code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaRamDecode.cpp rename to code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp diff --git a/code/ryzom/client/src/seven_zip/LzmaRamDecode.h b/code/nel/3rdparty/seven_zip/LzmaRamDecode.h similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaRamDecode.h rename to code/nel/3rdparty/seven_zip/LzmaRamDecode.h diff --git a/code/ryzom/client/src/seven_zip/LzmaTypes.h b/code/nel/3rdparty/seven_zip/LzmaTypes.h similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaTypes.h rename to code/nel/3rdparty/seven_zip/LzmaTypes.h diff --git a/code/ryzom/client/src/seven_zip/readme.txt b/code/nel/3rdparty/seven_zip/readme.txt similarity index 100% rename from code/ryzom/client/src/seven_zip/readme.txt rename to code/nel/3rdparty/seven_zip/readme.txt diff --git a/code/nel/CMakeLists.txt b/code/nel/CMakeLists.txt index 53bf071e3..d3f0a3e5c 100644 --- a/code/nel/CMakeLists.txt +++ b/code/nel/CMakeLists.txt @@ -70,6 +70,7 @@ IF(WITH_INSTALL_LIBRARIES) ADD_SUBDIRECTORY(include) ENDIF(WITH_INSTALL_LIBRARIES) +ADD_SUBDIRECTORY(3rdparty) ADD_SUBDIRECTORY(src) IF(WITH_NEL_SAMPLES) diff --git a/code/nel/src/misc/CMakeLists.txt b/code/nel/src/misc/CMakeLists.txt index 2d3ef9066..8494c23a6 100644 --- a/code/nel/src/misc/CMakeLists.txt +++ b/code/nel/src/misc/CMakeLists.txt @@ -39,7 +39,7 @@ ENDIF(UNIX) INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR} ${PNG_INCLUDE_DIR} config_file) -TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY}) +TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} nel_sevenzip) SET_TARGET_PROPERTIES(nelmisc PROPERTIES LINK_INTERFACE_LIBRARIES "") NL_DEFAULT_PROPS(nelmisc "NeL, Library: NeL Misc") NL_ADD_RUNTIME_FLAGS(nelmisc) diff --git a/code/ryzom/client/src/CMakeLists.txt b/code/ryzom/client/src/CMakeLists.txt index c20d9d0dc..3fbbfd0e2 100644 --- a/code/ryzom/client/src/CMakeLists.txt +++ b/code/ryzom/client/src/CMakeLists.txt @@ -5,9 +5,7 @@ ADD_SUBDIRECTORY(client_sheets) IF(WITH_RYZOM_CLIENT) # These are Windows/MFC apps -SET(SEVENZIP_LIBRARY "ryzom_sevenzip") -ADD_SUBDIRECTORY(seven_zip) IF(WITH_RYZOM_PATCH) ADD_DEFINITIONS(-DRZ_USE_PATCH) diff --git a/code/ryzom/client/src/seven_zip/CMakeLists.txt b/code/ryzom/client/src/seven_zip/CMakeLists.txt deleted file mode 100644 index cc152757b..000000000 --- a/code/ryzom/client/src/seven_zip/CMakeLists.txt +++ /dev/null @@ -1,27 +0,0 @@ -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) - -FILE(GLOB LIB_SRC *.cpp *.h) - -LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) - -NL_TARGET_LIB(ryzom_sevenzip ${LIB_SRC}) -# TARGET_LINK_LIBRARIES(ryzom_sevenzip ${PLATFORM_LINKFLAGS}) -NL_DEFAULT_PROPS(ryzom_sevenzip "Ryzom, Library: Seven Zip") -NL_ADD_RUNTIME_FLAGS(ryzom_sevenzip) -NL_ADD_LIB_SUFFIX(ryzom_sevenzip) - -ADD_DEFINITIONS(-D_SZ_ONE_DIRECTORY) - -IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) - INSTALL(TARGETS ryzom_sevenzip LIBRARY DESTINATION ${RYZOM_LIB_PREFIX} ARCHIVE DESTINATION ${RYZOM_LIB_PREFIX} COMPONENT libraries) -ENDIF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) - -IF(WITH_RYZOM_TOOLS) - ADD_EXECUTABLE(7zDec ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) - - TARGET_LINK_LIBRARIES(7zDec ryzom_sevenzip) - NL_DEFAULT_PROPS(7zDec "Ryzom, Tools, Misc: Seven Zip Decoder") - NL_ADD_RUNTIME_FLAGS(7zDec) - - INSTALL(TARGETS 7zDec RUNTIME DESTINATION ${RYZOM_BIN_PREFIX} COMPONENT tools) -ENDIF(WITH_RYZOM_TOOLS) diff --git a/code/ryzom/tools/client/client_patcher/CMakeLists.txt b/code/ryzom/tools/client/client_patcher/CMakeLists.txt index e5b845e43..69b14e5a1 100644 --- a/code/ryzom/tools/client/client_patcher/CMakeLists.txt +++ b/code/ryzom/tools/client/client_patcher/CMakeLists.txt @@ -19,7 +19,7 @@ TARGET_LINK_LIBRARIES(ryzom_client_patcher nelmisc nelnet ryzom_gameshare - ryzom_sevenzip + nel_sevenzip ${CURL_LIBRARIES}) IF(APPLE) From 4ba6b6c4bf3ce45f27d51ec00eb80045f9ac88d4 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 24 Sep 2014 20:30:11 +0200 Subject: [PATCH 03/20] Workaround '0' unpack directory bug, and don't choke on 0 size files when unpacking bnp files --HG-- branch : feature-streamed-package --- code/nel/tools/misc/bnp_make/main.cpp | 15 +++--- code/ryzom/client/src/login_patch.cpp | 69 +++++++++++++++++++-------- 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/code/nel/tools/misc/bnp_make/main.cpp b/code/nel/tools/misc/bnp_make/main.cpp index a3f34a6a4..3bb70f5aa 100644 --- a/code/nel/tools/misc/bnp_make/main.cpp +++ b/code/nel/tools/misc/bnp_make/main.cpp @@ -330,13 +330,16 @@ void unpack (const string &dirName) if (out != NULL) { nlfseek64 (bnp, rBNPFile.Pos, SEEK_SET); - uint8 *ptr = new uint8[rBNPFile.Size]; - if (fread (ptr, rBNPFile.Size, 1, bnp) != 1) - nlwarning("%s read error", filename.c_str()); - if (fwrite (ptr, rBNPFile.Size, 1, out) != 1) - nlwarning("%s write error", filename.c_str()); + if (rBNPFile.Size) + { + uint8 *ptr = new uint8[rBNPFile.Size]; + if (fread (ptr, rBNPFile.Size, 1, bnp) != 1) + nlwarning("%s read error", filename.c_str()); + if (fwrite (ptr, rBNPFile.Size, 1, out) != 1) + nlwarning("%s write error", filename.c_str()); + delete [] ptr; + } fclose (out); - delete [] ptr; } } fclose (bnp); diff --git a/code/ryzom/client/src/login_patch.cpp b/code/ryzom/client/src/login_patch.cpp index 5b9a83c83..d2f4d02e0 100644 --- a/code/ryzom/client/src/login_patch.cpp +++ b/code/ryzom/client/src/login_patch.cpp @@ -1176,13 +1176,35 @@ void CPatchManager::readDescFile(sint32 nVersion) std::string unpackTo = category.getUnpackTo(); - if (unpackTo.substr(0, 2) == "./") + if (unpackTo == "0") + { + nlwarning("BUG: unpackTo == '0'"); + unpackTo = ClientRootPath; + category.setUnpackTo(unpackTo); + } + else if (unpackTo.substr(0, 2) == "./") { unpackTo = ClientRootPath + unpackTo.substr(2); category.setUnpackTo(unpackTo); } } } + else + { + for (cat = 0; cat < DescFile.getCategories().categoryCount(); ++cat) + { + CBNPCategory &category = const_cast(DescFile.getCategories().getCategory(cat)); + + std::string unpackTo = category.getUnpackTo(); + + if (unpackTo == "0") + { + nlwarning("BUG: unpackTo == '0'"); + unpackTo = "./"; + category.setUnpackTo(unpackTo); + } + } + } // tmp for debug : flag some categories as 'Mainland' for (cat = 0; cat < DescFile.getCategories().categoryCount(); ++cat) @@ -1884,29 +1906,36 @@ bool CPatchManager::bnpUnpack(const string &srcBigfile, const string &dstPath, v if (out != NULL) { nlfseek64 (bnp, rBNPFile.Pos, SEEK_SET); - uint8 *ptr = new uint8[rBNPFile.Size]; - - if (fread (ptr, rBNPFile.Size, 1, bnp) != 1) + if (rBNPFile.Size) { - fclose(out); - return false; - } + uint8 *ptr = new uint8[rBNPFile.Size]; - bool writeError = fwrite (ptr, rBNPFile.Size, 1, out) != 1; - if (writeError) - { - nlwarning("errno = %d", errno); - } - bool diskFull = ferror(out) && errno == 28 /* ENOSPC*/; - fclose (out); - delete [] ptr; - if (diskFull) - { - throw NLMISC::EDiskFullError(filename); + if (fread (ptr, rBNPFile.Size, 1, bnp) != 1) + { + fclose(out); + return false; + } + + bool writeError = fwrite (ptr, rBNPFile.Size, 1, out) != 1; + if (writeError) + { + nlwarning("errno = %d", errno); + } + bool diskFull = ferror(out) && errno == 28 /* ENOSPC*/; + fclose (out); + delete [] ptr; + if (diskFull) + { + throw NLMISC::EDiskFullError(filename); + } + if (writeError) + { + throw NLMISC::EWriteError(filename); + } } - if (writeError) + else { - throw NLMISC::EWriteError(filename); + fclose (out); } } } From 7d09a6cb150f1aa9eda99a901653682658606ee5 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 24 Sep 2014 21:52:18 +0200 Subject: [PATCH 04/20] Add streamed package manager --HG-- branch : feature-streamed-package --- code/nel/include/nel/misc/app_context.h | 15 ++++ code/nel/include/nel/misc/path.h | 6 ++ code/nel/include/nel/misc/streamed_package.h | 1 + .../nel/misc/streamed_package_manager.h | 34 +++++++- code/nel/src/misc/file.cpp | 34 ++++++++ code/nel/src/misc/path.cpp | 82 +++++++++++++++++-- .../nel/src/misc/streamed_package_manager.cpp | 68 +++++++++++++++ 7 files changed, 230 insertions(+), 10 deletions(-) diff --git a/code/nel/include/nel/misc/app_context.h b/code/nel/include/nel/misc/app_context.h index 46d2a15c7..2388dffe1 100644 --- a/code/nel/include/nel/misc/app_context.h +++ b/code/nel/include/nel/misc/app_context.h @@ -229,6 +229,20 @@ namespace NLMISC } \ private: +#define NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(className) \ + NLMISC_SAFE_SINGLETON_DECL(className); \ + \ + void className::releaseInstance() \ + { \ + if (_Instance) \ + { \ + NLMISC::INelContext::getInstance().releaseSingletonPointer(#className, _Instance); \ + delete _Instance; \ + _Instance = NULL; \ + } \ + } \ + + /** The same as above, but generate a getInstance method that * return a pointer instead of a reference */ @@ -296,5 +310,6 @@ void initNelLibrary(CLibrary &lib); } // namespace NLMISC +#include #endif //APP_CONTEXT_H diff --git a/code/nel/include/nel/misc/path.h b/code/nel/include/nel/misc/path.h index f3120c907..1efe9a4c6 100644 --- a/code/nel/include/nel/misc/path.h +++ b/code/nel/include/nel/misc/path.h @@ -74,6 +74,9 @@ public: /** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included (the extention (Nel Big File) is used to know that it's a big file) */ void addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); + + /** Sale but for .snp (Streamed NeL Package) */ + void addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); /** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included */ void addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); @@ -371,6 +374,9 @@ public: /** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included (the extention (Nel Big File) is used to know that it's a big file) */ static void addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); + /** Same but Streamed Package */ + static void addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); + /** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included */ static void addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); diff --git a/code/nel/include/nel/misc/streamed_package.h b/code/nel/include/nel/misc/streamed_package.h index b27e6658f..c777d8a02 100644 --- a/code/nel/include/nel/misc/streamed_package.h +++ b/code/nel/include/nel/misc/streamed_package.h @@ -17,6 +17,7 @@ #ifndef NLMISC_STREAMED_PACKAGE_H #define NLMISC_STREAMED_PACKAGE_H +#include #include namespace NLMISC { diff --git a/code/nel/include/nel/misc/streamed_package_manager.h b/code/nel/include/nel/misc/streamed_package_manager.h index 0bec38584..b1c3b05e5 100644 --- a/code/nel/include/nel/misc/streamed_package_manager.h +++ b/code/nel/include/nel/misc/streamed_package_manager.h @@ -14,21 +14,49 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -#ifndef NLMISC_STREAMED_PACKAGE_H -#define NLMISC_STREAMED_PACKAGE_H +#ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H +#define NLMISC_STREAMED_PACKAGE_MANAGER_H + +#include +#include +#include namespace NLMISC { class CStreamedPackageManager { + NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(CStreamedPackageManager); + public: CStreamedPackageManager(); ~CStreamedPackageManager(); + + /// Loads a package into the package manager + /// package: ex. /games/nel/data/characters_maps_hr.snp + bool loadPackage(const std::string &package); + + /// Get a file list + void list(std::vector &fileNames, const std::string &package); + + /// Unload all packages + void unloadAll(); + + /// Get an existing file or download if necessary + /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. + /// fileName: ex. fy_hof_underwear.dds + bool getFile(std::string &filePath, const std::string &fileName); + + /// Get file size + bool getFileSize(uint32 &fileSize, const std::string &fileName); + +private: + std::map m_Packages; + std::map m_Entries; }; /* class CStreamedPackageManager */ } /* namespace NLMISC */ -#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */ +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */ /* end of file */ diff --git a/code/nel/src/misc/file.cpp b/code/nel/src/misc/file.cpp index 842d9ea51..a542a3dc2 100644 --- a/code/nel/src/misc/file.cpp +++ b/code/nel/src/misc/file.cpp @@ -23,6 +23,7 @@ #include "nel/misc/command.h" #include "nel/misc/sstring.h" #include "nel/misc/xml_pack.h" +#include "nel/misc/streamed_package_manager.h" #ifndef NL_OS_WINDOWS #include @@ -203,6 +204,39 @@ bool CIFile::open(const std::string &path, bool text) _F = CXMLPack::getInstance().getFile (path, _FileSize, _BigFileOffset, dummy, _AlwaysOpened); } } + else if (pos > 3 && path[pos-3] == 's' && path[pos-2] == 'n' && path[pos-1] == 'p') + { + nldebug("Opening a streamed package file"); + + _IsInXMLPackFile = false; + _IsInBigFile = false; + _BigFileOffset = 0; + _AlwaysOpened = false; + std::string filePath; + if (CStreamedPackageManager::getInstance().getFile (filePath, path)) + { + _F = fopen (filePath.c_str(), mode); + if (_F != NULL) + { + _FileSize=CFile::getFileSize(_F); + if (_FileSize == 0) + { + nlwarning("FILE: Size of file '%s' is 0", path.c_str()); + fclose(_F); + _F = NULL; + } + } + else + { + nlwarning("Failed to open file '%s', error %u : %s", path.c_str(), errno, strerror(errno)); + _FileSize = 0; + } + } + else + { + nlerror("File '%s' not in streamed package", path.c_str()); + } + } else { // bnp file diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index d47b8ce4a..7ed6cb54f 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -23,6 +23,7 @@ #include "nel/misc/progress_callback.h" #include "nel/misc/file.h" #include "nel/misc/xml_pack.h" +#include "nel/misc/streamed_package_manager.h" #ifdef NL_OS_WINDOWS # ifndef NL_COMP_MINGW @@ -292,6 +293,7 @@ void CFileContainer::clearMap () nlassert(!_MemoryCompressed); _Files.clear (); CBigFile::getInstance().removeAll (); + CStreamedPackageManager::getInstance().unloadAll (); NL_DISPLAY_PATH("PATH: CPath::clearMap(): map directory cleared"); } @@ -1161,16 +1163,26 @@ void CFileContainer::addSearchFile (const string &file, bool remap, const string return; } + std::string fileExtension = CFile::getExtension(newFile); + // check if it s a big file - if (CFile::getExtension(newFile) == "bnp") + if (fileExtension == "bnp") { NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is a big file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str()); addSearchBigFile(file, false, false, progressCallBack); return; } + // check if it s a streamed package + if (fileExtension == "snp") + { + NL_DISPLAY_PATH ("PATH: CPath::addSearchStreamedPackage(%s, %d, '%s'): '%s' is a streamed package, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str()); + addSearchStreamedPackage(file, false, false, progressCallBack); + return; + } + // check if it s an xml pack file - if (CFile::getExtension(newFile) == "xml_pack") + if (fileExtension == "xml_pack") { NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is an xml pack file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str()); addSearchXmlpackFile(file, false, false, progressCallBack); @@ -1391,6 +1403,52 @@ void CFileContainer::addSearchBigFile (const string &sBigFilename, bool recurse, fclose (Handle); } +void CPath::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack) +{ + getInstance()->_FileContainer.addSearchBigFile(filename, recurse, alternative, progressCallBack); +} + +void CFileContainer::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack) +{ + // Check if filename is not empty + if (filename.empty()) + { + nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): can't add empty file, skip it", filename.c_str(), recurse, alternative); + return; + } + // Check if the file exists + if (!CFile::isExists(filename)) + { + nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not found, skip it", filename.c_str(), recurse, alternative, filename.c_str()); + return; + } + // Check if it s a file + if (CFile::isDirectory(filename)) + { + nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not a file, skip it", filename.c_str(), recurse, alternative, filename.c_str()); + return; + } + + // Add the file itself + std::string packname = NLMISC::toLower(CFile::getFilename(filename)); + insertFileInMap(packname, filename, false, CFile::getExtension(filename)); + + // Add the package to the package manager + if (CStreamedPackageManager::getInstance().loadPackage(filename)) + { + std::vector fileNames; + CStreamedPackageManager::getInstance().list(fileNames, packname); + + for (std::vector::iterator it(fileNames.begin()), end(fileNames.end()); it != end; ++it) + { + // Add the file to the lookup + std::string filePackageName = packname + "@" + (*it); + nldebug("Insert '%s'", filePackageName.c_str()); + insertFileInMap((*it), filePackageName, false, CFile::getExtension(*it)); + } + } +} + // WARNING : recurse is not used void CPath::addSearchXmlpackFile (const string &sXmlpackFilename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack) { @@ -1983,6 +2041,7 @@ string CFile::findNewFile (const string &filename) // \warning doesn't work with big file uint32 CFile::getFileSize (const std::string &filename) { + std::string::size_type pos; if (filename.find("@@") != string::npos) { uint32 fs = 0, bfo; @@ -1990,12 +2049,21 @@ uint32 CFile::getFileSize (const std::string &filename) CXMLPack::getInstance().getFile (filename, fs, bfo, c, d); return fs; } - else if (filename.find('@') != string::npos) + else if ((pos = filename.find('@')) != string::npos) { - uint32 fs = 0, bfo; - bool c, d; - CBigFile::getInstance().getFile (filename, fs, bfo, c, d); - return fs; + if (pos > 3 && filename[pos-3] == 's' && filename[pos-2] == 'n' && filename[pos-1] == 'p') + { + uint32 fs = 0; + CStreamedPackageManager::getInstance().getFileSize (fs, filename); + return fs; + } + else + { + uint32 fs = 0, bfo; + bool c, d; + CBigFile::getInstance().getFile (filename, fs, bfo, c, d); + return fs; + } } else { diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 2d04cb95a..4a5862130 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -18,9 +18,13 @@ // Project includes #include +#include +#include namespace NLMISC { +NLMISC_SAFE_SINGLETON_IMPL(CStreamedPackageManager); + CStreamedPackageManager::CStreamedPackageManager() { // init @@ -31,6 +35,70 @@ CStreamedPackageManager::~CStreamedPackageManager() // release } +bool CStreamedPackageManager::loadPackage(const std::string &package) +{ + nldebug("Load package '%s'", package.c_str()); + + std::string packname = NLMISC::toLower(CFile::getFilename(package)); + m_Packages[packname] = CStreamedPackage(); + std::map::iterator it = m_Packages.find(packname); + CStreamedPackage &p = it->second; + + try + { + CIFile fi; + fi.open(package); + fi.serial(p); + } + catch (Exception &e) + { + nlerror("Package serial exception: '%s'", e.what()); + m_Packages.erase(it); + return false; + } + + for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + m_Entries[entry.Name] = &entry; + } + + return true; +} + +void CStreamedPackageManager::list(std::vector &fileNames, const std::string &package) +{ + nldebug("List package '%s'", package.c_str()); + + std::map::iterator it = m_Packages.find(package); + CStreamedPackage &p = it->second; + + for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + fileNames.push_back(entry.Name); + } +} + +void CStreamedPackageManager::unloadAll() +{ + m_Packages.clear(); + m_Entries.clear(); +} + +bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName) +{ + // .. :) + nldebug("Get file path for streamed file '%s'", fileName.c_str()); + return false; +} + +bool CStreamedPackageManager::getFileSize(uint32 &fileSize, const std::string &fileName) +{ + nldebug("Get file size for streamed file '%s'", fileName.c_str()); + return false; +} + } /* namespace NLMISC */ /* end of file */ From 105ecbff90bf1ea4b0ac5c0ba5c8635c10594038 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 24 Sep 2014 21:52:41 +0200 Subject: [PATCH 05/20] Add pipeline script for creating streamed packages --HG-- branch : feature-streamed-package --- .../build_gamedata/configuration/tools.py | 1 + .../tools/build_gamedata/d1_client_patch.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/code/nel/tools/build_gamedata/configuration/tools.py b/code/nel/tools/build_gamedata/configuration/tools.py index c0c962360..bf904072e 100755 --- a/code/nel/tools/build_gamedata/configuration/tools.py +++ b/code/nel/tools/build_gamedata/configuration/tools.py @@ -87,6 +87,7 @@ BuildClodBankTool = "build_clod_bank" SheetsPackerTool = "sheets_packer" SheetsPackerShardTool = "sheets_packer_shard" BnpMakeTool = "bnp_make" +SnpMakeTool = "snp_make" AiBuildWmapTool = "ai_build_wmap" TgaCutTool = "tga_cut" PatchGenTool = "patch_gen" diff --git a/code/nel/tools/build_gamedata/d1_client_patch.py b/code/nel/tools/build_gamedata/d1_client_patch.py index 600347d0b..4b4f37072 100755 --- a/code/nel/tools/build_gamedata/d1_client_patch.py +++ b/code/nel/tools/build_gamedata/d1_client_patch.py @@ -51,6 +51,7 @@ printLog(log, "") # Find tools BnpMake = findTool(log, ToolDirectories, BnpMakeTool, ToolSuffix) +SnpMake = findTool(log, ToolDirectories, SnpMakeTool, ToolSuffix); PatchGen = findTool(log, ToolDirectories, PatchGenTool, ToolSuffix) printLog(log, "") @@ -121,11 +122,14 @@ else: targetPath = ClientPatchDirectory + "/bnp" mkPath(log, targetPath) for category in InstallClientData: + packExt = ".bnp" + if (category["StreamedPackages"]): + packExt = ".snp" for package in category["Packages"]: printLog(log, "PACKAGE " + package[0]) sourcePath = InstallDirectory + "/" + package[0] mkPath(log, sourcePath) - targetBnp = targetPath + "/" + package[0] + ".bnp" + targetBnp = targetPath + "/" + package[0] + packExt if (len(package[1]) > 0): targetBnp = targetPath + "/" + package[1][0] printLog(log, "TARGET " + package[1][0]) @@ -135,8 +139,16 @@ else: else: needUpdateBnp = needUpdateDirNoSubdirFile(log, sourcePath, targetBnp) if (needUpdateBnp): - printLog(log, "BNP " + targetBnp) - subprocess.call([ BnpMake, "/p", sourcePath, targetPath ] + package[1]) + if (category["StreamedPackages"]): + printLog(log, "SNP " + targetBnp) + cwDir = os.getcwd().replace("\\", "/") + toolDir = os.path.dirname(Lzma).replace("\\", "/") + os.chdir(toolDir) + subprocess.call([ SnpMake, "/p", sourcePath, targetBnp, ClientPatchDirectory + "/stream" ] + package[1]) + os.chdir(cwDir) + else: + printLog(log, "BNP " + targetBnp) + subprocess.call([ BnpMake, "/p", sourcePath, targetPath, targetBnp ] + package[1]) else: printLog(log, "SKIP " + targetBnp) printLog(log, "") From e6a327fb1ab0deb02816fec8e8f39ec3efa2d58d Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 24 Sep 2014 23:40:43 +0200 Subject: [PATCH 06/20] Implementation fixes, ref #179 --HG-- branch : feature-streamed-package --- code/nel/include/nel/misc/app_context.h | 20 ++++++++++--------- code/nel/src/misc/file.cpp | 2 +- code/nel/src/misc/path.cpp | 12 ++++++++--- .../tools/build_gamedata/d2_client_install.py | 7 +++++-- code/ryzom/client/src/login.cpp | 2 ++ code/ryzom/client/src/release.cpp | 2 ++ .../screenshot_islands.cpp | 2 +- .../client/src/snowballs_client.cpp | 1 + 8 files changed, 32 insertions(+), 16 deletions(-) diff --git a/code/nel/include/nel/misc/app_context.h b/code/nel/include/nel/misc/app_context.h index 2388dffe1..3ab77aada 100644 --- a/code/nel/include/nel/misc/app_context.h +++ b/code/nel/include/nel/misc/app_context.h @@ -230,17 +230,19 @@ namespace NLMISC private: #define NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(className) \ - NLMISC_SAFE_SINGLETON_DECL(className); \ - \ - void className::releaseInstance() \ - { \ - if (_Instance) \ + NLMISC_SAFE_SINGLETON_DECL(className); \ + \ + public: \ + static void className::releaseInstance() \ { \ - NLMISC::INelContext::getInstance().releaseSingletonPointer(#className, _Instance); \ - delete _Instance; \ - _Instance = NULL; \ + if (_Instance) \ + { \ + NLMISC::INelContext::getInstance().releaseSingletonPointer(#className, _Instance); \ + delete _Instance; \ + _Instance = NULL; \ + } \ } \ - } \ + private: /** The same as above, but generate a getInstance method that diff --git a/code/nel/src/misc/file.cpp b/code/nel/src/misc/file.cpp index a542a3dc2..e2168def6 100644 --- a/code/nel/src/misc/file.cpp +++ b/code/nel/src/misc/file.cpp @@ -213,7 +213,7 @@ bool CIFile::open(const std::string &path, bool text) _BigFileOffset = 0; _AlwaysOpened = false; std::string filePath; - if (CStreamedPackageManager::getInstance().getFile (filePath, path)) + if (CStreamedPackageManager::getInstance().getFile (filePath, path.substr(pos+1))) { _F = fopen (filePath.c_str(), mode); if (_F != NULL) diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index 7ed6cb54f..b7d58df51 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -1736,7 +1736,8 @@ void CFileContainer::memoryCompress() while (it != _Files.end()) { string sTmp = SSMpath.get(it->second.idPath); - if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && !it->second.Remapped) + nldebug("A: %s", sTmp.c_str()); + if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && (sTmp.find("snp@") == string::npos) && !it->second.Remapped) { // This is a file included in a bigfile (so the name is in the bigfile manager) } @@ -1759,7 +1760,8 @@ void CFileContainer::memoryCompress() { CFileEntry &rFE = it->second; string sTmp = SSMpath.get(rFE.idPath); - if (sTmp.find("@") == string::npos || sTmp.find("@@") != string::npos || rFE.Remapped) + nldebug("B: %s", sTmp.c_str()); + if ((sTmp.find("@") == string::npos) || (sTmp.find("@@") != string::npos) || (sTmp.find("snp@") != string::npos) || rFE.Remapped) { strcpy(_AllFileNames+nSize, rFE.Name.c_str()); _MCFiles[nNb].Name = _AllFileNames+nSize; @@ -1784,8 +1786,12 @@ void CFileContainer::memoryCompress() it++; } + nldebug("Passed"); + contReset(_Files); _MemoryCompressed = true; + + nldebug("Done"); } void CPath::memoryUncompress() @@ -2054,7 +2060,7 @@ uint32 CFile::getFileSize (const std::string &filename) if (pos > 3 && filename[pos-3] == 's' && filename[pos-2] == 'n' && filename[pos-1] == 'p') { uint32 fs = 0; - CStreamedPackageManager::getInstance().getFileSize (fs, filename); + CStreamedPackageManager::getInstance().getFileSize (fs, filename.substr(pos+1)); return fs; } else diff --git a/code/nel/tools/build_gamedata/d2_client_install.py b/code/nel/tools/build_gamedata/d2_client_install.py index febaef656..bb399447f 100755 --- a/code/nel/tools/build_gamedata/d2_client_install.py +++ b/code/nel/tools/build_gamedata/d2_client_install.py @@ -47,6 +47,9 @@ printLog(log, "") for category in InstallClientData: printLog(log, "CATEGORY " + category["Name"]) + packExt = ".bnp" + if (category["StreamedPackages"]): + packExt = ".snp" if (category["UnpackTo"] != None): targetPath = ClientInstallDirectory if (category["UnpackTo"] != ""): @@ -62,8 +65,8 @@ for category in InstallClientData: mkPath(log, targetPath) for package in category["Packages"]: printLog(log, "PACKAGE " + package[0]) - sourceBnp = sourcePath + "/" + package[0] + ".bnp" - targetBnp = targetPath + "/" + package[0] + ".bnp" + sourceBnp = sourcePath + "/" + package[0] + packExt + targetBnp = targetPath + "/" + package[0] + packExt if (len(package[1]) > 0): sourceBnp = sourcePath + "/" + package[1][0] targetBnp = targetPath + "/" + package[1][0] diff --git a/code/ryzom/client/src/login.cpp b/code/ryzom/client/src/login.cpp index 091e24a80..e4d9b18b5 100644 --- a/code/ryzom/client/src/login.cpp +++ b/code/ryzom/client/src/login.cpp @@ -28,6 +28,7 @@ #include "nel/misc/thread.h" #include "nel/misc/big_file.h" #include "nel/misc/system_utils.h" +#include "nel/misc/streamed_package_manager.h" #include "nel/net/tcp_sock.h" #include "nel/3d/u_driver.h" @@ -1614,6 +1615,7 @@ void initPatch() CBGDownloaderAccess::getInstance().startTask(taskDesc, string(), true /* showDownloader */); // no command line since bg downloader should already be started // release lock on bnp, so that they can be written NLMISC::CBigFile::getInstance().removeAll(); + NLMISC::CStreamedPackageManager::getInstance().unloadAll(); } NLGUI::CDBManager::getInstance()->getDbProp("UI:VARIABLES:SCREEN")->setValue32(UI_VARIABLES_SCREEN_PATCHING); diff --git a/code/ryzom/client/src/release.cpp b/code/ryzom/client/src/release.cpp index f88849197..403b654e9 100644 --- a/code/ryzom/client/src/release.cpp +++ b/code/ryzom/client/src/release.cpp @@ -25,6 +25,7 @@ #include "nel/misc/debug.h" #include "nel/misc/async_file_manager.h" #include "nel/misc/system_utils.h" +#include "nel/misc/streamed_package_manager.h" // 3D Interface. #include "nel/3d/bloom_effect.h" #include "nel/3d/fxaa.h" @@ -644,6 +645,7 @@ void release() R2::CObjectSerializer::releaseInstance(); NLMISC::CBigFile::getInstance().removeAll(); NLMISC::CBigFile::releaseInstance(); + NLMISC::CStreamedPackageManager::releaseInstance(); NL3D::CFastHLSModifier::releaseInstance(); CLandscapePolyDrawer::releaseInstance(); NL3D::CParticleSystemShape::releaseInstance(); diff --git a/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp b/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp index 4f167ea9c..d678c0b17 100644 --- a/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp +++ b/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp @@ -295,7 +295,7 @@ void CScreenshotIslands::searchIslandsBorders() zonelFiles.clear(); string bnpFileName = itCont->first + ".bnp"; - CBigFile::getInstance().list(bnpFileName.c_str(), filenames); + CBigFile::getInstance().list(bnpFileName.c_str(), filenames); // FIXME FIXME NOT READING FROM BNP! for(uint i=0; i Date: Wed, 24 Sep 2014 23:41:40 +0200 Subject: [PATCH 07/20] Remove some debugging, ref #179 --HG-- branch : feature-streamed-package --- code/nel/src/misc/path.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index b7d58df51..56249f637 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -1736,7 +1736,6 @@ void CFileContainer::memoryCompress() while (it != _Files.end()) { string sTmp = SSMpath.get(it->second.idPath); - nldebug("A: %s", sTmp.c_str()); if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && (sTmp.find("snp@") == string::npos) && !it->second.Remapped) { // This is a file included in a bigfile (so the name is in the bigfile manager) @@ -1760,7 +1759,6 @@ void CFileContainer::memoryCompress() { CFileEntry &rFE = it->second; string sTmp = SSMpath.get(rFE.idPath); - nldebug("B: %s", sTmp.c_str()); if ((sTmp.find("@") == string::npos) || (sTmp.find("@@") != string::npos) || (sTmp.find("snp@") != string::npos) || rFE.Remapped) { strcpy(_AllFileNames+nSize, rFE.Name.c_str()); @@ -1786,12 +1784,8 @@ void CFileContainer::memoryCompress() it++; } - nldebug("Passed"); - contReset(_Files); _MemoryCompressed = true; - - nldebug("Done"); } void CPath::memoryUncompress() From fb7955103b99ae955c1d785347b9e058c0ba0bf6 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 25 Sep 2014 17:39:51 +0200 Subject: [PATCH 08/20] Implement streamed data download, ref #179 --HG-- branch : feature-streamed-package --- code/nel/include/nel/misc/path.h | 4 +- .../nel/misc/streamed_package_manager.h | 16 +- code/nel/src/misc/file.cpp | 1 + code/nel/src/misc/path.cpp | 14 +- .../nel/src/misc/streamed_package_manager.cpp | 174 +++++++++++++++++- .../tools/build_gamedata/d1_client_patch.py | 4 +- code/ryzom/client/src/client_cfg.cpp | 15 +- code/ryzom/client/src/client_cfg.h | 4 + code/ryzom/client/src/init.cpp | 10 + code/ryzom/client/src/init.h | 1 + code/ryzom/client/src/streamable_ig.cpp | 2 + 11 files changed, 230 insertions(+), 15 deletions(-) diff --git a/code/nel/include/nel/misc/path.h b/code/nel/include/nel/misc/path.h index 1efe9a4c6..b6694c7a1 100644 --- a/code/nel/include/nel/misc/path.h +++ b/code/nel/include/nel/misc/path.h @@ -279,8 +279,8 @@ private: struct CMCFileEntry { char *Name; // Normal case (the search is done by using nlstricmp) - uint32 idPath : 16; // Path (not with file at the end) - look in the SSMpath (65536 different path allowed) - uint32 idExt : 15; // real extension of the file if remapped - look in the SSMext (32768 different extension allowed) + uint32 idPath : 20; // Path (not with file at the end) - look in the SSMpath (1048576 different path allowed) + uint32 idExt : 11; // real extension of the file if remapped - look in the SSMext (2048 different extension allowed) uint32 Remapped : 1; // true if the file is remapped }; diff --git a/code/nel/include/nel/misc/streamed_package_manager.h b/code/nel/include/nel/misc/streamed_package_manager.h index b1c3b05e5..e23344a70 100644 --- a/code/nel/include/nel/misc/streamed_package_manager.h +++ b/code/nel/include/nel/misc/streamed_package_manager.h @@ -31,7 +31,6 @@ public: CStreamedPackageManager(); ~CStreamedPackageManager(); - /// Loads a package into the package manager /// package: ex. /games/nel/data/characters_maps_hr.snp bool loadPackage(const std::string &package); @@ -49,9 +48,20 @@ public: /// Get file size bool getFileSize(uint32 &fileSize, const std::string &fileName); +public: + /// Set storage path (ex. stream/) + std::string Path; + + /// Loads a package into the package manager (ex. http://patch.live.polyverse.org/stream/) + typedef std::vector THosts; + THosts Hosts; + private: - std::map m_Packages; - std::map m_Entries; + typedef std::map TPackages; + TPackages m_Packages; + + typedef std::map TEntries; + TEntries m_Entries; }; /* class CStreamedPackageManager */ diff --git a/code/nel/src/misc/file.cpp b/code/nel/src/misc/file.cpp index e2168def6..b53a660d9 100644 --- a/code/nel/src/misc/file.cpp +++ b/code/nel/src/misc/file.cpp @@ -234,6 +234,7 @@ bool CIFile::open(const std::string &path, bool text) } else { + // TEMPORARY ERROR nlerror("File '%s' not in streamed package", path.c_str()); } } diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index 56249f637..553f8e4ba 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -342,7 +342,7 @@ void CFileContainer::remapExtension (const string &ext1, const string &ext2, boo nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): can't remap empty extension", ext1lwr.c_str(), ext2lwr.c_str(), substitute); } - if (ext1lwr == "bnp" || ext2lwr == "bnp") + if (ext1lwr == "bnp" || ext2lwr == "bnp" || ext1lwr == "snp" || ext2lwr == "snp") { nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): you can't remap a big file", ext1lwr.c_str(), ext2lwr.c_str(), substitute); } @@ -1445,6 +1445,18 @@ void CFileContainer::addSearchStreamedPackage (const string &filename, bool recu std::string filePackageName = packname + "@" + (*it); nldebug("Insert '%s'", filePackageName.c_str()); insertFileInMap((*it), filePackageName, false, CFile::getExtension(*it)); + + // Remapped extensions + std::string ext = CFile::getExtension(*it); + for (uint j = 0; j < _Extensions.size (); j++) + { + if (_Extensions[j].first == ext) + { + // Need to remap + insertFileInMap(CFile::getFilenameWithoutExtension(*it) + "." + _Extensions[j].second, + filePackageName, true, _Extensions[j].first); + } + } } } } diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 4a5862130..250c92d07 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -16,6 +16,13 @@ #include "stdmisc.h" +// 3rd Party includes +#include +#include +#include +#include +#include + // Project includes #include #include @@ -88,15 +95,174 @@ void CStreamedPackageManager::unloadAll() bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName) { - // .. :) nldebug("Get file path for streamed file '%s'", fileName.c_str()); - return false; + + TEntries::iterator it = m_Entries.find(fileName); + if (it == m_Entries.end()) + return false; + const CStreamedPackage::CEntry *entry = it->second; + + CStreamedPackage::makePath(filePath, entry->Hash); + std::string downloadUrlFile = filePath + ".lzma"; + filePath = Path + filePath; + std::string downloadPath = filePath + ".download." + toString(rand()); + + std::string storageDirectory = CFile::getPath(downloadPath); + CFile::createDirectoryTree(storageDirectory); + /*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory)) + { + nldebug("Unable to create directory '%s'", storageDirectory.c_str()); + return false; + }*/ + + // download + for (; ; ) + { + if (CFile::fileExists(filePath)) + return true; + + std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile; + nldebug("Download streamed package '%s' from '%s'", fileName.c_str(), downloadUrl.c_str()); + + FILE *fp = fopen(downloadPath.c_str(), "wb"); + if (fp == NULL) + { + nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), fileName.c_str()); + return false; + } + + CURL *curl; + CURLcode res; + curl = curl_easy_init(); + if (curl) + { + curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_FILE, fp); + res = curl_easy_perform(curl); + long r; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r); + curl_easy_cleanup(curl); + + bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/; + fclose(fp); + + if (diskFull) + { + CFile::deleteFile(downloadPath); + throw EDiskFullError(downloadPath); + } + + if (res != CURLE_OK || r < 200 || r >= 300) + { + CFile::deleteFile(downloadPath); + nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str()); + nlSleep(1000); + continue; + } + } + else + { + nldebug("Curl initialize failed"); + fclose(fp); + CFile::deleteFile(downloadPath); + return false; + } + + // ok! + break; + } + + // read into memory :( + std::string unpackPath = filePath + ".extract." + toString(rand()); + + std::vector inBuffer; + { + CIFile inStream(downloadPath); + uint32 inSize = inStream.getFileSize(); + inBuffer.resize(inSize); + inStream.serialBuffer(&inBuffer[0], inSize); + } + CFile::deleteFile(downloadPath); + + if (inBuffer.size() < LZMA_PROPERTIES_SIZE + 8) + { + nlwarning("Invalid file size %u, too small file '%s'", inBuffer.size(), downloadPath.c_str()); + return false; + } + + // extract + { + CLzmaDecoderState state; + uint8 *pos = &inBuffer[0]; + int ret = LzmaDecodeProperties(&state.Properties, (unsigned char *)pos, LZMA_PROPERTIES_SIZE); + if (ret != 0) + { + nlwarning("Failed to decode lzma properies in file '%s'", downloadPath.c_str()); + return false; + } + + // FROM login_patch.cpp + // alloc the probs, making sure they are deleted in function exit + size_t nbProb = LzmaGetNumProbs(&state.Properties); + std::vector probs; + probs.resize(nbProb); + state.Probs = &probs[0]; + + pos += LZMA_PROPERTIES_SIZE; + + // read the output file size + size_t fileSize = 0; + for (int i = 0; i < 8; i++) + { + //Byte b; + if (pos >= &inBuffer[0] + inBuffer.size()) + { + nlerror("pos >= inBuffer.get() + inSize"); + return false; + } + fileSize |= ((UInt64)*pos++) << (8 * i); + } + + nlassert(fileSize == entry->Size); + + SizeT outProcessed = 0; + SizeT inProcessed = 0; + // allocate the output buffer :( + std::vector outBuffer; + outBuffer.resize(fileSize); + // decompress the file in memory + ret = LzmaDecode(&state, (unsigned char *)pos, (SizeT)(inBuffer.size() - (pos - &inBuffer[0])), &inProcessed, (unsigned char*)&outBuffer[0], (SizeT)fileSize, &outProcessed); + if (ret != 0 || outProcessed != fileSize) + { + nlwarning("Failed to decode lzma file '%s'", downloadPath.c_str()); + return false; + } + + { + COFile outStream(unpackPath); + outStream.serialBuffer(&outBuffer[0], (uint)fileSize); + } + + if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str())) + { + nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str()); + // in case downloaded from another thread + return CFile::fileExists(filePath); + } + + return true; + } } bool CStreamedPackageManager::getFileSize(uint32 &fileSize, const std::string &fileName) { - nldebug("Get file size for streamed file '%s'", fileName.c_str()); - return false; + // nldebug("Get file size for streamed file '%s'", fileName.c_str()); + TEntries::iterator it = m_Entries.find(fileName); + if (it == m_Entries.end()) + return false; + fileSize = it->second->Size; + return true; } } /* namespace NLMISC */ diff --git a/code/nel/tools/build_gamedata/d1_client_patch.py b/code/nel/tools/build_gamedata/d1_client_patch.py index 4b4f37072..080141e5f 100755 --- a/code/nel/tools/build_gamedata/d1_client_patch.py +++ b/code/nel/tools/build_gamedata/d1_client_patch.py @@ -97,7 +97,7 @@ else: if category["UnpackTo"] != "": cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./" + category["UnpackTo"] + "/\"/>\n") else: - cfg.write("\t\t\t<_UnpackTo type=\"SINT32\" value=\"./\"/>\n") + cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./\"/>\n") cfg.write("\t\t\t<_IsOptional type=\"SINT32\" value=\"" + str(category["IsOptional"]) + "\"/>\n") cfg.write("\t\t\t<_IsIncremental type=\"SINT32\" value=\"" + str(category["IsIncremental"]) + "\"/>\n") for package in category["Packages"]: @@ -148,7 +148,7 @@ else: os.chdir(cwDir) else: printLog(log, "BNP " + targetBnp) - subprocess.call([ BnpMake, "/p", sourcePath, targetPath, targetBnp ] + package[1]) + subprocess.call([ BnpMake, "/p", sourcePath, targetPath ] + package[1]) else: printLog(log, "SKIP " + targetBnp) printLog(log, "") diff --git a/code/ryzom/client/src/client_cfg.cpp b/code/ryzom/client/src/client_cfg.cpp index 6cde86cba..5b69b284e 100644 --- a/code/ryzom/client/src/client_cfg.cpp +++ b/code/ryzom/client/src/client_cfg.cpp @@ -464,6 +464,8 @@ CClientConfig::CClientConfig() ColorShout = CRGBA(150,0,0,255); // Default Shout color. ColorTalk = CRGBA(255,255,255,255); // Default Talk color. + StreamedPackagePath = "stream"; + // PreDataPath.push_back("data/gamedev/language/"); // Default Path for the language data // DataPath.push_back("data/"); // Default Path for the Data. @@ -1247,18 +1249,25 @@ void CClientConfig::setValues() ////////// // MISC // + // Pre Data Path. READ_STRINGVECTOR_FV(PreDataPath); // Data Path. READ_STRINGVECTOR_FV(DataPath); - // List of files that trigger R2ED reload when touched - READ_STRINGVECTOR_FV(R2EDReloadFiles); - // Data Path no recurse. READ_STRINGVECTOR_FV(DataPathNoRecurse); + // Streamed package path + READ_STRING_FV(StreamedPackagePath); + + // Streamed package hosts + READ_STRINGVECTOR_FV(StreamedPackageHosts); + + // List of files that trigger R2ED reload when touched + READ_STRINGVECTOR_FV(R2EDReloadFiles); + // Update packed sheet Path READ_STRINGVECTOR_FV(UpdatePackedSheetPath); diff --git a/code/ryzom/client/src/client_cfg.h b/code/ryzom/client/src/client_cfg.h index 7d7f1c65d..59b4f96a5 100644 --- a/code/ryzom/client/src/client_cfg.h +++ b/code/ryzom/client/src/client_cfg.h @@ -360,6 +360,10 @@ struct CClientConfig std::vector DataPath; /// Data Path no recurse. std::vector DataPathNoRecurse; + /// Streamed package path + std::string StreamedPackagePath; + /// Streamed package hosts + std::vector StreamedPackageHosts; // TODO: From 'domain' SQL table /// Update packed sheet Path. std::vector UpdatePackedSheetPath; /// True if we want the packed sheet to be updated if needed diff --git a/code/ryzom/client/src/init.cpp b/code/ryzom/client/src/init.cpp index 1644f7e2d..6a1cf5bcf 100644 --- a/code/ryzom/client/src/init.cpp +++ b/code/ryzom/client/src/init.cpp @@ -34,6 +34,7 @@ #include "nel/misc/system_info.h" #include "nel/misc/block_memory.h" #include "nel/misc/system_utils.h" +#include "nel/misc/streamed_package_manager.h" // 3D Interface. #include "nel/3d/bloom_effect.h" #include "nel/3d/u_driver.h" @@ -664,6 +665,14 @@ void initStereoDisplayDevice() IStereoDisplay::releaseUnusedLibraries(); } +void initStreamedPackageManager(NLMISC::IProgressCallback &progress) +{ + CStreamedPackageManager &spm = CStreamedPackageManager::getInstance(); + spm.Path = replaceApplicationDirToken(ClientCfg.StreamedPackagePath); + for (uint i = 0; i < ClientCfg.StreamedPackageHosts.size(); i++) + spm.Hosts.push_back(ClientCfg.StreamedPackageHosts[i]); +} + void addSearchPaths(IProgressCallback &progress) { // Add search path of UI addon. Allow only a subset of files. @@ -848,6 +857,7 @@ void prelogInit() CPath::remapExtension ("png", "tga", true); FPU_CHECKER_ONCE + initStreamedPackageManager(ProgressBar); addPreDataPaths(ProgressBar); FPU_CHECKER_ONCE diff --git a/code/ryzom/client/src/init.h b/code/ryzom/client/src/init.h index cdcdfb418..cf1e22f6e 100644 --- a/code/ryzom/client/src/init.h +++ b/code/ryzom/client/src/init.h @@ -33,6 +33,7 @@ void prelogInit(); // Initialize the application after login step void postlogInit(); +void initStreamedPackageManager(NLMISC::IProgressCallback &progress); void addSearchPaths(NLMISC::IProgressCallback &progress); void addPreDataPaths(NLMISC::IProgressCallback &progress); diff --git a/code/ryzom/client/src/streamable_ig.cpp b/code/ryzom/client/src/streamable_ig.cpp index f73df6126..99930b947 100644 --- a/code/ryzom/client/src/streamable_ig.cpp +++ b/code/ryzom/client/src/streamable_ig.cpp @@ -89,8 +89,10 @@ CStreamableIG::~CStreamableIG() H_AUTO_USE(RZ_StremableIG) if (!_Linked) { + #ifdef NL_DEBUG if(!ClientCfg.Light) nlwarning("Loading async %p", this); + #endif #ifdef NL_DEBUG //nlinfo("Loading async : %s", Name.c_str()); #endif From 85615992d784756999b45c7321cbdd2c093b6bed Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 25 Sep 2014 19:47:57 +0200 Subject: [PATCH 09/20] Remove some debugging --HG-- branch : feature-streamed-package --- code/nel/src/misc/CMakeLists.txt | 2 +- code/nel/src/misc/file.cpp | 2 +- code/nel/src/misc/streamed_package_manager.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/code/nel/src/misc/CMakeLists.txt b/code/nel/src/misc/CMakeLists.txt index 8494c23a6..7f172bbfe 100644 --- a/code/nel/src/misc/CMakeLists.txt +++ b/code/nel/src/misc/CMakeLists.txt @@ -39,7 +39,7 @@ ENDIF(UNIX) INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR} ${PNG_INCLUDE_DIR} config_file) -TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} nel_sevenzip) +TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARIES} nel_sevenzip) SET_TARGET_PROPERTIES(nelmisc PROPERTIES LINK_INTERFACE_LIBRARIES "") NL_DEFAULT_PROPS(nelmisc "NeL, Library: NeL Misc") NL_ADD_RUNTIME_FLAGS(nelmisc) diff --git a/code/nel/src/misc/file.cpp b/code/nel/src/misc/file.cpp index b53a660d9..9197b35a1 100644 --- a/code/nel/src/misc/file.cpp +++ b/code/nel/src/misc/file.cpp @@ -206,7 +206,7 @@ bool CIFile::open(const std::string &path, bool text) } else if (pos > 3 && path[pos-3] == 's' && path[pos-2] == 'n' && path[pos-1] == 'p') { - nldebug("Opening a streamed package file"); + // nldebug("Opening a streamed package file"); _IsInXMLPackFile = false; _IsInBigFile = false; diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 250c92d07..ffe33e76c 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -95,7 +95,7 @@ void CStreamedPackageManager::unloadAll() bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName) { - nldebug("Get file path for streamed file '%s'", fileName.c_str()); + // nldebug("Get file path for streamed file '%s'", fileName.c_str()); TEntries::iterator it = m_Entries.find(fileName); if (it == m_Entries.end()) From 40f4eb4ad2255e75ab3166a962732d4c2e019cba Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 25 Sep 2014 20:10:16 +0200 Subject: [PATCH 10/20] Fix package description generation --HG-- branch : feature-streamed-package --- .../tools/build_gamedata/d1_client_patch.py | 5 +++- code/ryzom/client/src/login_patch.cpp | 24 +------------------ 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/code/nel/tools/build_gamedata/d1_client_patch.py b/code/nel/tools/build_gamedata/d1_client_patch.py index 080141e5f..47a952812 100755 --- a/code/nel/tools/build_gamedata/d1_client_patch.py +++ b/code/nel/tools/build_gamedata/d1_client_patch.py @@ -91,6 +91,9 @@ else: inCategories = 1 cfg.write("\t<_Categories>\n") for category in InstallClientData: + packExt = ".bnp" + if (category["StreamedPackages"]): + packExt = ".snp" cfg.write("\t\t<_Category>\n") cfg.write("\t\t\t<_Name type=\"STRING\" value=\"" + category["Name"] + "\"/>\n") if category["UnpackTo"] != None: @@ -104,7 +107,7 @@ else: if (len(package[1]) > 0): cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[1][0] + "\"/>\n") else: - cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[0] + ".bnp\"/>\n") + cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[0] + packExt + "\"/>\n") for ref in category["Refs"]: cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + ref + "_.ref\"/>\n") cfg.write("\t\t\n") diff --git a/code/ryzom/client/src/login_patch.cpp b/code/ryzom/client/src/login_patch.cpp index d2f4d02e0..657eb8fb3 100644 --- a/code/ryzom/client/src/login_patch.cpp +++ b/code/ryzom/client/src/login_patch.cpp @@ -1176,35 +1176,13 @@ void CPatchManager::readDescFile(sint32 nVersion) std::string unpackTo = category.getUnpackTo(); - if (unpackTo == "0") - { - nlwarning("BUG: unpackTo == '0'"); - unpackTo = ClientRootPath; - category.setUnpackTo(unpackTo); - } - else if (unpackTo.substr(0, 2) == "./") + if (unpackTo.substr(0, 2) == "./") { unpackTo = ClientRootPath + unpackTo.substr(2); category.setUnpackTo(unpackTo); } } } - else - { - for (cat = 0; cat < DescFile.getCategories().categoryCount(); ++cat) - { - CBNPCategory &category = const_cast(DescFile.getCategories().getCategory(cat)); - - std::string unpackTo = category.getUnpackTo(); - - if (unpackTo == "0") - { - nlwarning("BUG: unpackTo == '0'"); - unpackTo = "./"; - category.setUnpackTo(unpackTo); - } - } - } // tmp for debug : flag some categories as 'Mainland' for (cat = 0; cat < DescFile.getCategories().categoryCount(); ++cat) From 9a1510caf07e25ca7bbd44ee5a151f37aae296e1 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 25 Sep 2014 20:25:54 +0200 Subject: [PATCH 11/20] Fix patching --HG-- branch : feature-streamed-package --- code/ryzom/client/src/login_patch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/ryzom/client/src/login_patch.cpp b/code/ryzom/client/src/login_patch.cpp index 657eb8fb3..949527e82 100644 --- a/code/ryzom/client/src/login_patch.cpp +++ b/code/ryzom/client/src/login_patch.cpp @@ -2688,7 +2688,7 @@ void CPatchThread::run() CPatchManager::SFileToPatch &rFTP = AllFilesToPatch[i]; string ext = NLMISC::CFile::getExtension(rFTP.FileName); - if (ext == "bnp") + if (ext == "bnp" || ext == "snp") { float oldCurrentFilePatched = CurrentFilePatched; processFile (rFTP); From 8d93e9308faea54c5775f25e42987c3f916ba04c Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 25 Sep 2014 23:57:50 +0200 Subject: [PATCH 12/20] Remove some debugging --HG-- branch : feature-streamed-package --- code/nel/src/misc/file.cpp | 2 +- code/nel/src/misc/path.cpp | 2 +- code/nel/src/misc/streamed_package_manager.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/code/nel/src/misc/file.cpp b/code/nel/src/misc/file.cpp index 9197b35a1..757f7bd49 100644 --- a/code/nel/src/misc/file.cpp +++ b/code/nel/src/misc/file.cpp @@ -235,7 +235,7 @@ bool CIFile::open(const std::string &path, bool text) else { // TEMPORARY ERROR - nlerror("File '%s' not in streamed package", path.c_str()); + // nlerror("File '%s' not in streamed package", path.c_str()); } } else diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index 553f8e4ba..4d8ad6969 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -1443,7 +1443,7 @@ void CFileContainer::addSearchStreamedPackage (const string &filename, bool recu { // Add the file to the lookup std::string filePackageName = packname + "@" + (*it); - nldebug("Insert '%s'", filePackageName.c_str()); + // nldebug("Insert '%s'", filePackageName.c_str()); insertFileInMap((*it), filePackageName, false, CFile::getExtension(*it)); // Remapped extensions diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index ffe33e76c..fbdfc1f5a 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -75,7 +75,7 @@ bool CStreamedPackageManager::loadPackage(const std::string &package) void CStreamedPackageManager::list(std::vector &fileNames, const std::string &package) { - nldebug("List package '%s'", package.c_str()); + // nldebug("List package '%s'", package.c_str()); std::map::iterator it = m_Packages.find(package); CStreamedPackage &p = it->second; From 242476e1f6294c5258d13dae5b88a9f3bbd95f4b Mon Sep 17 00:00:00 2001 From: kaetemi Date: Mon, 24 Nov 2014 14:54:15 +0100 Subject: [PATCH 13/20] Fix library linking --HG-- branch : feature-streamed-package --- code/nel/src/misc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nel/src/misc/CMakeLists.txt b/code/nel/src/misc/CMakeLists.txt index 7f172bbfe..8836bf566 100644 --- a/code/nel/src/misc/CMakeLists.txt +++ b/code/nel/src/misc/CMakeLists.txt @@ -39,7 +39,7 @@ ENDIF(UNIX) INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR} ${PNG_INCLUDE_DIR} config_file) -TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARIES} nel_sevenzip) +TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARY} nel_sevenzip) SET_TARGET_PROPERTIES(nelmisc PROPERTIES LINK_INTERFACE_LIBRARIES "") NL_DEFAULT_PROPS(nelmisc "NeL, Library: NeL Misc") NL_ADD_RUNTIME_FLAGS(nelmisc) From 6b5d7b66a499b820f6a18dcdb21052dab983c2fb Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 10 Dec 2014 16:51:42 +0100 Subject: [PATCH 14/20] Fix compile --HG-- branch : feature-streamed-package --- code/nel/include/nel/misc/app_context.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nel/include/nel/misc/app_context.h b/code/nel/include/nel/misc/app_context.h index 3ab77aada..e30642646 100644 --- a/code/nel/include/nel/misc/app_context.h +++ b/code/nel/include/nel/misc/app_context.h @@ -233,7 +233,7 @@ namespace NLMISC NLMISC_SAFE_SINGLETON_DECL(className); \ \ public: \ - static void className::releaseInstance() \ + static void releaseInstance() \ { \ if (_Instance) \ { \ From 153ea8169dfdd4c215d8bd323b051946d9b5cb83 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 10 Dec 2014 17:00:26 +0100 Subject: [PATCH 15/20] Fix link --HG-- branch : feature-streamed-package --- code/nel/src/misc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nel/src/misc/CMakeLists.txt b/code/nel/src/misc/CMakeLists.txt index 8836bf566..82a40616e 100644 --- a/code/nel/src/misc/CMakeLists.txt +++ b/code/nel/src/misc/CMakeLists.txt @@ -39,7 +39,7 @@ ENDIF(UNIX) INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR} ${PNG_INCLUDE_DIR} config_file) -TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARY} nel_sevenzip) +TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARY} ${CURL_LIBRARIES} nel_sevenzip) SET_TARGET_PROPERTIES(nelmisc PROPERTIES LINK_INTERFACE_LIBRARIES "") NL_DEFAULT_PROPS(nelmisc "NeL, Library: NeL Misc") NL_ADD_RUNTIME_FLAGS(nelmisc) From b19c685ace6e610500cef9324089b49f513a4526 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 10 Dec 2014 17:04:40 +0100 Subject: [PATCH 16/20] Fix link --HG-- branch : feature-streamed-package --- code/CMakeLists.txt | 36 ++++++++++++++++---------------- code/nel/src/misc/CMakeLists.txt | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index 84d903d89..a6b7e2727 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -150,31 +150,31 @@ IF(WITH_NEL) IF(WITH_GUI) FIND_PACKAGE(Libwww REQUIRED) FIND_PACKAGE(Luabind REQUIRED) + ENDIF(WITH_GUI) FIND_PACKAGE(CURL REQUIRED) - IF(WIN32 OR CURL_LIBRARIES MATCHES "\\.a") - SET(CURL_STATIC ON) - ENDIF(WIN32 OR CURL_LIBRARIES MATCHES "\\.a") + IF(WIN32 OR CURL_LIBRARIES MATCHES "\\.a") + SET(CURL_STATIC ON) + ENDIF(WIN32 OR CURL_LIBRARIES MATCHES "\\.a") - IF(CURL_STATIC) - SET(CURL_DEFINITIONS -DCURL_STATICLIB) + IF(CURL_STATIC) + SET(CURL_DEFINITIONS -DCURL_STATICLIB) - FIND_PACKAGE(OpenSSL QUIET) + FIND_PACKAGE(OpenSSL QUIET) - IF(OPENSSL_FOUND) - SET(CURL_INCLUDE_DIRS ${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR}) - SET(CURL_LIBRARIES ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES}) - ENDIF(OPENSSL_FOUND) + IF(OPENSSL_FOUND) + SET(CURL_INCLUDE_DIRS ${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR}) + SET(CURL_LIBRARIES ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES}) + ENDIF(OPENSSL_FOUND) - # CURL Macports version depends on libidn, libintl and libiconv too - IF(APPLE) - FIND_LIBRARY(IDN_LIBRARY idn) - FIND_LIBRARY(INTL_LIBRARY intl) + # CURL Macports version depends on libidn, libintl and libiconv too + IF(APPLE) + FIND_LIBRARY(IDN_LIBRARY idn) + FIND_LIBRARY(INTL_LIBRARY intl) - SET(CURL_LIBRARIES ${CURL_LIBRARIES} ${IDN_LIBRARY} ${INTL_LIBRARY}) - ENDIF(APPLE) - ENDIF(CURL_STATIC) - ENDIF(WITH_GUI) + SET(CURL_LIBRARIES ${CURL_LIBRARIES} ${IDN_LIBRARY} ${INTL_LIBRARY}) + ENDIF(APPLE) + ENDIF(CURL_STATIC) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/nel/include ${LIBXML2_INCLUDE_DIR}) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/nel/3rdparty) diff --git a/code/nel/src/misc/CMakeLists.txt b/code/nel/src/misc/CMakeLists.txt index 82a40616e..f5bfa6c7b 100644 --- a/code/nel/src/misc/CMakeLists.txt +++ b/code/nel/src/misc/CMakeLists.txt @@ -46,7 +46,7 @@ NL_ADD_RUNTIME_FLAGS(nelmisc) NL_ADD_LIB_SUFFIX(nelmisc) -ADD_DEFINITIONS(${LIBXML2_DEFINITIONS}) +ADD_DEFINITIONS(${LIBXML2_DEFINITIONS} ${CURL_DEFINITIONS}) IF(WITH_PCH) ADD_NATIVE_PRECOMPILED_HEADER(nelmisc ${CMAKE_CURRENT_SOURCE_DIR}/stdmisc.h ${CMAKE_CURRENT_SOURCE_DIR}/stdmisc.cpp) From a7af274ae0629bd2bfb937f5aa538e7285ad06f3 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Fri, 12 Dec 2014 01:21:07 +0100 Subject: [PATCH 17/20] Don't decompress empty file in stream package --HG-- branch : feature-streamed-package --- code/nel/src/misc/streamed_package_manager.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index fbdfc1f5a..8114da948 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -231,12 +231,15 @@ bool CStreamedPackageManager::getFile(std::string &filePath, const std::string & // allocate the output buffer :( std::vector outBuffer; outBuffer.resize(fileSize); - // decompress the file in memory - ret = LzmaDecode(&state, (unsigned char *)pos, (SizeT)(inBuffer.size() - (pos - &inBuffer[0])), &inProcessed, (unsigned char*)&outBuffer[0], (SizeT)fileSize, &outProcessed); - if (ret != 0 || outProcessed != fileSize) + if (fileSize) { - nlwarning("Failed to decode lzma file '%s'", downloadPath.c_str()); - return false; + // decompress the file in memory + ret = LzmaDecode(&state, (unsigned char *)pos, (SizeT)(inBuffer.size() - (pos - &inBuffer[0])), &inProcessed, (unsigned char*)&outBuffer[0], (SizeT)fileSize, &outProcessed); + if (ret != 0 || outProcessed != fileSize) + { + nlwarning("Failed to decode lzma file '%s'", downloadPath.c_str()); + return false; + } } { From 0f59cb4343b922b815113fadba70e1d6d7f813cb Mon Sep 17 00:00:00 2001 From: kaetemi Date: Fri, 12 Dec 2014 01:21:07 +0100 Subject: [PATCH 18/20] Validate downloaded files --HG-- branch : feature-streamed-package --- code/nel/src/misc/streamed_package_manager.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 8114da948..89e5724c3 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -27,6 +27,7 @@ #include #include #include +#include namespace NLMISC { @@ -242,6 +243,15 @@ bool CStreamedPackageManager::getFile(std::string &filePath, const std::string & } } + CHashKey outHash = outBuffer.size() ? getSHA1(&outBuffer[0], outBuffer.size()) : CHashKey(); + if (!(outHash == entry->Hash)) + { + std::string wantHashS = entry->Hash.toString(); + std::string outHashS = outHash.toString(); + nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str()); + return false; + } + { COFile outStream(unpackPath); outStream.serialBuffer(&outBuffer[0], (uint)fileSize); From e277fedfb8f43a201a59c8416e8471be3c6ecbf2 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Fri, 12 Dec 2014 01:21:07 +0100 Subject: [PATCH 19/20] Don't write empty buffer --HG-- branch : feature-streamed-package --- code/nel/src/misc/streamed_package_manager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 89e5724c3..23b77def7 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -254,7 +254,8 @@ bool CStreamedPackageManager::getFile(std::string &filePath, const std::string & { COFile outStream(unpackPath); - outStream.serialBuffer(&outBuffer[0], (uint)fileSize); + if (fileSize) + outStream.serialBuffer(&outBuffer[0], (uint)fileSize); } if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str())) From 5db42f1eb621ac33366aa4d7f7c82fe375c62fd8 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 16 Apr 2015 17:20:29 +0200 Subject: [PATCH 20/20] Changed path cfg --HG-- branch : feature-streamed-package --- code/ryzom/client/src/init.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/ryzom/client/src/init.cpp b/code/ryzom/client/src/init.cpp index c40cbd258..350a2496c 100644 --- a/code/ryzom/client/src/init.cpp +++ b/code/ryzom/client/src/init.cpp @@ -646,7 +646,7 @@ void initStereoDisplayDevice() void initStreamedPackageManager(NLMISC::IProgressCallback &progress) { CStreamedPackageManager &spm = CStreamedPackageManager::getInstance(); - spm.Path = replaceApplicationDirToken(ClientCfg.StreamedPackagePath); + spm.Path = ClientCfg.StreamedPackagePath; for (uint i = 0; i < ClientCfg.StreamedPackageHosts.size(); i++) spm.Hosts.push_back(ClientCfg.StreamedPackageHosts[i]); }