From 9c3e8674ae6113d0f99005a49881b0634c26bb3c Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 24 Sep 2014 16:08:16 +0200 Subject: [PATCH 01/27] Add tool to build streamed package, ref #179 --HG-- branch : feature-streamed-package --- code/nel/include/nel/misc/streamed_package.h | 56 +++ .../nel/misc/streamed_package_manager.h | 34 ++ code/nel/src/misc/streamed_package.cpp | 66 ++++ .../nel/src/misc/streamed_package_manager.cpp | 36 ++ code/nel/tools/misc/CMakeLists.txt | 2 +- code/nel/tools/misc/snp_make/CMakeLists.txt | 9 + code/nel/tools/misc/snp_make/main.cpp | 355 ++++++++++++++++++ .../tools/patch_gen/patch_gen_common.cpp | 3 +- 8 files changed, 559 insertions(+), 2 deletions(-) create mode 100644 code/nel/include/nel/misc/streamed_package.h create mode 100644 code/nel/include/nel/misc/streamed_package_manager.h create mode 100644 code/nel/src/misc/streamed_package.cpp create mode 100644 code/nel/src/misc/streamed_package_manager.cpp create mode 100644 code/nel/tools/misc/snp_make/CMakeLists.txt create mode 100644 code/nel/tools/misc/snp_make/main.cpp diff --git a/code/nel/include/nel/misc/streamed_package.h b/code/nel/include/nel/misc/streamed_package.h new file mode 100644 index 000000000..b27e6658f --- /dev/null +++ b/code/nel/include/nel/misc/streamed_package.h @@ -0,0 +1,56 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_H +#define NLMISC_STREAMED_PACKAGE_H + +#include + +namespace NLMISC { + +class CStreamedPackage +{ +public: + struct CEntry + { + std::string Name; + CHashKey Hash; + uint32 Size; + uint32 LastModified; + + void serial(NLMISC::IStream &f) throw(NLMISC::EStream); + + }; + +public: + CStreamedPackage(); + ~CStreamedPackage(); + + void serial(NLMISC::IStream &f) throw(NLMISC::EStream); + + static void makePath(std::string &result, const CHashKey &hash); + +public: + typedef std::vector TEntries; + TEntries Entries; + +}; /* class CStreamedPackage */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */ + +/* end of file */ diff --git a/code/nel/include/nel/misc/streamed_package_manager.h b/code/nel/include/nel/misc/streamed_package_manager.h new file mode 100644 index 000000000..0bec38584 --- /dev/null +++ b/code/nel/include/nel/misc/streamed_package_manager.h @@ -0,0 +1,34 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_H +#define NLMISC_STREAMED_PACKAGE_H + +namespace NLMISC { + +class CStreamedPackageManager +{ +public: + CStreamedPackageManager(); + ~CStreamedPackageManager(); + +}; /* class CStreamedPackageManager */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */ + +/* end of file */ diff --git a/code/nel/src/misc/streamed_package.cpp b/code/nel/src/misc/streamed_package.cpp new file mode 100644 index 000000000..fed7d1dc3 --- /dev/null +++ b/code/nel/src/misc/streamed_package.cpp @@ -0,0 +1,66 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// Project includes +#include +#include + +namespace NLMISC { + +CStreamedPackage::CStreamedPackage() +{ + // init +} + +CStreamedPackage::~CStreamedPackage() +{ + // release +} + +void CStreamedPackage::serial(NLMISC::IStream &f) throw(NLMISC::EStream) +{ + f.serialCheck(NELID("SNPK")); + + uint version = 1; + f.serialVersion(version); + + f.serialCont(Entries); +} + +void CStreamedPackage::CEntry::serial(NLMISC::IStream &f) throw(NLMISC::EStream) +{ + uint version = 1; + f.serialVersion(version); + + f.serial(Name); + f.serial(Hash); + f.serial(Size); + f.serial(LastModified); +} + +void CStreamedPackage::makePath(std::string &result, const CHashKey &hash) +{ + std::string lowerHash = NLMISC::toLower(hash.toString()); + result = std::string("/") + lowerHash.substr(0, 2) + + "/" + lowerHash.substr(2, 2) + + "/" + lowerHash.substr(4); +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp new file mode 100644 index 000000000..2d04cb95a --- /dev/null +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -0,0 +1,36 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// Project includes +#include + +namespace NLMISC { + +CStreamedPackageManager::CStreamedPackageManager() +{ + // init +} + +CStreamedPackageManager::~CStreamedPackageManager() +{ + // release +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/tools/misc/CMakeLists.txt b/code/nel/tools/misc/CMakeLists.txt index 5386cbbc6..76399f6a3 100644 --- a/code/nel/tools/misc/CMakeLists.txt +++ b/code/nel/tools/misc/CMakeLists.txt @@ -1,4 +1,4 @@ -SUBDIRS(bnp_make disp_sheet_id extract_filename lock make_sheet_id xml_packer) +SUBDIRS(bnp_make snp_make disp_sheet_id extract_filename lock make_sheet_id xml_packer) IF(WITH_QT) ADD_SUBDIRECTORY(words_dic_qt) diff --git a/code/nel/tools/misc/snp_make/CMakeLists.txt b/code/nel/tools/misc/snp_make/CMakeLists.txt new file mode 100644 index 000000000..a1b5f388d --- /dev/null +++ b/code/nel/tools/misc/snp_make/CMakeLists.txt @@ -0,0 +1,9 @@ +FILE(GLOB SRC *.cpp *.h) + +ADD_EXECUTABLE(snp_make ${SRC}) + +TARGET_LINK_LIBRARIES(snp_make nelmisc) +NL_DEFAULT_PROPS(snp_make "NeL, Tools, Misc: snp_make") +NL_ADD_RUNTIME_FLAGS(snp_make) + +INSTALL(TARGETS snp_make RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT toolsmisc) diff --git a/code/nel/tools/misc/snp_make/main.cpp b/code/nel/tools/misc/snp_make/main.cpp new file mode 100644 index 000000000..61fc11810 --- /dev/null +++ b/code/nel/tools/misc/snp_make/main.cpp @@ -0,0 +1,355 @@ +// NeL - MMORPG Framework +// Copyright (C) 2010 Winch Gate Property Limited +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "nel/misc/types_nl.h" + +#include +#include + +#ifdef NL_OS_WINDOWS +# include +# include +#endif + +#include +#include + +#include "nel/misc/debug.h" +#include "nel/misc/file.h" +#include "nel/misc/path.h" +#include "nel/misc/algo.h" +#include "nel/misc/common.h" +#include "nel/misc/streamed_package.h" + +using namespace std; +using namespace NLMISC; + +// --------------------------------------------------------------------------- + +class CWildCard +{ +public: + string Expression; + bool Not; +}; +std::vector WildCards; + +std::string SourceDirectory; +std::string PackageFileName; +std::string StreamDirectory; + +CStreamedPackage Package; + +// --------------------------------------------------------------------------- + +bool keepFile (const char *fileName) +{ + uint i; + bool ifPresent = false; + bool ifTrue = false; + string file = toLower(CFile::getFilename (fileName)); + for (i=0; i [option] ... [option]\n"); + printf (" option : \n"); + printf (" -if wildcard : add the file if it matches the wilcard (at least one 'if' conditions must be met for a file to be adding)\n"); + printf (" -ifnot wildcard : add the file if it doesn't match the wilcard (all the 'ifnot' conditions must be met for a file to be adding)\n"); + printf (" Pack the directory to a snp file\n"); + printf (" snp_make /l \n"); + printf (" List the files contained in the snp file\n"); +} + +// --------------------------------------------------------------------------- + +void generateLZMA(const std::string sourceFile, const std::string &outputFile) +{ + std::string cmd="lzma e "; + cmd+=" "+sourceFile+" "+outputFile; + nlinfo("executing system command: %s",cmd.c_str()); +#ifdef NL_OS_WINDOWS + _spawnlp(_P_WAIT, "lzma.exe","lzma.exe", "e", sourceFile.c_str(), outputFile.c_str(), NULL); +#else // NL_OS_WINDOWS + sint error = system (cmd.c_str()); + if (error) + nlwarning("'%s' failed with error code %d", cmd.c_str(), error); +#endif // NL_OS_WINDOWS +} + +// --------------------------------------------------------------------------- + +uint readOptions (int nNbArg, char **ppArgs) +{ + uint i; + uint optionCount = 0; + for (i=0; i<(uint)nNbArg; i++) + { + // If ? + if ((strcmp (ppArgs[i], "-if") == 0) && ((i+1)<(uint)nNbArg)) + { + CWildCard card; + card.Expression = toLower(string(ppArgs[i+1])); + card.Not = false; + WildCards.push_back (card); + optionCount += 2; + } + // If not ? + if ((strcmp (ppArgs[i], "-ifnot") == 0) && ((i+1)<(uint)nNbArg)) + { + CWildCard card; + card.Expression = toLower(string(ppArgs[i+1])); + card.Not = true; + WildCards.push_back (card); + optionCount += 2; + } + } + return optionCount; +} + +// --------------------------------------------------------------------------- +int main (int nNbArg, char **ppArgs) +{ + NLMISC::CApplicationContext myApplicationContext; + + if (nNbArg < 3) + { + usage(); + return -1; + } + + if ((strcmp(ppArgs[1], "/p") == 0) || (strcmp(ppArgs[1], "/P") == 0) || + (strcmp(ppArgs[1], "-p") == 0) || (strcmp(ppArgs[1], "-P") == 0)) + { + if (nNbArg < 5) + { + usage(); + return -1; + } + + SourceDirectory = ppArgs[2]; + PackageFileName = ppArgs[3]; + StreamDirectory = ppArgs[4]; + readOptions(nNbArg, ppArgs); + + nldebug("Make streamed package: '%s'", PackageFileName.c_str()); + + if (CFile::fileExists(PackageFileName)) + { + nldebug("Update existing package"); + try + { + CIFile fi; + fi.open(PackageFileName); + fi.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + } + else + { + nldebug("New package"); + } + + std::vector pathContent; // contains full pathnames + std::vector nameContent; // only filename + CPath::getPathContent(SourceDirectory, true, false, true, pathContent); + nameContent.reserve(pathContent.size()); + for (std::vector::size_type i = 0; i < pathContent.size(); ++i) + { + const std::string &file = pathContent[i]; + if (keepFile(file.c_str())) + { + std::string fileName = NLMISC::toLower(CFile::getFilename(file)); + // nldebug("File: '%s' ('%s')", file.c_str(), fileName.c_str()); + nameContent.push_back(fileName); + nlassert(nameContent.size() == (i + 1)); + } + else + { + // Not included in this package + pathContent.erase(pathContent.begin() + i); + --i; + } + } + + std::vector packageIndex; // index of file in package + packageIndex.resize(pathContent.size(), -1); + + for (CStreamedPackage::TEntries::size_type i = 0; i < Package.Entries.size(); ++i) + { + const CStreamedPackage::CEntry &entry = Package.Entries[i]; + + sint foundIndex = -1; // find index in found file list + for (std::vector::size_type j = 0; j < pathContent.size(); ++j) + { + if (nameContent[j] == entry.Name) + { + foundIndex = j; + break; + } + } + + if (foundIndex < 0) + { + nlinfo("File no longer exists: '%s'", entry.Name.c_str()); + Package.Entries.erase(Package.Entries.begin() + i); + --i; + } + else + { + // File still exists, map it + packageIndex[foundIndex] = i; + } + } + + for (std::vector::size_type i = 0; i < pathContent.size(); ++i) + { + sint pidx = packageIndex[i]; + const std::string &name = nameContent[i]; + const std::string &path = pathContent[i]; + + if (pidx < 0) + { + nlinfo("File added: '%s'", name.c_str()); + pidx = Package.Entries.size(); + Package.Entries.push_back(CStreamedPackage::CEntry()); + Package.Entries[pidx].Name = name; + Package.Entries[pidx].LastModified = 0; + Package.Entries[pidx].Size = 0; + } + else + { + nlinfo("File check for changes: '%s'", name.c_str()); + } + + CStreamedPackage::CEntry &entry = Package.Entries[pidx]; + + std::string targetLzmaOld; // in case lzma wasn't made make sure it exists a second run + CStreamedPackage::makePath(targetLzmaOld, entry.Hash); + targetLzmaOld = StreamDirectory + targetLzmaOld + ".lzma"; + + uint32 lastModified = CFile::getFileModificationDate(path); + uint32 fileSize = CFile::getFileSize(path); + if (lastModified > entry.LastModified || fileSize != entry.Size || !CFile::fileExists(targetLzmaOld)) + { + entry.LastModified = lastModified; + + nlinfo("Calculate file hash"); + CHashKey hash = getSHA1(path, true); + /*nldebug("%s", hash.toString().c_str()); + std::string hashPath; + CStreamedPackage::makePath(hashPath, hash); + nldebug("%s", hashPath.c_str());*/ + + if (hash == entry.Hash && fileSize == entry.Size) + { + // File has not changed + } + else + { + nlinfo("File changed"); + entry.Hash = hash; + entry.Size = fileSize; + } + + std::string targetLzma; // in case lzma wasn't made make sure it exists a second run + CStreamedPackage::makePath(targetLzma, entry.Hash); + targetLzma = StreamDirectory + targetLzma + ".lzma"; + + if (!CFile::fileExists(targetLzma)) + { + // make the compressed file + nlinfo("%s -> %s", path.c_str(), targetLzma.c_str()); + CFile::createDirectoryTree(CFile::getPath(targetLzma)); + generateLZMA(path, targetLzma); + } + } + } + + try + { + nldebug("Store package '%s'", PackageFileName.c_str()); + COFile fo; + fo.open(PackageFileName); + fo.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + + return 0; + } + + if ((strcmp(ppArgs[1], "/l") == 0) || (strcmp(ppArgs[1], "/L") == 0) || + (strcmp(ppArgs[1], "-l") == 0) || (strcmp(ppArgs[1], "-L") == 0)) + { + PackageFileName = ppArgs[2]; + if (!CFile::fileExists(PackageFileName)) + { + nlwarning("ERROR (snp_make) : package doesn't exist: '%s'", PackageFileName.c_str()); + return -1; + } + + try + { + CIFile fi; + fi.open(PackageFileName); + fi.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + + for (CStreamedPackage::TEntries::const_iterator it(Package.Entries.begin()), end(Package.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + + printf("List files in '%s'", PackageFileName.c_str()); + printf("%s { Hash: '%s', Size: '%u', LastModified: '%u' }", entry.Name.c_str(), entry.Hash.toString().c_str(), entry.Size, entry.LastModified); + } + + return 0; + } + + usage (); + return -1; +} diff --git a/code/ryzom/tools/patch_gen/patch_gen_common.cpp b/code/ryzom/tools/patch_gen/patch_gen_common.cpp index 3d28a2439..127be5053 100644 --- a/code/ryzom/tools/patch_gen/patch_gen_common.cpp +++ b/code/ryzom/tools/patch_gen/patch_gen_common.cpp @@ -442,7 +442,8 @@ void CPackageDescription::buildDefaultFileList() std::vector fileList; NLMISC::CPath::getPathContent(_BnpDirectory,false,false,true,fileList); for (uint32 i=0;i Date: Wed, 24 Sep 2014 17:51:05 +0200 Subject: [PATCH 02/27] Move 7zip library --HG-- branch : feature-streamed-package --- code/CMakeLists.txt | 1 + code/nel/3rdparty/CMakeLists.txt | 2 ++ .../3rdparty}/seven_zip/7zAlloc.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zAlloc.h | 0 .../3rdparty}/seven_zip/7zBuffer.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zBuffer.h | 0 .../src => nel/3rdparty}/seven_zip/7zCrc.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zCrc.h | 0 .../3rdparty}/seven_zip/7zDecode.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zDecode.h | 0 .../3rdparty}/seven_zip/7zExtract.cpp | 0 .../3rdparty}/seven_zip/7zExtract.h | 0 .../3rdparty}/seven_zip/7zHeader.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zHeader.h | 0 .../src => nel/3rdparty}/seven_zip/7zIn.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zIn.h | 0 .../src => nel/3rdparty}/seven_zip/7zItem.cpp | 0 .../src => nel/3rdparty}/seven_zip/7zItem.h | 0 .../src => nel/3rdparty}/seven_zip/7zMain.cpp | 0 .../3rdparty}/seven_zip/7zMethodID.cpp | 0 .../3rdparty}/seven_zip/7zMethodID.h | 0 .../src => nel/3rdparty}/seven_zip/7zTypes.h | 0 .../3rdparty}/seven_zip/BranchTypes.h | 0 .../3rdparty}/seven_zip/BranchX86.cpp | 0 .../3rdparty}/seven_zip/BranchX86.h | 0 code/nel/3rdparty/seven_zip/CMakeLists.txt | 27 +++++++++++++++++++ .../3rdparty}/seven_zip/LzmaDecode.cpp | 0 .../3rdparty}/seven_zip/LzmaDecode.h | 0 .../3rdparty}/seven_zip/LzmaRamDecode.cpp | 0 .../3rdparty}/seven_zip/LzmaRamDecode.h | 0 .../3rdparty}/seven_zip/LzmaTypes.h | 0 .../src => nel/3rdparty}/seven_zip/readme.txt | 0 code/nel/CMakeLists.txt | 1 + code/nel/src/misc/CMakeLists.txt | 2 +- code/ryzom/client/src/CMakeLists.txt | 2 -- .../ryzom/client/src/seven_zip/CMakeLists.txt | 27 ------------------- .../client/client_patcher/CMakeLists.txt | 2 +- 37 files changed, 33 insertions(+), 31 deletions(-) create mode 100644 code/nel/3rdparty/CMakeLists.txt rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zAlloc.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zAlloc.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zBuffer.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zBuffer.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zCrc.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zCrc.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zDecode.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zDecode.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zExtract.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zExtract.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zHeader.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zHeader.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zIn.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zIn.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zItem.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zItem.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zMain.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zMethodID.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zMethodID.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/7zTypes.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/BranchTypes.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/BranchX86.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/BranchX86.h (100%) create mode 100644 code/nel/3rdparty/seven_zip/CMakeLists.txt rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/LzmaDecode.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/LzmaDecode.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/LzmaRamDecode.cpp (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/LzmaRamDecode.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/LzmaTypes.h (100%) rename code/{ryzom/client/src => nel/3rdparty}/seven_zip/readme.txt (100%) delete mode 100644 code/ryzom/client/src/seven_zip/CMakeLists.txt diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index 071554e06..80bc52421 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -177,6 +177,7 @@ IF(WITH_NEL) ENDIF(WITH_GUI) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/nel/include ${LIBXML2_INCLUDE_DIR}) + INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/nel/3rdparty) ADD_DEFINITIONS(${LIBXML2_DEFINITIONS}) ADD_SUBDIRECTORY(nel) ENDIF(WITH_NEL) diff --git a/code/nel/3rdparty/CMakeLists.txt b/code/nel/3rdparty/CMakeLists.txt new file mode 100644 index 000000000..bdd3e5fcd --- /dev/null +++ b/code/nel/3rdparty/CMakeLists.txt @@ -0,0 +1,2 @@ +SET(SEVENZIP_LIBRARY "nel_sevenzip") +ADD_SUBDIRECTORY(seven_zip) diff --git a/code/ryzom/client/src/seven_zip/7zAlloc.cpp b/code/nel/3rdparty/seven_zip/7zAlloc.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zAlloc.cpp rename to code/nel/3rdparty/seven_zip/7zAlloc.cpp diff --git a/code/ryzom/client/src/seven_zip/7zAlloc.h b/code/nel/3rdparty/seven_zip/7zAlloc.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zAlloc.h rename to code/nel/3rdparty/seven_zip/7zAlloc.h diff --git a/code/ryzom/client/src/seven_zip/7zBuffer.cpp b/code/nel/3rdparty/seven_zip/7zBuffer.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zBuffer.cpp rename to code/nel/3rdparty/seven_zip/7zBuffer.cpp diff --git a/code/ryzom/client/src/seven_zip/7zBuffer.h b/code/nel/3rdparty/seven_zip/7zBuffer.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zBuffer.h rename to code/nel/3rdparty/seven_zip/7zBuffer.h diff --git a/code/ryzom/client/src/seven_zip/7zCrc.cpp b/code/nel/3rdparty/seven_zip/7zCrc.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zCrc.cpp rename to code/nel/3rdparty/seven_zip/7zCrc.cpp diff --git a/code/ryzom/client/src/seven_zip/7zCrc.h b/code/nel/3rdparty/seven_zip/7zCrc.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zCrc.h rename to code/nel/3rdparty/seven_zip/7zCrc.h diff --git a/code/ryzom/client/src/seven_zip/7zDecode.cpp b/code/nel/3rdparty/seven_zip/7zDecode.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zDecode.cpp rename to code/nel/3rdparty/seven_zip/7zDecode.cpp diff --git a/code/ryzom/client/src/seven_zip/7zDecode.h b/code/nel/3rdparty/seven_zip/7zDecode.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zDecode.h rename to code/nel/3rdparty/seven_zip/7zDecode.h diff --git a/code/ryzom/client/src/seven_zip/7zExtract.cpp b/code/nel/3rdparty/seven_zip/7zExtract.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zExtract.cpp rename to code/nel/3rdparty/seven_zip/7zExtract.cpp diff --git a/code/ryzom/client/src/seven_zip/7zExtract.h b/code/nel/3rdparty/seven_zip/7zExtract.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zExtract.h rename to code/nel/3rdparty/seven_zip/7zExtract.h diff --git a/code/ryzom/client/src/seven_zip/7zHeader.cpp b/code/nel/3rdparty/seven_zip/7zHeader.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zHeader.cpp rename to code/nel/3rdparty/seven_zip/7zHeader.cpp diff --git a/code/ryzom/client/src/seven_zip/7zHeader.h b/code/nel/3rdparty/seven_zip/7zHeader.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zHeader.h rename to code/nel/3rdparty/seven_zip/7zHeader.h diff --git a/code/ryzom/client/src/seven_zip/7zIn.cpp b/code/nel/3rdparty/seven_zip/7zIn.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zIn.cpp rename to code/nel/3rdparty/seven_zip/7zIn.cpp diff --git a/code/ryzom/client/src/seven_zip/7zIn.h b/code/nel/3rdparty/seven_zip/7zIn.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zIn.h rename to code/nel/3rdparty/seven_zip/7zIn.h diff --git a/code/ryzom/client/src/seven_zip/7zItem.cpp b/code/nel/3rdparty/seven_zip/7zItem.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zItem.cpp rename to code/nel/3rdparty/seven_zip/7zItem.cpp diff --git a/code/ryzom/client/src/seven_zip/7zItem.h b/code/nel/3rdparty/seven_zip/7zItem.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zItem.h rename to code/nel/3rdparty/seven_zip/7zItem.h diff --git a/code/ryzom/client/src/seven_zip/7zMain.cpp b/code/nel/3rdparty/seven_zip/7zMain.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zMain.cpp rename to code/nel/3rdparty/seven_zip/7zMain.cpp diff --git a/code/ryzom/client/src/seven_zip/7zMethodID.cpp b/code/nel/3rdparty/seven_zip/7zMethodID.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/7zMethodID.cpp rename to code/nel/3rdparty/seven_zip/7zMethodID.cpp diff --git a/code/ryzom/client/src/seven_zip/7zMethodID.h b/code/nel/3rdparty/seven_zip/7zMethodID.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zMethodID.h rename to code/nel/3rdparty/seven_zip/7zMethodID.h diff --git a/code/ryzom/client/src/seven_zip/7zTypes.h b/code/nel/3rdparty/seven_zip/7zTypes.h similarity index 100% rename from code/ryzom/client/src/seven_zip/7zTypes.h rename to code/nel/3rdparty/seven_zip/7zTypes.h diff --git a/code/ryzom/client/src/seven_zip/BranchTypes.h b/code/nel/3rdparty/seven_zip/BranchTypes.h similarity index 100% rename from code/ryzom/client/src/seven_zip/BranchTypes.h rename to code/nel/3rdparty/seven_zip/BranchTypes.h diff --git a/code/ryzom/client/src/seven_zip/BranchX86.cpp b/code/nel/3rdparty/seven_zip/BranchX86.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/BranchX86.cpp rename to code/nel/3rdparty/seven_zip/BranchX86.cpp diff --git a/code/ryzom/client/src/seven_zip/BranchX86.h b/code/nel/3rdparty/seven_zip/BranchX86.h similarity index 100% rename from code/ryzom/client/src/seven_zip/BranchX86.h rename to code/nel/3rdparty/seven_zip/BranchX86.h diff --git a/code/nel/3rdparty/seven_zip/CMakeLists.txt b/code/nel/3rdparty/seven_zip/CMakeLists.txt new file mode 100644 index 000000000..d2d2e690d --- /dev/null +++ b/code/nel/3rdparty/seven_zip/CMakeLists.txt @@ -0,0 +1,27 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +FILE(GLOB LIB_SRC *.cpp *.h) + +LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) + +NL_TARGET_LIB(nel_sevenzip ${LIB_SRC}) +# TARGET_LINK_LIBRARIES(nel_sevenzip ${PLATFORM_LINKFLAGS}) +NL_DEFAULT_PROPS(nel_sevenzip "NeL, 3rd Party: Seven Zip") +NL_ADD_RUNTIME_FLAGS(nel_sevenzip) +NL_ADD_LIB_SUFFIX(nel_sevenzip) + +ADD_DEFINITIONS(-D_SZ_ONE_DIRECTORY) + +IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) + INSTALL(TARGETS nel_sevenzip LIBRARY DESTINATION ${NL_LIB_PREFIX} ARCHIVE DESTINATION ${NL_LIB_PREFIX} COMPONENT libraries) +ENDIF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) + +IF(WITH_TOOLS) + ADD_EXECUTABLE(7zDec ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) + + TARGET_LINK_LIBRARIES(7zDec nel_sevenzip) + NL_DEFAULT_PROPS(7zDec "NeL, 3rd Party: Seven Zip Decoder") + NL_ADD_RUNTIME_FLAGS(7zDec) + + INSTALL(TARGETS 7zDec RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT tools) +ENDIF(WITH_TOOLS) diff --git a/code/ryzom/client/src/seven_zip/LzmaDecode.cpp b/code/nel/3rdparty/seven_zip/LzmaDecode.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaDecode.cpp rename to code/nel/3rdparty/seven_zip/LzmaDecode.cpp diff --git a/code/ryzom/client/src/seven_zip/LzmaDecode.h b/code/nel/3rdparty/seven_zip/LzmaDecode.h similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaDecode.h rename to code/nel/3rdparty/seven_zip/LzmaDecode.h diff --git a/code/ryzom/client/src/seven_zip/LzmaRamDecode.cpp b/code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaRamDecode.cpp rename to code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp diff --git a/code/ryzom/client/src/seven_zip/LzmaRamDecode.h b/code/nel/3rdparty/seven_zip/LzmaRamDecode.h similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaRamDecode.h rename to code/nel/3rdparty/seven_zip/LzmaRamDecode.h diff --git a/code/ryzom/client/src/seven_zip/LzmaTypes.h b/code/nel/3rdparty/seven_zip/LzmaTypes.h similarity index 100% rename from code/ryzom/client/src/seven_zip/LzmaTypes.h rename to code/nel/3rdparty/seven_zip/LzmaTypes.h diff --git a/code/ryzom/client/src/seven_zip/readme.txt b/code/nel/3rdparty/seven_zip/readme.txt similarity index 100% rename from code/ryzom/client/src/seven_zip/readme.txt rename to code/nel/3rdparty/seven_zip/readme.txt diff --git a/code/nel/CMakeLists.txt b/code/nel/CMakeLists.txt index 53bf071e3..d3f0a3e5c 100644 --- a/code/nel/CMakeLists.txt +++ b/code/nel/CMakeLists.txt @@ -70,6 +70,7 @@ IF(WITH_INSTALL_LIBRARIES) ADD_SUBDIRECTORY(include) ENDIF(WITH_INSTALL_LIBRARIES) +ADD_SUBDIRECTORY(3rdparty) ADD_SUBDIRECTORY(src) IF(WITH_NEL_SAMPLES) diff --git a/code/nel/src/misc/CMakeLists.txt b/code/nel/src/misc/CMakeLists.txt index 2d3ef9066..8494c23a6 100644 --- a/code/nel/src/misc/CMakeLists.txt +++ b/code/nel/src/misc/CMakeLists.txt @@ -39,7 +39,7 @@ ENDIF(UNIX) INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR} ${PNG_INCLUDE_DIR} config_file) -TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY}) +TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} nel_sevenzip) SET_TARGET_PROPERTIES(nelmisc PROPERTIES LINK_INTERFACE_LIBRARIES "") NL_DEFAULT_PROPS(nelmisc "NeL, Library: NeL Misc") NL_ADD_RUNTIME_FLAGS(nelmisc) diff --git a/code/ryzom/client/src/CMakeLists.txt b/code/ryzom/client/src/CMakeLists.txt index c20d9d0dc..3fbbfd0e2 100644 --- a/code/ryzom/client/src/CMakeLists.txt +++ b/code/ryzom/client/src/CMakeLists.txt @@ -5,9 +5,7 @@ ADD_SUBDIRECTORY(client_sheets) IF(WITH_RYZOM_CLIENT) # These are Windows/MFC apps -SET(SEVENZIP_LIBRARY "ryzom_sevenzip") -ADD_SUBDIRECTORY(seven_zip) IF(WITH_RYZOM_PATCH) ADD_DEFINITIONS(-DRZ_USE_PATCH) diff --git a/code/ryzom/client/src/seven_zip/CMakeLists.txt b/code/ryzom/client/src/seven_zip/CMakeLists.txt deleted file mode 100644 index cc152757b..000000000 --- a/code/ryzom/client/src/seven_zip/CMakeLists.txt +++ /dev/null @@ -1,27 +0,0 @@ -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) - -FILE(GLOB LIB_SRC *.cpp *.h) - -LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) - -NL_TARGET_LIB(ryzom_sevenzip ${LIB_SRC}) -# TARGET_LINK_LIBRARIES(ryzom_sevenzip ${PLATFORM_LINKFLAGS}) -NL_DEFAULT_PROPS(ryzom_sevenzip "Ryzom, Library: Seven Zip") -NL_ADD_RUNTIME_FLAGS(ryzom_sevenzip) -NL_ADD_LIB_SUFFIX(ryzom_sevenzip) - -ADD_DEFINITIONS(-D_SZ_ONE_DIRECTORY) - -IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) - INSTALL(TARGETS ryzom_sevenzip LIBRARY DESTINATION ${RYZOM_LIB_PREFIX} ARCHIVE DESTINATION ${RYZOM_LIB_PREFIX} COMPONENT libraries) -ENDIF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) - -IF(WITH_RYZOM_TOOLS) - ADD_EXECUTABLE(7zDec ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) - - TARGET_LINK_LIBRARIES(7zDec ryzom_sevenzip) - NL_DEFAULT_PROPS(7zDec "Ryzom, Tools, Misc: Seven Zip Decoder") - NL_ADD_RUNTIME_FLAGS(7zDec) - - INSTALL(TARGETS 7zDec RUNTIME DESTINATION ${RYZOM_BIN_PREFIX} COMPONENT tools) -ENDIF(WITH_RYZOM_TOOLS) diff --git a/code/ryzom/tools/client/client_patcher/CMakeLists.txt b/code/ryzom/tools/client/client_patcher/CMakeLists.txt index e5b845e43..69b14e5a1 100644 --- a/code/ryzom/tools/client/client_patcher/CMakeLists.txt +++ b/code/ryzom/tools/client/client_patcher/CMakeLists.txt @@ -19,7 +19,7 @@ TARGET_LINK_LIBRARIES(ryzom_client_patcher nelmisc nelnet ryzom_gameshare - ryzom_sevenzip + nel_sevenzip ${CURL_LIBRARIES}) IF(APPLE) From 4ba6b6c4bf3ce45f27d51ec00eb80045f9ac88d4 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 24 Sep 2014 20:30:11 +0200 Subject: [PATCH 03/27] Workaround '0' unpack directory bug, and don't choke on 0 size files when unpacking bnp files --HG-- branch : feature-streamed-package --- code/nel/tools/misc/bnp_make/main.cpp | 15 +++--- code/ryzom/client/src/login_patch.cpp | 69 +++++++++++++++++++-------- 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/code/nel/tools/misc/bnp_make/main.cpp b/code/nel/tools/misc/bnp_make/main.cpp index a3f34a6a4..3bb70f5aa 100644 --- a/code/nel/tools/misc/bnp_make/main.cpp +++ b/code/nel/tools/misc/bnp_make/main.cpp @@ -330,13 +330,16 @@ void unpack (const string &dirName) if (out != NULL) { nlfseek64 (bnp, rBNPFile.Pos, SEEK_SET); - uint8 *ptr = new uint8[rBNPFile.Size]; - if (fread (ptr, rBNPFile.Size, 1, bnp) != 1) - nlwarning("%s read error", filename.c_str()); - if (fwrite (ptr, rBNPFile.Size, 1, out) != 1) - nlwarning("%s write error", filename.c_str()); + if (rBNPFile.Size) + { + uint8 *ptr = new uint8[rBNPFile.Size]; + if (fread (ptr, rBNPFile.Size, 1, bnp) != 1) + nlwarning("%s read error", filename.c_str()); + if (fwrite (ptr, rBNPFile.Size, 1, out) != 1) + nlwarning("%s write error", filename.c_str()); + delete [] ptr; + } fclose (out); - delete [] ptr; } } fclose (bnp); diff --git a/code/ryzom/client/src/login_patch.cpp b/code/ryzom/client/src/login_patch.cpp index 5b9a83c83..d2f4d02e0 100644 --- a/code/ryzom/client/src/login_patch.cpp +++ b/code/ryzom/client/src/login_patch.cpp @@ -1176,13 +1176,35 @@ void CPatchManager::readDescFile(sint32 nVersion) std::string unpackTo = category.getUnpackTo(); - if (unpackTo.substr(0, 2) == "./") + if (unpackTo == "0") + { + nlwarning("BUG: unpackTo == '0'"); + unpackTo = ClientRootPath; + category.setUnpackTo(unpackTo); + } + else if (unpackTo.substr(0, 2) == "./") { unpackTo = ClientRootPath + unpackTo.substr(2); category.setUnpackTo(unpackTo); } } } + else + { + for (cat = 0; cat < DescFile.getCategories().categoryCount(); ++cat) + { + CBNPCategory &category = const_cast(DescFile.getCategories().getCategory(cat)); + + std::string unpackTo = category.getUnpackTo(); + + if (unpackTo == "0") + { + nlwarning("BUG: unpackTo == '0'"); + unpackTo = "./"; + category.setUnpackTo(unpackTo); + } + } + } // tmp for debug : flag some categories as 'Mainland' for (cat = 0; cat < DescFile.getCategories().categoryCount(); ++cat) @@ -1884,29 +1906,36 @@ bool CPatchManager::bnpUnpack(const string &srcBigfile, const string &dstPath, v if (out != NULL) { nlfseek64 (bnp, rBNPFile.Pos, SEEK_SET); - uint8 *ptr = new uint8[rBNPFile.Size]; - - if (fread (ptr, rBNPFile.Size, 1, bnp) != 1) + if (rBNPFile.Size) { - fclose(out); - return false; - } + uint8 *ptr = new uint8[rBNPFile.Size]; - bool writeError = fwrite (ptr, rBNPFile.Size, 1, out) != 1; - if (writeError) - { - nlwarning("errno = %d", errno); - } - bool diskFull = ferror(out) && errno == 28 /* ENOSPC*/; - fclose (out); - delete [] ptr; - if (diskFull) - { - throw NLMISC::EDiskFullError(filename); + if (fread (ptr, rBNPFile.Size, 1, bnp) != 1) + { + fclose(out); + return false; + } + + bool writeError = fwrite (ptr, rBNPFile.Size, 1, out) != 1; + if (writeError) + { + nlwarning("errno = %d", errno); + } + bool diskFull = ferror(out) && errno == 28 /* ENOSPC*/; + fclose (out); + delete [] ptr; + if (diskFull) + { + throw NLMISC::EDiskFullError(filename); + } + if (writeError) + { + throw NLMISC::EWriteError(filename); + } } - if (writeError) + else { - throw NLMISC::EWriteError(filename); + fclose (out); } } } From 7d09a6cb150f1aa9eda99a901653682658606ee5 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 24 Sep 2014 21:52:18 +0200 Subject: [PATCH 04/27] Add streamed package manager --HG-- branch : feature-streamed-package --- code/nel/include/nel/misc/app_context.h | 15 ++++ code/nel/include/nel/misc/path.h | 6 ++ code/nel/include/nel/misc/streamed_package.h | 1 + .../nel/misc/streamed_package_manager.h | 34 +++++++- code/nel/src/misc/file.cpp | 34 ++++++++ code/nel/src/misc/path.cpp | 82 +++++++++++++++++-- .../nel/src/misc/streamed_package_manager.cpp | 68 +++++++++++++++ 7 files changed, 230 insertions(+), 10 deletions(-) diff --git a/code/nel/include/nel/misc/app_context.h b/code/nel/include/nel/misc/app_context.h index 46d2a15c7..2388dffe1 100644 --- a/code/nel/include/nel/misc/app_context.h +++ b/code/nel/include/nel/misc/app_context.h @@ -229,6 +229,20 @@ namespace NLMISC } \ private: +#define NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(className) \ + NLMISC_SAFE_SINGLETON_DECL(className); \ + \ + void className::releaseInstance() \ + { \ + if (_Instance) \ + { \ + NLMISC::INelContext::getInstance().releaseSingletonPointer(#className, _Instance); \ + delete _Instance; \ + _Instance = NULL; \ + } \ + } \ + + /** The same as above, but generate a getInstance method that * return a pointer instead of a reference */ @@ -296,5 +310,6 @@ void initNelLibrary(CLibrary &lib); } // namespace NLMISC +#include #endif //APP_CONTEXT_H diff --git a/code/nel/include/nel/misc/path.h b/code/nel/include/nel/misc/path.h index f3120c907..1efe9a4c6 100644 --- a/code/nel/include/nel/misc/path.h +++ b/code/nel/include/nel/misc/path.h @@ -74,6 +74,9 @@ public: /** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included (the extention (Nel Big File) is used to know that it's a big file) */ void addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); + + /** Sale but for .snp (Streamed NeL Package) */ + void addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); /** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included */ void addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); @@ -371,6 +374,9 @@ public: /** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included (the extention (Nel Big File) is used to know that it's a big file) */ static void addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); + /** Same but Streamed Package */ + static void addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); + /** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included */ static void addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL); diff --git a/code/nel/include/nel/misc/streamed_package.h b/code/nel/include/nel/misc/streamed_package.h index b27e6658f..c777d8a02 100644 --- a/code/nel/include/nel/misc/streamed_package.h +++ b/code/nel/include/nel/misc/streamed_package.h @@ -17,6 +17,7 @@ #ifndef NLMISC_STREAMED_PACKAGE_H #define NLMISC_STREAMED_PACKAGE_H +#include #include namespace NLMISC { diff --git a/code/nel/include/nel/misc/streamed_package_manager.h b/code/nel/include/nel/misc/streamed_package_manager.h index 0bec38584..b1c3b05e5 100644 --- a/code/nel/include/nel/misc/streamed_package_manager.h +++ b/code/nel/include/nel/misc/streamed_package_manager.h @@ -14,21 +14,49 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -#ifndef NLMISC_STREAMED_PACKAGE_H -#define NLMISC_STREAMED_PACKAGE_H +#ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H +#define NLMISC_STREAMED_PACKAGE_MANAGER_H + +#include +#include +#include namespace NLMISC { class CStreamedPackageManager { + NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(CStreamedPackageManager); + public: CStreamedPackageManager(); ~CStreamedPackageManager(); + + /// Loads a package into the package manager + /// package: ex. /games/nel/data/characters_maps_hr.snp + bool loadPackage(const std::string &package); + + /// Get a file list + void list(std::vector &fileNames, const std::string &package); + + /// Unload all packages + void unloadAll(); + + /// Get an existing file or download if necessary + /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. + /// fileName: ex. fy_hof_underwear.dds + bool getFile(std::string &filePath, const std::string &fileName); + + /// Get file size + bool getFileSize(uint32 &fileSize, const std::string &fileName); + +private: + std::map m_Packages; + std::map m_Entries; }; /* class CStreamedPackageManager */ } /* namespace NLMISC */ -#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */ +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */ /* end of file */ diff --git a/code/nel/src/misc/file.cpp b/code/nel/src/misc/file.cpp index 842d9ea51..a542a3dc2 100644 --- a/code/nel/src/misc/file.cpp +++ b/code/nel/src/misc/file.cpp @@ -23,6 +23,7 @@ #include "nel/misc/command.h" #include "nel/misc/sstring.h" #include "nel/misc/xml_pack.h" +#include "nel/misc/streamed_package_manager.h" #ifndef NL_OS_WINDOWS #include @@ -203,6 +204,39 @@ bool CIFile::open(const std::string &path, bool text) _F = CXMLPack::getInstance().getFile (path, _FileSize, _BigFileOffset, dummy, _AlwaysOpened); } } + else if (pos > 3 && path[pos-3] == 's' && path[pos-2] == 'n' && path[pos-1] == 'p') + { + nldebug("Opening a streamed package file"); + + _IsInXMLPackFile = false; + _IsInBigFile = false; + _BigFileOffset = 0; + _AlwaysOpened = false; + std::string filePath; + if (CStreamedPackageManager::getInstance().getFile (filePath, path)) + { + _F = fopen (filePath.c_str(), mode); + if (_F != NULL) + { + _FileSize=CFile::getFileSize(_F); + if (_FileSize == 0) + { + nlwarning("FILE: Size of file '%s' is 0", path.c_str()); + fclose(_F); + _F = NULL; + } + } + else + { + nlwarning("Failed to open file '%s', error %u : %s", path.c_str(), errno, strerror(errno)); + _FileSize = 0; + } + } + else + { + nlerror("File '%s' not in streamed package", path.c_str()); + } + } else { // bnp file diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index d47b8ce4a..7ed6cb54f 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -23,6 +23,7 @@ #include "nel/misc/progress_callback.h" #include "nel/misc/file.h" #include "nel/misc/xml_pack.h" +#include "nel/misc/streamed_package_manager.h" #ifdef NL_OS_WINDOWS # ifndef NL_COMP_MINGW @@ -292,6 +293,7 @@ void CFileContainer::clearMap () nlassert(!_MemoryCompressed); _Files.clear (); CBigFile::getInstance().removeAll (); + CStreamedPackageManager::getInstance().unloadAll (); NL_DISPLAY_PATH("PATH: CPath::clearMap(): map directory cleared"); } @@ -1161,16 +1163,26 @@ void CFileContainer::addSearchFile (const string &file, bool remap, const string return; } + std::string fileExtension = CFile::getExtension(newFile); + // check if it s a big file - if (CFile::getExtension(newFile) == "bnp") + if (fileExtension == "bnp") { NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is a big file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str()); addSearchBigFile(file, false, false, progressCallBack); return; } + // check if it s a streamed package + if (fileExtension == "snp") + { + NL_DISPLAY_PATH ("PATH: CPath::addSearchStreamedPackage(%s, %d, '%s'): '%s' is a streamed package, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str()); + addSearchStreamedPackage(file, false, false, progressCallBack); + return; + } + // check if it s an xml pack file - if (CFile::getExtension(newFile) == "xml_pack") + if (fileExtension == "xml_pack") { NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is an xml pack file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str()); addSearchXmlpackFile(file, false, false, progressCallBack); @@ -1391,6 +1403,52 @@ void CFileContainer::addSearchBigFile (const string &sBigFilename, bool recurse, fclose (Handle); } +void CPath::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack) +{ + getInstance()->_FileContainer.addSearchBigFile(filename, recurse, alternative, progressCallBack); +} + +void CFileContainer::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack) +{ + // Check if filename is not empty + if (filename.empty()) + { + nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): can't add empty file, skip it", filename.c_str(), recurse, alternative); + return; + } + // Check if the file exists + if (!CFile::isExists(filename)) + { + nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not found, skip it", filename.c_str(), recurse, alternative, filename.c_str()); + return; + } + // Check if it s a file + if (CFile::isDirectory(filename)) + { + nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not a file, skip it", filename.c_str(), recurse, alternative, filename.c_str()); + return; + } + + // Add the file itself + std::string packname = NLMISC::toLower(CFile::getFilename(filename)); + insertFileInMap(packname, filename, false, CFile::getExtension(filename)); + + // Add the package to the package manager + if (CStreamedPackageManager::getInstance().loadPackage(filename)) + { + std::vector fileNames; + CStreamedPackageManager::getInstance().list(fileNames, packname); + + for (std::vector::iterator it(fileNames.begin()), end(fileNames.end()); it != end; ++it) + { + // Add the file to the lookup + std::string filePackageName = packname + "@" + (*it); + nldebug("Insert '%s'", filePackageName.c_str()); + insertFileInMap((*it), filePackageName, false, CFile::getExtension(*it)); + } + } +} + // WARNING : recurse is not used void CPath::addSearchXmlpackFile (const string &sXmlpackFilename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack) { @@ -1983,6 +2041,7 @@ string CFile::findNewFile (const string &filename) // \warning doesn't work with big file uint32 CFile::getFileSize (const std::string &filename) { + std::string::size_type pos; if (filename.find("@@") != string::npos) { uint32 fs = 0, bfo; @@ -1990,12 +2049,21 @@ uint32 CFile::getFileSize (const std::string &filename) CXMLPack::getInstance().getFile (filename, fs, bfo, c, d); return fs; } - else if (filename.find('@') != string::npos) + else if ((pos = filename.find('@')) != string::npos) { - uint32 fs = 0, bfo; - bool c, d; - CBigFile::getInstance().getFile (filename, fs, bfo, c, d); - return fs; + if (pos > 3 && filename[pos-3] == 's' && filename[pos-2] == 'n' && filename[pos-1] == 'p') + { + uint32 fs = 0; + CStreamedPackageManager::getInstance().getFileSize (fs, filename); + return fs; + } + else + { + uint32 fs = 0, bfo; + bool c, d; + CBigFile::getInstance().getFile (filename, fs, bfo, c, d); + return fs; + } } else { diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 2d04cb95a..4a5862130 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -18,9 +18,13 @@ // Project includes #include +#include +#include namespace NLMISC { +NLMISC_SAFE_SINGLETON_IMPL(CStreamedPackageManager); + CStreamedPackageManager::CStreamedPackageManager() { // init @@ -31,6 +35,70 @@ CStreamedPackageManager::~CStreamedPackageManager() // release } +bool CStreamedPackageManager::loadPackage(const std::string &package) +{ + nldebug("Load package '%s'", package.c_str()); + + std::string packname = NLMISC::toLower(CFile::getFilename(package)); + m_Packages[packname] = CStreamedPackage(); + std::map::iterator it = m_Packages.find(packname); + CStreamedPackage &p = it->second; + + try + { + CIFile fi; + fi.open(package); + fi.serial(p); + } + catch (Exception &e) + { + nlerror("Package serial exception: '%s'", e.what()); + m_Packages.erase(it); + return false; + } + + for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + m_Entries[entry.Name] = &entry; + } + + return true; +} + +void CStreamedPackageManager::list(std::vector &fileNames, const std::string &package) +{ + nldebug("List package '%s'", package.c_str()); + + std::map::iterator it = m_Packages.find(package); + CStreamedPackage &p = it->second; + + for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + fileNames.push_back(entry.Name); + } +} + +void CStreamedPackageManager::unloadAll() +{ + m_Packages.clear(); + m_Entries.clear(); +} + +bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName) +{ + // .. :) + nldebug("Get file path for streamed file '%s'", fileName.c_str()); + return false; +} + +bool CStreamedPackageManager::getFileSize(uint32 &fileSize, const std::string &fileName) +{ + nldebug("Get file size for streamed file '%s'", fileName.c_str()); + return false; +} + } /* namespace NLMISC */ /* end of file */ From 105ecbff90bf1ea4b0ac5c0ba5c8635c10594038 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 24 Sep 2014 21:52:41 +0200 Subject: [PATCH 05/27] Add pipeline script for creating streamed packages --HG-- branch : feature-streamed-package --- .../build_gamedata/configuration/tools.py | 1 + .../tools/build_gamedata/d1_client_patch.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/code/nel/tools/build_gamedata/configuration/tools.py b/code/nel/tools/build_gamedata/configuration/tools.py index c0c962360..bf904072e 100755 --- a/code/nel/tools/build_gamedata/configuration/tools.py +++ b/code/nel/tools/build_gamedata/configuration/tools.py @@ -87,6 +87,7 @@ BuildClodBankTool = "build_clod_bank" SheetsPackerTool = "sheets_packer" SheetsPackerShardTool = "sheets_packer_shard" BnpMakeTool = "bnp_make" +SnpMakeTool = "snp_make" AiBuildWmapTool = "ai_build_wmap" TgaCutTool = "tga_cut" PatchGenTool = "patch_gen" diff --git a/code/nel/tools/build_gamedata/d1_client_patch.py b/code/nel/tools/build_gamedata/d1_client_patch.py index 600347d0b..4b4f37072 100755 --- a/code/nel/tools/build_gamedata/d1_client_patch.py +++ b/code/nel/tools/build_gamedata/d1_client_patch.py @@ -51,6 +51,7 @@ printLog(log, "") # Find tools BnpMake = findTool(log, ToolDirectories, BnpMakeTool, ToolSuffix) +SnpMake = findTool(log, ToolDirectories, SnpMakeTool, ToolSuffix); PatchGen = findTool(log, ToolDirectories, PatchGenTool, ToolSuffix) printLog(log, "") @@ -121,11 +122,14 @@ else: targetPath = ClientPatchDirectory + "/bnp" mkPath(log, targetPath) for category in InstallClientData: + packExt = ".bnp" + if (category["StreamedPackages"]): + packExt = ".snp" for package in category["Packages"]: printLog(log, "PACKAGE " + package[0]) sourcePath = InstallDirectory + "/" + package[0] mkPath(log, sourcePath) - targetBnp = targetPath + "/" + package[0] + ".bnp" + targetBnp = targetPath + "/" + package[0] + packExt if (len(package[1]) > 0): targetBnp = targetPath + "/" + package[1][0] printLog(log, "TARGET " + package[1][0]) @@ -135,8 +139,16 @@ else: else: needUpdateBnp = needUpdateDirNoSubdirFile(log, sourcePath, targetBnp) if (needUpdateBnp): - printLog(log, "BNP " + targetBnp) - subprocess.call([ BnpMake, "/p", sourcePath, targetPath ] + package[1]) + if (category["StreamedPackages"]): + printLog(log, "SNP " + targetBnp) + cwDir = os.getcwd().replace("\\", "/") + toolDir = os.path.dirname(Lzma).replace("\\", "/") + os.chdir(toolDir) + subprocess.call([ SnpMake, "/p", sourcePath, targetBnp, ClientPatchDirectory + "/stream" ] + package[1]) + os.chdir(cwDir) + else: + printLog(log, "BNP " + targetBnp) + subprocess.call([ BnpMake, "/p", sourcePath, targetPath, targetBnp ] + package[1]) else: printLog(log, "SKIP " + targetBnp) printLog(log, "") From e6a327fb1ab0deb02816fec8e8f39ec3efa2d58d Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 24 Sep 2014 23:40:43 +0200 Subject: [PATCH 06/27] Implementation fixes, ref #179 --HG-- branch : feature-streamed-package --- code/nel/include/nel/misc/app_context.h | 20 ++++++++++--------- code/nel/src/misc/file.cpp | 2 +- code/nel/src/misc/path.cpp | 12 ++++++++--- .../tools/build_gamedata/d2_client_install.py | 7 +++++-- code/ryzom/client/src/login.cpp | 2 ++ code/ryzom/client/src/release.cpp | 2 ++ .../screenshot_islands.cpp | 2 +- .../client/src/snowballs_client.cpp | 1 + 8 files changed, 32 insertions(+), 16 deletions(-) diff --git a/code/nel/include/nel/misc/app_context.h b/code/nel/include/nel/misc/app_context.h index 2388dffe1..3ab77aada 100644 --- a/code/nel/include/nel/misc/app_context.h +++ b/code/nel/include/nel/misc/app_context.h @@ -230,17 +230,19 @@ namespace NLMISC private: #define NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(className) \ - NLMISC_SAFE_SINGLETON_DECL(className); \ - \ - void className::releaseInstance() \ - { \ - if (_Instance) \ + NLMISC_SAFE_SINGLETON_DECL(className); \ + \ + public: \ + static void className::releaseInstance() \ { \ - NLMISC::INelContext::getInstance().releaseSingletonPointer(#className, _Instance); \ - delete _Instance; \ - _Instance = NULL; \ + if (_Instance) \ + { \ + NLMISC::INelContext::getInstance().releaseSingletonPointer(#className, _Instance); \ + delete _Instance; \ + _Instance = NULL; \ + } \ } \ - } \ + private: /** The same as above, but generate a getInstance method that diff --git a/code/nel/src/misc/file.cpp b/code/nel/src/misc/file.cpp index a542a3dc2..e2168def6 100644 --- a/code/nel/src/misc/file.cpp +++ b/code/nel/src/misc/file.cpp @@ -213,7 +213,7 @@ bool CIFile::open(const std::string &path, bool text) _BigFileOffset = 0; _AlwaysOpened = false; std::string filePath; - if (CStreamedPackageManager::getInstance().getFile (filePath, path)) + if (CStreamedPackageManager::getInstance().getFile (filePath, path.substr(pos+1))) { _F = fopen (filePath.c_str(), mode); if (_F != NULL) diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index 7ed6cb54f..b7d58df51 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -1736,7 +1736,8 @@ void CFileContainer::memoryCompress() while (it != _Files.end()) { string sTmp = SSMpath.get(it->second.idPath); - if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && !it->second.Remapped) + nldebug("A: %s", sTmp.c_str()); + if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && (sTmp.find("snp@") == string::npos) && !it->second.Remapped) { // This is a file included in a bigfile (so the name is in the bigfile manager) } @@ -1759,7 +1760,8 @@ void CFileContainer::memoryCompress() { CFileEntry &rFE = it->second; string sTmp = SSMpath.get(rFE.idPath); - if (sTmp.find("@") == string::npos || sTmp.find("@@") != string::npos || rFE.Remapped) + nldebug("B: %s", sTmp.c_str()); + if ((sTmp.find("@") == string::npos) || (sTmp.find("@@") != string::npos) || (sTmp.find("snp@") != string::npos) || rFE.Remapped) { strcpy(_AllFileNames+nSize, rFE.Name.c_str()); _MCFiles[nNb].Name = _AllFileNames+nSize; @@ -1784,8 +1786,12 @@ void CFileContainer::memoryCompress() it++; } + nldebug("Passed"); + contReset(_Files); _MemoryCompressed = true; + + nldebug("Done"); } void CPath::memoryUncompress() @@ -2054,7 +2060,7 @@ uint32 CFile::getFileSize (const std::string &filename) if (pos > 3 && filename[pos-3] == 's' && filename[pos-2] == 'n' && filename[pos-1] == 'p') { uint32 fs = 0; - CStreamedPackageManager::getInstance().getFileSize (fs, filename); + CStreamedPackageManager::getInstance().getFileSize (fs, filename.substr(pos+1)); return fs; } else diff --git a/code/nel/tools/build_gamedata/d2_client_install.py b/code/nel/tools/build_gamedata/d2_client_install.py index febaef656..bb399447f 100755 --- a/code/nel/tools/build_gamedata/d2_client_install.py +++ b/code/nel/tools/build_gamedata/d2_client_install.py @@ -47,6 +47,9 @@ printLog(log, "") for category in InstallClientData: printLog(log, "CATEGORY " + category["Name"]) + packExt = ".bnp" + if (category["StreamedPackages"]): + packExt = ".snp" if (category["UnpackTo"] != None): targetPath = ClientInstallDirectory if (category["UnpackTo"] != ""): @@ -62,8 +65,8 @@ for category in InstallClientData: mkPath(log, targetPath) for package in category["Packages"]: printLog(log, "PACKAGE " + package[0]) - sourceBnp = sourcePath + "/" + package[0] + ".bnp" - targetBnp = targetPath + "/" + package[0] + ".bnp" + sourceBnp = sourcePath + "/" + package[0] + packExt + targetBnp = targetPath + "/" + package[0] + packExt if (len(package[1]) > 0): sourceBnp = sourcePath + "/" + package[1][0] targetBnp = targetPath + "/" + package[1][0] diff --git a/code/ryzom/client/src/login.cpp b/code/ryzom/client/src/login.cpp index 091e24a80..e4d9b18b5 100644 --- a/code/ryzom/client/src/login.cpp +++ b/code/ryzom/client/src/login.cpp @@ -28,6 +28,7 @@ #include "nel/misc/thread.h" #include "nel/misc/big_file.h" #include "nel/misc/system_utils.h" +#include "nel/misc/streamed_package_manager.h" #include "nel/net/tcp_sock.h" #include "nel/3d/u_driver.h" @@ -1614,6 +1615,7 @@ void initPatch() CBGDownloaderAccess::getInstance().startTask(taskDesc, string(), true /* showDownloader */); // no command line since bg downloader should already be started // release lock on bnp, so that they can be written NLMISC::CBigFile::getInstance().removeAll(); + NLMISC::CStreamedPackageManager::getInstance().unloadAll(); } NLGUI::CDBManager::getInstance()->getDbProp("UI:VARIABLES:SCREEN")->setValue32(UI_VARIABLES_SCREEN_PATCHING); diff --git a/code/ryzom/client/src/release.cpp b/code/ryzom/client/src/release.cpp index f88849197..403b654e9 100644 --- a/code/ryzom/client/src/release.cpp +++ b/code/ryzom/client/src/release.cpp @@ -25,6 +25,7 @@ #include "nel/misc/debug.h" #include "nel/misc/async_file_manager.h" #include "nel/misc/system_utils.h" +#include "nel/misc/streamed_package_manager.h" // 3D Interface. #include "nel/3d/bloom_effect.h" #include "nel/3d/fxaa.h" @@ -644,6 +645,7 @@ void release() R2::CObjectSerializer::releaseInstance(); NLMISC::CBigFile::getInstance().removeAll(); NLMISC::CBigFile::releaseInstance(); + NLMISC::CStreamedPackageManager::releaseInstance(); NL3D::CFastHLSModifier::releaseInstance(); CLandscapePolyDrawer::releaseInstance(); NL3D::CParticleSystemShape::releaseInstance(); diff --git a/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp b/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp index 4f167ea9c..d678c0b17 100644 --- a/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp +++ b/code/ryzom/tools/client/r2_islands_textures/screenshot_islands.cpp @@ -295,7 +295,7 @@ void CScreenshotIslands::searchIslandsBorders() zonelFiles.clear(); string bnpFileName = itCont->first + ".bnp"; - CBigFile::getInstance().list(bnpFileName.c_str(), filenames); + CBigFile::getInstance().list(bnpFileName.c_str(), filenames); // FIXME FIXME NOT READING FROM BNP! for(uint i=0; i Date: Wed, 24 Sep 2014 23:41:40 +0200 Subject: [PATCH 07/27] Remove some debugging, ref #179 --HG-- branch : feature-streamed-package --- code/nel/src/misc/path.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index b7d58df51..56249f637 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -1736,7 +1736,6 @@ void CFileContainer::memoryCompress() while (it != _Files.end()) { string sTmp = SSMpath.get(it->second.idPath); - nldebug("A: %s", sTmp.c_str()); if ((sTmp.find("@@") == string::npos) && (sTmp.find('@') != string::npos) && (sTmp.find("snp@") == string::npos) && !it->second.Remapped) { // This is a file included in a bigfile (so the name is in the bigfile manager) @@ -1760,7 +1759,6 @@ void CFileContainer::memoryCompress() { CFileEntry &rFE = it->second; string sTmp = SSMpath.get(rFE.idPath); - nldebug("B: %s", sTmp.c_str()); if ((sTmp.find("@") == string::npos) || (sTmp.find("@@") != string::npos) || (sTmp.find("snp@") != string::npos) || rFE.Remapped) { strcpy(_AllFileNames+nSize, rFE.Name.c_str()); @@ -1786,12 +1784,8 @@ void CFileContainer::memoryCompress() it++; } - nldebug("Passed"); - contReset(_Files); _MemoryCompressed = true; - - nldebug("Done"); } void CPath::memoryUncompress() From fb7955103b99ae955c1d785347b9e058c0ba0bf6 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 25 Sep 2014 17:39:51 +0200 Subject: [PATCH 08/27] Implement streamed data download, ref #179 --HG-- branch : feature-streamed-package --- code/nel/include/nel/misc/path.h | 4 +- .../nel/misc/streamed_package_manager.h | 16 +- code/nel/src/misc/file.cpp | 1 + code/nel/src/misc/path.cpp | 14 +- .../nel/src/misc/streamed_package_manager.cpp | 174 +++++++++++++++++- .../tools/build_gamedata/d1_client_patch.py | 4 +- code/ryzom/client/src/client_cfg.cpp | 15 +- code/ryzom/client/src/client_cfg.h | 4 + code/ryzom/client/src/init.cpp | 10 + code/ryzom/client/src/init.h | 1 + code/ryzom/client/src/streamable_ig.cpp | 2 + 11 files changed, 230 insertions(+), 15 deletions(-) diff --git a/code/nel/include/nel/misc/path.h b/code/nel/include/nel/misc/path.h index 1efe9a4c6..b6694c7a1 100644 --- a/code/nel/include/nel/misc/path.h +++ b/code/nel/include/nel/misc/path.h @@ -279,8 +279,8 @@ private: struct CMCFileEntry { char *Name; // Normal case (the search is done by using nlstricmp) - uint32 idPath : 16; // Path (not with file at the end) - look in the SSMpath (65536 different path allowed) - uint32 idExt : 15; // real extension of the file if remapped - look in the SSMext (32768 different extension allowed) + uint32 idPath : 20; // Path (not with file at the end) - look in the SSMpath (1048576 different path allowed) + uint32 idExt : 11; // real extension of the file if remapped - look in the SSMext (2048 different extension allowed) uint32 Remapped : 1; // true if the file is remapped }; diff --git a/code/nel/include/nel/misc/streamed_package_manager.h b/code/nel/include/nel/misc/streamed_package_manager.h index b1c3b05e5..e23344a70 100644 --- a/code/nel/include/nel/misc/streamed_package_manager.h +++ b/code/nel/include/nel/misc/streamed_package_manager.h @@ -31,7 +31,6 @@ public: CStreamedPackageManager(); ~CStreamedPackageManager(); - /// Loads a package into the package manager /// package: ex. /games/nel/data/characters_maps_hr.snp bool loadPackage(const std::string &package); @@ -49,9 +48,20 @@ public: /// Get file size bool getFileSize(uint32 &fileSize, const std::string &fileName); +public: + /// Set storage path (ex. stream/) + std::string Path; + + /// Loads a package into the package manager (ex. http://patch.live.polyverse.org/stream/) + typedef std::vector THosts; + THosts Hosts; + private: - std::map m_Packages; - std::map m_Entries; + typedef std::map TPackages; + TPackages m_Packages; + + typedef std::map TEntries; + TEntries m_Entries; }; /* class CStreamedPackageManager */ diff --git a/code/nel/src/misc/file.cpp b/code/nel/src/misc/file.cpp index e2168def6..b53a660d9 100644 --- a/code/nel/src/misc/file.cpp +++ b/code/nel/src/misc/file.cpp @@ -234,6 +234,7 @@ bool CIFile::open(const std::string &path, bool text) } else { + // TEMPORARY ERROR nlerror("File '%s' not in streamed package", path.c_str()); } } diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index 56249f637..553f8e4ba 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -342,7 +342,7 @@ void CFileContainer::remapExtension (const string &ext1, const string &ext2, boo nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): can't remap empty extension", ext1lwr.c_str(), ext2lwr.c_str(), substitute); } - if (ext1lwr == "bnp" || ext2lwr == "bnp") + if (ext1lwr == "bnp" || ext2lwr == "bnp" || ext1lwr == "snp" || ext2lwr == "snp") { nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): you can't remap a big file", ext1lwr.c_str(), ext2lwr.c_str(), substitute); } @@ -1445,6 +1445,18 @@ void CFileContainer::addSearchStreamedPackage (const string &filename, bool recu std::string filePackageName = packname + "@" + (*it); nldebug("Insert '%s'", filePackageName.c_str()); insertFileInMap((*it), filePackageName, false, CFile::getExtension(*it)); + + // Remapped extensions + std::string ext = CFile::getExtension(*it); + for (uint j = 0; j < _Extensions.size (); j++) + { + if (_Extensions[j].first == ext) + { + // Need to remap + insertFileInMap(CFile::getFilenameWithoutExtension(*it) + "." + _Extensions[j].second, + filePackageName, true, _Extensions[j].first); + } + } } } } diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 4a5862130..250c92d07 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -16,6 +16,13 @@ #include "stdmisc.h" +// 3rd Party includes +#include +#include +#include +#include +#include + // Project includes #include #include @@ -88,15 +95,174 @@ void CStreamedPackageManager::unloadAll() bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName) { - // .. :) nldebug("Get file path for streamed file '%s'", fileName.c_str()); - return false; + + TEntries::iterator it = m_Entries.find(fileName); + if (it == m_Entries.end()) + return false; + const CStreamedPackage::CEntry *entry = it->second; + + CStreamedPackage::makePath(filePath, entry->Hash); + std::string downloadUrlFile = filePath + ".lzma"; + filePath = Path + filePath; + std::string downloadPath = filePath + ".download." + toString(rand()); + + std::string storageDirectory = CFile::getPath(downloadPath); + CFile::createDirectoryTree(storageDirectory); + /*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory)) + { + nldebug("Unable to create directory '%s'", storageDirectory.c_str()); + return false; + }*/ + + // download + for (; ; ) + { + if (CFile::fileExists(filePath)) + return true; + + std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile; + nldebug("Download streamed package '%s' from '%s'", fileName.c_str(), downloadUrl.c_str()); + + FILE *fp = fopen(downloadPath.c_str(), "wb"); + if (fp == NULL) + { + nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), fileName.c_str()); + return false; + } + + CURL *curl; + CURLcode res; + curl = curl_easy_init(); + if (curl) + { + curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_FILE, fp); + res = curl_easy_perform(curl); + long r; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r); + curl_easy_cleanup(curl); + + bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/; + fclose(fp); + + if (diskFull) + { + CFile::deleteFile(downloadPath); + throw EDiskFullError(downloadPath); + } + + if (res != CURLE_OK || r < 200 || r >= 300) + { + CFile::deleteFile(downloadPath); + nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str()); + nlSleep(1000); + continue; + } + } + else + { + nldebug("Curl initialize failed"); + fclose(fp); + CFile::deleteFile(downloadPath); + return false; + } + + // ok! + break; + } + + // read into memory :( + std::string unpackPath = filePath + ".extract." + toString(rand()); + + std::vector inBuffer; + { + CIFile inStream(downloadPath); + uint32 inSize = inStream.getFileSize(); + inBuffer.resize(inSize); + inStream.serialBuffer(&inBuffer[0], inSize); + } + CFile::deleteFile(downloadPath); + + if (inBuffer.size() < LZMA_PROPERTIES_SIZE + 8) + { + nlwarning("Invalid file size %u, too small file '%s'", inBuffer.size(), downloadPath.c_str()); + return false; + } + + // extract + { + CLzmaDecoderState state; + uint8 *pos = &inBuffer[0]; + int ret = LzmaDecodeProperties(&state.Properties, (unsigned char *)pos, LZMA_PROPERTIES_SIZE); + if (ret != 0) + { + nlwarning("Failed to decode lzma properies in file '%s'", downloadPath.c_str()); + return false; + } + + // FROM login_patch.cpp + // alloc the probs, making sure they are deleted in function exit + size_t nbProb = LzmaGetNumProbs(&state.Properties); + std::vector probs; + probs.resize(nbProb); + state.Probs = &probs[0]; + + pos += LZMA_PROPERTIES_SIZE; + + // read the output file size + size_t fileSize = 0; + for (int i = 0; i < 8; i++) + { + //Byte b; + if (pos >= &inBuffer[0] + inBuffer.size()) + { + nlerror("pos >= inBuffer.get() + inSize"); + return false; + } + fileSize |= ((UInt64)*pos++) << (8 * i); + } + + nlassert(fileSize == entry->Size); + + SizeT outProcessed = 0; + SizeT inProcessed = 0; + // allocate the output buffer :( + std::vector outBuffer; + outBuffer.resize(fileSize); + // decompress the file in memory + ret = LzmaDecode(&state, (unsigned char *)pos, (SizeT)(inBuffer.size() - (pos - &inBuffer[0])), &inProcessed, (unsigned char*)&outBuffer[0], (SizeT)fileSize, &outProcessed); + if (ret != 0 || outProcessed != fileSize) + { + nlwarning("Failed to decode lzma file '%s'", downloadPath.c_str()); + return false; + } + + { + COFile outStream(unpackPath); + outStream.serialBuffer(&outBuffer[0], (uint)fileSize); + } + + if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str())) + { + nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str()); + // in case downloaded from another thread + return CFile::fileExists(filePath); + } + + return true; + } } bool CStreamedPackageManager::getFileSize(uint32 &fileSize, const std::string &fileName) { - nldebug("Get file size for streamed file '%s'", fileName.c_str()); - return false; + // nldebug("Get file size for streamed file '%s'", fileName.c_str()); + TEntries::iterator it = m_Entries.find(fileName); + if (it == m_Entries.end()) + return false; + fileSize = it->second->Size; + return true; } } /* namespace NLMISC */ diff --git a/code/nel/tools/build_gamedata/d1_client_patch.py b/code/nel/tools/build_gamedata/d1_client_patch.py index 4b4f37072..080141e5f 100755 --- a/code/nel/tools/build_gamedata/d1_client_patch.py +++ b/code/nel/tools/build_gamedata/d1_client_patch.py @@ -97,7 +97,7 @@ else: if category["UnpackTo"] != "": cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./" + category["UnpackTo"] + "/\"/>\n") else: - cfg.write("\t\t\t<_UnpackTo type=\"SINT32\" value=\"./\"/>\n") + cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./\"/>\n") cfg.write("\t\t\t<_IsOptional type=\"SINT32\" value=\"" + str(category["IsOptional"]) + "\"/>\n") cfg.write("\t\t\t<_IsIncremental type=\"SINT32\" value=\"" + str(category["IsIncremental"]) + "\"/>\n") for package in category["Packages"]: @@ -148,7 +148,7 @@ else: os.chdir(cwDir) else: printLog(log, "BNP " + targetBnp) - subprocess.call([ BnpMake, "/p", sourcePath, targetPath, targetBnp ] + package[1]) + subprocess.call([ BnpMake, "/p", sourcePath, targetPath ] + package[1]) else: printLog(log, "SKIP " + targetBnp) printLog(log, "") diff --git a/code/ryzom/client/src/client_cfg.cpp b/code/ryzom/client/src/client_cfg.cpp index 6cde86cba..5b69b284e 100644 --- a/code/ryzom/client/src/client_cfg.cpp +++ b/code/ryzom/client/src/client_cfg.cpp @@ -464,6 +464,8 @@ CClientConfig::CClientConfig() ColorShout = CRGBA(150,0,0,255); // Default Shout color. ColorTalk = CRGBA(255,255,255,255); // Default Talk color. + StreamedPackagePath = "stream"; + // PreDataPath.push_back("data/gamedev/language/"); // Default Path for the language data // DataPath.push_back("data/"); // Default Path for the Data. @@ -1247,18 +1249,25 @@ void CClientConfig::setValues() ////////// // MISC // + // Pre Data Path. READ_STRINGVECTOR_FV(PreDataPath); // Data Path. READ_STRINGVECTOR_FV(DataPath); - // List of files that trigger R2ED reload when touched - READ_STRINGVECTOR_FV(R2EDReloadFiles); - // Data Path no recurse. READ_STRINGVECTOR_FV(DataPathNoRecurse); + // Streamed package path + READ_STRING_FV(StreamedPackagePath); + + // Streamed package hosts + READ_STRINGVECTOR_FV(StreamedPackageHosts); + + // List of files that trigger R2ED reload when touched + READ_STRINGVECTOR_FV(R2EDReloadFiles); + // Update packed sheet Path READ_STRINGVECTOR_FV(UpdatePackedSheetPath); diff --git a/code/ryzom/client/src/client_cfg.h b/code/ryzom/client/src/client_cfg.h index 7d7f1c65d..59b4f96a5 100644 --- a/code/ryzom/client/src/client_cfg.h +++ b/code/ryzom/client/src/client_cfg.h @@ -360,6 +360,10 @@ struct CClientConfig std::vector DataPath; /// Data Path no recurse. std::vector DataPathNoRecurse; + /// Streamed package path + std::string StreamedPackagePath; + /// Streamed package hosts + std::vector StreamedPackageHosts; // TODO: From 'domain' SQL table /// Update packed sheet Path. std::vector UpdatePackedSheetPath; /// True if we want the packed sheet to be updated if needed diff --git a/code/ryzom/client/src/init.cpp b/code/ryzom/client/src/init.cpp index 1644f7e2d..6a1cf5bcf 100644 --- a/code/ryzom/client/src/init.cpp +++ b/code/ryzom/client/src/init.cpp @@ -34,6 +34,7 @@ #include "nel/misc/system_info.h" #include "nel/misc/block_memory.h" #include "nel/misc/system_utils.h" +#include "nel/misc/streamed_package_manager.h" // 3D Interface. #include "nel/3d/bloom_effect.h" #include "nel/3d/u_driver.h" @@ -664,6 +665,14 @@ void initStereoDisplayDevice() IStereoDisplay::releaseUnusedLibraries(); } +void initStreamedPackageManager(NLMISC::IProgressCallback &progress) +{ + CStreamedPackageManager &spm = CStreamedPackageManager::getInstance(); + spm.Path = replaceApplicationDirToken(ClientCfg.StreamedPackagePath); + for (uint i = 0; i < ClientCfg.StreamedPackageHosts.size(); i++) + spm.Hosts.push_back(ClientCfg.StreamedPackageHosts[i]); +} + void addSearchPaths(IProgressCallback &progress) { // Add search path of UI addon. Allow only a subset of files. @@ -848,6 +857,7 @@ void prelogInit() CPath::remapExtension ("png", "tga", true); FPU_CHECKER_ONCE + initStreamedPackageManager(ProgressBar); addPreDataPaths(ProgressBar); FPU_CHECKER_ONCE diff --git a/code/ryzom/client/src/init.h b/code/ryzom/client/src/init.h index cdcdfb418..cf1e22f6e 100644 --- a/code/ryzom/client/src/init.h +++ b/code/ryzom/client/src/init.h @@ -33,6 +33,7 @@ void prelogInit(); // Initialize the application after login step void postlogInit(); +void initStreamedPackageManager(NLMISC::IProgressCallback &progress); void addSearchPaths(NLMISC::IProgressCallback &progress); void addPreDataPaths(NLMISC::IProgressCallback &progress); diff --git a/code/ryzom/client/src/streamable_ig.cpp b/code/ryzom/client/src/streamable_ig.cpp index f73df6126..99930b947 100644 --- a/code/ryzom/client/src/streamable_ig.cpp +++ b/code/ryzom/client/src/streamable_ig.cpp @@ -89,8 +89,10 @@ CStreamableIG::~CStreamableIG() H_AUTO_USE(RZ_StremableIG) if (!_Linked) { + #ifdef NL_DEBUG if(!ClientCfg.Light) nlwarning("Loading async %p", this); + #endif #ifdef NL_DEBUG //nlinfo("Loading async : %s", Name.c_str()); #endif From 85615992d784756999b45c7321cbdd2c093b6bed Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 25 Sep 2014 19:47:57 +0200 Subject: [PATCH 09/27] Remove some debugging --HG-- branch : feature-streamed-package --- code/nel/src/misc/CMakeLists.txt | 2 +- code/nel/src/misc/file.cpp | 2 +- code/nel/src/misc/streamed_package_manager.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/code/nel/src/misc/CMakeLists.txt b/code/nel/src/misc/CMakeLists.txt index 8494c23a6..7f172bbfe 100644 --- a/code/nel/src/misc/CMakeLists.txt +++ b/code/nel/src/misc/CMakeLists.txt @@ -39,7 +39,7 @@ ENDIF(UNIX) INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR} ${PNG_INCLUDE_DIR} config_file) -TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} nel_sevenzip) +TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARIES} nel_sevenzip) SET_TARGET_PROPERTIES(nelmisc PROPERTIES LINK_INTERFACE_LIBRARIES "") NL_DEFAULT_PROPS(nelmisc "NeL, Library: NeL Misc") NL_ADD_RUNTIME_FLAGS(nelmisc) diff --git a/code/nel/src/misc/file.cpp b/code/nel/src/misc/file.cpp index b53a660d9..9197b35a1 100644 --- a/code/nel/src/misc/file.cpp +++ b/code/nel/src/misc/file.cpp @@ -206,7 +206,7 @@ bool CIFile::open(const std::string &path, bool text) } else if (pos > 3 && path[pos-3] == 's' && path[pos-2] == 'n' && path[pos-1] == 'p') { - nldebug("Opening a streamed package file"); + // nldebug("Opening a streamed package file"); _IsInXMLPackFile = false; _IsInBigFile = false; diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 250c92d07..ffe33e76c 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -95,7 +95,7 @@ void CStreamedPackageManager::unloadAll() bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName) { - nldebug("Get file path for streamed file '%s'", fileName.c_str()); + // nldebug("Get file path for streamed file '%s'", fileName.c_str()); TEntries::iterator it = m_Entries.find(fileName); if (it == m_Entries.end()) From 40f4eb4ad2255e75ab3166a962732d4c2e019cba Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 25 Sep 2014 20:10:16 +0200 Subject: [PATCH 10/27] Fix package description generation --HG-- branch : feature-streamed-package --- .../tools/build_gamedata/d1_client_patch.py | 5 +++- code/ryzom/client/src/login_patch.cpp | 24 +------------------ 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/code/nel/tools/build_gamedata/d1_client_patch.py b/code/nel/tools/build_gamedata/d1_client_patch.py index 080141e5f..47a952812 100755 --- a/code/nel/tools/build_gamedata/d1_client_patch.py +++ b/code/nel/tools/build_gamedata/d1_client_patch.py @@ -91,6 +91,9 @@ else: inCategories = 1 cfg.write("\t<_Categories>\n") for category in InstallClientData: + packExt = ".bnp" + if (category["StreamedPackages"]): + packExt = ".snp" cfg.write("\t\t<_Category>\n") cfg.write("\t\t\t<_Name type=\"STRING\" value=\"" + category["Name"] + "\"/>\n") if category["UnpackTo"] != None: @@ -104,7 +107,7 @@ else: if (len(package[1]) > 0): cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[1][0] + "\"/>\n") else: - cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[0] + ".bnp\"/>\n") + cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + package[0] + packExt + "\"/>\n") for ref in category["Refs"]: cfg.write("\t\t\t<_Files type=\"STRING\" value=\"" + ref + "_.ref\"/>\n") cfg.write("\t\t\n") diff --git a/code/ryzom/client/src/login_patch.cpp b/code/ryzom/client/src/login_patch.cpp index d2f4d02e0..657eb8fb3 100644 --- a/code/ryzom/client/src/login_patch.cpp +++ b/code/ryzom/client/src/login_patch.cpp @@ -1176,35 +1176,13 @@ void CPatchManager::readDescFile(sint32 nVersion) std::string unpackTo = category.getUnpackTo(); - if (unpackTo == "0") - { - nlwarning("BUG: unpackTo == '0'"); - unpackTo = ClientRootPath; - category.setUnpackTo(unpackTo); - } - else if (unpackTo.substr(0, 2) == "./") + if (unpackTo.substr(0, 2) == "./") { unpackTo = ClientRootPath + unpackTo.substr(2); category.setUnpackTo(unpackTo); } } } - else - { - for (cat = 0; cat < DescFile.getCategories().categoryCount(); ++cat) - { - CBNPCategory &category = const_cast(DescFile.getCategories().getCategory(cat)); - - std::string unpackTo = category.getUnpackTo(); - - if (unpackTo == "0") - { - nlwarning("BUG: unpackTo == '0'"); - unpackTo = "./"; - category.setUnpackTo(unpackTo); - } - } - } // tmp for debug : flag some categories as 'Mainland' for (cat = 0; cat < DescFile.getCategories().categoryCount(); ++cat) From 9a1510caf07e25ca7bbd44ee5a151f37aae296e1 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 25 Sep 2014 20:25:54 +0200 Subject: [PATCH 11/27] Fix patching --HG-- branch : feature-streamed-package --- code/ryzom/client/src/login_patch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/ryzom/client/src/login_patch.cpp b/code/ryzom/client/src/login_patch.cpp index 657eb8fb3..949527e82 100644 --- a/code/ryzom/client/src/login_patch.cpp +++ b/code/ryzom/client/src/login_patch.cpp @@ -2688,7 +2688,7 @@ void CPatchThread::run() CPatchManager::SFileToPatch &rFTP = AllFilesToPatch[i]; string ext = NLMISC::CFile::getExtension(rFTP.FileName); - if (ext == "bnp") + if (ext == "bnp" || ext == "snp") { float oldCurrentFilePatched = CurrentFilePatched; processFile (rFTP); From 8d93e9308faea54c5775f25e42987c3f916ba04c Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 25 Sep 2014 23:57:50 +0200 Subject: [PATCH 12/27] Remove some debugging --HG-- branch : feature-streamed-package --- code/nel/src/misc/file.cpp | 2 +- code/nel/src/misc/path.cpp | 2 +- code/nel/src/misc/streamed_package_manager.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/code/nel/src/misc/file.cpp b/code/nel/src/misc/file.cpp index 9197b35a1..757f7bd49 100644 --- a/code/nel/src/misc/file.cpp +++ b/code/nel/src/misc/file.cpp @@ -235,7 +235,7 @@ bool CIFile::open(const std::string &path, bool text) else { // TEMPORARY ERROR - nlerror("File '%s' not in streamed package", path.c_str()); + // nlerror("File '%s' not in streamed package", path.c_str()); } } else diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index 553f8e4ba..4d8ad6969 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -1443,7 +1443,7 @@ void CFileContainer::addSearchStreamedPackage (const string &filename, bool recu { // Add the file to the lookup std::string filePackageName = packname + "@" + (*it); - nldebug("Insert '%s'", filePackageName.c_str()); + // nldebug("Insert '%s'", filePackageName.c_str()); insertFileInMap((*it), filePackageName, false, CFile::getExtension(*it)); // Remapped extensions diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index ffe33e76c..fbdfc1f5a 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -75,7 +75,7 @@ bool CStreamedPackageManager::loadPackage(const std::string &package) void CStreamedPackageManager::list(std::vector &fileNames, const std::string &package) { - nldebug("List package '%s'", package.c_str()); + // nldebug("List package '%s'", package.c_str()); std::map::iterator it = m_Packages.find(package); CStreamedPackage &p = it->second; From 242476e1f6294c5258d13dae5b88a9f3bbd95f4b Mon Sep 17 00:00:00 2001 From: kaetemi Date: Mon, 24 Nov 2014 14:54:15 +0100 Subject: [PATCH 13/27] Fix library linking --HG-- branch : feature-streamed-package --- code/nel/src/misc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nel/src/misc/CMakeLists.txt b/code/nel/src/misc/CMakeLists.txt index 7f172bbfe..8836bf566 100644 --- a/code/nel/src/misc/CMakeLists.txt +++ b/code/nel/src/misc/CMakeLists.txt @@ -39,7 +39,7 @@ ENDIF(UNIX) INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR} ${PNG_INCLUDE_DIR} config_file) -TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARIES} nel_sevenzip) +TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARY} nel_sevenzip) SET_TARGET_PROPERTIES(nelmisc PROPERTIES LINK_INTERFACE_LIBRARIES "") NL_DEFAULT_PROPS(nelmisc "NeL, Library: NeL Misc") NL_ADD_RUNTIME_FLAGS(nelmisc) From 6b5d7b66a499b820f6a18dcdb21052dab983c2fb Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 10 Dec 2014 16:51:42 +0100 Subject: [PATCH 14/27] Fix compile --HG-- branch : feature-streamed-package --- code/nel/include/nel/misc/app_context.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nel/include/nel/misc/app_context.h b/code/nel/include/nel/misc/app_context.h index 3ab77aada..e30642646 100644 --- a/code/nel/include/nel/misc/app_context.h +++ b/code/nel/include/nel/misc/app_context.h @@ -233,7 +233,7 @@ namespace NLMISC NLMISC_SAFE_SINGLETON_DECL(className); \ \ public: \ - static void className::releaseInstance() \ + static void releaseInstance() \ { \ if (_Instance) \ { \ From 153ea8169dfdd4c215d8bd323b051946d9b5cb83 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 10 Dec 2014 17:00:26 +0100 Subject: [PATCH 15/27] Fix link --HG-- branch : feature-streamed-package --- code/nel/src/misc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nel/src/misc/CMakeLists.txt b/code/nel/src/misc/CMakeLists.txt index 8836bf566..82a40616e 100644 --- a/code/nel/src/misc/CMakeLists.txt +++ b/code/nel/src/misc/CMakeLists.txt @@ -39,7 +39,7 @@ ENDIF(UNIX) INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR} ${PNG_INCLUDE_DIR} config_file) -TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARY} nel_sevenzip) +TARGET_LINK_LIBRARIES(nelmisc ${CMAKE_THREAD_LIBS_INIT} ${LIBXML2_LIBRARIES} ${ZLIB_LIBRARY} ${CURL_LIBRARY} ${CURL_LIBRARIES} nel_sevenzip) SET_TARGET_PROPERTIES(nelmisc PROPERTIES LINK_INTERFACE_LIBRARIES "") NL_DEFAULT_PROPS(nelmisc "NeL, Library: NeL Misc") NL_ADD_RUNTIME_FLAGS(nelmisc) From b19c685ace6e610500cef9324089b49f513a4526 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 10 Dec 2014 17:04:40 +0100 Subject: [PATCH 16/27] Fix link --HG-- branch : feature-streamed-package --- code/CMakeLists.txt | 36 ++++++++++++++++---------------- code/nel/src/misc/CMakeLists.txt | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index 84d903d89..a6b7e2727 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -150,31 +150,31 @@ IF(WITH_NEL) IF(WITH_GUI) FIND_PACKAGE(Libwww REQUIRED) FIND_PACKAGE(Luabind REQUIRED) + ENDIF(WITH_GUI) FIND_PACKAGE(CURL REQUIRED) - IF(WIN32 OR CURL_LIBRARIES MATCHES "\\.a") - SET(CURL_STATIC ON) - ENDIF(WIN32 OR CURL_LIBRARIES MATCHES "\\.a") + IF(WIN32 OR CURL_LIBRARIES MATCHES "\\.a") + SET(CURL_STATIC ON) + ENDIF(WIN32 OR CURL_LIBRARIES MATCHES "\\.a") - IF(CURL_STATIC) - SET(CURL_DEFINITIONS -DCURL_STATICLIB) + IF(CURL_STATIC) + SET(CURL_DEFINITIONS -DCURL_STATICLIB) - FIND_PACKAGE(OpenSSL QUIET) + FIND_PACKAGE(OpenSSL QUIET) - IF(OPENSSL_FOUND) - SET(CURL_INCLUDE_DIRS ${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR}) - SET(CURL_LIBRARIES ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES}) - ENDIF(OPENSSL_FOUND) + IF(OPENSSL_FOUND) + SET(CURL_INCLUDE_DIRS ${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR}) + SET(CURL_LIBRARIES ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES}) + ENDIF(OPENSSL_FOUND) - # CURL Macports version depends on libidn, libintl and libiconv too - IF(APPLE) - FIND_LIBRARY(IDN_LIBRARY idn) - FIND_LIBRARY(INTL_LIBRARY intl) + # CURL Macports version depends on libidn, libintl and libiconv too + IF(APPLE) + FIND_LIBRARY(IDN_LIBRARY idn) + FIND_LIBRARY(INTL_LIBRARY intl) - SET(CURL_LIBRARIES ${CURL_LIBRARIES} ${IDN_LIBRARY} ${INTL_LIBRARY}) - ENDIF(APPLE) - ENDIF(CURL_STATIC) - ENDIF(WITH_GUI) + SET(CURL_LIBRARIES ${CURL_LIBRARIES} ${IDN_LIBRARY} ${INTL_LIBRARY}) + ENDIF(APPLE) + ENDIF(CURL_STATIC) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/nel/include ${LIBXML2_INCLUDE_DIR}) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/nel/3rdparty) diff --git a/code/nel/src/misc/CMakeLists.txt b/code/nel/src/misc/CMakeLists.txt index 82a40616e..f5bfa6c7b 100644 --- a/code/nel/src/misc/CMakeLists.txt +++ b/code/nel/src/misc/CMakeLists.txt @@ -46,7 +46,7 @@ NL_ADD_RUNTIME_FLAGS(nelmisc) NL_ADD_LIB_SUFFIX(nelmisc) -ADD_DEFINITIONS(${LIBXML2_DEFINITIONS}) +ADD_DEFINITIONS(${LIBXML2_DEFINITIONS} ${CURL_DEFINITIONS}) IF(WITH_PCH) ADD_NATIVE_PRECOMPILED_HEADER(nelmisc ${CMAKE_CURRENT_SOURCE_DIR}/stdmisc.h ${CMAKE_CURRENT_SOURCE_DIR}/stdmisc.cpp) From a7af274ae0629bd2bfb937f5aa538e7285ad06f3 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Fri, 12 Dec 2014 01:21:07 +0100 Subject: [PATCH 17/27] Don't decompress empty file in stream package --HG-- branch : feature-streamed-package --- code/nel/src/misc/streamed_package_manager.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index fbdfc1f5a..8114da948 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -231,12 +231,15 @@ bool CStreamedPackageManager::getFile(std::string &filePath, const std::string & // allocate the output buffer :( std::vector outBuffer; outBuffer.resize(fileSize); - // decompress the file in memory - ret = LzmaDecode(&state, (unsigned char *)pos, (SizeT)(inBuffer.size() - (pos - &inBuffer[0])), &inProcessed, (unsigned char*)&outBuffer[0], (SizeT)fileSize, &outProcessed); - if (ret != 0 || outProcessed != fileSize) + if (fileSize) { - nlwarning("Failed to decode lzma file '%s'", downloadPath.c_str()); - return false; + // decompress the file in memory + ret = LzmaDecode(&state, (unsigned char *)pos, (SizeT)(inBuffer.size() - (pos - &inBuffer[0])), &inProcessed, (unsigned char*)&outBuffer[0], (SizeT)fileSize, &outProcessed); + if (ret != 0 || outProcessed != fileSize) + { + nlwarning("Failed to decode lzma file '%s'", downloadPath.c_str()); + return false; + } } { From 0f59cb4343b922b815113fadba70e1d6d7f813cb Mon Sep 17 00:00:00 2001 From: kaetemi Date: Fri, 12 Dec 2014 01:21:07 +0100 Subject: [PATCH 18/27] Validate downloaded files --HG-- branch : feature-streamed-package --- code/nel/src/misc/streamed_package_manager.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 8114da948..89e5724c3 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -27,6 +27,7 @@ #include #include #include +#include namespace NLMISC { @@ -242,6 +243,15 @@ bool CStreamedPackageManager::getFile(std::string &filePath, const std::string & } } + CHashKey outHash = outBuffer.size() ? getSHA1(&outBuffer[0], outBuffer.size()) : CHashKey(); + if (!(outHash == entry->Hash)) + { + std::string wantHashS = entry->Hash.toString(); + std::string outHashS = outHash.toString(); + nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str()); + return false; + } + { COFile outStream(unpackPath); outStream.serialBuffer(&outBuffer[0], (uint)fileSize); From e277fedfb8f43a201a59c8416e8471be3c6ecbf2 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Fri, 12 Dec 2014 01:21:07 +0100 Subject: [PATCH 19/27] Don't write empty buffer --HG-- branch : feature-streamed-package --- code/nel/src/misc/streamed_package_manager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 89e5724c3..23b77def7 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -254,7 +254,8 @@ bool CStreamedPackageManager::getFile(std::string &filePath, const std::string & { COFile outStream(unpackPath); - outStream.serialBuffer(&outBuffer[0], (uint)fileSize); + if (fileSize) + outStream.serialBuffer(&outBuffer[0], (uint)fileSize); } if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str())) From 5db42f1eb621ac33366aa4d7f7c82fe375c62fd8 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Thu, 16 Apr 2015 17:20:29 +0200 Subject: [PATCH 20/27] Changed path cfg --HG-- branch : feature-streamed-package --- code/ryzom/client/src/init.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/ryzom/client/src/init.cpp b/code/ryzom/client/src/init.cpp index c40cbd258..350a2496c 100644 --- a/code/ryzom/client/src/init.cpp +++ b/code/ryzom/client/src/init.cpp @@ -646,7 +646,7 @@ void initStereoDisplayDevice() void initStreamedPackageManager(NLMISC::IProgressCallback &progress) { CStreamedPackageManager &spm = CStreamedPackageManager::getInstance(); - spm.Path = replaceApplicationDirToken(ClientCfg.StreamedPackagePath); + spm.Path = ClientCfg.StreamedPackagePath; for (uint i = 0; i < ClientCfg.StreamedPackageHosts.size(); i++) spm.Hosts.push_back(ClientCfg.StreamedPackageHosts[i]); } From ce9fac523308938c953f35d8e43a316ae03a5f66 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 13 Nov 2019 16:13:31 +0800 Subject: [PATCH 21/27] Cleanup streamed package merge --- code/CMakeLists.txt | 6 +++--- code/nel/tools/misc/CMakeLists.txt | 10 +++++++++- code/ryzom/client/src/login_patch.cpp | 1 + 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt index e263da274..46900daa6 100644 --- a/code/CMakeLists.txt +++ b/code/CMakeLists.txt @@ -48,15 +48,15 @@ CHECK_OUT_OF_SOURCE() IF(CMAKE_VERSION VERSION_GREATER "2.8.10") STRING(TIMESTAMP CURRENT_YEAR "%Y") ELSE() - SET(CURRENT_YEAR "2016") + SET(CURRENT_YEAR "2019") ENDIF() CMAKE_MINIMUM_REQUIRED(VERSION 2.6) PROJECT(RyzomCore CXX C) SET(NL_VERSION_MAJOR 1) SET(NL_VERSION_MINOR 0) -SET(NL_VERSION_PATCH 1) -SET(YEAR "2004-${CURRENT_YEAR}") +SET(NL_VERSION_PATCH 2) +SET(YEAR "2001-${CURRENT_YEAR}") SET(AUTHOR "Winch Gate and The Ryzom Core Community") SET(RYZOM_VERSION_MAJOR 3) diff --git a/code/nel/tools/misc/CMakeLists.txt b/code/nel/tools/misc/CMakeLists.txt index 8f962d1f2..f9112a823 100644 --- a/code/nel/tools/misc/CMakeLists.txt +++ b/code/nel/tools/misc/CMakeLists.txt @@ -4,7 +4,15 @@ IF(WITH_QT OR WITH_QT5) ENDIF() IF(WITH_NEL_TOOLS) - SUBDIRS(bnp_make disp_sheet_id extract_filename lock make_sheet_id xml_packer) + SUBDIRS( + bnp_make + snp_make + disp_sheet_id + extract_filename + lock + make_sheet_id + xml_packer + ) IF(WITH_QT OR WITH_QT5) ADD_SUBDIRECTORY(words_dic_qt) diff --git a/code/ryzom/client/src/login_patch.cpp b/code/ryzom/client/src/login_patch.cpp index 84d5247fc..908f12977 100644 --- a/code/ryzom/client/src/login_patch.cpp +++ b/code/ryzom/client/src/login_patch.cpp @@ -1882,6 +1882,7 @@ bool CPatchManager::bnpUnpack(const string &srcBigfile, const string &dstPath, v return false; } + // Unpack if (!bnpFile.unpack(DestPath)) return false; From 3fe0ad30a3306c08eed09bf6623090c228aef867 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 13 Nov 2019 18:20:39 +0800 Subject: [PATCH 22/27] Merge streamed package feature with latest 7z --- code/nel/3rdparty/seven_zip/7zAlloc.cpp | 80 - code/nel/3rdparty/seven_zip/7zBuffer.cpp | 29 - code/nel/3rdparty/seven_zip/7zBuffer.h | 19 - code/nel/3rdparty/seven_zip/7zCrc.cpp | 128 - code/nel/3rdparty/seven_zip/7zDecode.cpp | 150 - code/nel/3rdparty/seven_zip/7zDecode.h | 21 - code/nel/3rdparty/seven_zip/7zExtract.cpp | 116 - code/nel/3rdparty/seven_zip/7zExtract.h | 40 - code/nel/3rdparty/seven_zip/7zHeader.cpp | 5 - code/nel/3rdparty/seven_zip/7zHeader.h | 55 - code/nel/3rdparty/seven_zip/7zIn.cpp | 1281 ------- code/nel/3rdparty/seven_zip/7zIn.h | 55 - code/nel/3rdparty/seven_zip/7zItem.cpp | 133 - code/nel/3rdparty/seven_zip/7zItem.h | 90 - code/nel/3rdparty/seven_zip/7zMain.cpp | 686 ---- code/nel/3rdparty/seven_zip/7zMethodID.cpp | 14 - code/nel/3rdparty/seven_zip/7zMethodID.h | 18 - code/nel/3rdparty/seven_zip/BranchTypes.h | 25 - code/nel/3rdparty/seven_zip/BranchX86.cpp | 101 - code/nel/3rdparty/seven_zip/BranchX86.h | 13 - code/nel/3rdparty/seven_zip/CMakeLists.txt | 20 +- code/nel/3rdparty/seven_zip/LzmaDecode.cpp | 584 ---- code/nel/3rdparty/seven_zip/LzmaDecode.h | 113 - code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp | 79 - code/nel/3rdparty/seven_zip/LzmaRamDecode.h | 55 - code/nel/3rdparty/seven_zip/LzmaTypes.h | 45 - code/nel/3rdparty/seven_zip/readme.txt | 22 - .../include/nel/misc}/seven_zip.h | 8 + code/nel/include/nel/misc/sha1.h | 34 + code/nel/src/misc/CMakeLists.txt | 9 +- .../seven_zip => nel/src/misc}/seven_zip.cpp | 101 +- code/nel/src/misc/sha1.cpp | 71 +- .../nel/src/misc/streamed_package_manager.cpp | 110 +- .../tools/build_gamedata/d1_client_patch.py | 12 +- code/nel/tools/misc/snp_make/main.cpp | 11 +- code/ryzom/client/CMakeLists.txt | 1 - code/ryzom/client/src/CMakeLists.txt | 9 +- code/ryzom/client/src/seven_zip/7z.h | 202 -- code/ryzom/client/src/seven_zip/7zArcIn.cpp | 1771 ---------- code/ryzom/client/src/seven_zip/7zBuf.cpp | 36 - code/ryzom/client/src/seven_zip/7zBuf.h | 35 - code/ryzom/client/src/seven_zip/7zBuf2.cpp | 52 - code/ryzom/client/src/seven_zip/7zCrcOpt.cpp | 115 - code/ryzom/client/src/seven_zip/7zDec.cpp | 591 ---- code/ryzom/client/src/seven_zip/7zFile.cpp | 286 -- code/ryzom/client/src/seven_zip/7zFile.h | 83 - code/ryzom/client/src/seven_zip/7zStream.cpp | 176 - code/ryzom/client/src/seven_zip/7zVersion.h | 27 - code/ryzom/client/src/seven_zip/7zVersion.rc | 55 - code/ryzom/client/src/seven_zip/Aes.cpp | 306 -- code/ryzom/client/src/seven_zip/Aes.h | 38 - code/ryzom/client/src/seven_zip/AesOpt.cpp | 184 - code/ryzom/client/src/seven_zip/Alloc.cpp | 455 --- code/ryzom/client/src/seven_zip/Alloc.h | 51 - code/ryzom/client/src/seven_zip/Bcj2.cpp | 257 -- code/ryzom/client/src/seven_zip/Bcj2.h | 146 - code/ryzom/client/src/seven_zip/Bcj2Enc.cpp | 311 -- code/ryzom/client/src/seven_zip/Bra.cpp | 230 -- code/ryzom/client/src/seven_zip/Bra.h | 64 - code/ryzom/client/src/seven_zip/Bra86.cpp | 82 - code/ryzom/client/src/seven_zip/BraIA64.cpp | 53 - .../ryzom/client/src/seven_zip/CMakeLists.txt | 31 - code/ryzom/client/src/seven_zip/Compiler.h | 33 - code/ryzom/client/src/seven_zip/CpuArch.cpp | 218 -- code/ryzom/client/src/seven_zip/CpuArch.h | 336 -- code/ryzom/client/src/seven_zip/Delta.cpp | 64 - code/ryzom/client/src/seven_zip/Delta.h | 19 - code/ryzom/client/src/seven_zip/LzFind.cpp | 1127 ------- code/ryzom/client/src/seven_zip/LzFind.h | 121 - code/ryzom/client/src/seven_zip/LzHash.h | 57 - code/ryzom/client/src/seven_zip/Lzma2Dec.cpp | 488 --- code/ryzom/client/src/seven_zip/Lzma2Dec.h | 120 - code/ryzom/client/src/seven_zip/Lzma2Enc.cpp | 803 ----- code/ryzom/client/src/seven_zip/Lzma2Enc.h | 55 - code/ryzom/client/src/seven_zip/Lzma86.h | 111 - code/ryzom/client/src/seven_zip/Lzma86Dec.cpp | 54 - code/ryzom/client/src/seven_zip/Lzma86Enc.cpp | 106 - code/ryzom/client/src/seven_zip/LzmaDec.cpp | 1185 ------- code/ryzom/client/src/seven_zip/LzmaDec.h | 234 -- code/ryzom/client/src/seven_zip/LzmaEnc.cpp | 2978 ----------------- code/ryzom/client/src/seven_zip/LzmaEnc.h | 76 - code/ryzom/client/src/seven_zip/LzmaLib.cpp | 40 - code/ryzom/client/src/seven_zip/LzmaLib.h | 131 - code/ryzom/client/src/seven_zip/Ppmd.h | 85 - code/ryzom/client/src/seven_zip/Ppmd7.cpp | 712 ---- code/ryzom/client/src/seven_zip/Ppmd7.h | 142 - code/ryzom/client/src/seven_zip/Ppmd7Dec.cpp | 191 -- code/ryzom/client/src/seven_zip/Ppmd7Enc.cpp | 187 -- code/ryzom/client/src/seven_zip/Precomp.cpp | 4 - code/ryzom/client/src/seven_zip/Precomp.h | 19 - code/ryzom/client/src/seven_zip/RotateDefs.h | 30 - code/ryzom/client/src/seven_zip/Sha256.cpp | 248 -- code/ryzom/client/src/seven_zip/Sha256.h | 26 - code/ryzom/client/src/seven_zip/Sort.cpp | 141 - code/ryzom/client/src/seven_zip/Sort.h | 18 - code/ryzom/client/src/seven_zip/Xz.cpp | 90 - code/ryzom/client/src/seven_zip/Xz.h | 460 --- code/ryzom/client/src/seven_zip/XzCrc64.cpp | 86 - code/ryzom/client/src/seven_zip/XzCrc64.h | 26 - .../ryzom/client/src/seven_zip/XzCrc64Opt.cpp | 69 - code/ryzom/client/src/seven_zip/XzDec.cpp | 2766 --------------- code/ryzom/client/src/seven_zip/XzEnc.cpp | 1329 -------- code/ryzom/client/src/seven_zip/XzEnc.h | 60 - code/ryzom/client/src/seven_zip/XzIn.cpp | 319 -- .../client/client_patcher/CMakeLists.txt | 2 +- .../client/ryzom_installer/CMakeLists.txt | 4 +- code/ryzom/tools/patch_gen/CMakeLists.txt | 6 +- 107 files changed, 242 insertions(+), 25114 deletions(-) delete mode 100644 code/nel/3rdparty/seven_zip/7zAlloc.cpp delete mode 100644 code/nel/3rdparty/seven_zip/7zBuffer.cpp delete mode 100644 code/nel/3rdparty/seven_zip/7zBuffer.h delete mode 100644 code/nel/3rdparty/seven_zip/7zCrc.cpp delete mode 100644 code/nel/3rdparty/seven_zip/7zDecode.cpp delete mode 100644 code/nel/3rdparty/seven_zip/7zDecode.h delete mode 100644 code/nel/3rdparty/seven_zip/7zExtract.cpp delete mode 100644 code/nel/3rdparty/seven_zip/7zExtract.h delete mode 100644 code/nel/3rdparty/seven_zip/7zHeader.cpp delete mode 100644 code/nel/3rdparty/seven_zip/7zHeader.h delete mode 100644 code/nel/3rdparty/seven_zip/7zIn.cpp delete mode 100644 code/nel/3rdparty/seven_zip/7zIn.h delete mode 100644 code/nel/3rdparty/seven_zip/7zItem.cpp delete mode 100644 code/nel/3rdparty/seven_zip/7zItem.h delete mode 100644 code/nel/3rdparty/seven_zip/7zMain.cpp delete mode 100644 code/nel/3rdparty/seven_zip/7zMethodID.cpp delete mode 100644 code/nel/3rdparty/seven_zip/7zMethodID.h delete mode 100644 code/nel/3rdparty/seven_zip/BranchTypes.h delete mode 100644 code/nel/3rdparty/seven_zip/BranchX86.cpp delete mode 100644 code/nel/3rdparty/seven_zip/BranchX86.h delete mode 100644 code/nel/3rdparty/seven_zip/LzmaDecode.cpp delete mode 100644 code/nel/3rdparty/seven_zip/LzmaDecode.h delete mode 100644 code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp delete mode 100644 code/nel/3rdparty/seven_zip/LzmaRamDecode.h delete mode 100644 code/nel/3rdparty/seven_zip/LzmaTypes.h delete mode 100644 code/nel/3rdparty/seven_zip/readme.txt rename code/{ryzom/client/src/seven_zip => nel/include/nel/misc}/seven_zip.h (86%) rename code/{ryzom/client/src/seven_zip => nel/src/misc}/seven_zip.cpp (80%) delete mode 100644 code/ryzom/client/src/seven_zip/7z.h delete mode 100644 code/ryzom/client/src/seven_zip/7zArcIn.cpp delete mode 100644 code/ryzom/client/src/seven_zip/7zBuf.cpp delete mode 100644 code/ryzom/client/src/seven_zip/7zBuf.h delete mode 100644 code/ryzom/client/src/seven_zip/7zBuf2.cpp delete mode 100644 code/ryzom/client/src/seven_zip/7zCrcOpt.cpp delete mode 100644 code/ryzom/client/src/seven_zip/7zDec.cpp delete mode 100644 code/ryzom/client/src/seven_zip/7zFile.cpp delete mode 100644 code/ryzom/client/src/seven_zip/7zFile.h delete mode 100644 code/ryzom/client/src/seven_zip/7zStream.cpp delete mode 100644 code/ryzom/client/src/seven_zip/7zVersion.h delete mode 100644 code/ryzom/client/src/seven_zip/7zVersion.rc delete mode 100644 code/ryzom/client/src/seven_zip/Aes.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Aes.h delete mode 100644 code/ryzom/client/src/seven_zip/AesOpt.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Alloc.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Alloc.h delete mode 100644 code/ryzom/client/src/seven_zip/Bcj2.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Bcj2.h delete mode 100644 code/ryzom/client/src/seven_zip/Bcj2Enc.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Bra.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Bra.h delete mode 100644 code/ryzom/client/src/seven_zip/Bra86.cpp delete mode 100644 code/ryzom/client/src/seven_zip/BraIA64.cpp delete mode 100644 code/ryzom/client/src/seven_zip/CMakeLists.txt delete mode 100644 code/ryzom/client/src/seven_zip/Compiler.h delete mode 100644 code/ryzom/client/src/seven_zip/CpuArch.cpp delete mode 100644 code/ryzom/client/src/seven_zip/CpuArch.h delete mode 100644 code/ryzom/client/src/seven_zip/Delta.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Delta.h delete mode 100644 code/ryzom/client/src/seven_zip/LzFind.cpp delete mode 100644 code/ryzom/client/src/seven_zip/LzFind.h delete mode 100644 code/ryzom/client/src/seven_zip/LzHash.h delete mode 100644 code/ryzom/client/src/seven_zip/Lzma2Dec.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Lzma2Dec.h delete mode 100644 code/ryzom/client/src/seven_zip/Lzma2Enc.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Lzma2Enc.h delete mode 100644 code/ryzom/client/src/seven_zip/Lzma86.h delete mode 100644 code/ryzom/client/src/seven_zip/Lzma86Dec.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Lzma86Enc.cpp delete mode 100644 code/ryzom/client/src/seven_zip/LzmaDec.cpp delete mode 100644 code/ryzom/client/src/seven_zip/LzmaDec.h delete mode 100644 code/ryzom/client/src/seven_zip/LzmaEnc.cpp delete mode 100644 code/ryzom/client/src/seven_zip/LzmaEnc.h delete mode 100644 code/ryzom/client/src/seven_zip/LzmaLib.cpp delete mode 100644 code/ryzom/client/src/seven_zip/LzmaLib.h delete mode 100644 code/ryzom/client/src/seven_zip/Ppmd.h delete mode 100644 code/ryzom/client/src/seven_zip/Ppmd7.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Ppmd7.h delete mode 100644 code/ryzom/client/src/seven_zip/Ppmd7Dec.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Ppmd7Enc.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Precomp.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Precomp.h delete mode 100644 code/ryzom/client/src/seven_zip/RotateDefs.h delete mode 100644 code/ryzom/client/src/seven_zip/Sha256.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Sha256.h delete mode 100644 code/ryzom/client/src/seven_zip/Sort.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Sort.h delete mode 100644 code/ryzom/client/src/seven_zip/Xz.cpp delete mode 100644 code/ryzom/client/src/seven_zip/Xz.h delete mode 100644 code/ryzom/client/src/seven_zip/XzCrc64.cpp delete mode 100644 code/ryzom/client/src/seven_zip/XzCrc64.h delete mode 100644 code/ryzom/client/src/seven_zip/XzCrc64Opt.cpp delete mode 100644 code/ryzom/client/src/seven_zip/XzDec.cpp delete mode 100644 code/ryzom/client/src/seven_zip/XzEnc.cpp delete mode 100644 code/ryzom/client/src/seven_zip/XzEnc.h delete mode 100644 code/ryzom/client/src/seven_zip/XzIn.cpp diff --git a/code/nel/3rdparty/seven_zip/7zAlloc.cpp b/code/nel/3rdparty/seven_zip/7zAlloc.cpp deleted file mode 100644 index c924a529f..000000000 --- a/code/nel/3rdparty/seven_zip/7zAlloc.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* 7zAlloc.c -- Allocation functions -2017-04-03 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include - -#include "7zAlloc.h" - -/* #define _SZ_ALLOC_DEBUG */ -/* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ - -#ifdef _SZ_ALLOC_DEBUG - -#ifdef _WIN32 -#include -#endif - -#include -int g_allocCount = 0; -int g_allocCountTemp = 0; - -#endif - -void *SzAlloc(ISzAllocPtr p, size_t size) -{ - UNUSED_VAR(p); - if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc %10u bytes; count = %10d", (unsigned)size, g_allocCount); - g_allocCount++; - #endif - return malloc(size); -} - -void SzFree(ISzAllocPtr p, void *address) -{ - UNUSED_VAR(p); - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - { - g_allocCount--; - fprintf(stderr, "\nFree; count = %10d", g_allocCount); - } - #endif - free(address); -} - -void *SzAllocTemp(ISzAllocPtr p, size_t size) -{ - UNUSED_VAR(p); - if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_temp %10u bytes; count = %10d", (unsigned)size, g_allocCountTemp); - g_allocCountTemp++; - #ifdef _WIN32 - return HeapAlloc(GetProcessHeap(), 0, size); - #endif - #endif - return malloc(size); -} - -void SzFreeTemp(ISzAllocPtr p, void *address) -{ - UNUSED_VAR(p); - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - { - g_allocCountTemp--; - fprintf(stderr, "\nFree_temp; count = %10d", g_allocCountTemp); - } - #ifdef _WIN32 - HeapFree(GetProcessHeap(), 0, address); - return; - #endif - #endif - free(address); -} diff --git a/code/nel/3rdparty/seven_zip/7zBuffer.cpp b/code/nel/3rdparty/seven_zip/7zBuffer.cpp deleted file mode 100644 index 3c4b71e80..000000000 --- a/code/nel/3rdparty/seven_zip/7zBuffer.cpp +++ /dev/null @@ -1,29 +0,0 @@ -/* 7zBuffer.c */ - -#include "7zBuffer.h" -#include "7zAlloc.h" - -void SzByteBufferInit(CSzByteBuffer *buffer) -{ - buffer->Capacity = 0; - buffer->Items = 0; -} - -int SzByteBufferCreate(CSzByteBuffer *buffer, size_t newCapacity, void * (*allocFunc)(size_t size)) -{ - buffer->Capacity = newCapacity; - if (newCapacity == 0) - { - buffer->Items = 0; - return 1; - } - buffer->Items = (Byte *)allocFunc(newCapacity); - return (buffer->Items != 0); -} - -void SzByteBufferFree(CSzByteBuffer *buffer, void (*freeFunc)(void *)) -{ - freeFunc(buffer->Items); - buffer->Items = 0; - buffer->Capacity = 0; -} diff --git a/code/nel/3rdparty/seven_zip/7zBuffer.h b/code/nel/3rdparty/seven_zip/7zBuffer.h deleted file mode 100644 index 17e59060f..000000000 --- a/code/nel/3rdparty/seven_zip/7zBuffer.h +++ /dev/null @@ -1,19 +0,0 @@ -/* 7zBuffer.h */ - -#ifndef __7Z_BUFFER_H -#define __7Z_BUFFER_H - -#include -#include "7zTypes.h" - -typedef struct _CSzByteBuffer -{ - size_t Capacity; - Byte *Items; -}CSzByteBuffer; - -void SzByteBufferInit(CSzByteBuffer *buffer); -int SzByteBufferCreate(CSzByteBuffer *buffer, size_t newCapacity, void * (*allocFunc)(size_t size)); -void SzByteBufferFree(CSzByteBuffer *buffer, void (*freeFunc)(void *)); - -#endif diff --git a/code/nel/3rdparty/seven_zip/7zCrc.cpp b/code/nel/3rdparty/seven_zip/7zCrc.cpp deleted file mode 100644 index b4d84f023..000000000 --- a/code/nel/3rdparty/seven_zip/7zCrc.cpp +++ /dev/null @@ -1,128 +0,0 @@ -/* 7zCrc.c -- CRC32 init -2017-06-06 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "7zCrc.h" -#include "CpuArch.h" - -#define kCrcPoly 0xEDB88320 - -#ifdef MY_CPU_LE - #define CRC_NUM_TABLES 8 -#else - #define CRC_NUM_TABLES 9 - - #define CRC_UINT32_SWAP(v) ((v >> 24) | ((v >> 8) & 0xFF00) | ((v << 8) & 0xFF0000) | (v << 24)) - - UInt32 MY_FAST_CALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, const UInt32 *table); - UInt32 MY_FAST_CALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, const UInt32 *table); -#endif - -#ifndef MY_CPU_BE - UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table); - UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table); -#endif - -typedef UInt32 (MY_FAST_CALL *CRC_FUNC)(UInt32 v, const void *data, size_t size, const UInt32 *table); - -CRC_FUNC g_CrcUpdateT4; -CRC_FUNC g_CrcUpdateT8; -CRC_FUNC g_CrcUpdate; - -UInt32 g_CrcTable[256 * CRC_NUM_TABLES]; - -UInt32 MY_FAST_CALL CrcUpdate(UInt32 v, const void *data, size_t size) -{ - return g_CrcUpdate(v, data, size, g_CrcTable); -} - -UInt32 MY_FAST_CALL CrcCalc(const void *data, size_t size) -{ - return g_CrcUpdate(CRC_INIT_VAL, data, size, g_CrcTable) ^ CRC_INIT_VAL; -} - -#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) - -UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table) -{ - const Byte *p = (const Byte *)data; - const Byte *pEnd = p + size; - for (; p != pEnd; p++) - v = CRC_UPDATE_BYTE_2(v, *p); - return v; -} - -void MY_FAST_CALL CrcGenerateTable() -{ - UInt32 i; - for (i = 0; i < 256; i++) - { - UInt32 r = i; - unsigned j; - for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); - g_CrcTable[i] = r; - } - for (i = 256; i < 256 * CRC_NUM_TABLES; i++) - { - UInt32 r = g_CrcTable[(size_t)i - 256]; - g_CrcTable[i] = g_CrcTable[r & 0xFF] ^ (r >> 8); - } - - #if CRC_NUM_TABLES < 4 - - g_CrcUpdate = CrcUpdateT1; - - #else - - #ifdef MY_CPU_LE - - g_CrcUpdateT4 = CrcUpdateT4; - g_CrcUpdate = CrcUpdateT4; - - #if CRC_NUM_TABLES >= 8 - g_CrcUpdateT8 = CrcUpdateT8; - - #ifdef MY_CPU_X86_OR_AMD64 - if (!CPU_Is_InOrder()) - #endif - g_CrcUpdate = CrcUpdateT8; - #endif - - #else - { - #ifndef MY_CPU_BE - UInt32 k = 0x01020304; - const Byte *p = (const Byte *)&k; - if (p[0] == 4 && p[1] == 3) - { - g_CrcUpdateT4 = CrcUpdateT4; - g_CrcUpdate = CrcUpdateT4; - #if CRC_NUM_TABLES >= 8 - g_CrcUpdateT8 = CrcUpdateT8; - g_CrcUpdate = CrcUpdateT8; - #endif - } - else if (p[0] != 1 || p[1] != 2) - g_CrcUpdate = CrcUpdateT1; - else - #endif - { - for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--) - { - UInt32 x = g_CrcTable[(size_t)i - 256]; - g_CrcTable[i] = CRC_UINT32_SWAP(x); - } - g_CrcUpdateT4 = CrcUpdateT1_BeT4; - g_CrcUpdate = CrcUpdateT1_BeT4; - #if CRC_NUM_TABLES >= 8 - g_CrcUpdateT8 = CrcUpdateT1_BeT8; - g_CrcUpdate = CrcUpdateT1_BeT8; - #endif - } - } - #endif - - #endif -} diff --git a/code/nel/3rdparty/seven_zip/7zDecode.cpp b/code/nel/3rdparty/seven_zip/7zDecode.cpp deleted file mode 100644 index 50d6b8a1f..000000000 --- a/code/nel/3rdparty/seven_zip/7zDecode.cpp +++ /dev/null @@ -1,150 +0,0 @@ -/* 7zDecode.c */ - -#include "7zDecode.h" -#ifdef _SZ_ONE_DIRECTORY -#include "LzmaDecode.h" -#else -#include "../../Compress/LZMA_C/LzmaDecode.h" -#endif - -CMethodID k_Copy = { { 0x0 }, 1 }; -CMethodID k_LZMA = { { 0x3, 0x1, 0x1 }, 3 }; - -#ifdef _LZMA_IN_CB - -typedef struct _CLzmaInCallbackImp -{ - ILzmaInCallback InCallback; - ISzInStream *InStream; - size_t Size; -} CLzmaInCallbackImp; - -int LzmaReadImp(void *object, const unsigned char **buffer, SizeT *size) -{ - CLzmaInCallbackImp *cb = (CLzmaInCallbackImp *)object; - size_t processedSize; - SZ_RESULT res; - *size = 0; - res = cb->InStream->Read((void *)cb->InStream, (void **)buffer, cb->Size, &processedSize); - *size = (SizeT)processedSize; - if (processedSize > cb->Size) - return (int)SZE_FAIL; - cb->Size -= processedSize; - if (res == SZ_OK) - return 0; - return (int)res; -} - -#endif - -SZ_RESULT SzDecode(const CFileSize *packSizes, const CFolder *folder, - #ifdef _LZMA_IN_CB - ISzInStream *inStream, - #else - const Byte *inBuffer, - #endif - Byte *outBuffer, size_t outSize, - size_t *outSizeProcessed, ISzAlloc *allocMain) -{ - UInt32 si; - size_t inSize = 0; - CCoderInfo *coder; - if (folder->NumPackStreams != 1) - return SZE_NOTIMPL; - if (folder->NumCoders != 1) - return SZE_NOTIMPL; - coder = folder->Coders; - *outSizeProcessed = 0; - - for (si = 0; si < folder->NumPackStreams; si++) - inSize += (size_t)packSizes[si]; - - if (AreMethodsEqual(&coder->MethodID, &k_Copy)) - { - size_t i; - if (inSize != outSize) - return SZE_DATA_ERROR; - #ifdef _LZMA_IN_CB - for (i = 0; i < inSize;) - { - size_t j; - Byte *inBuffer; - size_t bufferSize; - RINOK(inStream->Read((void *)inStream, (void **)&inBuffer, inSize - i, &bufferSize)); - if (bufferSize == 0) - return SZE_DATA_ERROR; - if (bufferSize > inSize - i) - return SZE_FAIL; - *outSizeProcessed += bufferSize; - for (j = 0; j < bufferSize && i < inSize; j++, i++) - outBuffer[i] = inBuffer[j]; - } - #else - for (i = 0; i < inSize; i++) - outBuffer[i] = inBuffer[i]; - *outSizeProcessed = inSize; - #endif - return SZ_OK; - } - - if (AreMethodsEqual(&coder->MethodID, &k_LZMA)) - { - #ifdef _LZMA_IN_CB - CLzmaInCallbackImp lzmaCallback; - #else - SizeT inProcessed; - #endif - - CLzmaDecoderState state; /* it's about 24-80 bytes structure, if int is 32-bit */ - int result; - SizeT outSizeProcessedLoc; - - #ifdef _LZMA_IN_CB - lzmaCallback.Size = inSize; - lzmaCallback.InStream = inStream; - lzmaCallback.InCallback.Read = LzmaReadImp; - #endif - - if (LzmaDecodeProperties(&state.Properties, coder->Properties.Items, - (SizeT)coder->Properties.Capacity) != LZMA_RESULT_OK) - return SZE_FAIL; - - state.Probs = (CProb *)allocMain->Alloc(LzmaGetNumProbs(&state.Properties) * sizeof(CProb)); - if (state.Probs == 0) - return SZE_OUTOFMEMORY; - - #ifdef _LZMA_OUT_READ - if (state.Properties.DictionarySize == 0) - state.Dictionary = 0; - else - { - state.Dictionary = (unsigned char *)allocMain->Alloc(state.Properties.DictionarySize); - if (state.Dictionary == 0) - { - allocMain->Free(state.Probs); - return SZE_OUTOFMEMORY; - } - } - LzmaDecoderInit(&state); - #endif - - result = LzmaDecode(&state, - #ifdef _LZMA_IN_CB - &lzmaCallback.InCallback, - #else - inBuffer, (SizeT)inSize, &inProcessed, - #endif - outBuffer, (SizeT)outSize, &outSizeProcessedLoc); - *outSizeProcessed = (size_t)outSizeProcessedLoc; - allocMain->Free(state.Probs); - #ifdef _LZMA_OUT_READ - allocMain->Free(state.Dictionary); - #endif - if (result == LZMA_RESULT_DATA_ERROR) - return SZE_DATA_ERROR; - if (result != LZMA_RESULT_OK) - return SZE_FAIL; - return SZ_OK; - } - return SZE_NOTIMPL; -} diff --git a/code/nel/3rdparty/seven_zip/7zDecode.h b/code/nel/3rdparty/seven_zip/7zDecode.h deleted file mode 100644 index 74bb180fd..000000000 --- a/code/nel/3rdparty/seven_zip/7zDecode.h +++ /dev/null @@ -1,21 +0,0 @@ -/* 7zDecode.h */ - -#ifndef __7Z_DECODE_H -#define __7Z_DECODE_H - -#include "7zItem.h" -#include "7zAlloc.h" -#ifdef _LZMA_IN_CB -#include "7zIn.h" -#endif - -SZ_RESULT SzDecode(const CFileSize *packSizes, const CFolder *folder, - #ifdef _LZMA_IN_CB - ISzInStream *stream, - #else - const Byte *inBuffer, - #endif - Byte *outBuffer, size_t outSize, - size_t *outSizeProcessed, ISzAlloc *allocMain); - -#endif diff --git a/code/nel/3rdparty/seven_zip/7zExtract.cpp b/code/nel/3rdparty/seven_zip/7zExtract.cpp deleted file mode 100644 index 6ef872c31..000000000 --- a/code/nel/3rdparty/seven_zip/7zExtract.cpp +++ /dev/null @@ -1,116 +0,0 @@ -/* 7zExtract.c */ - -#include "7zExtract.h" -#include "7zDecode.h" -#include "7zCrc.h" - -SZ_RESULT SzExtract( - ISzInStream *inStream, - CArchiveDatabaseEx *db, - UInt32 fileIndex, - UInt32 *blockIndex, - Byte **outBuffer, - size_t *outBufferSize, - size_t *offset, - size_t *outSizeProcessed, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) -{ - UInt32 folderIndex = db->FileIndexToFolderIndexMap[fileIndex]; - SZ_RESULT res = SZ_OK; - *offset = 0; - *outSizeProcessed = 0; - if (folderIndex == (UInt32)-1) - { - allocMain->Free(*outBuffer); - *blockIndex = folderIndex; - *outBuffer = 0; - *outBufferSize = 0; - return SZ_OK; - } - - if (*outBuffer == 0 || *blockIndex != folderIndex) - { - CFolder *folder = db->Database.Folders + folderIndex; - CFileSize unPackSize = SzFolderGetUnPackSize(folder); - #ifndef _LZMA_IN_CB - CFileSize packSize = SzArDbGetFolderFullPackSize(db, folderIndex); - Byte *inBuffer = 0; - size_t processedSize; - #endif - *blockIndex = folderIndex; - allocMain->Free(*outBuffer); - *outBuffer = 0; - - RINOK(inStream->Seek(inStream, SzArDbGetFolderStreamPos(db, folderIndex, 0))); - - #ifndef _LZMA_IN_CB - if (packSize != 0) - { - inBuffer = (Byte *)allocTemp->Alloc((size_t)packSize); - if (inBuffer == 0) - return SZE_OUTOFMEMORY; - } - res = inStream->Read(inStream, inBuffer, (size_t)packSize, &processedSize); - if (res == SZ_OK && processedSize != (size_t)packSize) - res = SZE_FAIL; - #endif - if (res == SZ_OK) - { - *outBufferSize = (size_t)unPackSize; - if (unPackSize != 0) - { - *outBuffer = (Byte *)allocMain->Alloc((size_t)unPackSize); - if (*outBuffer == 0) - res = SZE_OUTOFMEMORY; - } - if (res == SZ_OK) - { - size_t outRealSize; - res = SzDecode(db->Database.PackSizes + - db->FolderStartPackStreamIndex[folderIndex], folder, - #ifdef _LZMA_IN_CB - inStream, - #else - inBuffer, - #endif - *outBuffer, (size_t)unPackSize, &outRealSize, allocTemp); - if (res == SZ_OK) - { - if (outRealSize == (size_t)unPackSize) - { - if (folder->UnPackCRCDefined) - { - if (!CrcVerifyDigest(folder->UnPackCRC, *outBuffer, (size_t)unPackSize)) - res = SZE_FAIL; - } - } - else - res = SZE_FAIL; - } - } - } - #ifndef _LZMA_IN_CB - allocTemp->Free(inBuffer); - #endif - } - if (res == SZ_OK) - { - UInt32 i; - CFileItem *fileItem = db->Database.Files + fileIndex; - *offset = 0; - for(i = db->FolderStartFileIndex[folderIndex]; i < fileIndex; i++) - *offset += (UInt32)db->Database.Files[i].Size; - *outSizeProcessed = (size_t)fileItem->Size; - if (*offset + *outSizeProcessed > *outBufferSize) - return SZE_FAIL; - { - if (fileItem->IsFileCRCDefined) - { - if (!CrcVerifyDigest(fileItem->FileCRC, *outBuffer + *offset, *outSizeProcessed)) - res = SZE_FAIL; - } - } - } - return res; -} diff --git a/code/nel/3rdparty/seven_zip/7zExtract.h b/code/nel/3rdparty/seven_zip/7zExtract.h deleted file mode 100644 index e9a4fb4e3..000000000 --- a/code/nel/3rdparty/seven_zip/7zExtract.h +++ /dev/null @@ -1,40 +0,0 @@ -/* 7zExtract.h */ - -#ifndef __7Z_EXTRACT_H -#define __7Z_EXTRACT_H - -#include "7zIn.h" - -/* - SzExtract extracts file from archive - - *outBuffer must be 0 before first call for each new archive. - - Extracting cache: - If you need to decompress more than one file, you can send - these values from previous call: - *blockIndex, - *outBuffer, - *outBufferSize - You can consider "*outBuffer" as cache of solid block. If your archive is solid, - it will increase decompression speed. - - If you use external function, you can declare these 3 cache variables - (blockIndex, outBuffer, outBufferSize) as static in that external function. - - Free *outBuffer and set *outBuffer to 0, if you want to flush cache. -*/ - -SZ_RESULT SzExtract( - ISzInStream *inStream, - CArchiveDatabaseEx *db, - UInt32 fileIndex, /* index of file */ - UInt32 *blockIndex, /* index of solid block */ - Byte **outBuffer, /* pointer to pointer to output buffer (allocated with allocMain) */ - size_t *outBufferSize, /* buffer size for output buffer */ - size_t *offset, /* offset of stream for required file in *outBuffer */ - size_t *outSizeProcessed, /* size of file in *outBuffer */ - ISzAlloc *allocMain, - ISzAlloc *allocTemp); - -#endif diff --git a/code/nel/3rdparty/seven_zip/7zHeader.cpp b/code/nel/3rdparty/seven_zip/7zHeader.cpp deleted file mode 100644 index 3be4bc27e..000000000 --- a/code/nel/3rdparty/seven_zip/7zHeader.cpp +++ /dev/null @@ -1,5 +0,0 @@ -/* 7zHeader.c */ - -#include "7zHeader.h" - -Byte k7zSignature[k7zSignatureSize] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C}; diff --git a/code/nel/3rdparty/seven_zip/7zHeader.h b/code/nel/3rdparty/seven_zip/7zHeader.h deleted file mode 100644 index 0356aaa6b..000000000 --- a/code/nel/3rdparty/seven_zip/7zHeader.h +++ /dev/null @@ -1,55 +0,0 @@ -/* 7zHeader.h */ - -#ifndef __7Z_HEADER_H -#define __7Z_HEADER_H - -#include "7zTypes.h" - -#define k7zSignatureSize 6 -extern Byte k7zSignature[k7zSignatureSize]; - -#define k7zMajorVersion 0 - -#define k7zStartHeaderSize 0x20 - -enum EIdEnum -{ - k7zIdEnd, - - k7zIdHeader, - - k7zIdArchiveProperties, - - k7zIdAdditionalStreamsInfo, - k7zIdMainStreamsInfo, - k7zIdFilesInfo, - - k7zIdPackInfo, - k7zIdUnPackInfo, - k7zIdSubStreamsInfo, - - k7zIdSize, - k7zIdCRC, - - k7zIdFolder, - - k7zIdCodersUnPackSize, - k7zIdNumUnPackStream, - - k7zIdEmptyStream, - k7zIdEmptyFile, - k7zIdAnti, - - k7zIdName, - k7zIdCreationTime, - k7zIdLastAccessTime, - k7zIdLastWriteTime, - k7zIdWinAttributes, - k7zIdComment, - - k7zIdEncodedHeader, - - k7zIdStartPos -}; - -#endif diff --git a/code/nel/3rdparty/seven_zip/7zIn.cpp b/code/nel/3rdparty/seven_zip/7zIn.cpp deleted file mode 100644 index b9dbf4695..000000000 --- a/code/nel/3rdparty/seven_zip/7zIn.cpp +++ /dev/null @@ -1,1281 +0,0 @@ -/* 7zIn.c */ - -#include "7zIn.h" -#include "7zCrc.h" -#include "7zDecode.h" - -#define RINOM(x) { if((x) == 0) return SZE_OUTOFMEMORY; } - -void SzArDbExInit(CArchiveDatabaseEx *db) -{ - SzArchiveDatabaseInit(&db->Database); - db->FolderStartPackStreamIndex = 0; - db->PackStreamStartPositions = 0; - db->FolderStartFileIndex = 0; - db->FileIndexToFolderIndexMap = 0; -} - -void SzArDbExFree(CArchiveDatabaseEx *db, void (*freeFunc)(void *)) -{ - freeFunc(db->FolderStartPackStreamIndex); - freeFunc(db->PackStreamStartPositions); - freeFunc(db->FolderStartFileIndex); - freeFunc(db->FileIndexToFolderIndexMap); - SzArchiveDatabaseFree(&db->Database, freeFunc); - SzArDbExInit(db); -} - -/* -CFileSize GetFolderPackStreamSize(int folderIndex, int streamIndex) const -{ - return PackSizes[FolderStartPackStreamIndex[folderIndex] + streamIndex]; -} - -CFileSize GetFilePackSize(int fileIndex) const -{ - int folderIndex = FileIndexToFolderIndexMap[fileIndex]; - if (folderIndex >= 0) - { - const CFolder &folderInfo = Folders[folderIndex]; - if (FolderStartFileIndex[folderIndex] == fileIndex) - return GetFolderFullPackSize(folderIndex); - } - return 0; -} -*/ - -#define MY_ALLOC(T, p, size, allocFunc) { if ((size) == 0) p = 0; else \ - if ((p = (T *)allocFunc((size) * sizeof(T))) == 0) return SZE_OUTOFMEMORY; } - -SZ_RESULT SzArDbExFill(CArchiveDatabaseEx *db, void * (*allocFunc)(size_t size)) -{ - UInt32 startPos = 0; - CFileSize startPosSize = 0; - UInt32 i; - UInt32 folderIndex = 0; - UInt32 indexInFolder = 0; - MY_ALLOC(UInt32, db->FolderStartPackStreamIndex, db->Database.NumFolders, allocFunc); - for(i = 0; i < db->Database.NumFolders; i++) - { - db->FolderStartPackStreamIndex[i] = startPos; - startPos += db->Database.Folders[i].NumPackStreams; - } - - MY_ALLOC(CFileSize, db->PackStreamStartPositions, db->Database.NumPackStreams, allocFunc); - - for(i = 0; i < db->Database.NumPackStreams; i++) - { - db->PackStreamStartPositions[i] = startPosSize; - startPosSize += db->Database.PackSizes[i]; - } - - MY_ALLOC(UInt32, db->FolderStartFileIndex, db->Database.NumFolders, allocFunc); - MY_ALLOC(UInt32, db->FileIndexToFolderIndexMap, db->Database.NumFiles, allocFunc); - - for (i = 0; i < db->Database.NumFiles; i++) - { - CFileItem *file = db->Database.Files + i; - int emptyStream = !file->HasStream; - if (emptyStream && indexInFolder == 0) - { - db->FileIndexToFolderIndexMap[i] = (UInt32)-1; - continue; - } - if (indexInFolder == 0) - { - /* - v3.13 incorrectly worked with empty folders - v4.07: Loop for skipping empty folders - */ - for(;;) - { - if (folderIndex >= db->Database.NumFolders) - return SZE_ARCHIVE_ERROR; - db->FolderStartFileIndex[folderIndex] = i; - if (db->Database.Folders[folderIndex].NumUnPackStreams != 0) - break; - folderIndex++; - } - } - db->FileIndexToFolderIndexMap[i] = folderIndex; - if (emptyStream) - continue; - indexInFolder++; - if (indexInFolder >= db->Database.Folders[folderIndex].NumUnPackStreams) - { - folderIndex++; - indexInFolder = 0; - } - } - return SZ_OK; -} - - -CFileSize SzArDbGetFolderStreamPos(CArchiveDatabaseEx *db, UInt32 folderIndex, UInt32 indexInFolder) -{ - return db->ArchiveInfo.DataStartPosition + - db->PackStreamStartPositions[db->FolderStartPackStreamIndex[folderIndex] + indexInFolder]; -} - -CFileSize SzArDbGetFolderFullPackSize(CArchiveDatabaseEx *db, UInt32 folderIndex) -{ - UInt32 packStreamIndex = db->FolderStartPackStreamIndex[folderIndex]; - CFolder *folder = db->Database.Folders + folderIndex; - CFileSize size = 0; - UInt32 i; - for (i = 0; i < folder->NumPackStreams; i++) - size += db->Database.PackSizes[packStreamIndex + i]; - return size; -} - - -/* -SZ_RESULT SzReadTime(const CObjectVector &dataVector, - CObjectVector &files, UInt64 type) -{ - CBoolVector boolVector; - RINOK(ReadBoolVector2(files.Size(), boolVector)) - - CStreamSwitch streamSwitch; - RINOK(streamSwitch.Set(this, &dataVector)); - - for(int i = 0; i < files.Size(); i++) - { - CFileItem &file = files[i]; - CArchiveFileTime fileTime; - bool defined = boolVector[i]; - if (defined) - { - UInt32 low, high; - RINOK(SzReadUInt32(low)); - RINOK(SzReadUInt32(high)); - fileTime.dwLowDateTime = low; - fileTime.dwHighDateTime = high; - } - switch(type) - { - case k7zIdCreationTime: - file.IsCreationTimeDefined = defined; - if (defined) - file.CreationTime = fileTime; - break; - case k7zIdLastWriteTime: - file.IsLastWriteTimeDefined = defined; - if (defined) - file.LastWriteTime = fileTime; - break; - case k7zIdLastAccessTime: - file.IsLastAccessTimeDefined = defined; - if (defined) - file.LastAccessTime = fileTime; - break; - } - } - return SZ_OK; -} -*/ - -SZ_RESULT SafeReadDirect(ISzInStream *inStream, Byte *data, size_t size) -{ - #ifdef _LZMA_IN_CB - while (size > 0) - { - Byte *inBuffer; - size_t processedSize; - RINOK(inStream->Read(inStream, (void **)&inBuffer, size, &processedSize)); - if (processedSize == 0 || processedSize > size) - return SZE_FAIL; - size -= processedSize; - do - { - *data++ = *inBuffer++; - } - while (--processedSize != 0); - } - #else - size_t processedSize; - RINOK(inStream->Read(inStream, data, size, &processedSize)); - if (processedSize != size) - return SZE_FAIL; - #endif - return SZ_OK; -} - -SZ_RESULT SafeReadDirectByte(ISzInStream *inStream, Byte *data) -{ - return SafeReadDirect(inStream, data, 1); -} - -SZ_RESULT SafeReadDirectUInt32(ISzInStream *inStream, UInt32 *value) -{ - int i; - *value = 0; - for (i = 0; i < 4; i++) - { - Byte b; - RINOK(SafeReadDirectByte(inStream, &b)); - *value |= ((UInt32)b << (8 * i)); - } - return SZ_OK; -} - -SZ_RESULT SafeReadDirectUInt64(ISzInStream *inStream, UInt64 *value) -{ - int i; - *value = 0; - for (i = 0; i < 8; i++) - { - Byte b; - RINOK(SafeReadDirectByte(inStream, &b)); - *value |= ((UInt32)b << (8 * i)); - } - return SZ_OK; -} - -int TestSignatureCandidate(Byte *testBytes) -{ - size_t i; - for (i = 0; i < k7zSignatureSize; i++) - if (testBytes[i] != k7zSignature[i]) - return 0; - return 1; -} - -typedef struct _CSzState -{ - Byte *Data; - size_t Size; -}CSzData; - -SZ_RESULT SzReadByte(CSzData *sd, Byte *b) -{ - if (sd->Size == 0) - return SZE_ARCHIVE_ERROR; - sd->Size--; - *b = *sd->Data++; - return SZ_OK; -} - -SZ_RESULT SzReadBytes(CSzData *sd, Byte *data, size_t size) -{ - size_t i; - for (i = 0; i < size; i++) - { - RINOK(SzReadByte(sd, data + i)); - } - return SZ_OK; -} - -SZ_RESULT SzReadUInt32(CSzData *sd, UInt32 *value) -{ - int i; - *value = 0; - for (i = 0; i < 4; i++) - { - Byte b; - RINOK(SzReadByte(sd, &b)); - *value |= ((UInt32)(b) << (8 * i)); - } - return SZ_OK; -} - -SZ_RESULT SzReadNumber(CSzData *sd, UInt64 *value) -{ - Byte firstByte; - Byte mask = 0x80; - int i; - RINOK(SzReadByte(sd, &firstByte)); - *value = 0; - for (i = 0; i < 8; i++) - { - Byte b; - if ((firstByte & mask) == 0) - { - UInt64 highPart = firstByte & (mask - 1); - *value += (highPart << (8 * i)); - return SZ_OK; - } - RINOK(SzReadByte(sd, &b)); - *value |= ((UInt64)b << (8 * i)); - mask >>= 1; - } - return SZ_OK; -} - -SZ_RESULT SzReadSize(CSzData *sd, CFileSize *value) -{ - UInt64 value64; - RINOK(SzReadNumber(sd, &value64)); - *value = (CFileSize)value64; - return SZ_OK; -} - -SZ_RESULT SzReadNumber32(CSzData *sd, UInt32 *value) -{ - UInt64 value64; - RINOK(SzReadNumber(sd, &value64)); - if (value64 >= 0x80000000) - return SZE_NOTIMPL; - if (value64 >= ((UInt64)(1) << ((sizeof(size_t) - 1) * 8 + 2))) - return SZE_NOTIMPL; - *value = (UInt32)value64; - return SZ_OK; -} - -SZ_RESULT SzReadID(CSzData *sd, UInt64 *value) -{ - return SzReadNumber(sd, value); -} - -SZ_RESULT SzSkeepDataSize(CSzData *sd, UInt64 size) -{ - if (size > sd->Size) - return SZE_ARCHIVE_ERROR; - sd->Size -= (size_t)size; - sd->Data += (size_t)size; - return SZ_OK; -} - -SZ_RESULT SzSkeepData(CSzData *sd) -{ - UInt64 size; - RINOK(SzReadNumber(sd, &size)); - return SzSkeepDataSize(sd, size); -} - -SZ_RESULT SzReadArchiveProperties(CSzData *sd) -{ - for(;;) - { - UInt64 type; - RINOK(SzReadID(sd, &type)); - if (type == k7zIdEnd) - break; - SzSkeepData(sd); - } - return SZ_OK; -} - -SZ_RESULT SzWaitAttribute(CSzData *sd, UInt64 attribute) -{ - for(;;) - { - UInt64 type; - RINOK(SzReadID(sd, &type)); - if (type == attribute) - return SZ_OK; - if (type == k7zIdEnd) - return SZE_ARCHIVE_ERROR; - RINOK(SzSkeepData(sd)); - } -} - -SZ_RESULT SzReadBoolVector(CSzData *sd, size_t numItems, Byte **v, void * (*allocFunc)(size_t size)) -{ - Byte b = 0; - Byte mask = 0; - size_t i; - MY_ALLOC(Byte, *v, numItems, allocFunc); - for(i = 0; i < numItems; i++) - { - if (mask == 0) - { - RINOK(SzReadByte(sd, &b)); - mask = 0x80; - } - (*v)[i] = (Byte)(((b & mask) != 0) ? 1 : 0); - mask >>= 1; - } - return SZ_OK; -} - -SZ_RESULT SzReadBoolVector2(CSzData *sd, size_t numItems, Byte **v, void * (*allocFunc)(size_t size)) -{ - Byte allAreDefined; - size_t i; - RINOK(SzReadByte(sd, &allAreDefined)); - if (allAreDefined == 0) - return SzReadBoolVector(sd, numItems, v, allocFunc); - MY_ALLOC(Byte, *v, numItems, allocFunc); - for(i = 0; i < numItems; i++) - (*v)[i] = 1; - return SZ_OK; -} - -SZ_RESULT SzReadHashDigests( - CSzData *sd, - size_t numItems, - Byte **digestsDefined, - UInt32 **digests, - void * (*allocFunc)(size_t size)) -{ - size_t i; - RINOK(SzReadBoolVector2(sd, numItems, digestsDefined, allocFunc)); - MY_ALLOC(UInt32, *digests, numItems, allocFunc); - for(i = 0; i < numItems; i++) - if ((*digestsDefined)[i]) - { - RINOK(SzReadUInt32(sd, (*digests) + i)); - } - return SZ_OK; -} - -SZ_RESULT SzReadPackInfo( - CSzData *sd, - CFileSize *dataOffset, - UInt32 *numPackStreams, - CFileSize **packSizes, - Byte **packCRCsDefined, - UInt32 **packCRCs, - void * (*allocFunc)(size_t size)) -{ - UInt32 i; - RINOK(SzReadSize(sd, dataOffset)); - RINOK(SzReadNumber32(sd, numPackStreams)); - - RINOK(SzWaitAttribute(sd, k7zIdSize)); - - MY_ALLOC(CFileSize, *packSizes, (size_t)*numPackStreams, allocFunc); - - for(i = 0; i < *numPackStreams; i++) - { - RINOK(SzReadSize(sd, (*packSizes) + i)); - } - - for(;;) - { - UInt64 type; - RINOK(SzReadID(sd, &type)); - if (type == k7zIdEnd) - break; - if (type == k7zIdCRC) - { - RINOK(SzReadHashDigests(sd, (size_t)*numPackStreams, packCRCsDefined, packCRCs, allocFunc)); - continue; - } - RINOK(SzSkeepData(sd)); - } - if (*packCRCsDefined == 0) - { - MY_ALLOC(Byte, *packCRCsDefined, (size_t)*numPackStreams, allocFunc); - MY_ALLOC(UInt32, *packCRCs, (size_t)*numPackStreams, allocFunc); - for(i = 0; i < *numPackStreams; i++) - { - (*packCRCsDefined)[i] = 0; - (*packCRCs)[i] = 0; - } - } - return SZ_OK; -} - -SZ_RESULT SzReadSwitch(CSzData *sd) -{ - Byte external; - RINOK(SzReadByte(sd, &external)); - return (external == 0) ? SZ_OK: SZE_ARCHIVE_ERROR; -} - -SZ_RESULT SzGetNextFolderItem(CSzData *sd, CFolder *folder, void * (*allocFunc)(size_t size)) -{ - UInt32 numCoders; - UInt32 numBindPairs; - UInt32 numPackedStreams; - UInt32 i; - UInt32 numInStreams = 0; - UInt32 numOutStreams = 0; - RINOK(SzReadNumber32(sd, &numCoders)); - folder->NumCoders = numCoders; - - MY_ALLOC(CCoderInfo, folder->Coders, (size_t)numCoders, allocFunc); - - for (i = 0; i < numCoders; i++) - SzCoderInfoInit(folder->Coders + i); - - for (i = 0; i < numCoders; i++) - { - Byte mainByte; - CCoderInfo *coder = folder->Coders + i; - { - RINOK(SzReadByte(sd, &mainByte)); - coder->MethodID.IDSize = (Byte)(mainByte & 0xF); - RINOK(SzReadBytes(sd, coder->MethodID.ID, coder->MethodID.IDSize)); - if ((mainByte & 0x10) != 0) - { - RINOK(SzReadNumber32(sd, &coder->NumInStreams)); - RINOK(SzReadNumber32(sd, &coder->NumOutStreams)); - } - else - { - coder->NumInStreams = 1; - coder->NumOutStreams = 1; - } - if ((mainByte & 0x20) != 0) - { - UInt64 propertiesSize = 0; - RINOK(SzReadNumber(sd, &propertiesSize)); - if (!SzByteBufferCreate(&coder->Properties, (size_t)propertiesSize, allocFunc)) - return SZE_OUTOFMEMORY; - RINOK(SzReadBytes(sd, coder->Properties.Items, (size_t)propertiesSize)); - } - } - while ((mainByte & 0x80) != 0) - { - RINOK(SzReadByte(sd, &mainByte)); - RINOK(SzSkeepDataSize(sd, (mainByte & 0xF))); - if ((mainByte & 0x10) != 0) - { - UInt32 n; - RINOK(SzReadNumber32(sd, &n)); - RINOK(SzReadNumber32(sd, &n)); - } - if ((mainByte & 0x20) != 0) - { - UInt64 propertiesSize = 0; - RINOK(SzReadNumber(sd, &propertiesSize)); - RINOK(SzSkeepDataSize(sd, propertiesSize)); - } - } - numInStreams += (UInt32)coder->NumInStreams; - numOutStreams += (UInt32)coder->NumOutStreams; - } - - numBindPairs = numOutStreams - 1; - folder->NumBindPairs = numBindPairs; - - - MY_ALLOC(CBindPair, folder->BindPairs, (size_t)numBindPairs, allocFunc); - - for (i = 0; i < numBindPairs; i++) - { - CBindPair *bindPair = folder->BindPairs + i;; - RINOK(SzReadNumber32(sd, &bindPair->InIndex)); - RINOK(SzReadNumber32(sd, &bindPair->OutIndex)); - } - - numPackedStreams = numInStreams - (UInt32)numBindPairs; - - folder->NumPackStreams = numPackedStreams; - MY_ALLOC(UInt32, folder->PackStreams, (size_t)numPackedStreams, allocFunc); - - if (numPackedStreams == 1) - { - UInt32 j; - UInt32 pi = 0; - for (j = 0; j < numInStreams; j++) - if (SzFolderFindBindPairForInStream(folder, j) < 0) - { - folder->PackStreams[pi++] = j; - break; - } - } - else - for(i = 0; i < numPackedStreams; i++) - { - RINOK(SzReadNumber32(sd, folder->PackStreams + i)); - } - return SZ_OK; -} - -SZ_RESULT SzReadUnPackInfo( - CSzData *sd, - UInt32 *numFolders, - CFolder **folders, /* for allocFunc */ - void * (*allocFunc)(size_t size), - ISzAlloc *allocTemp) -{ - UInt32 i; - RINOK(SzWaitAttribute(sd, k7zIdFolder)); - RINOK(SzReadNumber32(sd, numFolders)); - { - RINOK(SzReadSwitch(sd)); - - MY_ALLOC(CFolder, *folders, (size_t)*numFolders, allocFunc); - - for(i = 0; i < *numFolders; i++) - SzFolderInit((*folders) + i); - - for(i = 0; i < *numFolders; i++) - { - RINOK(SzGetNextFolderItem(sd, (*folders) + i, allocFunc)); - } - } - - RINOK(SzWaitAttribute(sd, k7zIdCodersUnPackSize)); - - for(i = 0; i < *numFolders; i++) - { - UInt32 j; - CFolder *folder = (*folders) + i; - UInt32 numOutStreams = SzFolderGetNumOutStreams(folder); - - MY_ALLOC(CFileSize, folder->UnPackSizes, (size_t)numOutStreams, allocFunc); - - for(j = 0; j < numOutStreams; j++) - { - RINOK(SzReadSize(sd, folder->UnPackSizes + j)); - } - } - - for(;;) - { - UInt64 type; - RINOK(SzReadID(sd, &type)); - if (type == k7zIdEnd) - return SZ_OK; - if (type == k7zIdCRC) - { - SZ_RESULT res; - Byte *crcsDefined = 0; - UInt32 *crcs = 0; - res = SzReadHashDigests(sd, *numFolders, &crcsDefined, &crcs, allocTemp->Alloc); - if (res == SZ_OK) - { - for(i = 0; i < *numFolders; i++) - { - CFolder *folder = (*folders) + i; - folder->UnPackCRCDefined = crcsDefined[i]; - folder->UnPackCRC = crcs[i]; - } - } - allocTemp->Free(crcs); - allocTemp->Free(crcsDefined); - RINOK(res); - continue; - } - RINOK(SzSkeepData(sd)); - } -} - -SZ_RESULT SzReadSubStreamsInfo( - CSzData *sd, - UInt32 numFolders, - CFolder *folders, - UInt32 *numUnPackStreams, - CFileSize **unPackSizes, - Byte **digestsDefined, - UInt32 **digests, - ISzAlloc *allocTemp) -{ - UInt64 type = 0; - UInt32 i; - UInt32 si = 0; - UInt32 numDigests = 0; - - for(i = 0; i < numFolders; i++) - folders[i].NumUnPackStreams = 1; - *numUnPackStreams = numFolders; - - for(;;) - { - RINOK(SzReadID(sd, &type)); - if (type == k7zIdNumUnPackStream) - { - *numUnPackStreams = 0; - for(i = 0; i < numFolders; i++) - { - UInt32 numStreams; - RINOK(SzReadNumber32(sd, &numStreams)); - folders[i].NumUnPackStreams = numStreams; - *numUnPackStreams += numStreams; - } - continue; - } - if (type == k7zIdCRC || type == k7zIdSize) - break; - if (type == k7zIdEnd) - break; - RINOK(SzSkeepData(sd)); - } - - if (*numUnPackStreams == 0) - { - *unPackSizes = 0; - *digestsDefined = 0; - *digests = 0; - } - else - { - *unPackSizes = (CFileSize *)allocTemp->Alloc((size_t)*numUnPackStreams * sizeof(CFileSize)); - RINOM(*unPackSizes); - *digestsDefined = (Byte *)allocTemp->Alloc((size_t)*numUnPackStreams * sizeof(Byte)); - RINOM(*digestsDefined); - *digests = (UInt32 *)allocTemp->Alloc((size_t)*numUnPackStreams * sizeof(UInt32)); - RINOM(*digests); - } - - for(i = 0; i < numFolders; i++) - { - /* - v3.13 incorrectly worked with empty folders - v4.07: we check that folder is empty - */ - CFileSize sum = 0; - UInt32 j; - UInt32 numSubstreams = folders[i].NumUnPackStreams; - if (numSubstreams == 0) - continue; - if (type == k7zIdSize) - for (j = 1; j < numSubstreams; j++) - { - CFileSize size; - RINOK(SzReadSize(sd, &size)); - (*unPackSizes)[si++] = size; - sum += size; - } - (*unPackSizes)[si++] = SzFolderGetUnPackSize(folders + i) - sum; - } - if (type == k7zIdSize) - { - RINOK(SzReadID(sd, &type)); - } - - for(i = 0; i < *numUnPackStreams; i++) - { - (*digestsDefined)[i] = 0; - (*digests)[i] = 0; - } - - - for(i = 0; i < numFolders; i++) - { - UInt32 numSubstreams = folders[i].NumUnPackStreams; - if (numSubstreams != 1 || !folders[i].UnPackCRCDefined) - numDigests += numSubstreams; - } - - - si = 0; - for(;;) - { - if (type == k7zIdCRC) - { - int digestIndex = 0; - Byte *digestsDefined2 = 0; - UInt32 *digests2 = 0; - SZ_RESULT res = SzReadHashDigests(sd, numDigests, &digestsDefined2, &digests2, allocTemp->Alloc); - if (res == SZ_OK) - { - for (i = 0; i < numFolders; i++) - { - CFolder *folder = folders + i; - UInt32 numSubstreams = folder->NumUnPackStreams; - if (numSubstreams == 1 && folder->UnPackCRCDefined) - { - (*digestsDefined)[si] = 1; - (*digests)[si] = folder->UnPackCRC; - si++; - } - else - { - UInt32 j; - for (j = 0; j < numSubstreams; j++, digestIndex++) - { - (*digestsDefined)[si] = digestsDefined2[digestIndex]; - (*digests)[si] = digests2[digestIndex]; - si++; - } - } - } - } - allocTemp->Free(digestsDefined2); - allocTemp->Free(digests2); - RINOK(res); - } - else if (type == k7zIdEnd) - return SZ_OK; - else - { - RINOK(SzSkeepData(sd)); - } - RINOK(SzReadID(sd, &type)); - } -} - - -SZ_RESULT SzReadStreamsInfo( - CSzData *sd, - CFileSize *dataOffset, - CArchiveDatabase *db, - UInt32 *numUnPackStreams, - CFileSize **unPackSizes, /* allocTemp */ - Byte **digestsDefined, /* allocTemp */ - UInt32 **digests, /* allocTemp */ - void * (*allocFunc)(size_t size), - ISzAlloc *allocTemp) -{ - for(;;) - { - UInt64 type; - RINOK(SzReadID(sd, &type)); - if ((UInt64)(int)type != type) - return SZE_FAIL; - switch((int)type) - { - case k7zIdEnd: - return SZ_OK; - case k7zIdPackInfo: - { - RINOK(SzReadPackInfo(sd, dataOffset, &db->NumPackStreams, - &db->PackSizes, &db->PackCRCsDefined, &db->PackCRCs, allocFunc)); - break; - } - case k7zIdUnPackInfo: - { - RINOK(SzReadUnPackInfo(sd, &db->NumFolders, &db->Folders, allocFunc, allocTemp)); - break; - } - case k7zIdSubStreamsInfo: - { - RINOK(SzReadSubStreamsInfo(sd, db->NumFolders, db->Folders, - numUnPackStreams, unPackSizes, digestsDefined, digests, allocTemp)); - break; - } - default: - return SZE_FAIL; - } - } -} - -Byte kUtf8Limits[5] = { 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; - -SZ_RESULT SzReadFileNames(CSzData *sd, UInt32 numFiles, CFileItem *files, - void * (*allocFunc)(size_t size)) -{ - UInt32 i; - for(i = 0; i < numFiles; i++) - { - UInt32 len = 0; - UInt32 pos = 0; - CFileItem *file = files + i; - while(pos + 2 <= sd->Size) - { - int numAdds; - UInt32 value = (UInt32)(sd->Data[pos] | (((UInt32)sd->Data[pos + 1]) << 8)); - pos += 2; - len++; - if (value == 0) - break; - if (value < 0x80) - continue; - if (value >= 0xD800 && value < 0xE000) - { - UInt32 c2; - if (value >= 0xDC00) - return SZE_ARCHIVE_ERROR; - if (pos + 2 > sd->Size) - return SZE_ARCHIVE_ERROR; - c2 = (UInt32)(sd->Data[pos] | (((UInt32)sd->Data[pos + 1]) << 8)); - pos += 2; - if (c2 < 0xDC00 || c2 >= 0xE000) - return SZE_ARCHIVE_ERROR; - value = ((value - 0xD800) << 10) | (c2 - 0xDC00); - } - for (numAdds = 1; numAdds < 5; numAdds++) - if (value < (((UInt32)1) << (numAdds * 5 + 6))) - break; - len += numAdds; - } - - MY_ALLOC(char, file->Name, (size_t)len, allocFunc); - - len = 0; - while(2 <= sd->Size) - { - int numAdds; - UInt32 value = (UInt32)(sd->Data[0] | (((UInt32)sd->Data[1]) << 8)); - SzSkeepDataSize(sd, 2); - if (value < 0x80) - { - file->Name[len++] = (char)value; - if (value == 0) - break; - continue; - } - if (value >= 0xD800 && value < 0xE000) - { - UInt32 c2 = (UInt32)(sd->Data[0] | (((UInt32)sd->Data[1]) << 8)); - SzSkeepDataSize(sd, 2); - value = ((value - 0xD800) << 10) | (c2 - 0xDC00); - } - for (numAdds = 1; numAdds < 5; numAdds++) - if (value < (((UInt32)1) << (numAdds * 5 + 6))) - break; - file->Name[len++] = (char)(kUtf8Limits[numAdds - 1] + (value >> (6 * numAdds))); - do - { - numAdds--; - file->Name[len++] = (char)(0x80 + ((value >> (6 * numAdds)) & 0x3F)); - } - while(numAdds > 0); - - len += numAdds; - } - } - return SZ_OK; -} - -SZ_RESULT SzReadHeader2( - CSzData *sd, - CArchiveDatabaseEx *db, /* allocMain */ - CFileSize **unPackSizes, /* allocTemp */ - Byte **digestsDefined, /* allocTemp */ - UInt32 **digests, /* allocTemp */ - Byte **emptyStreamVector, /* allocTemp */ - Byte **emptyFileVector, /* allocTemp */ - ISzAlloc *allocMain, - ISzAlloc *allocTemp) -{ - UInt64 type; - UInt32 numUnPackStreams = 0; - UInt32 numFiles = 0; - CFileItem *files = 0; - UInt32 numEmptyStreams = 0; - UInt32 i; - - RINOK(SzReadID(sd, &type)); - - if (type == k7zIdArchiveProperties) - { - RINOK(SzReadArchiveProperties(sd)); - RINOK(SzReadID(sd, &type)); - } - - - if (type == k7zIdMainStreamsInfo) - { - RINOK(SzReadStreamsInfo(sd, - &db->ArchiveInfo.DataStartPosition, - &db->Database, - &numUnPackStreams, - unPackSizes, - digestsDefined, - digests, allocMain->Alloc, allocTemp)); - db->ArchiveInfo.DataStartPosition += db->ArchiveInfo.StartPositionAfterHeader; - RINOK(SzReadID(sd, &type)); - } - - if (type == k7zIdEnd) - return SZ_OK; - if (type != k7zIdFilesInfo) - return SZE_ARCHIVE_ERROR; - - RINOK(SzReadNumber32(sd, &numFiles)); - db->Database.NumFiles = numFiles; - - MY_ALLOC(CFileItem, files, (size_t)numFiles, allocMain->Alloc); - - db->Database.Files = files; - for(i = 0; i < numFiles; i++) - SzFileInit(files + i); - - for(;;) - { - UInt64 type; - UInt64 size; - RINOK(SzReadID(sd, &type)); - if (type == k7zIdEnd) - break; - RINOK(SzReadNumber(sd, &size)); - - if ((UInt64)(int)type != type) - { - RINOK(SzSkeepDataSize(sd, size)); - } - else - switch((int)type) - { - case k7zIdName: - { - RINOK(SzReadSwitch(sd)); - RINOK(SzReadFileNames(sd, numFiles, files, allocMain->Alloc)) - break; - } - case k7zIdEmptyStream: - { - RINOK(SzReadBoolVector(sd, numFiles, emptyStreamVector, allocTemp->Alloc)); - numEmptyStreams = 0; - for (i = 0; i < numFiles; i++) - if ((*emptyStreamVector)[i]) - numEmptyStreams++; - break; - } - case k7zIdEmptyFile: - { - RINOK(SzReadBoolVector(sd, numEmptyStreams, emptyFileVector, allocTemp->Alloc)); - break; - } - default: - { - RINOK(SzSkeepDataSize(sd, size)); - } - } - } - - { - UInt32 emptyFileIndex = 0; - UInt32 sizeIndex = 0; - for(i = 0; i < numFiles; i++) - { - CFileItem *file = files + i; - file->IsAnti = 0; - if (*emptyStreamVector == 0) - file->HasStream = 1; - else - file->HasStream = (Byte)((*emptyStreamVector)[i] ? 0 : 1); - if(file->HasStream) - { - file->IsDirectory = 0; - file->Size = (*unPackSizes)[sizeIndex]; - file->FileCRC = (*digests)[sizeIndex]; - file->IsFileCRCDefined = (Byte)(*digestsDefined)[sizeIndex]; - sizeIndex++; - } - else - { - if (*emptyFileVector == 0) - file->IsDirectory = 1; - else - file->IsDirectory = (Byte)((*emptyFileVector)[emptyFileIndex] ? 0 : 1); - emptyFileIndex++; - file->Size = 0; - file->IsFileCRCDefined = 0; - } - } - } - return SzArDbExFill(db, allocMain->Alloc); -} - -SZ_RESULT SzReadHeader( - CSzData *sd, - CArchiveDatabaseEx *db, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) -{ - CFileSize *unPackSizes = 0; - Byte *digestsDefined = 0; - UInt32 *digests = 0; - Byte *emptyStreamVector = 0; - Byte *emptyFileVector = 0; - SZ_RESULT res = SzReadHeader2(sd, db, - &unPackSizes, &digestsDefined, &digests, - &emptyStreamVector, &emptyFileVector, - allocMain, allocTemp); - allocTemp->Free(unPackSizes); - allocTemp->Free(digestsDefined); - allocTemp->Free(digests); - allocTemp->Free(emptyStreamVector); - allocTemp->Free(emptyFileVector); - return res; -} - -SZ_RESULT SzReadAndDecodePackedStreams2( - ISzInStream *inStream, - CSzData *sd, - CSzByteBuffer *outBuffer, - CFileSize baseOffset, - CArchiveDatabase *db, - CFileSize **unPackSizes, - Byte **digestsDefined, - UInt32 **digests, - #ifndef _LZMA_IN_CB - Byte **inBuffer, - #endif - ISzAlloc *allocTemp) -{ - - UInt32 numUnPackStreams = 0; - CFileSize dataStartPos; - CFolder *folder; - #ifndef _LZMA_IN_CB - CFileSize packSize = 0; - UInt32 i = 0; - #endif - CFileSize unPackSize; - size_t outRealSize; - SZ_RESULT res; - - RINOK(SzReadStreamsInfo(sd, &dataStartPos, db, - &numUnPackStreams, unPackSizes, digestsDefined, digests, - allocTemp->Alloc, allocTemp)); - - dataStartPos += baseOffset; - if (db->NumFolders != 1) - return SZE_ARCHIVE_ERROR; - - folder = db->Folders; - unPackSize = SzFolderGetUnPackSize(folder); - - RINOK(inStream->Seek(inStream, dataStartPos)); - - #ifndef _LZMA_IN_CB - for (i = 0; i < db->NumPackStreams; i++) - packSize += db->PackSizes[i]; - - MY_ALLOC(Byte, *inBuffer, (size_t)packSize, allocTemp->Alloc); - - RINOK(SafeReadDirect(inStream, *inBuffer, (size_t)packSize)); - #endif - - if (!SzByteBufferCreate(outBuffer, (size_t)unPackSize, allocTemp->Alloc)) - return SZE_OUTOFMEMORY; - - res = SzDecode(db->PackSizes, folder, - #ifdef _LZMA_IN_CB - inStream, - #else - *inBuffer, - #endif - outBuffer->Items, (size_t)unPackSize, - &outRealSize, allocTemp); - RINOK(res) - if (outRealSize != (UInt32)unPackSize) - return SZE_FAIL; - if (folder->UnPackCRCDefined) - if (!CrcVerifyDigest(folder->UnPackCRC, outBuffer->Items, (size_t)unPackSize)) - return SZE_FAIL; - return SZ_OK; -} - -SZ_RESULT SzReadAndDecodePackedStreams( - ISzInStream *inStream, - CSzData *sd, - CSzByteBuffer *outBuffer, - CFileSize baseOffset, - ISzAlloc *allocTemp) -{ - CArchiveDatabase db; - CFileSize *unPackSizes = 0; - Byte *digestsDefined = 0; - UInt32 *digests = 0; - #ifndef _LZMA_IN_CB - Byte *inBuffer = 0; - #endif - SZ_RESULT res; - SzArchiveDatabaseInit(&db); - res = SzReadAndDecodePackedStreams2(inStream, sd, outBuffer, baseOffset, - &db, &unPackSizes, &digestsDefined, &digests, - #ifndef _LZMA_IN_CB - &inBuffer, - #endif - allocTemp); - SzArchiveDatabaseFree(&db, allocTemp->Free); - allocTemp->Free(unPackSizes); - allocTemp->Free(digestsDefined); - allocTemp->Free(digests); - #ifndef _LZMA_IN_CB - allocTemp->Free(inBuffer); - #endif - return res; -} - -SZ_RESULT SzArchiveOpen2( - ISzInStream *inStream, - CArchiveDatabaseEx *db, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) -{ - Byte signature[k7zSignatureSize]; - Byte version; - UInt32 crcFromArchive; - UInt64 nextHeaderOffset; - UInt64 nextHeaderSize; - UInt32 nextHeaderCRC; - UInt32 crc; - CFileSize pos = 0; - CSzByteBuffer buffer; - CSzData sd; - SZ_RESULT res; - - RINOK(SafeReadDirect(inStream, signature, k7zSignatureSize)); - - if (!TestSignatureCandidate(signature)) - return SZE_ARCHIVE_ERROR; - - /* - db.Clear(); - db.ArchiveInfo.StartPosition = _arhiveBeginStreamPosition; - */ - RINOK(SafeReadDirectByte(inStream, &version)); - if (version != k7zMajorVersion) - return SZE_ARCHIVE_ERROR; - RINOK(SafeReadDirectByte(inStream, &version)); - - RINOK(SafeReadDirectUInt32(inStream, &crcFromArchive)); - - CrcInit(&crc); - RINOK(SafeReadDirectUInt64(inStream, &nextHeaderOffset)); - CrcUpdateUInt64(&crc, nextHeaderOffset); - RINOK(SafeReadDirectUInt64(inStream, &nextHeaderSize)); - CrcUpdateUInt64(&crc, nextHeaderSize); - RINOK(SafeReadDirectUInt32(inStream, &nextHeaderCRC)); - CrcUpdateUInt32(&crc, nextHeaderCRC); - - pos = k7zStartHeaderSize; - db->ArchiveInfo.StartPositionAfterHeader = pos; - - if (CrcGetDigest(&crc) != crcFromArchive) - return SZE_ARCHIVE_ERROR; - - if (nextHeaderSize == 0) - return SZ_OK; - - RINOK(inStream->Seek(inStream, (CFileSize)(pos + nextHeaderOffset))); - - if (!SzByteBufferCreate(&buffer, (size_t)nextHeaderSize, allocTemp->Alloc)) - return SZE_OUTOFMEMORY; - - res = SafeReadDirect(inStream, buffer.Items, (size_t)nextHeaderSize); - if (res == SZ_OK) - { - if (CrcVerifyDigest(nextHeaderCRC, buffer.Items, (UInt32)nextHeaderSize)) - { - for(;;) - { - UInt64 type; - sd.Data = buffer.Items; - sd.Size = buffer.Capacity; - res = SzReadID(&sd, &type); - if (res != SZ_OK) - break; - if (type == k7zIdHeader) - { - res = SzReadHeader(&sd, db, allocMain, allocTemp); - break; - } - if (type != k7zIdEncodedHeader) - { - res = SZE_ARCHIVE_ERROR; - break; - } - { - CSzByteBuffer outBuffer; - res = SzReadAndDecodePackedStreams(inStream, &sd, &outBuffer, - db->ArchiveInfo.StartPositionAfterHeader, - allocTemp); - if (res != SZ_OK) - { - SzByteBufferFree(&outBuffer, allocTemp->Free); - break; - } - SzByteBufferFree(&buffer, allocTemp->Free); - buffer.Items = outBuffer.Items; - buffer.Capacity = outBuffer.Capacity; - } - } - } - } - SzByteBufferFree(&buffer, allocTemp->Free); - return res; -} - -SZ_RESULT SzArchiveOpen( - ISzInStream *inStream, - CArchiveDatabaseEx *db, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) -{ - SZ_RESULT res = SzArchiveOpen2(inStream, db, allocMain, allocTemp); - if (res != SZ_OK) - SzArDbExFree(db, allocMain->Free); - return res; -} diff --git a/code/nel/3rdparty/seven_zip/7zIn.h b/code/nel/3rdparty/seven_zip/7zIn.h deleted file mode 100644 index 6bfa2a709..000000000 --- a/code/nel/3rdparty/seven_zip/7zIn.h +++ /dev/null @@ -1,55 +0,0 @@ -/* 7zIn.h */ - -#ifndef __7Z_IN_H -#define __7Z_IN_H - -#include "7zHeader.h" -#include "7zItem.h" -#include "7zAlloc.h" - -typedef struct _CInArchiveInfo -{ - CFileSize StartPositionAfterHeader; - CFileSize DataStartPosition; -}CInArchiveInfo; - -typedef struct _CArchiveDatabaseEx -{ - CArchiveDatabase Database; - CInArchiveInfo ArchiveInfo; - UInt32 *FolderStartPackStreamIndex; - CFileSize *PackStreamStartPositions; - UInt32 *FolderStartFileIndex; - UInt32 *FileIndexToFolderIndexMap; -}CArchiveDatabaseEx; - -void SzArDbExInit(CArchiveDatabaseEx *db); -void SzArDbExFree(CArchiveDatabaseEx *db, void (*freeFunc)(void *)); -CFileSize SzArDbGetFolderStreamPos(CArchiveDatabaseEx *db, UInt32 folderIndex, UInt32 indexInFolder); -CFileSize SzArDbGetFolderFullPackSize(CArchiveDatabaseEx *db, UInt32 folderIndex); - -typedef struct _ISzInStream -{ - #ifdef _LZMA_IN_CB - SZ_RESULT (*Read)( - void *object, /* pointer to ISzInStream itself */ - void **buffer, /* out: pointer to buffer with data */ - size_t maxRequiredSize, /* max required size to read */ - size_t *processedSize); /* real processed size. - processedSize can be less than maxRequiredSize. - If processedSize == 0, then there are no more - bytes in stream. */ - #else - SZ_RESULT (*Read)(void *object, void *buffer, size_t size, size_t *processedSize); - #endif - SZ_RESULT (*Seek)(void *object, CFileSize pos); -} ISzInStream; - - -int SzArchiveOpen( - ISzInStream *inStream, - CArchiveDatabaseEx *db, - ISzAlloc *allocMain, - ISzAlloc *allocTemp); - -#endif diff --git a/code/nel/3rdparty/seven_zip/7zItem.cpp b/code/nel/3rdparty/seven_zip/7zItem.cpp deleted file mode 100644 index 2a408050f..000000000 --- a/code/nel/3rdparty/seven_zip/7zItem.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/* 7zItem.c */ - -#include "7zItem.h" -#include "7zAlloc.h" - -void SzCoderInfoInit(CCoderInfo *coder) -{ - SzByteBufferInit(&coder->Properties); -} - -void SzCoderInfoFree(CCoderInfo *coder, void (*freeFunc)(void *p)) -{ - SzByteBufferFree(&coder->Properties, freeFunc); - SzCoderInfoInit(coder); -} - -void SzFolderInit(CFolder *folder) -{ - folder->NumCoders = 0; - folder->Coders = 0; - folder->NumBindPairs = 0; - folder->BindPairs = 0; - folder->NumPackStreams = 0; - folder->PackStreams = 0; - folder->UnPackSizes = 0; - folder->UnPackCRCDefined = 0; - folder->UnPackCRC = 0; - folder->NumUnPackStreams = 0; -} - -void SzFolderFree(CFolder *folder, void (*freeFunc)(void *p)) -{ - UInt32 i; - for (i = 0; i < folder->NumCoders; i++) - SzCoderInfoFree(&folder->Coders[i], freeFunc); - freeFunc(folder->Coders); - freeFunc(folder->BindPairs); - freeFunc(folder->PackStreams); - freeFunc(folder->UnPackSizes); - SzFolderInit(folder); -} - -UInt32 SzFolderGetNumOutStreams(CFolder *folder) -{ - UInt32 result = 0; - UInt32 i; - for (i = 0; i < folder->NumCoders; i++) - result += folder->Coders[i].NumOutStreams; - return result; -} - -int SzFolderFindBindPairForInStream(CFolder *folder, UInt32 inStreamIndex) -{ - UInt32 i; - for(i = 0; i < folder->NumBindPairs; i++) - if (folder->BindPairs[i].InIndex == inStreamIndex) - return i; - return -1; -} - - -int SzFolderFindBindPairForOutStream(CFolder *folder, UInt32 outStreamIndex) -{ - UInt32 i; - for(i = 0; i < folder->NumBindPairs; i++) - if (folder->BindPairs[i].OutIndex == outStreamIndex) - return i; - return -1; -} - -CFileSize SzFolderGetUnPackSize(CFolder *folder) -{ - int i = (int)SzFolderGetNumOutStreams(folder); - if (i == 0) - return 0; - for (i--; i >= 0; i--) - if (SzFolderFindBindPairForOutStream(folder, i) < 0) - return folder->UnPackSizes[i]; - /* throw 1; */ - return 0; -} - -/* -int FindPackStreamArrayIndex(int inStreamIndex) const -{ - for(int i = 0; i < PackStreams.Size(); i++) - if (PackStreams[i] == inStreamIndex) - return i; - return -1; -} -*/ - -void SzFileInit(CFileItem *fileItem) -{ - fileItem->IsFileCRCDefined = 0; - fileItem->HasStream = 1; - fileItem->IsDirectory = 0; - fileItem->IsAnti = 0; - fileItem->Name = 0; -} - -void SzFileFree(CFileItem *fileItem, void (*freeFunc)(void *p)) -{ - freeFunc(fileItem->Name); - SzFileInit(fileItem); -} - -void SzArchiveDatabaseInit(CArchiveDatabase *db) -{ - db->NumPackStreams = 0; - db->PackSizes = 0; - db->PackCRCsDefined = 0; - db->PackCRCs = 0; - db->NumFolders = 0; - db->Folders = 0; - db->NumFiles = 0; - db->Files = 0; -} - -void SzArchiveDatabaseFree(CArchiveDatabase *db, void (*freeFunc)(void *)) -{ - UInt32 i; - for (i = 0; i < db->NumFolders; i++) - SzFolderFree(&db->Folders[i], freeFunc); - for (i = 0; i < db->NumFiles; i++) - SzFileFree(&db->Files[i], freeFunc); - freeFunc(db->PackSizes); - freeFunc(db->PackCRCsDefined); - freeFunc(db->PackCRCs); - freeFunc(db->Folders); - freeFunc(db->Files); - SzArchiveDatabaseInit(db); -} diff --git a/code/nel/3rdparty/seven_zip/7zItem.h b/code/nel/3rdparty/seven_zip/7zItem.h deleted file mode 100644 index 876539a98..000000000 --- a/code/nel/3rdparty/seven_zip/7zItem.h +++ /dev/null @@ -1,90 +0,0 @@ -/* 7zItem.h */ - -#ifndef __7Z_ITEM_H -#define __7Z_ITEM_H - -#include "7zMethodID.h" -#include "7zHeader.h" -#include "7zBuffer.h" - -typedef struct _CCoderInfo -{ - UInt32 NumInStreams; - UInt32 NumOutStreams; - CMethodID MethodID; - CSzByteBuffer Properties; -}CCoderInfo; - -void SzCoderInfoInit(CCoderInfo *coder); -void SzCoderInfoFree(CCoderInfo *coder, void (*freeFunc)(void *p)); - -typedef struct _CBindPair -{ - UInt32 InIndex; - UInt32 OutIndex; -}CBindPair; - -typedef struct _CFolder -{ - UInt32 NumCoders; - CCoderInfo *Coders; - UInt32 NumBindPairs; - CBindPair *BindPairs; - UInt32 NumPackStreams; - UInt32 *PackStreams; - CFileSize *UnPackSizes; - int UnPackCRCDefined; - UInt32 UnPackCRC; - - UInt32 NumUnPackStreams; -}CFolder; - -void SzFolderInit(CFolder *folder); -CFileSize SzFolderGetUnPackSize(CFolder *folder); -int SzFolderFindBindPairForInStream(CFolder *folder, UInt32 inStreamIndex); -UInt32 SzFolderGetNumOutStreams(CFolder *folder); -CFileSize SzFolderGetUnPackSize(CFolder *folder); - -/* #define CArchiveFileTime UInt64 */ - -typedef struct _CFileItem -{ - /* - CArchiveFileTime LastWriteTime; - CFileSize StartPos; - UInt32 Attributes; - */ - CFileSize Size; - UInt32 FileCRC; - char *Name; - - Byte IsFileCRCDefined; - Byte HasStream; - Byte IsDirectory; - Byte IsAnti; - /* - int AreAttributesDefined; - int IsLastWriteTimeDefined; - int IsStartPosDefined; - */ -}CFileItem; - -void SzFileInit(CFileItem *fileItem); - -typedef struct _CArchiveDatabase -{ - UInt32 NumPackStreams; - CFileSize *PackSizes; - Byte *PackCRCsDefined; - UInt32 *PackCRCs; - UInt32 NumFolders; - CFolder *Folders; - UInt32 NumFiles; - CFileItem *Files; -}CArchiveDatabase; - -void SzArchiveDatabaseInit(CArchiveDatabase *db); -void SzArchiveDatabaseFree(CArchiveDatabase *db, void (*freeFunc)(void *)); - - -#endif diff --git a/code/nel/3rdparty/seven_zip/7zMain.cpp b/code/nel/3rdparty/seven_zip/7zMain.cpp deleted file mode 100644 index 59ecd2885..000000000 --- a/code/nel/3rdparty/seven_zip/7zMain.cpp +++ /dev/null @@ -1,686 +0,0 @@ -/* 7zMain.c - Test application for 7z Decoder -2019-02-02 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include -#include - -#include "CpuArch.h" - -#include "7z.h" -#include "7zAlloc.h" -#include "7zBuf.h" -#include "7zCrc.h" -#include "7zFile.h" -#include "7zVersion.h" - -#ifndef USE_WINDOWS_FILE -/* for mkdir */ -#ifdef _WIN32 -#include -#else -#include -#include -#endif -#endif - - -#define kInputBufSize ((size_t)1 << 18) - -static const ISzAlloc g_Alloc = { SzAlloc, SzFree }; - - -static void Print(const char *s) -{ - fputs(s, stdout); -} - - -static int Buf_EnsureSize(CBuf *dest, size_t size) -{ - if (dest->size >= size) - return 1; - Buf_Free(dest, &g_Alloc); - return Buf_Create(dest, size, &g_Alloc); -} - -#ifndef _WIN32 -#define _USE_UTF8 -#endif - -/* #define _USE_UTF8 */ - -#ifdef _USE_UTF8 - -#define _UTF8_START(n) (0x100 - (1 << (7 - (n)))) - -#define _UTF8_RANGE(n) (((UInt32)1) << ((n) * 5 + 6)) - -#define _UTF8_HEAD(n, val) ((Byte)(_UTF8_START(n) + (val >> (6 * (n))))) -#define _UTF8_CHAR(n, val) ((Byte)(0x80 + (((val) >> (6 * (n))) & 0x3F))) - -static size_t Utf16_To_Utf8_Calc(const UInt16 *src, const UInt16 *srcLim) -{ - size_t size = 0; - for (;;) - { - UInt32 val; - if (src == srcLim) - return size; - - size++; - val = *src++; - - if (val < 0x80) - continue; - - if (val < _UTF8_RANGE(1)) - { - size++; - continue; - } - - if (val >= 0xD800 && val < 0xDC00 && src != srcLim) - { - UInt32 c2 = *src; - if (c2 >= 0xDC00 && c2 < 0xE000) - { - src++; - size += 3; - continue; - } - } - - size += 2; - } -} - -static Byte *Utf16_To_Utf8(Byte *dest, const UInt16 *src, const UInt16 *srcLim) -{ - for (;;) - { - UInt32 val; - if (src == srcLim) - return dest; - - val = *src++; - - if (val < 0x80) - { - *dest++ = (char)val; - continue; - } - - if (val < _UTF8_RANGE(1)) - { - dest[0] = _UTF8_HEAD(1, val); - dest[1] = _UTF8_CHAR(0, val); - dest += 2; - continue; - } - - if (val >= 0xD800 && val < 0xDC00 && src != srcLim) - { - UInt32 c2 = *src; - if (c2 >= 0xDC00 && c2 < 0xE000) - { - src++; - val = (((val - 0xD800) << 10) | (c2 - 0xDC00)) + 0x10000; - dest[0] = _UTF8_HEAD(3, val); - dest[1] = _UTF8_CHAR(2, val); - dest[2] = _UTF8_CHAR(1, val); - dest[3] = _UTF8_CHAR(0, val); - dest += 4; - continue; - } - } - - dest[0] = _UTF8_HEAD(2, val); - dest[1] = _UTF8_CHAR(1, val); - dest[2] = _UTF8_CHAR(0, val); - dest += 3; - } -} - -static SRes Utf16_To_Utf8Buf(CBuf *dest, const UInt16 *src, size_t srcLen) -{ - size_t destLen = Utf16_To_Utf8_Calc(src, src + srcLen); - destLen += 1; - if (!Buf_EnsureSize(dest, destLen)) - return SZ_ERROR_MEM; - *Utf16_To_Utf8(dest->data, src, src + srcLen) = 0; - return SZ_OK; -} - -#endif - -static SRes Utf16_To_Char(CBuf *buf, const UInt16 *s - #ifndef _USE_UTF8 - , UINT codePage - #endif - ) -{ - unsigned len = 0; - for (len = 0; s[len] != 0; len++); - - #ifndef _USE_UTF8 - { - unsigned size = len * 3 + 100; - if (!Buf_EnsureSize(buf, size)) - return SZ_ERROR_MEM; - { - buf->data[0] = 0; - if (len != 0) - { - char defaultChar = '_'; - BOOL defUsed; - unsigned numChars = 0; - numChars = WideCharToMultiByte(codePage, 0, (LPCWSTR)s, len, (char *)buf->data, size, &defaultChar, &defUsed); - if (numChars == 0 || numChars >= size) - return SZ_ERROR_FAIL; - buf->data[numChars] = 0; - } - return SZ_OK; - } - } - #else - return Utf16_To_Utf8Buf(buf, s, len); - #endif -} - -#ifdef _WIN32 - #ifndef USE_WINDOWS_FILE - static UINT g_FileCodePage = CP_ACP; - #endif - #define MY_FILE_CODE_PAGE_PARAM ,g_FileCodePage -#else - #define MY_FILE_CODE_PAGE_PARAM -#endif - -static WRes MyCreateDir(const UInt16 *name) -{ - #ifdef USE_WINDOWS_FILE - - return CreateDirectoryW((LPCWSTR)name, NULL) ? 0 : GetLastError(); - - #else - - CBuf buf; - WRes res; - Buf_Init(&buf); - RINOK(Utf16_To_Char(&buf, name MY_FILE_CODE_PAGE_PARAM)); - - res = - #ifdef _WIN32 - _mkdir((const char *)buf.data) - #else - mkdir((const char *)buf.data, 0777) - #endif - == 0 ? 0 : errno; - Buf_Free(&buf, &g_Alloc); - return res; - - #endif -} - -static WRes OutFile_OpenUtf16(CSzFile *p, const UInt16 *name) -{ - #ifdef USE_WINDOWS_FILE - return OutFile_OpenW(p, (LPCWSTR)name); - #else - CBuf buf; - WRes res; - Buf_Init(&buf); - RINOK(Utf16_To_Char(&buf, name MY_FILE_CODE_PAGE_PARAM)); - res = OutFile_Open(p, (const char *)buf.data); - Buf_Free(&buf, &g_Alloc); - return res; - #endif -} - - -static SRes PrintString(const UInt16 *s) -{ - CBuf buf; - SRes res; - Buf_Init(&buf); - res = Utf16_To_Char(&buf, s - #ifndef _USE_UTF8 - , CP_OEMCP - #endif - ); - if (res == SZ_OK) - Print((const char *)buf.data); - Buf_Free(&buf, &g_Alloc); - return res; -} - -static void UInt64ToStr(UInt64 value, char *s, int numDigits) -{ - char temp[32]; - int pos = 0; - do - { - temp[pos++] = (char)('0' + (unsigned)(value % 10)); - value /= 10; - } - while (value != 0); - - for (numDigits -= pos; numDigits > 0; numDigits--) - *s++ = ' '; - - do - *s++ = temp[--pos]; - while (pos); - *s = '\0'; -} - -static char *UIntToStr(char *s, unsigned value, int numDigits) -{ - char temp[16]; - int pos = 0; - do - temp[pos++] = (char)('0' + (value % 10)); - while (value /= 10); - - for (numDigits -= pos; numDigits > 0; numDigits--) - *s++ = '0'; - - do - *s++ = temp[--pos]; - while (pos); - *s = '\0'; - return s; -} - -static void UIntToStr_2(char *s, unsigned value) -{ - s[0] = (char)('0' + (value / 10)); - s[1] = (char)('0' + (value % 10)); -} - -#define PERIOD_4 (4 * 365 + 1) -#define PERIOD_100 (PERIOD_4 * 25 - 1) -#define PERIOD_400 (PERIOD_100 * 4 + 1) - -static void ConvertFileTimeToString(const CNtfsFileTime *nt, char *s) -{ - unsigned year, mon, hour, min, sec; - Byte ms[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; - unsigned t; - UInt32 v; - UInt64 v64 = nt->Low | ((UInt64)nt->High << 32); - v64 /= 10000000; - sec = (unsigned)(v64 % 60); v64 /= 60; - min = (unsigned)(v64 % 60); v64 /= 60; - hour = (unsigned)(v64 % 24); v64 /= 24; - - v = (UInt32)v64; - - year = (unsigned)(1601 + v / PERIOD_400 * 400); - v %= PERIOD_400; - - t = v / PERIOD_100; if (t == 4) t = 3; year += t * 100; v -= t * PERIOD_100; - t = v / PERIOD_4; if (t == 25) t = 24; year += t * 4; v -= t * PERIOD_4; - t = v / 365; if (t == 4) t = 3; year += t; v -= t * 365; - - if (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) - ms[1] = 29; - for (mon = 0;; mon++) - { - unsigned d = ms[mon]; - if (v < d) - break; - v -= d; - } - s = UIntToStr(s, year, 4); *s++ = '-'; - UIntToStr_2(s, mon + 1); s[2] = '-'; s += 3; - UIntToStr_2(s, (unsigned)v + 1); s[2] = ' '; s += 3; - UIntToStr_2(s, hour); s[2] = ':'; s += 3; - UIntToStr_2(s, min); s[2] = ':'; s += 3; - UIntToStr_2(s, sec); s[2] = 0; -} - -static void PrintLF() -{ - Print("\n"); -} - -static void PrintError(char *s) -{ - Print("\nERROR: "); - Print(s); - PrintLF(); -} - -static void GetAttribString(UInt32 wa, BoolInt isDir, char *s) -{ - #ifdef USE_WINDOWS_FILE - s[0] = (char)(((wa & FILE_ATTRIBUTE_DIRECTORY) != 0 || isDir) ? 'D' : '.'); - s[1] = (char)(((wa & FILE_ATTRIBUTE_READONLY ) != 0) ? 'R': '.'); - s[2] = (char)(((wa & FILE_ATTRIBUTE_HIDDEN ) != 0) ? 'H': '.'); - s[3] = (char)(((wa & FILE_ATTRIBUTE_SYSTEM ) != 0) ? 'S': '.'); - s[4] = (char)(((wa & FILE_ATTRIBUTE_ARCHIVE ) != 0) ? 'A': '.'); - s[5] = 0; - #else - s[0] = (char)(((wa & (1 << 4)) != 0 || isDir) ? 'D' : '.'); - s[1] = 0; - #endif -} - - -// #define NUM_PARENTS_MAX 128 - -int MY_CDECL main(int numargs, char *args[]) -{ - ISzAlloc allocImp; - ISzAlloc allocTempImp; - - CFileInStream archiveStream; - CLookToRead2 lookStream; - CSzArEx db; - SRes res; - UInt16 *temp = NULL; - size_t tempSize = 0; - // UInt32 parents[NUM_PARENTS_MAX]; - - Print("\n7z Decoder " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n"); - - if (numargs == 1) - { - Print( - "Usage: 7zDec \n\n" - "\n" - " e: Extract files from archive (without using directory names)\n" - " l: List contents of archive\n" - " t: Test integrity of archive\n" - " x: eXtract files with full paths\n"); - return 0; - } - - if (numargs < 3) - { - PrintError("incorrect command"); - return 1; - } - - #if defined(_WIN32) && !defined(USE_WINDOWS_FILE) && !defined(UNDER_CE) - g_FileCodePage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; - #endif - - - allocImp = g_Alloc; - allocTempImp = g_Alloc; - - #ifdef UNDER_CE - if (InFile_OpenW(&archiveStream.file, L"\test.7z")) - #else - if (InFile_Open(&archiveStream.file, args[2])) - #endif - { - PrintError("can not open input file"); - return 1; - } - - FileInStream_CreateVTable(&archiveStream); - LookToRead2_CreateVTable(&lookStream, False); - lookStream.buf = NULL; - - res = SZ_OK; - - { - lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); - if (!lookStream.buf) - res = SZ_ERROR_MEM; - else - { - lookStream.bufSize = kInputBufSize; - lookStream.realStream = &archiveStream.vt; - LookToRead2_Init(&lookStream); - } - } - - CrcGenerateTable(); - - SzArEx_Init(&db); - - if (res == SZ_OK) - { - res = SzArEx_Open(&db, &lookStream.vt, &allocImp, &allocTempImp); - } - - if (res == SZ_OK) - { - char *command = args[1]; - int listCommand = 0, testCommand = 0, fullPaths = 0; - - if (strcmp(command, "l") == 0) listCommand = 1; - else if (strcmp(command, "t") == 0) testCommand = 1; - else if (strcmp(command, "e") == 0) { } - else if (strcmp(command, "x") == 0) { fullPaths = 1; } - else - { - PrintError("incorrect command"); - res = SZ_ERROR_FAIL; - } - - if (res == SZ_OK) - { - UInt32 i; - - /* - if you need cache, use these 3 variables. - if you use external function, you can make these variable as static. - */ - UInt32 blockIndex = 0xFFFFFFFF; /* it can have any value before first call (if outBuffer = 0) */ - Byte *outBuffer = 0; /* it must be 0 before first call for each new archive. */ - size_t outBufferSize = 0; /* it can have any value before first call (if outBuffer = 0) */ - - for (i = 0; i < db.NumFiles; i++) - { - size_t offset = 0; - size_t outSizeProcessed = 0; - // const CSzFileItem *f = db.Files + i; - size_t len; - unsigned isDir = SzArEx_IsDir(&db, i); - if (listCommand == 0 && isDir && !fullPaths) - continue; - len = SzArEx_GetFileNameUtf16(&db, i, NULL); - // len = SzArEx_GetFullNameLen(&db, i); - - if (len > tempSize) - { - SzFree(NULL, temp); - tempSize = len; - temp = (UInt16 *)SzAlloc(NULL, tempSize * sizeof(temp[0])); - if (!temp) - { - res = SZ_ERROR_MEM; - break; - } - } - - SzArEx_GetFileNameUtf16(&db, i, temp); - /* - if (SzArEx_GetFullNameUtf16_Back(&db, i, temp + len) != temp) - { - res = SZ_ERROR_FAIL; - break; - } - */ - - if (listCommand) - { - char attr[8], s[32], t[32]; - UInt64 fileSize; - - GetAttribString(SzBitWithVals_Check(&db.Attribs, i) ? db.Attribs.Vals[i] : 0, isDir, attr); - - fileSize = SzArEx_GetFileSize(&db, i); - UInt64ToStr(fileSize, s, 10); - - if (SzBitWithVals_Check(&db.MTime, i)) - ConvertFileTimeToString(&db.MTime.Vals[i], t); - else - { - size_t j; - for (j = 0; j < 19; j++) - t[j] = ' '; - t[j] = '\0'; - } - - Print(t); - Print(" "); - Print(attr); - Print(" "); - Print(s); - Print(" "); - res = PrintString(temp); - if (res != SZ_OK) - break; - if (isDir) - Print("/"); - PrintLF(); - continue; - } - - Print(testCommand ? - "Testing ": - "Extracting "); - res = PrintString(temp); - if (res != SZ_OK) - break; - - if (isDir) - Print("/"); - else - { - res = SzArEx_Extract(&db, &lookStream.vt, i, - &blockIndex, &outBuffer, &outBufferSize, - &offset, &outSizeProcessed, - &allocImp, &allocTempImp); - if (res != SZ_OK) - break; - } - - if (!testCommand) - { - CSzFile outFile; - size_t processedSize; - size_t j; - UInt16 *name = (UInt16 *)temp; - const UInt16 *destPath = (const UInt16 *)name; - - for (j = 0; name[j] != 0; j++) - if (name[j] == '/') - { - if (fullPaths) - { - name[j] = 0; - MyCreateDir(name); - name[j] = CHAR_PATH_SEPARATOR; - } - else - destPath = name + j + 1; - } - - if (isDir) - { - MyCreateDir(destPath); - PrintLF(); - continue; - } - else if (OutFile_OpenUtf16(&outFile, destPath)) - { - PrintError("can not open output file"); - res = SZ_ERROR_FAIL; - break; - } - - processedSize = outSizeProcessed; - - if (File_Write(&outFile, outBuffer + offset, &processedSize) != 0 || processedSize != outSizeProcessed) - { - PrintError("can not write output file"); - res = SZ_ERROR_FAIL; - break; - } - - #ifdef USE_WINDOWS_FILE - { - FILETIME mtime, ctime; - FILETIME *mtimePtr = NULL; - FILETIME *ctimePtr = NULL; - - if (SzBitWithVals_Check(&db.MTime, i)) - { - const CNtfsFileTime *t = &db.MTime.Vals[i]; - mtime.dwLowDateTime = (DWORD)(t->Low); - mtime.dwHighDateTime = (DWORD)(t->High); - mtimePtr = &mtime; - } - if (SzBitWithVals_Check(&db.CTime, i)) - { - const CNtfsFileTime *t = &db.CTime.Vals[i]; - ctime.dwLowDateTime = (DWORD)(t->Low); - ctime.dwHighDateTime = (DWORD)(t->High); - ctimePtr = &ctime; - } - if (mtimePtr || ctimePtr) - SetFileTime(outFile.handle, ctimePtr, NULL, mtimePtr); - } - #endif - - if (File_Close(&outFile)) - { - PrintError("can not close output file"); - res = SZ_ERROR_FAIL; - break; - } - - #ifdef USE_WINDOWS_FILE - if (SzBitWithVals_Check(&db.Attribs, i)) - { - UInt32 attrib = db.Attribs.Vals[i]; - /* p7zip stores posix attributes in high 16 bits and adds 0x8000 as marker. - We remove posix bits, if we detect posix mode field */ - if ((attrib & 0xF0000000) != 0) - attrib &= 0x7FFF; - SetFileAttributesW((LPCWSTR)destPath, attrib); - } - #endif - } - PrintLF(); - } - ISzAlloc_Free(&allocImp, outBuffer); - } - } - - SzFree(NULL, temp); - SzArEx_Free(&db, &allocImp); - ISzAlloc_Free(&allocImp, lookStream.buf); - - File_Close(&archiveStream.file); - - if (res == SZ_OK) - { - Print("\nEverything is Ok\n"); - return 0; - } - - if (res == SZ_ERROR_UNSUPPORTED) - PrintError("decoder doesn't support this archive"); - else if (res == SZ_ERROR_MEM) - PrintError("can not allocate memory"); - else if (res == SZ_ERROR_CRC) - PrintError("CRC error"); - else - { - char s[32]; - UInt64ToStr(res, s, 0); - PrintError(s); - } - - return 1; -} diff --git a/code/nel/3rdparty/seven_zip/7zMethodID.cpp b/code/nel/3rdparty/seven_zip/7zMethodID.cpp deleted file mode 100644 index 5047359f3..000000000 --- a/code/nel/3rdparty/seven_zip/7zMethodID.cpp +++ /dev/null @@ -1,14 +0,0 @@ -/* 7zMethodID.c */ - -#include "7zMethodID.h" - -int AreMethodsEqual(CMethodID *a1, CMethodID *a2) -{ - int i; - if (a1->IDSize != a2->IDSize) - return 0; - for (i = 0; i < a1->IDSize; i++) - if (a1->ID[i] != a2->ID[i]) - return 0; - return 1; -} diff --git a/code/nel/3rdparty/seven_zip/7zMethodID.h b/code/nel/3rdparty/seven_zip/7zMethodID.h deleted file mode 100644 index 162fcd15b..000000000 --- a/code/nel/3rdparty/seven_zip/7zMethodID.h +++ /dev/null @@ -1,18 +0,0 @@ -/* 7zMethodID.h */ - -#ifndef __7Z_METHOD_ID_H -#define __7Z_METHOD_ID_H - -#include "7zTypes.h" - -#define kMethodIDSize 15 - -typedef struct _CMethodID -{ - Byte ID[kMethodIDSize]; - Byte IDSize; -} CMethodID; - -int AreMethodsEqual(CMethodID *a1, CMethodID *a2); - -#endif diff --git a/code/nel/3rdparty/seven_zip/BranchTypes.h b/code/nel/3rdparty/seven_zip/BranchTypes.h deleted file mode 100644 index f7ad3abc5..000000000 --- a/code/nel/3rdparty/seven_zip/BranchTypes.h +++ /dev/null @@ -1,25 +0,0 @@ -/* BranchTypes.h */ - -#ifndef __BRANCHTYPES_H -#define __BRANCHTYPES_H - -#ifndef _7ZIP_BYTE_DEFINED -#define _7ZIP_BYTE_DEFINED -typedef unsigned char Byte; -#endif - -#ifndef _7ZIP_UINT16_DEFINED -#define _7ZIP_UINT16_DEFINED -typedef unsigned short UInt16; -#endif - -#ifndef _7ZIP_UINT32_DEFINED -#define _7ZIP_UINT32_DEFINED -#ifdef _LZMA_UINT32_IS_ULONG -typedef unsigned long UInt32; -#else -typedef unsigned int UInt32; -#endif -#endif - -#endif diff --git a/code/nel/3rdparty/seven_zip/BranchX86.cpp b/code/nel/3rdparty/seven_zip/BranchX86.cpp deleted file mode 100644 index 1be3149b8..000000000 --- a/code/nel/3rdparty/seven_zip/BranchX86.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/* BranchX86.c */ - -#include "BranchX86.h" - -/* -static int inline Test86MSByte(Byte b) -{ - return (b == 0 || b == 0xFF); -} -*/ -#define Test86MSByte(b) ((b) == 0 || (b) == 0xFF) - -const int kMaskToAllowedStatus[8] = {1, 1, 1, 0, 1, 0, 0, 0}; -const Byte kMaskToBitNumber[8] = {0, 1, 2, 2, 3, 3, 3, 3}; - -/* -void x86_Convert_Init(UInt32 *prevMask, UInt32 *prevPos) -{ - *prevMask = 0; - *prevPos = (UInt32)(-5); -} -*/ - -UInt32 x86_Convert(Byte *buffer, UInt32 endPos, UInt32 nowPos, - UInt32 *prevMask, UInt32 *prevPos, int encoding) -{ - UInt32 bufferPos = 0; - UInt32 limit; - - if (endPos < 5) - return 0; - - if (nowPos - *prevPos > 5) - *prevPos = nowPos - 5; - - limit = endPos - 5; - while(bufferPos <= limit) - { - Byte b = buffer[bufferPos]; - UInt32 offset; - if (b != 0xE8 && b != 0xE9) - { - bufferPos++; - continue; - } - offset = (nowPos + bufferPos - *prevPos); - *prevPos = (nowPos + bufferPos); - if (offset > 5) - *prevMask = 0; - else - { - UInt32 i; - for (i = 0; i < offset; i++) - { - *prevMask &= 0x77; - *prevMask <<= 1; - } - } - b = buffer[bufferPos + 4]; - if (Test86MSByte(b) && kMaskToAllowedStatus[(*prevMask >> 1) & 0x7] && - (*prevMask >> 1) < 0x10) - { - UInt32 src = - ((UInt32)(b) << 24) | - ((UInt32)(buffer[bufferPos + 3]) << 16) | - ((UInt32)(buffer[bufferPos + 2]) << 8) | - (buffer[bufferPos + 1]); - - UInt32 dest; - for(;;) - { - UInt32 index; - if (encoding) - dest = (nowPos + bufferPos + 5) + src; - else - dest = src - (nowPos + bufferPos + 5); - if (*prevMask == 0) - break; - index = kMaskToBitNumber[*prevMask >> 1]; - b = (Byte)(dest >> (24 - index * 8)); - if (!Test86MSByte(b)) - break; - src = dest ^ ((1 << (32 - index * 8)) - 1); - } - buffer[bufferPos + 4] = (Byte)(~(((dest >> 24) & 1) - 1)); - buffer[bufferPos + 3] = (Byte)(dest >> 16); - buffer[bufferPos + 2] = (Byte)(dest >> 8); - buffer[bufferPos + 1] = (Byte)dest; - bufferPos += 5; - *prevMask = 0; - } - else - { - bufferPos++; - *prevMask |= 1; - if (Test86MSByte(b)) - *prevMask |= 0x10; - } - } - return bufferPos; -} diff --git a/code/nel/3rdparty/seven_zip/BranchX86.h b/code/nel/3rdparty/seven_zip/BranchX86.h deleted file mode 100644 index 25c1ae51b..000000000 --- a/code/nel/3rdparty/seven_zip/BranchX86.h +++ /dev/null @@ -1,13 +0,0 @@ -/* BranchX86.h */ - -#ifndef __BRANCHX86_H -#define __BRANCHX86_H - -#include "BranchTypes.h" - -#define x86_Convert_Init(prevMask, prevPos) { prevMask = 0; prevPos = (UInt32)(-5); } - -UInt32 x86_Convert(Byte *buffer, UInt32 endPos, UInt32 nowPos, - UInt32 *prevMask, UInt32 *prevPos, int encoding); - -#endif diff --git a/code/nel/3rdparty/seven_zip/CMakeLists.txt b/code/nel/3rdparty/seven_zip/CMakeLists.txt index d2d2e690d..95e93438d 100644 --- a/code/nel/3rdparty/seven_zip/CMakeLists.txt +++ b/code/nel/3rdparty/seven_zip/CMakeLists.txt @@ -1,8 +1,8 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -FILE(GLOB LIB_SRC *.cpp *.h) +FILE(GLOB LIB_SRC *.cpp *.c *.h) -LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) +LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/LzmaUtil.c) NL_TARGET_LIB(nel_sevenzip ${LIB_SRC}) # TARGET_LINK_LIBRARIES(nel_sevenzip ${PLATFORM_LINKFLAGS}) @@ -14,14 +14,14 @@ ADD_DEFINITIONS(-D_SZ_ONE_DIRECTORY) IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) INSTALL(TARGETS nel_sevenzip LIBRARY DESTINATION ${NL_LIB_PREFIX} ARCHIVE DESTINATION ${NL_LIB_PREFIX} COMPONENT libraries) -ENDIF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) +ENDIF() -IF(WITH_TOOLS) - ADD_EXECUTABLE(7zDec ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) +IF(WITH_NEL_TOOLS) + ADD_EXECUTABLE(lzma ${CMAKE_CURRENT_SOURCE_DIR}/LzmaUtil.c) - TARGET_LINK_LIBRARIES(7zDec nel_sevenzip) - NL_DEFAULT_PROPS(7zDec "NeL, 3rd Party: Seven Zip Decoder") - NL_ADD_RUNTIME_FLAGS(7zDec) + TARGET_LINK_LIBRARIES(lzma nel_sevenzip) + NL_DEFAULT_PROPS(lzma "NeL, 3rd Party: LZMA") + NL_ADD_RUNTIME_FLAGS(lzma) - INSTALL(TARGETS 7zDec RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT tools) -ENDIF(WITH_TOOLS) + INSTALL(TARGETS lzma RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT tools) +ENDIF() diff --git a/code/nel/3rdparty/seven_zip/LzmaDecode.cpp b/code/nel/3rdparty/seven_zip/LzmaDecode.cpp deleted file mode 100644 index c148a5a65..000000000 --- a/code/nel/3rdparty/seven_zip/LzmaDecode.cpp +++ /dev/null @@ -1,584 +0,0 @@ -/* - LzmaDecode.c - LZMA Decoder (optimized for Speed version) - - LZMA SDK 4.40 Copyright (c) 1999-2006 Igor Pavlov (2006-05-01) - http://www.7-zip.org/ - - LZMA SDK is licensed under two licenses: - 1) GNU Lesser General Public License (GNU LGPL) - 2) Common Public License (CPL) - It means that you can select one of these two licenses and - follow rules of that license. - - SPECIAL EXCEPTION: - Igor Pavlov, as the author of this Code, expressly permits you to - statically or dynamically link your Code (or bind by name) to the - interfaces of this file without subjecting your linked Code to the - terms of the CPL or GNU LGPL. Any modifications or additions - to this file, however, are subject to the LGPL or CPL terms. -*/ - -#include "LzmaDecode.h" - -#define kNumTopBits 24 -#define kTopValue ((UInt32)1 << kNumTopBits) - -#define kNumBitModelTotalBits 11 -#define kBitModelTotal (1 << kNumBitModelTotalBits) -#define kNumMoveBits 5 - -#define RC_READ_BYTE (*Buffer++) - -#define RC_INIT2 Code = 0; Range = 0xFFFFFFFF; \ - { int i; for(i = 0; i < 5; i++) { RC_TEST; Code = (Code << 8) | RC_READ_BYTE; }} - -#ifdef _LZMA_IN_CB - -#define RC_TEST { if (Buffer == BufferLim) \ - { SizeT size; int result = InCallback->Read(InCallback, &Buffer, &size); if (result != LZMA_RESULT_OK) return result; \ - BufferLim = Buffer + size; if (size == 0) return LZMA_RESULT_DATA_ERROR; }} - -#define RC_INIT Buffer = BufferLim = 0; RC_INIT2 - -#else - -#define RC_TEST { if (Buffer == BufferLim) return LZMA_RESULT_DATA_ERROR; } - -#define RC_INIT(buffer, bufferSize) Buffer = buffer; BufferLim = buffer + bufferSize; RC_INIT2 - -#endif - -#define RC_NORMALIZE if (Range < kTopValue) { RC_TEST; Range <<= 8; Code = (Code << 8) | RC_READ_BYTE; } - -#define IfBit0(p) RC_NORMALIZE; bound = (Range >> kNumBitModelTotalBits) * *(p); if (Code < bound) -#define UpdateBit0(p) Range = bound; *(p) += (kBitModelTotal - *(p)) >> kNumMoveBits; -#define UpdateBit1(p) Range -= bound; Code -= bound; *(p) -= (*(p)) >> kNumMoveBits; - -#define RC_GET_BIT2(p, mi, A0, A1) IfBit0(p) \ - { UpdateBit0(p); mi <<= 1; A0; } else \ - { UpdateBit1(p); mi = (mi + mi) + 1; A1; } - -#define RC_GET_BIT(p, mi) RC_GET_BIT2(p, mi, ; , ;) - -#define RangeDecoderBitTreeDecode(probs, numLevels, res) \ - { int i = numLevels; res = 1; \ - do { CProb *p = probs + res; RC_GET_BIT(p, res) } while(--i != 0); \ - res -= (1 << numLevels); } - - -#define kNumPosBitsMax 4 -#define kNumPosStatesMax (1 << kNumPosBitsMax) - -#define kLenNumLowBits 3 -#define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) -#define kLenNumHighBits 8 -#define kLenNumHighSymbols (1 << kLenNumHighBits) - -#define LenChoice 0 -#define LenChoice2 (LenChoice + 1) -#define LenLow (LenChoice2 + 1) -#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) -#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) -#define kNumLenProbs (LenHigh + kLenNumHighSymbols) - - -#define kNumStates 12 -#define kNumLitStates 7 - -#define kStartPosModelIndex 4 -#define kEndPosModelIndex 14 -#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) - -#define kNumPosSlotBits 6 -#define kNumLenToPosStates 4 - -#define kNumAlignBits 4 -#define kAlignTableSize (1 << kNumAlignBits) - -#define kMatchMinLen 2 - -#define IsMatch 0 -#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) -#define IsRepG0 (IsRep + kNumStates) -#define IsRepG1 (IsRepG0 + kNumStates) -#define IsRepG2 (IsRepG1 + kNumStates) -#define IsRep0Long (IsRepG2 + kNumStates) -#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) -#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) -#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) -#define LenCoder (Align + kAlignTableSize) -#define RepLenCoder (LenCoder + kNumLenProbs) -#define Literal (RepLenCoder + kNumLenProbs) - -#if Literal != LZMA_BASE_SIZE -StopCompilingDueBUG -#endif - -int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, SizeT size) -{ - unsigned char prop0; - if (size < LZMA_PROPERTIES_SIZE) - return LZMA_RESULT_DATA_ERROR; - prop0 = propsData[0]; - if (prop0 >= (9 * 5 * 5)) - return LZMA_RESULT_DATA_ERROR; - { - for (propsRes->pb = 0; prop0 >= (9 * 5); propsRes->pb++, prop0 -= (9 * 5)) ; - for (propsRes->lp = 0; prop0 >= 9; propsRes->lp++, prop0 -= 9) ; - propsRes->lc = prop0; - /* - unsigned char remainder = (unsigned char)(prop0 / 9); - propsRes->lc = prop0 % 9; - propsRes->pb = remainder / 5; - propsRes->lp = remainder % 5; - */ - } - - #ifdef _LZMA_OUT_READ - { - int i; - propsRes->DictionarySize = 0; - for (i = 0; i < 4; i++) - propsRes->DictionarySize += (UInt32)(propsData[1 + i]) << (i * 8); - if (propsRes->DictionarySize == 0) - propsRes->DictionarySize = 1; - } - #endif - return LZMA_RESULT_OK; -} - -#define kLzmaStreamWasFinishedId (-1) - -int LzmaDecode(CLzmaDecoderState *vs, - #ifdef _LZMA_IN_CB - ILzmaInCallback *InCallback, - #else - const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, - #endif - unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed) -{ - CProb *p = vs->Probs; - SizeT nowPos = 0; - Byte previousByte = 0; - UInt32 posStateMask = (1 << (vs->Properties.pb)) - 1; - UInt32 literalPosMask = (1 << (vs->Properties.lp)) - 1; - int lc = vs->Properties.lc; - - #ifdef _LZMA_OUT_READ - - UInt32 Range = vs->Range; - UInt32 Code = vs->Code; - #ifdef _LZMA_IN_CB - const Byte *Buffer = vs->Buffer; - const Byte *BufferLim = vs->BufferLim; - #else - const Byte *Buffer = inStream; - const Byte *BufferLim = inStream + inSize; - #endif - int state = vs->State; - UInt32 rep0 = vs->Reps[0], rep1 = vs->Reps[1], rep2 = vs->Reps[2], rep3 = vs->Reps[3]; - int len = vs->RemainLen; - UInt32 globalPos = vs->GlobalPos; - UInt32 distanceLimit = vs->DistanceLimit; - - Byte *dictionary = vs->Dictionary; - UInt32 dictionarySize = vs->Properties.DictionarySize; - UInt32 dictionaryPos = vs->DictionaryPos; - - Byte tempDictionary[4]; - - #ifndef _LZMA_IN_CB - *inSizeProcessed = 0; - #endif - *outSizeProcessed = 0; - if (len == kLzmaStreamWasFinishedId) - return LZMA_RESULT_OK; - - if (dictionarySize == 0) - { - dictionary = tempDictionary; - dictionarySize = 1; - tempDictionary[0] = vs->TempDictionary[0]; - } - - if (len == kLzmaNeedInitId) - { - { - UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); - UInt32 i; - for (i = 0; i < numProbs; i++) - p[i] = kBitModelTotal >> 1; - rep0 = rep1 = rep2 = rep3 = 1; - state = 0; - globalPos = 0; - distanceLimit = 0; - dictionaryPos = 0; - dictionary[dictionarySize - 1] = 0; - #ifdef _LZMA_IN_CB - RC_INIT; - #else - RC_INIT(inStream, inSize); - #endif - } - len = 0; - } - while(len != 0 && nowPos < outSize) - { - UInt32 pos = dictionaryPos - rep0; - if (pos >= dictionarySize) - pos += dictionarySize; - outStream[nowPos++] = dictionary[dictionaryPos] = dictionary[pos]; - if (++dictionaryPos == dictionarySize) - dictionaryPos = 0; - len--; - } - if (dictionaryPos == 0) - previousByte = dictionary[dictionarySize - 1]; - else - previousByte = dictionary[dictionaryPos - 1]; - - #else /* if !_LZMA_OUT_READ */ - - int state = 0; - UInt32 rep0 = 1, rep1 = 1, rep2 = 1, rep3 = 1; - int len = 0; - const Byte *Buffer; - const Byte *BufferLim; - UInt32 Range; - UInt32 Code; - - #ifndef _LZMA_IN_CB - *inSizeProcessed = 0; - #endif - *outSizeProcessed = 0; - - { - UInt32 i; - UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (lc + vs->Properties.lp)); - for (i = 0; i < numProbs; i++) - p[i] = kBitModelTotal >> 1; - } - - #ifdef _LZMA_IN_CB - RC_INIT; - #else - RC_INIT(inStream, inSize); - #endif - - #endif /* _LZMA_OUT_READ */ - - while(nowPos < outSize) - { - CProb *prob; - UInt32 bound; - int posState = (int)( - (nowPos - #ifdef _LZMA_OUT_READ - + globalPos - #endif - ) - & posStateMask); - - prob = p + IsMatch + (state << kNumPosBitsMax) + posState; - IfBit0(prob) - { - int symbol = 1; - UpdateBit0(prob) - prob = p + Literal + (LZMA_LIT_SIZE * - ((( - (nowPos - #ifdef _LZMA_OUT_READ - + globalPos - #endif - ) - & literalPosMask) << lc) + (previousByte >> (8 - lc)))); - - if (state >= kNumLitStates) - { - int matchByte; - #ifdef _LZMA_OUT_READ - UInt32 pos = dictionaryPos - rep0; - if (pos >= dictionarySize) - pos += dictionarySize; - matchByte = dictionary[pos]; - #else - matchByte = outStream[nowPos - rep0]; - #endif - do - { - int bit; - CProb *probLit; - matchByte <<= 1; - bit = (matchByte & 0x100); - probLit = prob + 0x100 + bit + symbol; - RC_GET_BIT2(probLit, symbol, if (bit != 0) break, if (bit == 0) break) - } - while (symbol < 0x100); - } - while (symbol < 0x100) - { - CProb *probLit = prob + symbol; - RC_GET_BIT(probLit, symbol) - } - previousByte = (Byte)(symbol & 0xff); - - outStream[nowPos++] = previousByte; - #ifdef _LZMA_OUT_READ - if (distanceLimit < dictionarySize) - distanceLimit++; - - dictionary[dictionaryPos] = previousByte; - if (++dictionaryPos == dictionarySize) - dictionaryPos = 0; - #endif - if (state < 4) state = 0; - else if (state < 10) state -= 3; - else state -= 6; - } - else - { - UpdateBit1(prob); - prob = p + IsRep + state; - IfBit0(prob) - { - UpdateBit0(prob); - rep3 = rep2; - rep2 = rep1; - rep1 = rep0; - state = state < kNumLitStates ? 0 : 3; - prob = p + LenCoder; - } - else - { - UpdateBit1(prob); - prob = p + IsRepG0 + state; - IfBit0(prob) - { - UpdateBit0(prob); - prob = p + IsRep0Long + (state << kNumPosBitsMax) + posState; - IfBit0(prob) - { - #ifdef _LZMA_OUT_READ - UInt32 pos; - #endif - UpdateBit0(prob); - - #ifdef _LZMA_OUT_READ - if (distanceLimit == 0) - #else - if (nowPos == 0) - #endif - return LZMA_RESULT_DATA_ERROR; - - state = state < kNumLitStates ? 9 : 11; - #ifdef _LZMA_OUT_READ - pos = dictionaryPos - rep0; - if (pos >= dictionarySize) - pos += dictionarySize; - previousByte = dictionary[pos]; - dictionary[dictionaryPos] = previousByte; - if (++dictionaryPos == dictionarySize) - dictionaryPos = 0; - #else - previousByte = outStream[nowPos - rep0]; - #endif - outStream[nowPos++] = previousByte; - #ifdef _LZMA_OUT_READ - if (distanceLimit < dictionarySize) - distanceLimit++; - #endif - - continue; - } - else - { - UpdateBit1(prob); - } - } - else - { - UInt32 distance; - UpdateBit1(prob); - prob = p + IsRepG1 + state; - IfBit0(prob) - { - UpdateBit0(prob); - distance = rep1; - } - else - { - UpdateBit1(prob); - prob = p + IsRepG2 + state; - IfBit0(prob) - { - UpdateBit0(prob); - distance = rep2; - } - else - { - UpdateBit1(prob); - distance = rep3; - rep3 = rep2; - } - rep2 = rep1; - } - rep1 = rep0; - rep0 = distance; - } - state = state < kNumLitStates ? 8 : 11; - prob = p + RepLenCoder; - } - { - int numBits, offset; - CProb *probLen = prob + LenChoice; - IfBit0(probLen) - { - UpdateBit0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); - offset = 0; - numBits = kLenNumLowBits; - } - else - { - UpdateBit1(probLen); - probLen = prob + LenChoice2; - IfBit0(probLen) - { - UpdateBit0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); - offset = kLenNumLowSymbols; - numBits = kLenNumMidBits; - } - else - { - UpdateBit1(probLen); - probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; - numBits = kLenNumHighBits; - } - } - RangeDecoderBitTreeDecode(probLen, numBits, len); - len += offset; - } - - if (state < 4) - { - int posSlot; - state += kNumLitStates; - prob = p + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << - kNumPosSlotBits); - RangeDecoderBitTreeDecode(prob, kNumPosSlotBits, posSlot); - if (posSlot >= kStartPosModelIndex) - { - int numDirectBits = ((posSlot >> 1) - 1); - rep0 = (2 | ((UInt32)posSlot & 1)); - if (posSlot < kEndPosModelIndex) - { - rep0 <<= numDirectBits; - prob = p + SpecPos + rep0 - posSlot - 1; - } - else - { - numDirectBits -= kNumAlignBits; - do - { - RC_NORMALIZE - Range >>= 1; - rep0 <<= 1; - if (Code >= Range) - { - Code -= Range; - rep0 |= 1; - } - } - while (--numDirectBits != 0); - prob = p + Align; - rep0 <<= kNumAlignBits; - numDirectBits = kNumAlignBits; - } - { - int i = 1; - int mi = 1; - do - { - CProb *prob3 = prob + mi; - RC_GET_BIT2(prob3, mi, ; , rep0 |= i); - i <<= 1; - } - while(--numDirectBits != 0); - } - } - else - rep0 = posSlot; - if (++rep0 == (UInt32)(0)) - { - /* it's for stream version */ - len = kLzmaStreamWasFinishedId; - break; - } - } - - len += kMatchMinLen; - #ifdef _LZMA_OUT_READ - if (rep0 > distanceLimit) - #else - if (rep0 > nowPos) - #endif - return LZMA_RESULT_DATA_ERROR; - - #ifdef _LZMA_OUT_READ - if (dictionarySize - distanceLimit > (UInt32)len) - distanceLimit += len; - else - distanceLimit = dictionarySize; - #endif - - do - { - #ifdef _LZMA_OUT_READ - UInt32 pos = dictionaryPos - rep0; - if (pos >= dictionarySize) - pos += dictionarySize; - previousByte = dictionary[pos]; - dictionary[dictionaryPos] = previousByte; - if (++dictionaryPos == dictionarySize) - dictionaryPos = 0; - #else - previousByte = outStream[nowPos - rep0]; - #endif - len--; - outStream[nowPos++] = previousByte; - } - while(len != 0 && nowPos < outSize); - } - } - RC_NORMALIZE; - - #ifdef _LZMA_OUT_READ - vs->Range = Range; - vs->Code = Code; - vs->DictionaryPos = dictionaryPos; - vs->GlobalPos = globalPos + (UInt32)nowPos; - vs->DistanceLimit = distanceLimit; - vs->Reps[0] = rep0; - vs->Reps[1] = rep1; - vs->Reps[2] = rep2; - vs->Reps[3] = rep3; - vs->State = state; - vs->RemainLen = len; - vs->TempDictionary[0] = tempDictionary[0]; - #endif - - #ifdef _LZMA_IN_CB - vs->Buffer = Buffer; - vs->BufferLim = BufferLim; - #else - *inSizeProcessed = (SizeT)(Buffer - inStream); - #endif - *outSizeProcessed = nowPos; - return LZMA_RESULT_OK; -} diff --git a/code/nel/3rdparty/seven_zip/LzmaDecode.h b/code/nel/3rdparty/seven_zip/LzmaDecode.h deleted file mode 100644 index 748fd0a68..000000000 --- a/code/nel/3rdparty/seven_zip/LzmaDecode.h +++ /dev/null @@ -1,113 +0,0 @@ -/* - LzmaDecode.h - LZMA Decoder interface - - LZMA SDK 4.40 Copyright (c) 1999-2006 Igor Pavlov (2006-05-01) - http://www.7-zip.org/ - - LZMA SDK is licensed under two licenses: - 1) GNU Lesser General Public License (GNU LGPL) - 2) Common Public License (CPL) - It means that you can select one of these two licenses and - follow rules of that license. - - SPECIAL EXCEPTION: - Igor Pavlov, as the author of this code, expressly permits you to - statically or dynamically link your code (or bind by name) to the - interfaces of this file without subjecting your linked code to the - terms of the CPL or GNU LGPL. Any modifications or additions - to this file, however, are subject to the LGPL or CPL terms. -*/ - -#ifndef __LZMADECODE_H -#define __LZMADECODE_H - -#include "LzmaTypes.h" - -/* #define _LZMA_IN_CB */ -/* Use callback for input data */ - -/* #define _LZMA_OUT_READ */ -/* Use read function for output data */ - -/* #define _LZMA_PROB32 */ -/* It can increase speed on some 32-bit CPUs, - but memory usage will be doubled in that case */ - -/* #define _LZMA_LOC_OPT */ -/* Enable local speed optimizations inside code */ - -#ifdef _LZMA_PROB32 -#define CProb UInt32 -#else -#define CProb UInt16 -#endif - -#define LZMA_RESULT_OK 0 -#define LZMA_RESULT_DATA_ERROR 1 - -#ifdef _LZMA_IN_CB -typedef struct _ILzmaInCallback -{ - int (*Read)(void *object, const unsigned char **buffer, SizeT *bufferSize); -} ILzmaInCallback; -#endif - -#define LZMA_BASE_SIZE 1846 -#define LZMA_LIT_SIZE 768 - -#define LZMA_PROPERTIES_SIZE 5 - -typedef struct _CLzmaProperties -{ - int lc; - int lp; - int pb; - #ifdef _LZMA_OUT_READ - UInt32 DictionarySize; - #endif -}CLzmaProperties; - -int LzmaDecodeProperties(CLzmaProperties *propsRes, const unsigned char *propsData, SizeT size); - -#define LzmaGetNumProbs(Properties) (LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((Properties)->lc + (Properties)->lp))) - -#define kLzmaNeedInitId (-2) - -typedef struct _CLzmaDecoderState -{ - CLzmaProperties Properties; - CProb *Probs; - - #ifdef _LZMA_IN_CB - const unsigned char *Buffer; - const unsigned char *BufferLim; - #endif - - #ifdef _LZMA_OUT_READ - unsigned char *Dictionary; - UInt32 Range; - UInt32 Code; - UInt32 DictionaryPos; - UInt32 GlobalPos; - UInt32 DistanceLimit; - UInt32 Reps[4]; - int State; - int RemainLen; - unsigned char TempDictionary[4]; - #endif -} CLzmaDecoderState; - -#ifdef _LZMA_OUT_READ -#define LzmaDecoderInit(vs) { (vs)->RemainLen = kLzmaNeedInitId; } -#endif - -int LzmaDecode(CLzmaDecoderState *vs, - #ifdef _LZMA_IN_CB - ILzmaInCallback *inCallback, - #else - const unsigned char *inStream, SizeT inSize, SizeT *inSizeProcessed, - #endif - unsigned char *outStream, SizeT outSize, SizeT *outSizeProcessed); - -#endif diff --git a/code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp b/code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp deleted file mode 100644 index ed1784d6d..000000000 --- a/code/nel/3rdparty/seven_zip/LzmaRamDecode.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/* LzmaRamDecode.c */ - -#include "LzmaRamDecode.h" -#ifdef _SZ_ONE_DIRECTORY -#include "LzmaDecode.h" -#include "BranchX86.h" -#else -#include "../LZMA_C/LzmaDecode.h" -#include "../Branch/BranchX86.h" -#endif - -#define LZMA_PROPS_SIZE 14 -#define LZMA_SIZE_OFFSET 6 - -int LzmaRamGetUncompressedSize( - const unsigned char *inBuffer, - size_t inSize, - size_t *outSize) -{ - unsigned int i; - if (inSize < LZMA_PROPS_SIZE) - return 1; - *outSize = 0; - for(i = 0; i < sizeof(size_t); i++) - *outSize += ((size_t)inBuffer[LZMA_SIZE_OFFSET + i]) << (8 * i); - for(; i < 8; i++) - if (inBuffer[LZMA_SIZE_OFFSET + i] != 0) - return 1; - return 0; -} - -#define SZE_DATA_ERROR (1) -#define SZE_OUTOFMEMORY (2) - -int LzmaRamDecompress( - const unsigned char *inBuffer, - size_t inSize, - unsigned char *outBuffer, - size_t outSize, - size_t *outSizeProcessed, - void * (*allocFunc)(size_t size), - void (*freeFunc)(void *)) -{ - CLzmaDecoderState state; /* it's about 24 bytes structure, if int is 32-bit */ - int result; - SizeT outSizeProcessedLoc; - SizeT inProcessed; - int useFilter; - - if (inSize < LZMA_PROPS_SIZE) - return 1; - useFilter = inBuffer[0]; - - *outSizeProcessed = 0; - if (useFilter > 1) - return 1; - - if (LzmaDecodeProperties(&state.Properties, inBuffer + 1, LZMA_PROPERTIES_SIZE) != LZMA_RESULT_OK) - return 1; - state.Probs = (CProb *)allocFunc(LzmaGetNumProbs(&state.Properties) * sizeof(CProb)); - if (state.Probs == 0) - return SZE_OUTOFMEMORY; - - result = LzmaDecode(&state, - inBuffer + LZMA_PROPS_SIZE, (SizeT)inSize - LZMA_PROPS_SIZE, &inProcessed, - outBuffer, (SizeT)outSize, &outSizeProcessedLoc); - freeFunc(state.Probs); - if (result != LZMA_RESULT_OK) - return 1; - *outSizeProcessed = (size_t)outSizeProcessedLoc; - if (useFilter == 1) - { - UInt32 _prevMask; - UInt32 _prevPos; - x86_Convert_Init(_prevMask, _prevPos); - x86_Convert(outBuffer, (UInt32)outSizeProcessedLoc, 0, &_prevMask, &_prevPos, 0); - } - return 0; -} diff --git a/code/nel/3rdparty/seven_zip/LzmaRamDecode.h b/code/nel/3rdparty/seven_zip/LzmaRamDecode.h deleted file mode 100644 index 7e641c553..000000000 --- a/code/nel/3rdparty/seven_zip/LzmaRamDecode.h +++ /dev/null @@ -1,55 +0,0 @@ -/* LzmaRamDecode.h */ - -#ifndef __LzmaRamDecode_h -#define __LzmaRamDecode_h - -#include - -/* -LzmaRamGetUncompressedSize: - In: - inBuffer - input data - inSize - input data size - Out: - outSize - uncompressed size - Return code: - 0 - OK - 1 - Error in headers -*/ - -int LzmaRamGetUncompressedSize( - const unsigned char *inBuffer, - size_t inSize, - size_t *outSize); - - -/* -LzmaRamDecompress: - In: - inBuffer - input data - inSize - input data size - outBuffer - output data - outSize - output size - allocFunc - alloc function (can be malloc) - freeFunc - free function (can be free) - Out: - outSizeProcessed - processed size - Return code: - 0 - OK - 1 - Error in headers / data stream - 2 - Memory allocating error - -Memory requirements depend from properties of LZMA stream. -With default lzma settings it's about 16 KB. -*/ - -int LzmaRamDecompress( - const unsigned char *inBuffer, - size_t inSize, - unsigned char *outBuffer, - size_t outSize, - size_t *outSizeProcessed, - void * (*allocFunc)(size_t size), - void (*freeFunc)(void *)); - -#endif diff --git a/code/nel/3rdparty/seven_zip/LzmaTypes.h b/code/nel/3rdparty/seven_zip/LzmaTypes.h deleted file mode 100644 index 288c5e45d..000000000 --- a/code/nel/3rdparty/seven_zip/LzmaTypes.h +++ /dev/null @@ -1,45 +0,0 @@ -/* -LzmaTypes.h - -Types for LZMA Decoder - -This file written and distributed to public domain by Igor Pavlov. -This file is part of LZMA SDK 4.40 (2006-05-01) -*/ - -#ifndef __LZMATYPES_H -#define __LZMATYPES_H - -#ifndef _7ZIP_BYTE_DEFINED -#define _7ZIP_BYTE_DEFINED -typedef unsigned char Byte; -#endif - -#ifndef _7ZIP_UINT16_DEFINED -#define _7ZIP_UINT16_DEFINED -typedef unsigned short UInt16; -#endif - -#ifndef _7ZIP_UINT32_DEFINED -#define _7ZIP_UINT32_DEFINED -#ifdef _LZMA_UINT32_IS_ULONG -typedef unsigned long UInt32; -#else -typedef unsigned int UInt32; -#endif -#endif - -/* #define _LZMA_SYSTEM_SIZE_T */ -/* Use system's size_t. You can use it to enable 64-bit sizes supporting */ - -#ifndef _7ZIP_SIZET_DEFINED -#define _7ZIP_SIZET_DEFINED -#ifdef _LZMA_SYSTEM_SIZE_T -#include -typedef size_t SizeT; -#else -typedef UInt32 SizeT; -#endif -#endif - -#endif diff --git a/code/nel/3rdparty/seven_zip/readme.txt b/code/nel/3rdparty/seven_zip/readme.txt deleted file mode 100644 index e9798c1df..000000000 --- a/code/nel/3rdparty/seven_zip/readme.txt +++ /dev/null @@ -1,22 +0,0 @@ - - -This is an extract of some files from the 7zip SDK. - -At time of writing (2007-01-12) there is no dll or library project in the 7zip -distribution (lzma443), so I build a custum project and copied in the 7zip -files needed to do 7zip/lzma extraction. - -To update this library, download the latest 7zip SDK and copy all file from - /C/7zip/Archive/7z_C -for the 7zip archive file format support - -and the 3 files needed from - /C/7zip/Compress/LZMA_C - (lzmaDecode.h lzmaDecode.cpp lzmaTypes.h) -for the lzma decrompression lib. - - -NB : If callback support must be enabled, add "#define _LZMA_IN_CB" in - 7zTypes.h - - diff --git a/code/ryzom/client/src/seven_zip/seven_zip.h b/code/nel/include/nel/misc/seven_zip.h similarity index 86% rename from code/ryzom/client/src/seven_zip/seven_zip.h rename to code/nel/include/nel/misc/seven_zip.h index e471a9ac0..469b24bb9 100644 --- a/code/ryzom/client/src/seven_zip/seven_zip.h +++ b/code/nel/include/nel/misc/seven_zip.h @@ -18,6 +18,9 @@ #define SEVEN_ZIP_H #include +#include + +namespace NLMISC { // utility func to decompress a monofile 7zip archive bool unpack7Zip(const std::string &sevenZipFileName, const std::string &destFileName); @@ -25,7 +28,12 @@ bool unpack7Zip(const std::string &sevenZipFileName, const std::string &destFile // utility func to decompress a single LZMA packed file bool unpackLZMA(const std::string &lzmaFileName, const std::string &destFileName); +// utility func to decompress a single LZMA packed file +bool unpackLZMA(const std::string &lzmaFileName, const std::string &destFileName, CHashKey &sha1); + // utility func to compress a single file to LZMA packed file bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName); +} + #endif diff --git a/code/nel/include/nel/misc/sha1.h b/code/nel/include/nel/misc/sha1.h index ae42c3c1e..9b7fe937b 100644 --- a/code/nel/include/nel/misc/sha1.h +++ b/code/nel/include/nel/misc/sha1.h @@ -23,6 +23,8 @@ #include +namespace NLMISC { + struct CHashKey { CHashKey () { HashKeyString.resize(20); } @@ -99,6 +101,27 @@ inline bool operator <(const struct CHashKey &a,const struct CHashKey &b) return a.HashKeyString. -#include "seven_zip.h" +#include -#include "nel/misc/types_nl.h" -#include "nel/misc/file.h" -#include "nel/misc/path.h" +#include +#include +#include -#include "7z.h" -#include "7zAlloc.h" -#include "7zCrc.h" -#include "7zVersion.h" -#include "LzmaLib.h" +#include +#include +#include +#include +#include +#include #include @@ -39,6 +40,7 @@ using namespace NLMISC; #define new DEBUG_NEW #endif +namespace NLMISC { /// Input stream class for 7zip archive class CNel7ZipInStream : public ISeekInStream @@ -273,6 +275,85 @@ bool unpackLZMA(const std::string &lzmaFile, const std::string &destFileName) return true; } +bool unpackLZMA(const std::string &lzmaFile, const std::string &destFileName, CHashKey &sha1) +{ + nldebug("unpackLZMA: decompress LZMA file '%s' to '%s", lzmaFile.c_str(), destFileName.c_str()); + + // open input file + CIFile inStream(lzmaFile); + uint32 inSize = inStream.getFileSize(); + + if (inSize < LZMA_PROPS_SIZE + 8) + { + nlwarning("unpackLZMA: Invalid file size, too small file '%s'", lzmaFile.c_str()); + return false; + } + + // allocate input buffer for props + std::vector propsBuffer(LZMA_PROPS_SIZE); + + // size of LZMA content + inSize -= LZMA_PROPS_SIZE + 8; + + // allocate input buffer for lzma data + std::vector inBuffer(inSize); + + uint64 fileSize = 0; + + try + { + // read props + inStream.serialBuffer(&propsBuffer[0], LZMA_PROPS_SIZE); + + // read uncompressed size + inStream.serial(fileSize); + + // read lzma content + inStream.serialBuffer(&inBuffer[0], inSize); + } + catch(const EReadError &e) + { + nlwarning("unpackLZMA: Error while reading '%s': %s", lzmaFile.c_str(), e.what()); + return false; + } + + // allocate the output buffer + std::vector outBuffer(fileSize); + + // in and out file sizes + SizeT outProcessed = (SizeT)fileSize; + SizeT inProcessed = (SizeT)inSize; + + // decompress the file in memory + sint res = LzmaUncompress(&outBuffer[0], &outProcessed, &inBuffer[0], &inProcessed, &propsBuffer[0], LZMA_PROPS_SIZE); + + if (res != 0 || outProcessed != fileSize) + { + nlwarning("unpackLZMA: Failed to decode lzma file '%s' with status %d", lzmaFile.c_str(), res); + return false; + } + + // get hash + sha1 = getSHA1(&outBuffer[0], outBuffer.size()); + + // store on output buffer + COFile outStream(destFileName); + + try + { + // write content + outStream.serialBuffer(&outBuffer[0], (uint)fileSize); + } + catch(const EFile &e) + { + nlwarning("unpackLZMA: Error while writing '%s': %s", destFileName.c_str(), e.what()); + CFile::deleteFile(destFileName); + return false; + } + + return true; +} + bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName) { nldebug("packLZMA: compress '%s' to LZMA file '%s", srcFileName.c_str(), lzmaFileName.c_str()); @@ -371,3 +452,5 @@ bool packLZMA(const std::string &srcFileName, const std::string &lzmaFileName) return false; } + +} diff --git a/code/nel/src/misc/sha1.cpp b/code/nel/src/misc/sha1.cpp index 8afa0c039..e4c3396a5 100644 --- a/code/nel/src/misc/sha1.cpp +++ b/code/nel/src/misc/sha1.cpp @@ -53,6 +53,8 @@ typedef unsigned int uint32_t; typedef unsigned char uint8_t; typedef short int_least16_t; +namespace NLMISC { + // // Constantes // @@ -68,27 +70,6 @@ enum shaStateError /* called Input after Result */ }; -#define SHA1HashSize 20 - -/* - * This structure will hold context information for the SHA-1 - * hashing operation - */ -typedef struct SHA1Context -{ - uint32_t Intermediate_Hash[SHA1HashSize/4]; /* Message Digest */ - - uint32_t Length_Low; /* Message length in bits */ - uint32_t Length_High; /* Message length in bits */ - - /* Index into message block array */ - int_least16_t Message_Block_Index; - uint8_t Message_Block[64]; /* 512-bit message blocks */ - - int Computed; /* Is the digest computed? */ - int Corrupted; /* Is the message digest corrupted? */ -} SHA1Context; - // // Function Prototypes // @@ -187,7 +168,50 @@ CHashKey getSHA1(const string &filename, bool forcePath) return CHashKey(); } - CHashKey hk (Message_Digest); + CHashKey hk(Message_Digest); + return hk; +} + +// Begin calculating a SHA1 hash +bool resetSHA1(SHA1Context *context) +{ + int err; + err = SHA1Reset(context); + if (err) + { + nlwarning("SHA: SHA1Reset Error %d.\n", err ); + return false; + } + return true; +} + +// Add data to a SHA1 hash calculation +bool inputSHA1(SHA1Context *context, const uint8 *buffer, uint32 size) +{ + int err; + err = SHA1Input(context, buffer, size); + if (err) + { + nlwarning ("SHA: SHA1Input Error %d.\n", err); + return false; + } + return true; +} + +// Get the SHA1 result +CHashKey getSHA1(SHA1Context *context) +{ + int err; + uint8_t messageDigest[20]; + + err = SHA1Result(context, messageDigest); + if (err) + { + nlwarning("SHA: SHA1Result Error %d, could not compute message digest.\n", err ); + return CHashKey(); + } + + CHashKey hk(messageDigest); return hk; } @@ -618,3 +642,6 @@ void SHA1PadMessage(SHA1Context *context) SHA1ProcessMessageBlock(context); } + +} + diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 23b77def7..483df6a52 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -18,18 +18,16 @@ // 3rd Party includes #include -#include -#include -#include -#include // Project includes #include #include #include +#include #include -namespace NLMISC { +namespace NLMISC +{ NLMISC_SAFE_SINGLETON_IMPL(CStreamedPackageManager); @@ -106,7 +104,7 @@ bool CStreamedPackageManager::getFile(std::string &filePath, const std::string & CStreamedPackage::makePath(filePath, entry->Hash); std::string downloadUrlFile = filePath + ".lzma"; filePath = Path + filePath; - std::string downloadPath = filePath + ".download." + toString(rand()); + std::string downloadPath = filePath + ".download." + toString(rand() * rand()); std::string storageDirectory = CFile::getPath(downloadPath); CFile::createDirectoryTree(storageDirectory); @@ -117,7 +115,7 @@ bool CStreamedPackageManager::getFile(std::string &filePath, const std::string & }*/ // download - for (; ; ) + for (;;) { if (CFile::fileExists(filePath)) return true; @@ -174,99 +172,31 @@ bool CStreamedPackageManager::getFile(std::string &filePath, const std::string & break; } - // read into memory :( - std::string unpackPath = filePath + ".extract." + toString(rand()); + // extract into file + std::string unpackPath = filePath + ".extract." + toString(rand() * rand()); - std::vector inBuffer; + CHashKey outHash; + if (!unpackLZMA(downloadPath, unpackPath, outHash)) { - CIFile inStream(downloadPath); - uint32 inSize = inStream.getFileSize(); - inBuffer.resize(inSize); - inStream.serialBuffer(&inBuffer[0], inSize); + return false; } - CFile::deleteFile(downloadPath); - if (inBuffer.size() < LZMA_PROPERTIES_SIZE + 8) + if (!(outHash == entry->Hash)) { - nlwarning("Invalid file size %u, too small file '%s'", inBuffer.size(), downloadPath.c_str()); + std::string wantHashS = entry->Hash.toString(); + std::string outHashS = outHash.toString(); + nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str()); return false; } - // extract + if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str())) { - CLzmaDecoderState state; - uint8 *pos = &inBuffer[0]; - int ret = LzmaDecodeProperties(&state.Properties, (unsigned char *)pos, LZMA_PROPERTIES_SIZE); - if (ret != 0) - { - nlwarning("Failed to decode lzma properies in file '%s'", downloadPath.c_str()); - return false; - } - - // FROM login_patch.cpp - // alloc the probs, making sure they are deleted in function exit - size_t nbProb = LzmaGetNumProbs(&state.Properties); - std::vector probs; - probs.resize(nbProb); - state.Probs = &probs[0]; - - pos += LZMA_PROPERTIES_SIZE; - - // read the output file size - size_t fileSize = 0; - for (int i = 0; i < 8; i++) - { - //Byte b; - if (pos >= &inBuffer[0] + inBuffer.size()) - { - nlerror("pos >= inBuffer.get() + inSize"); - return false; - } - fileSize |= ((UInt64)*pos++) << (8 * i); - } - - nlassert(fileSize == entry->Size); - - SizeT outProcessed = 0; - SizeT inProcessed = 0; - // allocate the output buffer :( - std::vector outBuffer; - outBuffer.resize(fileSize); - if (fileSize) - { - // decompress the file in memory - ret = LzmaDecode(&state, (unsigned char *)pos, (SizeT)(inBuffer.size() - (pos - &inBuffer[0])), &inProcessed, (unsigned char*)&outBuffer[0], (SizeT)fileSize, &outProcessed); - if (ret != 0 || outProcessed != fileSize) - { - nlwarning("Failed to decode lzma file '%s'", downloadPath.c_str()); - return false; - } - } - - CHashKey outHash = outBuffer.size() ? getSHA1(&outBuffer[0], outBuffer.size()) : CHashKey(); - if (!(outHash == entry->Hash)) - { - std::string wantHashS = entry->Hash.toString(); - std::string outHashS = outHash.toString(); - nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str()); - return false; - } - - { - COFile outStream(unpackPath); - if (fileSize) - outStream.serialBuffer(&outBuffer[0], (uint)fileSize); - } - - if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str())) - { - nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str()); - // in case downloaded from another thread - return CFile::fileExists(filePath); - } - - return true; + nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str()); + // in case downloaded from another thread + return CFile::fileExists(filePath); } + + return true; } bool CStreamedPackageManager::getFileSize(uint32 &fileSize, const std::string &fileName) diff --git a/code/nel/tools/build_gamedata/d1_client_patch.py b/code/nel/tools/build_gamedata/d1_client_patch.py index 00eaf1aaa..ea4cc742c 100755 --- a/code/nel/tools/build_gamedata/d1_client_patch.py +++ b/code/nel/tools/build_gamedata/d1_client_patch.py @@ -144,14 +144,14 @@ else: if (needUpdateBnp): if (category["StreamedPackages"]): printLog(log, "SNP " + targetBnp) - cwDir = os.getcwd().replace("\\", "/") - toolDir = os.path.dirname(Lzma).replace("\\", "/") - os.chdir(toolDir) - subprocess.call([ SnpMake, "/p", sourcePath, targetBnp, ClientPatchDirectory + "/stream" ] + package[1][1:]) - os.chdir(cwDir) + # cwDir = os.getcwd().replace("\\", "/") + # toolDir = os.path.dirname(Lzma).replace("\\", "/") + # os.chdir(toolDir) + subprocess.call([ SnpMake, "-p", sourcePath, targetBnp, ClientPatchDirectory + "/stream" ] + package[1][1:]) + # os.chdir(cwDir) else: printLog(log, "BNP " + targetBnp) - subprocess.call([ BnpMake, "-p", sourcePath, "-o", targetPath ] + package[1][1:]) + subprocess.call([ BnpMake, "-p", sourcePath, "-o", targetBnp ] + package[1][1:]) else: printLog(log, "SKIP " + targetBnp) printLog(log, "") diff --git a/code/nel/tools/misc/snp_make/main.cpp b/code/nel/tools/misc/snp_make/main.cpp index 61fc11810..a86d497d4 100644 --- a/code/nel/tools/misc/snp_make/main.cpp +++ b/code/nel/tools/misc/snp_make/main.cpp @@ -33,6 +33,7 @@ #include "nel/misc/algo.h" #include "nel/misc/common.h" #include "nel/misc/streamed_package.h" +#include "nel/misc/seven_zip.h" using namespace std; using namespace NLMISC; @@ -83,19 +84,22 @@ bool keepFile (const char *fileName) void usage() { printf ("USAGE : \n"); - printf (" snp_make /p [option] ... [option]\n"); + printf (" snp_make -p [option] ... [option]\n"); printf (" option : \n"); printf (" -if wildcard : add the file if it matches the wilcard (at least one 'if' conditions must be met for a file to be adding)\n"); printf (" -ifnot wildcard : add the file if it doesn't match the wilcard (all the 'ifnot' conditions must be met for a file to be adding)\n"); printf (" Pack the directory to a snp file\n"); - printf (" snp_make /l \n"); + printf (" snp_make -l \n"); printf (" List the files contained in the snp file\n"); } // --------------------------------------------------------------------------- -void generateLZMA(const std::string sourceFile, const std::string &outputFile) +void generateLZMA(const std::string &sourceFile, const std::string &outputFile) { + NLMISC::packLZMA(sourceFile, outputFile); + + /* std::string cmd="lzma e "; cmd+=" "+sourceFile+" "+outputFile; nlinfo("executing system command: %s",cmd.c_str()); @@ -106,6 +110,7 @@ void generateLZMA(const std::string sourceFile, const std::string &outputFile) if (error) nlwarning("'%s' failed with error code %d", cmd.c_str(), error); #endif // NL_OS_WINDOWS + */ } // --------------------------------------------------------------------------- diff --git a/code/ryzom/client/CMakeLists.txt b/code/ryzom/client/CMakeLists.txt index 43897c905..7a6cf7424 100644 --- a/code/ryzom/client/CMakeLists.txt +++ b/code/ryzom/client/CMakeLists.txt @@ -1,5 +1,4 @@ # Need clientsheets lib for sheets packer tool -# Need seven_zip for patch_gen and ryzom_installer ADD_SUBDIRECTORY(src) IF(WITH_RYZOM_CLIENT) diff --git a/code/ryzom/client/src/CMakeLists.txt b/code/ryzom/client/src/CMakeLists.txt index 4116fc75b..99b9a6773 100644 --- a/code/ryzom/client/src/CMakeLists.txt +++ b/code/ryzom/client/src/CMakeLists.txt @@ -3,11 +3,6 @@ IF(WITH_RYZOM_TOOLS OR WITH_RYZOM_CLIENT) ADD_SUBDIRECTORY(client_sheets) ENDIF() -IF(WITH_RYZOM_TOOLS OR WITH_RYZOM_CLIENT OR WITH_RYZOM_INSTALLER) - # Need seven_zip lib for patch_gen tool - ADD_SUBDIRECTORY(seven_zip) -ENDIF() - IF(WITH_RYZOM_CLIENT) IF(NOT WITH_GUI) MESSAGE(FATAL_ERROR "The client cannot be built without the NeL GUI Library (WITH_GUI)") @@ -158,6 +153,8 @@ IF(WITH_RYZOM_CLIENT) ${OPENSSL_INCLUDE_DIR} ) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/nel/3rdparty) + IF(STEAM_FOUND) INCLUDE_DIRECTORIES(${STEAM_INCLUDE_DIRS}) ENDIF() @@ -170,7 +167,7 @@ IF(WITH_RYZOM_CLIENT) nel3d nelgui nelsound - ryzom_sevenzip + nel_sevenzip ryzom_clientsheets ryzom_gameshare nelpacs diff --git a/code/ryzom/client/src/seven_zip/7z.h b/code/ryzom/client/src/seven_zip/7z.h deleted file mode 100644 index 6c7886e38..000000000 --- a/code/ryzom/client/src/seven_zip/7z.h +++ /dev/null @@ -1,202 +0,0 @@ -/* 7z.h -- 7z interface -2017-04-03 : Igor Pavlov : Public domain */ - -#ifndef __7Z_H -#define __7Z_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -#define k7zStartHeaderSize 0x20 -#define k7zSignatureSize 6 - -extern const Byte k7zSignature[k7zSignatureSize]; - -typedef struct -{ - const Byte *Data; - size_t Size; -} CSzData; - -/* CSzCoderInfo & CSzFolder support only default methods */ - -typedef struct -{ - size_t PropsOffset; - UInt32 MethodID; - Byte NumStreams; - Byte PropsSize; -} CSzCoderInfo; - -typedef struct -{ - UInt32 InIndex; - UInt32 OutIndex; -} CSzBond; - -#define SZ_NUM_CODERS_IN_FOLDER_MAX 4 -#define SZ_NUM_BONDS_IN_FOLDER_MAX 3 -#define SZ_NUM_PACK_STREAMS_IN_FOLDER_MAX 4 - -typedef struct -{ - UInt32 NumCoders; - UInt32 NumBonds; - UInt32 NumPackStreams; - UInt32 UnpackStream; - UInt32 PackStreams[SZ_NUM_PACK_STREAMS_IN_FOLDER_MAX]; - CSzBond Bonds[SZ_NUM_BONDS_IN_FOLDER_MAX]; - CSzCoderInfo Coders[SZ_NUM_CODERS_IN_FOLDER_MAX]; -} CSzFolder; - - -SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd); - -typedef struct -{ - UInt32 Low; - UInt32 High; -} CNtfsFileTime; - -typedef struct -{ - Byte *Defs; /* MSB 0 bit numbering */ - UInt32 *Vals; -} CSzBitUi32s; - -typedef struct -{ - Byte *Defs; /* MSB 0 bit numbering */ - // UInt64 *Vals; - CNtfsFileTime *Vals; -} CSzBitUi64s; - -#define SzBitArray_Check(p, i) (((p)[(i) >> 3] & (0x80 >> ((i) & 7))) != 0) - -#define SzBitWithVals_Check(p, i) ((p)->Defs && ((p)->Defs[(i) >> 3] & (0x80 >> ((i) & 7))) != 0) - -typedef struct -{ - UInt32 NumPackStreams; - UInt32 NumFolders; - - UInt64 *PackPositions; // NumPackStreams + 1 - CSzBitUi32s FolderCRCs; // NumFolders - - size_t *FoCodersOffsets; // NumFolders + 1 - UInt32 *FoStartPackStreamIndex; // NumFolders + 1 - UInt32 *FoToCoderUnpackSizes; // NumFolders + 1 - Byte *FoToMainUnpackSizeIndex; // NumFolders - UInt64 *CoderUnpackSizes; // for all coders in all folders - - Byte *CodersData; -} CSzAr; - -UInt64 SzAr_GetFolderUnpackSize(const CSzAr *p, UInt32 folderIndex); - -SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, - ILookInStream *stream, UInt64 startPos, - Byte *outBuffer, size_t outSize, - ISzAllocPtr allocMain); - -typedef struct -{ - CSzAr db; - - UInt64 startPosAfterHeader; - UInt64 dataPos; - - UInt32 NumFiles; - - UInt64 *UnpackPositions; // NumFiles + 1 - // Byte *IsEmptyFiles; - Byte *IsDirs; - CSzBitUi32s CRCs; - - CSzBitUi32s Attribs; - // CSzBitUi32s Parents; - CSzBitUi64s MTime; - CSzBitUi64s CTime; - - UInt32 *FolderToFile; // NumFolders + 1 - UInt32 *FileToFolder; // NumFiles - - size_t *FileNameOffsets; /* in 2-byte steps */ - Byte *FileNames; /* UTF-16-LE */ -} CSzArEx; - -#define SzArEx_IsDir(p, i) (SzBitArray_Check((p)->IsDirs, i)) - -#define SzArEx_GetFileSize(p, i) ((p)->UnpackPositions[(i) + 1] - (p)->UnpackPositions[i]) - -void SzArEx_Init(CSzArEx *p); -void SzArEx_Free(CSzArEx *p, ISzAllocPtr alloc); -UInt64 SzArEx_GetFolderStreamPos(const CSzArEx *p, UInt32 folderIndex, UInt32 indexInFolder); -int SzArEx_GetFolderFullPackSize(const CSzArEx *p, UInt32 folderIndex, UInt64 *resSize); - -/* -if dest == NULL, the return value specifies the required size of the buffer, - in 16-bit characters, including the null-terminating character. -if dest != NULL, the return value specifies the number of 16-bit characters that - are written to the dest, including the null-terminating character. */ - -size_t SzArEx_GetFileNameUtf16(const CSzArEx *p, size_t fileIndex, UInt16 *dest); - -/* -size_t SzArEx_GetFullNameLen(const CSzArEx *p, size_t fileIndex); -UInt16 *SzArEx_GetFullNameUtf16_Back(const CSzArEx *p, size_t fileIndex, UInt16 *dest); -*/ - - - -/* - SzArEx_Extract extracts file from archive - - *outBuffer must be 0 before first call for each new archive. - - Extracting cache: - If you need to decompress more than one file, you can send - these values from previous call: - *blockIndex, - *outBuffer, - *outBufferSize - You can consider "*outBuffer" as cache of solid block. If your archive is solid, - it will increase decompression speed. - - If you use external function, you can declare these 3 cache variables - (blockIndex, outBuffer, outBufferSize) as static in that external function. - - Free *outBuffer and set *outBuffer to 0, if you want to flush cache. -*/ - -SRes SzArEx_Extract( - const CSzArEx *db, - ILookInStream *inStream, - UInt32 fileIndex, /* index of file */ - UInt32 *blockIndex, /* index of solid block */ - Byte **outBuffer, /* pointer to pointer to output buffer (allocated with allocMain) */ - size_t *outBufferSize, /* buffer size for output buffer */ - size_t *offset, /* offset of stream for required file in *outBuffer */ - size_t *outSizeProcessed, /* size of file in *outBuffer */ - ISzAllocPtr allocMain, - ISzAllocPtr allocTemp); - - -/* -SzArEx_Open Errors: -SZ_ERROR_NO_ARCHIVE -SZ_ERROR_ARCHIVE -SZ_ERROR_UNSUPPORTED -SZ_ERROR_MEM -SZ_ERROR_CRC -SZ_ERROR_INPUT_EOF -SZ_ERROR_FAIL -*/ - -SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, - ISzAllocPtr allocMain, ISzAllocPtr allocTemp); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/7zArcIn.cpp b/code/ryzom/client/src/seven_zip/7zArcIn.cpp deleted file mode 100644 index f74d0fad5..000000000 --- a/code/ryzom/client/src/seven_zip/7zArcIn.cpp +++ /dev/null @@ -1,1771 +0,0 @@ -/* 7zArcIn.c -- 7z Input functions -2018-12-31 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include - -#include "7z.h" -#include "7zBuf.h" -#include "7zCrc.h" -#include "CpuArch.h" - -#define MY_ALLOC(T, p, size, alloc) { \ - if ((p = (T *)ISzAlloc_Alloc(alloc, (size) * sizeof(T))) == NULL) return SZ_ERROR_MEM; } - -#define MY_ALLOC_ZE(T, p, size, alloc) { if ((size) == 0) p = NULL; else MY_ALLOC(T, p, size, alloc) } - -#define MY_ALLOC_AND_CPY(to, size, from, alloc) \ - { MY_ALLOC(Byte, to, size, alloc); memcpy(to, from, size); } - -#define MY_ALLOC_ZE_AND_CPY(to, size, from, alloc) \ - { if ((size) == 0) to = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } } - -#define k7zMajorVersion 0 - -enum EIdEnum -{ - k7zIdEnd, - k7zIdHeader, - k7zIdArchiveProperties, - k7zIdAdditionalStreamsInfo, - k7zIdMainStreamsInfo, - k7zIdFilesInfo, - k7zIdPackInfo, - k7zIdUnpackInfo, - k7zIdSubStreamsInfo, - k7zIdSize, - k7zIdCRC, - k7zIdFolder, - k7zIdCodersUnpackSize, - k7zIdNumUnpackStream, - k7zIdEmptyStream, - k7zIdEmptyFile, - k7zIdAnti, - k7zIdName, - k7zIdCTime, - k7zIdATime, - k7zIdMTime, - k7zIdWinAttrib, - k7zIdComment, - k7zIdEncodedHeader, - k7zIdStartPos, - k7zIdDummy - // k7zNtSecure, - // k7zParent, - // k7zIsReal -}; - -const Byte k7zSignature[k7zSignatureSize] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C}; - -#define SzBitUi32s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } - -static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAllocPtr alloc) -{ - if (num == 0) - { - p->Defs = NULL; - p->Vals = NULL; - } - else - { - MY_ALLOC(Byte, p->Defs, (num + 7) >> 3, alloc); - MY_ALLOC(UInt32, p->Vals, num, alloc); - } - return SZ_OK; -} - -void SzBitUi32s_Free(CSzBitUi32s *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->Defs); p->Defs = NULL; - ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; -} - -#define SzBitUi64s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } - -void SzBitUi64s_Free(CSzBitUi64s *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->Defs); p->Defs = NULL; - ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; -} - - -static void SzAr_Init(CSzAr *p) -{ - p->NumPackStreams = 0; - p->NumFolders = 0; - - p->PackPositions = NULL; - SzBitUi32s_Init(&p->FolderCRCs); - - p->FoCodersOffsets = NULL; - p->FoStartPackStreamIndex = NULL; - p->FoToCoderUnpackSizes = NULL; - p->FoToMainUnpackSizeIndex = NULL; - p->CoderUnpackSizes = NULL; - - p->CodersData = NULL; -} - -static void SzAr_Free(CSzAr *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->PackPositions); - SzBitUi32s_Free(&p->FolderCRCs, alloc); - - ISzAlloc_Free(alloc, p->FoCodersOffsets); - ISzAlloc_Free(alloc, p->FoStartPackStreamIndex); - ISzAlloc_Free(alloc, p->FoToCoderUnpackSizes); - ISzAlloc_Free(alloc, p->FoToMainUnpackSizeIndex); - ISzAlloc_Free(alloc, p->CoderUnpackSizes); - - ISzAlloc_Free(alloc, p->CodersData); - - SzAr_Init(p); -} - - -void SzArEx_Init(CSzArEx *p) -{ - SzAr_Init(&p->db); - - p->NumFiles = 0; - p->dataPos = 0; - - p->UnpackPositions = NULL; - p->IsDirs = NULL; - - p->FolderToFile = NULL; - p->FileToFolder = NULL; - - p->FileNameOffsets = NULL; - p->FileNames = NULL; - - SzBitUi32s_Init(&p->CRCs); - SzBitUi32s_Init(&p->Attribs); - // SzBitUi32s_Init(&p->Parents); - SzBitUi64s_Init(&p->MTime); - SzBitUi64s_Init(&p->CTime); -} - -void SzArEx_Free(CSzArEx *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->UnpackPositions); - ISzAlloc_Free(alloc, p->IsDirs); - - ISzAlloc_Free(alloc, p->FolderToFile); - ISzAlloc_Free(alloc, p->FileToFolder); - - ISzAlloc_Free(alloc, p->FileNameOffsets); - ISzAlloc_Free(alloc, p->FileNames); - - SzBitUi32s_Free(&p->CRCs, alloc); - SzBitUi32s_Free(&p->Attribs, alloc); - // SzBitUi32s_Free(&p->Parents, alloc); - SzBitUi64s_Free(&p->MTime, alloc); - SzBitUi64s_Free(&p->CTime, alloc); - - SzAr_Free(&p->db, alloc); - SzArEx_Init(p); -} - - -static int TestSignatureCandidate(const Byte *testBytes) -{ - unsigned i; - for (i = 0; i < k7zSignatureSize; i++) - if (testBytes[i] != k7zSignature[i]) - return 0; - return 1; -} - -#define SzData_Clear(p) { (p)->Data = NULL; (p)->Size = 0; } - -#define SZ_READ_BYTE_SD(_sd_, dest) if ((_sd_)->Size == 0) return SZ_ERROR_ARCHIVE; (_sd_)->Size--; dest = *(_sd_)->Data++; -#define SZ_READ_BYTE(dest) SZ_READ_BYTE_SD(sd, dest) -#define SZ_READ_BYTE_2(dest) if (sd.Size == 0) return SZ_ERROR_ARCHIVE; sd.Size--; dest = *sd.Data++; - -#define SKIP_DATA(sd, size) { sd->Size -= (size_t)(size); sd->Data += (size_t)(size); } -#define SKIP_DATA2(sd, size) { sd.Size -= (size_t)(size); sd.Data += (size_t)(size); } - -#define SZ_READ_32(dest) if (sd.Size < 4) return SZ_ERROR_ARCHIVE; \ - dest = GetUi32(sd.Data); SKIP_DATA2(sd, 4); - -static MY_NO_INLINE SRes ReadNumber(CSzData *sd, UInt64 *value) -{ - Byte firstByte, mask; - unsigned i; - UInt32 v; - - SZ_READ_BYTE(firstByte); - if ((firstByte & 0x80) == 0) - { - *value = firstByte; - return SZ_OK; - } - SZ_READ_BYTE(v); - if ((firstByte & 0x40) == 0) - { - *value = (((UInt32)firstByte & 0x3F) << 8) | v; - return SZ_OK; - } - SZ_READ_BYTE(mask); - *value = v | ((UInt32)mask << 8); - mask = 0x20; - for (i = 2; i < 8; i++) - { - Byte b; - if ((firstByte & mask) == 0) - { - UInt64 highPart = (unsigned)firstByte & (unsigned)(mask - 1); - *value |= (highPart << (8 * i)); - return SZ_OK; - } - SZ_READ_BYTE(b); - *value |= ((UInt64)b << (8 * i)); - mask >>= 1; - } - return SZ_OK; -} - - -static MY_NO_INLINE SRes SzReadNumber32(CSzData *sd, UInt32 *value) -{ - Byte firstByte; - UInt64 value64; - if (sd->Size == 0) - return SZ_ERROR_ARCHIVE; - firstByte = *sd->Data; - if ((firstByte & 0x80) == 0) - { - *value = firstByte; - sd->Data++; - sd->Size--; - return SZ_OK; - } - RINOK(ReadNumber(sd, &value64)); - if (value64 >= (UInt32)0x80000000 - 1) - return SZ_ERROR_UNSUPPORTED; - if (value64 >= ((UInt64)(1) << ((sizeof(size_t) - 1) * 8 + 4))) - return SZ_ERROR_UNSUPPORTED; - *value = (UInt32)value64; - return SZ_OK; -} - -#define ReadID(sd, value) ReadNumber(sd, value) - -static SRes SkipData(CSzData *sd) -{ - UInt64 size; - RINOK(ReadNumber(sd, &size)); - if (size > sd->Size) - return SZ_ERROR_ARCHIVE; - SKIP_DATA(sd, size); - return SZ_OK; -} - -static SRes WaitId(CSzData *sd, UInt32 id) -{ - for (;;) - { - UInt64 type; - RINOK(ReadID(sd, &type)); - if (type == id) - return SZ_OK; - if (type == k7zIdEnd) - return SZ_ERROR_ARCHIVE; - RINOK(SkipData(sd)); - } -} - -static SRes RememberBitVector(CSzData *sd, UInt32 numItems, const Byte **v) -{ - UInt32 numBytes = (numItems + 7) >> 3; - if (numBytes > sd->Size) - return SZ_ERROR_ARCHIVE; - *v = sd->Data; - SKIP_DATA(sd, numBytes); - return SZ_OK; -} - -static UInt32 CountDefinedBits(const Byte *bits, UInt32 numItems) -{ - Byte b = 0; - unsigned m = 0; - UInt32 sum = 0; - for (; numItems != 0; numItems--) - { - if (m == 0) - { - b = *bits++; - m = 8; - } - m--; - sum += ((b >> m) & 1); - } - return sum; -} - -static MY_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, ISzAllocPtr alloc) -{ - Byte allAreDefined; - Byte *v2; - UInt32 numBytes = (numItems + 7) >> 3; - *v = NULL; - SZ_READ_BYTE(allAreDefined); - if (numBytes == 0) - return SZ_OK; - if (allAreDefined == 0) - { - if (numBytes > sd->Size) - return SZ_ERROR_ARCHIVE; - MY_ALLOC_AND_CPY(*v, numBytes, sd->Data, alloc); - SKIP_DATA(sd, numBytes); - return SZ_OK; - } - MY_ALLOC(Byte, *v, numBytes, alloc); - v2 = *v; - memset(v2, 0xFF, (size_t)numBytes); - { - unsigned numBits = (unsigned)numItems & 7; - if (numBits != 0) - v2[(size_t)numBytes - 1] = (Byte)((((UInt32)1 << numBits) - 1) << (8 - numBits)); - } - return SZ_OK; -} - -static MY_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) -{ - UInt32 i; - CSzData sd; - UInt32 *vals; - const Byte *defs; - MY_ALLOC_ZE(UInt32, crcs->Vals, numItems, alloc); - sd = *sd2; - defs = crcs->Defs; - vals = crcs->Vals; - for (i = 0; i < numItems; i++) - if (SzBitArray_Check(defs, i)) - { - SZ_READ_32(vals[i]); - } - else - vals[i] = 0; - *sd2 = sd; - return SZ_OK; -} - -static SRes ReadBitUi32s(CSzData *sd, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) -{ - SzBitUi32s_Free(crcs, alloc); - RINOK(ReadBitVector(sd, numItems, &crcs->Defs, alloc)); - return ReadUi32s(sd, numItems, crcs, alloc); -} - -static SRes SkipBitUi32s(CSzData *sd, UInt32 numItems) -{ - Byte allAreDefined; - UInt32 numDefined = numItems; - SZ_READ_BYTE(allAreDefined); - if (!allAreDefined) - { - size_t numBytes = (numItems + 7) >> 3; - if (numBytes > sd->Size) - return SZ_ERROR_ARCHIVE; - numDefined = CountDefinedBits(sd->Data, numItems); - SKIP_DATA(sd, numBytes); - } - if (numDefined > (sd->Size >> 2)) - return SZ_ERROR_ARCHIVE; - SKIP_DATA(sd, (size_t)numDefined * 4); - return SZ_OK; -} - -static SRes ReadPackInfo(CSzAr *p, CSzData *sd, ISzAllocPtr alloc) -{ - RINOK(SzReadNumber32(sd, &p->NumPackStreams)); - - RINOK(WaitId(sd, k7zIdSize)); - MY_ALLOC(UInt64, p->PackPositions, (size_t)p->NumPackStreams + 1, alloc); - { - UInt64 sum = 0; - UInt32 i; - UInt32 numPackStreams = p->NumPackStreams; - for (i = 0; i < numPackStreams; i++) - { - UInt64 packSize; - p->PackPositions[i] = sum; - RINOK(ReadNumber(sd, &packSize)); - sum += packSize; - if (sum < packSize) - return SZ_ERROR_ARCHIVE; - } - p->PackPositions[i] = sum; - } - - for (;;) - { - UInt64 type; - RINOK(ReadID(sd, &type)); - if (type == k7zIdEnd) - return SZ_OK; - if (type == k7zIdCRC) - { - /* CRC of packed streams is unused now */ - RINOK(SkipBitUi32s(sd, p->NumPackStreams)); - continue; - } - RINOK(SkipData(sd)); - } -} - -/* -static SRes SzReadSwitch(CSzData *sd) -{ - Byte external; - RINOK(SzReadByte(sd, &external)); - return (external == 0) ? SZ_OK: SZ_ERROR_UNSUPPORTED; -} -*/ - -#define k_NumCodersStreams_in_Folder_MAX (SZ_NUM_BONDS_IN_FOLDER_MAX + SZ_NUM_PACK_STREAMS_IN_FOLDER_MAX) - -SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd) -{ - UInt32 numCoders, i; - UInt32 numInStreams = 0; - const Byte *dataStart = sd->Data; - - f->NumCoders = 0; - f->NumBonds = 0; - f->NumPackStreams = 0; - f->UnpackStream = 0; - - RINOK(SzReadNumber32(sd, &numCoders)); - if (numCoders == 0 || numCoders > SZ_NUM_CODERS_IN_FOLDER_MAX) - return SZ_ERROR_UNSUPPORTED; - - for (i = 0; i < numCoders; i++) - { - Byte mainByte; - CSzCoderInfo *coder = f->Coders + i; - unsigned idSize, j; - UInt64 id; - - SZ_READ_BYTE(mainByte); - if ((mainByte & 0xC0) != 0) - return SZ_ERROR_UNSUPPORTED; - - idSize = (unsigned)(mainByte & 0xF); - if (idSize > sizeof(id)) - return SZ_ERROR_UNSUPPORTED; - if (idSize > sd->Size) - return SZ_ERROR_ARCHIVE; - id = 0; - for (j = 0; j < idSize; j++) - { - id = ((id << 8) | *sd->Data); - sd->Data++; - sd->Size--; - } - if (id > (UInt32)0xFFFFFFFF) - return SZ_ERROR_UNSUPPORTED; - coder->MethodID = (UInt32)id; - - coder->NumStreams = 1; - coder->PropsOffset = 0; - coder->PropsSize = 0; - - if ((mainByte & 0x10) != 0) - { - UInt32 numStreams; - - RINOK(SzReadNumber32(sd, &numStreams)); - if (numStreams > k_NumCodersStreams_in_Folder_MAX) - return SZ_ERROR_UNSUPPORTED; - coder->NumStreams = (Byte)numStreams; - - RINOK(SzReadNumber32(sd, &numStreams)); - if (numStreams != 1) - return SZ_ERROR_UNSUPPORTED; - } - - numInStreams += coder->NumStreams; - - if (numInStreams > k_NumCodersStreams_in_Folder_MAX) - return SZ_ERROR_UNSUPPORTED; - - if ((mainByte & 0x20) != 0) - { - UInt32 propsSize = 0; - RINOK(SzReadNumber32(sd, &propsSize)); - if (propsSize > sd->Size) - return SZ_ERROR_ARCHIVE; - if (propsSize >= 0x80) - return SZ_ERROR_UNSUPPORTED; - coder->PropsOffset = sd->Data - dataStart; - coder->PropsSize = (Byte)propsSize; - sd->Data += (size_t)propsSize; - sd->Size -= (size_t)propsSize; - } - } - - /* - if (numInStreams == 1 && numCoders == 1) - { - f->NumPackStreams = 1; - f->PackStreams[0] = 0; - } - else - */ - { - Byte streamUsed[k_NumCodersStreams_in_Folder_MAX]; - UInt32 numBonds, numPackStreams; - - numBonds = numCoders - 1; - if (numInStreams < numBonds) - return SZ_ERROR_ARCHIVE; - if (numBonds > SZ_NUM_BONDS_IN_FOLDER_MAX) - return SZ_ERROR_UNSUPPORTED; - f->NumBonds = numBonds; - - numPackStreams = numInStreams - numBonds; - if (numPackStreams > SZ_NUM_PACK_STREAMS_IN_FOLDER_MAX) - return SZ_ERROR_UNSUPPORTED; - f->NumPackStreams = numPackStreams; - - for (i = 0; i < numInStreams; i++) - streamUsed[i] = False; - - if (numBonds != 0) - { - Byte coderUsed[SZ_NUM_CODERS_IN_FOLDER_MAX]; - - for (i = 0; i < numCoders; i++) - coderUsed[i] = False; - - for (i = 0; i < numBonds; i++) - { - CSzBond *bp = f->Bonds + i; - - RINOK(SzReadNumber32(sd, &bp->InIndex)); - if (bp->InIndex >= numInStreams || streamUsed[bp->InIndex]) - return SZ_ERROR_ARCHIVE; - streamUsed[bp->InIndex] = True; - - RINOK(SzReadNumber32(sd, &bp->OutIndex)); - if (bp->OutIndex >= numCoders || coderUsed[bp->OutIndex]) - return SZ_ERROR_ARCHIVE; - coderUsed[bp->OutIndex] = True; - } - - for (i = 0; i < numCoders; i++) - if (!coderUsed[i]) - { - f->UnpackStream = i; - break; - } - - if (i == numCoders) - return SZ_ERROR_ARCHIVE; - } - - if (numPackStreams == 1) - { - for (i = 0; i < numInStreams; i++) - if (!streamUsed[i]) - break; - if (i == numInStreams) - return SZ_ERROR_ARCHIVE; - f->PackStreams[0] = i; - } - else - for (i = 0; i < numPackStreams; i++) - { - UInt32 index; - RINOK(SzReadNumber32(sd, &index)); - if (index >= numInStreams || streamUsed[index]) - return SZ_ERROR_ARCHIVE; - streamUsed[index] = True; - f->PackStreams[i] = index; - } - } - - f->NumCoders = numCoders; - - return SZ_OK; -} - - -static MY_NO_INLINE SRes SkipNumbers(CSzData *sd2, UInt32 num) -{ - CSzData sd; - sd = *sd2; - for (; num != 0; num--) - { - Byte firstByte, mask; - unsigned i; - SZ_READ_BYTE_2(firstByte); - if ((firstByte & 0x80) == 0) - continue; - if ((firstByte & 0x40) == 0) - { - if (sd.Size == 0) - return SZ_ERROR_ARCHIVE; - sd.Size--; - sd.Data++; - continue; - } - mask = 0x20; - for (i = 2; i < 8 && (firstByte & mask) != 0; i++) - mask >>= 1; - if (i > sd.Size) - return SZ_ERROR_ARCHIVE; - SKIP_DATA2(sd, i); - } - *sd2 = sd; - return SZ_OK; -} - - -#define k_Scan_NumCoders_MAX 64 -#define k_Scan_NumCodersStreams_in_Folder_MAX 64 - - -static SRes ReadUnpackInfo(CSzAr *p, - CSzData *sd2, - UInt32 numFoldersMax, - const CBuf *tempBufs, UInt32 numTempBufs, - ISzAllocPtr alloc) -{ - CSzData sd; - - UInt32 fo, numFolders, numCodersOutStreams, packStreamIndex; - const Byte *startBufPtr; - Byte external; - - RINOK(WaitId(sd2, k7zIdFolder)); - - RINOK(SzReadNumber32(sd2, &numFolders)); - if (numFolders > numFoldersMax) - return SZ_ERROR_UNSUPPORTED; - p->NumFolders = numFolders; - - SZ_READ_BYTE_SD(sd2, external); - if (external == 0) - sd = *sd2; - else - { - UInt32 index; - RINOK(SzReadNumber32(sd2, &index)); - if (index >= numTempBufs) - return SZ_ERROR_ARCHIVE; - sd.Data = tempBufs[index].data; - sd.Size = tempBufs[index].size; - } - - MY_ALLOC(size_t, p->FoCodersOffsets, (size_t)numFolders + 1, alloc); - MY_ALLOC(UInt32, p->FoStartPackStreamIndex, (size_t)numFolders + 1, alloc); - MY_ALLOC(UInt32, p->FoToCoderUnpackSizes, (size_t)numFolders + 1, alloc); - MY_ALLOC_ZE(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); - - startBufPtr = sd.Data; - - packStreamIndex = 0; - numCodersOutStreams = 0; - - for (fo = 0; fo < numFolders; fo++) - { - UInt32 numCoders, ci, numInStreams = 0; - - p->FoCodersOffsets[fo] = sd.Data - startBufPtr; - - RINOK(SzReadNumber32(&sd, &numCoders)); - if (numCoders == 0 || numCoders > k_Scan_NumCoders_MAX) - return SZ_ERROR_UNSUPPORTED; - - for (ci = 0; ci < numCoders; ci++) - { - Byte mainByte; - unsigned idSize; - UInt32 coderInStreams; - - SZ_READ_BYTE_2(mainByte); - if ((mainByte & 0xC0) != 0) - return SZ_ERROR_UNSUPPORTED; - idSize = (mainByte & 0xF); - if (idSize > 8) - return SZ_ERROR_UNSUPPORTED; - if (idSize > sd.Size) - return SZ_ERROR_ARCHIVE; - SKIP_DATA2(sd, idSize); - - coderInStreams = 1; - - if ((mainByte & 0x10) != 0) - { - UInt32 coderOutStreams; - RINOK(SzReadNumber32(&sd, &coderInStreams)); - RINOK(SzReadNumber32(&sd, &coderOutStreams)); - if (coderInStreams > k_Scan_NumCodersStreams_in_Folder_MAX || coderOutStreams != 1) - return SZ_ERROR_UNSUPPORTED; - } - - numInStreams += coderInStreams; - - if ((mainByte & 0x20) != 0) - { - UInt32 propsSize; - RINOK(SzReadNumber32(&sd, &propsSize)); - if (propsSize > sd.Size) - return SZ_ERROR_ARCHIVE; - SKIP_DATA2(sd, propsSize); - } - } - - { - UInt32 indexOfMainStream = 0; - UInt32 numPackStreams = 1; - - if (numCoders != 1 || numInStreams != 1) - { - Byte streamUsed[k_Scan_NumCodersStreams_in_Folder_MAX]; - Byte coderUsed[k_Scan_NumCoders_MAX]; - - UInt32 i; - UInt32 numBonds = numCoders - 1; - if (numInStreams < numBonds) - return SZ_ERROR_ARCHIVE; - - if (numInStreams > k_Scan_NumCodersStreams_in_Folder_MAX) - return SZ_ERROR_UNSUPPORTED; - - for (i = 0; i < numInStreams; i++) - streamUsed[i] = False; - for (i = 0; i < numCoders; i++) - coderUsed[i] = False; - - for (i = 0; i < numBonds; i++) - { - UInt32 index; - - RINOK(SzReadNumber32(&sd, &index)); - if (index >= numInStreams || streamUsed[index]) - return SZ_ERROR_ARCHIVE; - streamUsed[index] = True; - - RINOK(SzReadNumber32(&sd, &index)); - if (index >= numCoders || coderUsed[index]) - return SZ_ERROR_ARCHIVE; - coderUsed[index] = True; - } - - numPackStreams = numInStreams - numBonds; - - if (numPackStreams != 1) - for (i = 0; i < numPackStreams; i++) - { - UInt32 index; - RINOK(SzReadNumber32(&sd, &index)); - if (index >= numInStreams || streamUsed[index]) - return SZ_ERROR_ARCHIVE; - streamUsed[index] = True; - } - - for (i = 0; i < numCoders; i++) - if (!coderUsed[i]) - { - indexOfMainStream = i; - break; - } - - if (i == numCoders) - return SZ_ERROR_ARCHIVE; - } - - p->FoStartPackStreamIndex[fo] = packStreamIndex; - p->FoToCoderUnpackSizes[fo] = numCodersOutStreams; - p->FoToMainUnpackSizeIndex[fo] = (Byte)indexOfMainStream; - numCodersOutStreams += numCoders; - if (numCodersOutStreams < numCoders) - return SZ_ERROR_UNSUPPORTED; - if (numPackStreams > p->NumPackStreams - packStreamIndex) - return SZ_ERROR_ARCHIVE; - packStreamIndex += numPackStreams; - } - } - - p->FoToCoderUnpackSizes[fo] = numCodersOutStreams; - - { - size_t dataSize = sd.Data - startBufPtr; - p->FoStartPackStreamIndex[fo] = packStreamIndex; - p->FoCodersOffsets[fo] = dataSize; - MY_ALLOC_ZE_AND_CPY(p->CodersData, dataSize, startBufPtr, alloc); - } - - if (external != 0) - { - if (sd.Size != 0) - return SZ_ERROR_ARCHIVE; - sd = *sd2; - } - - RINOK(WaitId(&sd, k7zIdCodersUnpackSize)); - - MY_ALLOC_ZE(UInt64, p->CoderUnpackSizes, (size_t)numCodersOutStreams, alloc); - { - UInt32 i; - for (i = 0; i < numCodersOutStreams; i++) - { - RINOK(ReadNumber(&sd, p->CoderUnpackSizes + i)); - } - } - - for (;;) - { - UInt64 type; - RINOK(ReadID(&sd, &type)); - if (type == k7zIdEnd) - { - *sd2 = sd; - return SZ_OK; - } - if (type == k7zIdCRC) - { - RINOK(ReadBitUi32s(&sd, numFolders, &p->FolderCRCs, alloc)); - continue; - } - RINOK(SkipData(&sd)); - } -} - - -UInt64 SzAr_GetFolderUnpackSize(const CSzAr *p, UInt32 folderIndex) -{ - return p->CoderUnpackSizes[p->FoToCoderUnpackSizes[folderIndex] + p->FoToMainUnpackSizeIndex[folderIndex]]; -} - - -typedef struct -{ - UInt32 NumTotalSubStreams; - UInt32 NumSubDigests; - CSzData sdNumSubStreams; - CSzData sdSizes; - CSzData sdCRCs; -} CSubStreamInfo; - - -static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) -{ - UInt64 type = 0; - UInt32 numSubDigests = 0; - UInt32 numFolders = p->NumFolders; - UInt32 numUnpackStreams = numFolders; - UInt32 numUnpackSizesInData = 0; - - for (;;) - { - RINOK(ReadID(sd, &type)); - if (type == k7zIdNumUnpackStream) - { - UInt32 i; - ssi->sdNumSubStreams.Data = sd->Data; - numUnpackStreams = 0; - numSubDigests = 0; - for (i = 0; i < numFolders; i++) - { - UInt32 numStreams; - RINOK(SzReadNumber32(sd, &numStreams)); - if (numUnpackStreams > numUnpackStreams + numStreams) - return SZ_ERROR_UNSUPPORTED; - numUnpackStreams += numStreams; - if (numStreams != 0) - numUnpackSizesInData += (numStreams - 1); - if (numStreams != 1 || !SzBitWithVals_Check(&p->FolderCRCs, i)) - numSubDigests += numStreams; - } - ssi->sdNumSubStreams.Size = sd->Data - ssi->sdNumSubStreams.Data; - continue; - } - if (type == k7zIdCRC || type == k7zIdSize || type == k7zIdEnd) - break; - RINOK(SkipData(sd)); - } - - if (!ssi->sdNumSubStreams.Data) - { - numSubDigests = numFolders; - if (p->FolderCRCs.Defs) - numSubDigests = numFolders - CountDefinedBits(p->FolderCRCs.Defs, numFolders); - } - - ssi->NumTotalSubStreams = numUnpackStreams; - ssi->NumSubDigests = numSubDigests; - - if (type == k7zIdSize) - { - ssi->sdSizes.Data = sd->Data; - RINOK(SkipNumbers(sd, numUnpackSizesInData)); - ssi->sdSizes.Size = sd->Data - ssi->sdSizes.Data; - RINOK(ReadID(sd, &type)); - } - - for (;;) - { - if (type == k7zIdEnd) - return SZ_OK; - if (type == k7zIdCRC) - { - ssi->sdCRCs.Data = sd->Data; - RINOK(SkipBitUi32s(sd, numSubDigests)); - ssi->sdCRCs.Size = sd->Data - ssi->sdCRCs.Data; - } - else - { - RINOK(SkipData(sd)); - } - RINOK(ReadID(sd, &type)); - } -} - -static SRes SzReadStreamsInfo(CSzAr *p, - CSzData *sd, - UInt32 numFoldersMax, const CBuf *tempBufs, UInt32 numTempBufs, - UInt64 *dataOffset, - CSubStreamInfo *ssi, - ISzAllocPtr alloc) -{ - UInt64 type; - - SzData_Clear(&ssi->sdSizes); - SzData_Clear(&ssi->sdCRCs); - SzData_Clear(&ssi->sdNumSubStreams); - - *dataOffset = 0; - RINOK(ReadID(sd, &type)); - if (type == k7zIdPackInfo) - { - RINOK(ReadNumber(sd, dataOffset)); - RINOK(ReadPackInfo(p, sd, alloc)); - RINOK(ReadID(sd, &type)); - } - if (type == k7zIdUnpackInfo) - { - RINOK(ReadUnpackInfo(p, sd, numFoldersMax, tempBufs, numTempBufs, alloc)); - RINOK(ReadID(sd, &type)); - } - if (type == k7zIdSubStreamsInfo) - { - RINOK(ReadSubStreamsInfo(p, sd, ssi)); - RINOK(ReadID(sd, &type)); - } - else - { - ssi->NumTotalSubStreams = p->NumFolders; - // ssi->NumSubDigests = 0; - } - - return (type == k7zIdEnd ? SZ_OK : SZ_ERROR_UNSUPPORTED); -} - -static SRes SzReadAndDecodePackedStreams( - ILookInStream *inStream, - CSzData *sd, - CBuf *tempBufs, - UInt32 numFoldersMax, - UInt64 baseOffset, - CSzAr *p, - ISzAllocPtr allocTemp) -{ - UInt64 dataStartPos; - UInt32 fo; - CSubStreamInfo ssi; - - RINOK(SzReadStreamsInfo(p, sd, numFoldersMax, NULL, 0, &dataStartPos, &ssi, allocTemp)); - - dataStartPos += baseOffset; - if (p->NumFolders == 0) - return SZ_ERROR_ARCHIVE; - - for (fo = 0; fo < p->NumFolders; fo++) - Buf_Init(tempBufs + fo); - - for (fo = 0; fo < p->NumFolders; fo++) - { - CBuf *tempBuf = tempBufs + fo; - UInt64 unpackSize = SzAr_GetFolderUnpackSize(p, fo); - if ((size_t)unpackSize != unpackSize) - return SZ_ERROR_MEM; - if (!Buf_Create(tempBuf, (size_t)unpackSize, allocTemp)) - return SZ_ERROR_MEM; - } - - for (fo = 0; fo < p->NumFolders; fo++) - { - const CBuf *tempBuf = tempBufs + fo; - RINOK(LookInStream_SeekTo(inStream, dataStartPos)); - RINOK(SzAr_DecodeFolder(p, fo, inStream, dataStartPos, tempBuf->data, tempBuf->size, allocTemp)); - } - - return SZ_OK; -} - -static SRes SzReadFileNames(const Byte *data, size_t size, UInt32 numFiles, size_t *offsets) -{ - size_t pos = 0; - *offsets++ = 0; - if (numFiles == 0) - return (size == 0) ? SZ_OK : SZ_ERROR_ARCHIVE; - if (size < 2) - return SZ_ERROR_ARCHIVE; - if (data[size - 2] != 0 || data[size - 1] != 0) - return SZ_ERROR_ARCHIVE; - do - { - const Byte *p; - if (pos == size) - return SZ_ERROR_ARCHIVE; - for (p = data + pos; - #ifdef _WIN32 - *(const UInt16 *)p != 0 - #else - p[0] != 0 || p[1] != 0 - #endif - ; p += 2); - pos = p - data + 2; - *offsets++ = (pos >> 1); - } - while (--numFiles); - return (pos == size) ? SZ_OK : SZ_ERROR_ARCHIVE; -} - -static MY_NO_INLINE SRes ReadTime(CSzBitUi64s *p, UInt32 num, - CSzData *sd2, - const CBuf *tempBufs, UInt32 numTempBufs, - ISzAllocPtr alloc) -{ - CSzData sd; - UInt32 i; - CNtfsFileTime *vals; - Byte *defs; - Byte external; - - RINOK(ReadBitVector(sd2, num, &p->Defs, alloc)); - - SZ_READ_BYTE_SD(sd2, external); - if (external == 0) - sd = *sd2; - else - { - UInt32 index; - RINOK(SzReadNumber32(sd2, &index)); - if (index >= numTempBufs) - return SZ_ERROR_ARCHIVE; - sd.Data = tempBufs[index].data; - sd.Size = tempBufs[index].size; - } - - MY_ALLOC_ZE(CNtfsFileTime, p->Vals, num, alloc); - vals = p->Vals; - defs = p->Defs; - for (i = 0; i < num; i++) - if (SzBitArray_Check(defs, i)) - { - if (sd.Size < 8) - return SZ_ERROR_ARCHIVE; - vals[i].Low = GetUi32(sd.Data); - vals[i].High = GetUi32(sd.Data + 4); - SKIP_DATA2(sd, 8); - } - else - vals[i].High = vals[i].Low = 0; - - if (external == 0) - *sd2 = sd; - - return SZ_OK; -} - - -#define NUM_ADDITIONAL_STREAMS_MAX 8 - - -static SRes SzReadHeader2( - CSzArEx *p, /* allocMain */ - CSzData *sd, - ILookInStream *inStream, - CBuf *tempBufs, UInt32 *numTempBufs, - ISzAllocPtr allocMain, - ISzAllocPtr allocTemp - ) -{ - CSubStreamInfo ssi; - -{ - UInt64 type; - - SzData_Clear(&ssi.sdSizes); - SzData_Clear(&ssi.sdCRCs); - SzData_Clear(&ssi.sdNumSubStreams); - - ssi.NumSubDigests = 0; - ssi.NumTotalSubStreams = 0; - - RINOK(ReadID(sd, &type)); - - if (type == k7zIdArchiveProperties) - { - for (;;) - { - UInt64 type2; - RINOK(ReadID(sd, &type2)); - if (type2 == k7zIdEnd) - break; - RINOK(SkipData(sd)); - } - RINOK(ReadID(sd, &type)); - } - - if (type == k7zIdAdditionalStreamsInfo) - { - CSzAr tempAr; - SRes res; - - SzAr_Init(&tempAr); - res = SzReadAndDecodePackedStreams(inStream, sd, tempBufs, NUM_ADDITIONAL_STREAMS_MAX, - p->startPosAfterHeader, &tempAr, allocTemp); - *numTempBufs = tempAr.NumFolders; - SzAr_Free(&tempAr, allocTemp); - - if (res != SZ_OK) - return res; - RINOK(ReadID(sd, &type)); - } - - if (type == k7zIdMainStreamsInfo) - { - RINOK(SzReadStreamsInfo(&p->db, sd, (UInt32)1 << 30, tempBufs, *numTempBufs, - &p->dataPos, &ssi, allocMain)); - p->dataPos += p->startPosAfterHeader; - RINOK(ReadID(sd, &type)); - } - - if (type == k7zIdEnd) - { - return SZ_OK; - } - - if (type != k7zIdFilesInfo) - return SZ_ERROR_ARCHIVE; -} - -{ - UInt32 numFiles = 0; - UInt32 numEmptyStreams = 0; - const Byte *emptyStreams = NULL; - const Byte *emptyFiles = NULL; - - RINOK(SzReadNumber32(sd, &numFiles)); - p->NumFiles = numFiles; - - for (;;) - { - UInt64 type; - UInt64 size; - RINOK(ReadID(sd, &type)); - if (type == k7zIdEnd) - break; - RINOK(ReadNumber(sd, &size)); - if (size > sd->Size) - return SZ_ERROR_ARCHIVE; - - if (type >= ((UInt32)1 << 8)) - { - SKIP_DATA(sd, size); - } - else switch ((unsigned)type) - { - case k7zIdName: - { - size_t namesSize; - const Byte *namesData; - Byte external; - - SZ_READ_BYTE(external); - if (external == 0) - { - namesSize = (size_t)size - 1; - namesData = sd->Data; - } - else - { - UInt32 index; - RINOK(SzReadNumber32(sd, &index)); - if (index >= *numTempBufs) - return SZ_ERROR_ARCHIVE; - namesData = (tempBufs)[index].data; - namesSize = (tempBufs)[index].size; - } - - if ((namesSize & 1) != 0) - return SZ_ERROR_ARCHIVE; - MY_ALLOC(size_t, p->FileNameOffsets, numFiles + 1, allocMain); - MY_ALLOC_ZE_AND_CPY(p->FileNames, namesSize, namesData, allocMain); - RINOK(SzReadFileNames(p->FileNames, namesSize, numFiles, p->FileNameOffsets)) - if (external == 0) - { - SKIP_DATA(sd, namesSize); - } - break; - } - case k7zIdEmptyStream: - { - RINOK(RememberBitVector(sd, numFiles, &emptyStreams)); - numEmptyStreams = CountDefinedBits(emptyStreams, numFiles); - emptyFiles = NULL; - break; - } - case k7zIdEmptyFile: - { - RINOK(RememberBitVector(sd, numEmptyStreams, &emptyFiles)); - break; - } - case k7zIdWinAttrib: - { - Byte external; - CSzData sdSwitch; - CSzData *sdPtr; - SzBitUi32s_Free(&p->Attribs, allocMain); - RINOK(ReadBitVector(sd, numFiles, &p->Attribs.Defs, allocMain)); - - SZ_READ_BYTE(external); - if (external == 0) - sdPtr = sd; - else - { - UInt32 index; - RINOK(SzReadNumber32(sd, &index)); - if (index >= *numTempBufs) - return SZ_ERROR_ARCHIVE; - sdSwitch.Data = (tempBufs)[index].data; - sdSwitch.Size = (tempBufs)[index].size; - sdPtr = &sdSwitch; - } - RINOK(ReadUi32s(sdPtr, numFiles, &p->Attribs, allocMain)); - break; - } - /* - case k7zParent: - { - SzBitUi32s_Free(&p->Parents, allocMain); - RINOK(ReadBitVector(sd, numFiles, &p->Parents.Defs, allocMain)); - RINOK(SzReadSwitch(sd)); - RINOK(ReadUi32s(sd, numFiles, &p->Parents, allocMain)); - break; - } - */ - case k7zIdMTime: RINOK(ReadTime(&p->MTime, numFiles, sd, tempBufs, *numTempBufs, allocMain)); break; - case k7zIdCTime: RINOK(ReadTime(&p->CTime, numFiles, sd, tempBufs, *numTempBufs, allocMain)); break; - default: - { - SKIP_DATA(sd, size); - } - } - } - - if (numFiles - numEmptyStreams != ssi.NumTotalSubStreams) - return SZ_ERROR_ARCHIVE; - - for (;;) - { - UInt64 type; - RINOK(ReadID(sd, &type)); - if (type == k7zIdEnd) - break; - RINOK(SkipData(sd)); - } - - { - UInt32 i; - UInt32 emptyFileIndex = 0; - UInt32 folderIndex = 0; - UInt32 remSubStreams = 0; - UInt32 numSubStreams = 0; - UInt64 unpackPos = 0; - const Byte *digestsDefs = NULL; - const Byte *digestsVals = NULL; - UInt32 digestsValsIndex = 0; - UInt32 digestIndex; - Byte allDigestsDefined = 0; - Byte isDirMask = 0; - Byte crcMask = 0; - Byte mask = 0x80; - - MY_ALLOC(UInt32, p->FolderToFile, p->db.NumFolders + 1, allocMain); - MY_ALLOC_ZE(UInt32, p->FileToFolder, p->NumFiles, allocMain); - MY_ALLOC(UInt64, p->UnpackPositions, p->NumFiles + 1, allocMain); - MY_ALLOC_ZE(Byte, p->IsDirs, (p->NumFiles + 7) >> 3, allocMain); - - RINOK(SzBitUi32s_Alloc(&p->CRCs, p->NumFiles, allocMain)); - - if (ssi.sdCRCs.Size != 0) - { - SZ_READ_BYTE_SD(&ssi.sdCRCs, allDigestsDefined); - if (allDigestsDefined) - digestsVals = ssi.sdCRCs.Data; - else - { - size_t numBytes = (ssi.NumSubDigests + 7) >> 3; - digestsDefs = ssi.sdCRCs.Data; - digestsVals = digestsDefs + numBytes; - } - } - - digestIndex = 0; - - for (i = 0; i < numFiles; i++, mask >>= 1) - { - if (mask == 0) - { - UInt32 byteIndex = (i - 1) >> 3; - p->IsDirs[byteIndex] = isDirMask; - p->CRCs.Defs[byteIndex] = crcMask; - isDirMask = 0; - crcMask = 0; - mask = 0x80; - } - - p->UnpackPositions[i] = unpackPos; - p->CRCs.Vals[i] = 0; - - if (emptyStreams && SzBitArray_Check(emptyStreams, i)) - { - if (emptyFiles) - { - if (!SzBitArray_Check(emptyFiles, emptyFileIndex)) - isDirMask |= mask; - emptyFileIndex++; - } - else - isDirMask |= mask; - if (remSubStreams == 0) - { - p->FileToFolder[i] = (UInt32)-1; - continue; - } - } - - if (remSubStreams == 0) - { - for (;;) - { - if (folderIndex >= p->db.NumFolders) - return SZ_ERROR_ARCHIVE; - p->FolderToFile[folderIndex] = i; - numSubStreams = 1; - if (ssi.sdNumSubStreams.Data) - { - RINOK(SzReadNumber32(&ssi.sdNumSubStreams, &numSubStreams)); - } - remSubStreams = numSubStreams; - if (numSubStreams != 0) - break; - { - UInt64 folderUnpackSize = SzAr_GetFolderUnpackSize(&p->db, folderIndex); - unpackPos += folderUnpackSize; - if (unpackPos < folderUnpackSize) - return SZ_ERROR_ARCHIVE; - } - - folderIndex++; - } - } - - p->FileToFolder[i] = folderIndex; - - if (emptyStreams && SzBitArray_Check(emptyStreams, i)) - continue; - - if (--remSubStreams == 0) - { - UInt64 folderUnpackSize = SzAr_GetFolderUnpackSize(&p->db, folderIndex); - UInt64 startFolderUnpackPos = p->UnpackPositions[p->FolderToFile[folderIndex]]; - if (folderUnpackSize < unpackPos - startFolderUnpackPos) - return SZ_ERROR_ARCHIVE; - unpackPos = startFolderUnpackPos + folderUnpackSize; - if (unpackPos < folderUnpackSize) - return SZ_ERROR_ARCHIVE; - - if (numSubStreams == 1 && SzBitWithVals_Check(&p->db.FolderCRCs, i)) - { - p->CRCs.Vals[i] = p->db.FolderCRCs.Vals[folderIndex]; - crcMask |= mask; - } - else if (allDigestsDefined || (digestsDefs && SzBitArray_Check(digestsDefs, digestIndex))) - { - p->CRCs.Vals[i] = GetUi32(digestsVals + (size_t)digestsValsIndex * 4); - digestsValsIndex++; - crcMask |= mask; - } - - folderIndex++; - } - else - { - UInt64 v; - RINOK(ReadNumber(&ssi.sdSizes, &v)); - unpackPos += v; - if (unpackPos < v) - return SZ_ERROR_ARCHIVE; - if (allDigestsDefined || (digestsDefs && SzBitArray_Check(digestsDefs, digestIndex))) - { - p->CRCs.Vals[i] = GetUi32(digestsVals + (size_t)digestsValsIndex * 4); - digestsValsIndex++; - crcMask |= mask; - } - } - } - - if (mask != 0x80) - { - UInt32 byteIndex = (i - 1) >> 3; - p->IsDirs[byteIndex] = isDirMask; - p->CRCs.Defs[byteIndex] = crcMask; - } - - p->UnpackPositions[i] = unpackPos; - - if (remSubStreams != 0) - return SZ_ERROR_ARCHIVE; - - for (;;) - { - p->FolderToFile[folderIndex] = i; - if (folderIndex >= p->db.NumFolders) - break; - if (!ssi.sdNumSubStreams.Data) - return SZ_ERROR_ARCHIVE; - RINOK(SzReadNumber32(&ssi.sdNumSubStreams, &numSubStreams)); - if (numSubStreams != 0) - return SZ_ERROR_ARCHIVE; - /* - { - UInt64 folderUnpackSize = SzAr_GetFolderUnpackSize(&p->db, folderIndex); - unpackPos += folderUnpackSize; - if (unpackPos < folderUnpackSize) - return SZ_ERROR_ARCHIVE; - } - */ - folderIndex++; - } - - if (ssi.sdNumSubStreams.Data && ssi.sdNumSubStreams.Size != 0) - return SZ_ERROR_ARCHIVE; - } -} - return SZ_OK; -} - - -static SRes SzReadHeader( - CSzArEx *p, - CSzData *sd, - ILookInStream *inStream, - ISzAllocPtr allocMain, - ISzAllocPtr allocTemp) -{ - UInt32 i; - UInt32 numTempBufs = 0; - SRes res; - CBuf tempBufs[NUM_ADDITIONAL_STREAMS_MAX]; - - for (i = 0; i < NUM_ADDITIONAL_STREAMS_MAX; i++) - Buf_Init(tempBufs + i); - - res = SzReadHeader2(p, sd, inStream, - tempBufs, &numTempBufs, - allocMain, allocTemp); - - for (i = 0; i < NUM_ADDITIONAL_STREAMS_MAX; i++) - Buf_Free(tempBufs + i, allocTemp); - - RINOK(res); - - if (sd->Size != 0) - return SZ_ERROR_FAIL; - - return res; -} - -static SRes SzArEx_Open2( - CSzArEx *p, - ILookInStream *inStream, - ISzAllocPtr allocMain, - ISzAllocPtr allocTemp) -{ - Byte header[k7zStartHeaderSize]; - Int64 startArcPos; - UInt64 nextHeaderOffset, nextHeaderSize; - size_t nextHeaderSizeT; - UInt32 nextHeaderCRC; - CBuf buf; - SRes res; - - startArcPos = 0; - RINOK(ILookInStream_Seek(inStream, &startArcPos, SZ_SEEK_CUR)); - - RINOK(LookInStream_Read2(inStream, header, k7zStartHeaderSize, SZ_ERROR_NO_ARCHIVE)); - - if (!TestSignatureCandidate(header)) - return SZ_ERROR_NO_ARCHIVE; - if (header[6] != k7zMajorVersion) - return SZ_ERROR_UNSUPPORTED; - - nextHeaderOffset = GetUi64(header + 12); - nextHeaderSize = GetUi64(header + 20); - nextHeaderCRC = GetUi32(header + 28); - - p->startPosAfterHeader = startArcPos + k7zStartHeaderSize; - - if (CrcCalc(header + 12, 20) != GetUi32(header + 8)) - return SZ_ERROR_CRC; - - nextHeaderSizeT = (size_t)nextHeaderSize; - if (nextHeaderSizeT != nextHeaderSize) - return SZ_ERROR_MEM; - if (nextHeaderSizeT == 0) - return SZ_OK; - if (nextHeaderOffset > nextHeaderOffset + nextHeaderSize || - nextHeaderOffset > nextHeaderOffset + nextHeaderSize + k7zStartHeaderSize) - return SZ_ERROR_NO_ARCHIVE; - - { - Int64 pos = 0; - RINOK(ILookInStream_Seek(inStream, &pos, SZ_SEEK_END)); - if ((UInt64)pos < startArcPos + nextHeaderOffset || - (UInt64)pos < startArcPos + k7zStartHeaderSize + nextHeaderOffset || - (UInt64)pos < startArcPos + k7zStartHeaderSize + nextHeaderOffset + nextHeaderSize) - return SZ_ERROR_INPUT_EOF; - } - - RINOK(LookInStream_SeekTo(inStream, startArcPos + k7zStartHeaderSize + nextHeaderOffset)); - - if (!Buf_Create(&buf, nextHeaderSizeT, allocTemp)) - return SZ_ERROR_MEM; - - res = LookInStream_Read(inStream, buf.data, nextHeaderSizeT); - - if (res == SZ_OK) - { - res = SZ_ERROR_ARCHIVE; - if (CrcCalc(buf.data, nextHeaderSizeT) == nextHeaderCRC) - { - CSzData sd; - UInt64 type; - sd.Data = buf.data; - sd.Size = buf.size; - - res = ReadID(&sd, &type); - - if (res == SZ_OK && type == k7zIdEncodedHeader) - { - CSzAr tempAr; - CBuf tempBuf; - Buf_Init(&tempBuf); - - SzAr_Init(&tempAr); - res = SzReadAndDecodePackedStreams(inStream, &sd, &tempBuf, 1, p->startPosAfterHeader, &tempAr, allocTemp); - SzAr_Free(&tempAr, allocTemp); - - if (res != SZ_OK) - { - Buf_Free(&tempBuf, allocTemp); - } - else - { - Buf_Free(&buf, allocTemp); - buf.data = tempBuf.data; - buf.size = tempBuf.size; - sd.Data = buf.data; - sd.Size = buf.size; - res = ReadID(&sd, &type); - } - } - - if (res == SZ_OK) - { - if (type == k7zIdHeader) - { - /* - CSzData sd2; - unsigned ttt; - for (ttt = 0; ttt < 40000; ttt++) - { - SzArEx_Free(p, allocMain); - sd2 = sd; - res = SzReadHeader(p, &sd2, inStream, allocMain, allocTemp); - if (res != SZ_OK) - break; - } - */ - res = SzReadHeader(p, &sd, inStream, allocMain, allocTemp); - } - else - res = SZ_ERROR_UNSUPPORTED; - } - } - } - - Buf_Free(&buf, allocTemp); - return res; -} - - -SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, - ISzAllocPtr allocMain, ISzAllocPtr allocTemp) -{ - SRes res = SzArEx_Open2(p, inStream, allocMain, allocTemp); - if (res != SZ_OK) - SzArEx_Free(p, allocMain); - return res; -} - - -SRes SzArEx_Extract( - const CSzArEx *p, - ILookInStream *inStream, - UInt32 fileIndex, - UInt32 *blockIndex, - Byte **tempBuf, - size_t *outBufferSize, - size_t *offset, - size_t *outSizeProcessed, - ISzAllocPtr allocMain, - ISzAllocPtr allocTemp) -{ - UInt32 folderIndex = p->FileToFolder[fileIndex]; - SRes res = SZ_OK; - - *offset = 0; - *outSizeProcessed = 0; - - if (folderIndex == (UInt32)-1) - { - ISzAlloc_Free(allocMain, *tempBuf); - *blockIndex = folderIndex; - *tempBuf = NULL; - *outBufferSize = 0; - return SZ_OK; - } - - if (*tempBuf == NULL || *blockIndex != folderIndex) - { - UInt64 unpackSizeSpec = SzAr_GetFolderUnpackSize(&p->db, folderIndex); - /* - UInt64 unpackSizeSpec = - p->UnpackPositions[p->FolderToFile[(size_t)folderIndex + 1]] - - p->UnpackPositions[p->FolderToFile[folderIndex]]; - */ - size_t unpackSize = (size_t)unpackSizeSpec; - - if (unpackSize != unpackSizeSpec) - return SZ_ERROR_MEM; - *blockIndex = folderIndex; - ISzAlloc_Free(allocMain, *tempBuf); - *tempBuf = NULL; - - if (res == SZ_OK) - { - *outBufferSize = unpackSize; - if (unpackSize != 0) - { - *tempBuf = (Byte *)ISzAlloc_Alloc(allocMain, unpackSize); - if (*tempBuf == NULL) - res = SZ_ERROR_MEM; - } - - if (res == SZ_OK) - { - res = SzAr_DecodeFolder(&p->db, folderIndex, - inStream, p->dataPos, *tempBuf, unpackSize, allocTemp); - } - } - } - - if (res == SZ_OK) - { - UInt64 unpackPos = p->UnpackPositions[fileIndex]; - *offset = (size_t)(unpackPos - p->UnpackPositions[p->FolderToFile[folderIndex]]); - *outSizeProcessed = (size_t)(p->UnpackPositions[(size_t)fileIndex + 1] - unpackPos); - if (*offset + *outSizeProcessed > *outBufferSize) - return SZ_ERROR_FAIL; - if (SzBitWithVals_Check(&p->CRCs, fileIndex)) - if (CrcCalc(*tempBuf + *offset, *outSizeProcessed) != p->CRCs.Vals[fileIndex]) - res = SZ_ERROR_CRC; - } - - return res; -} - - -size_t SzArEx_GetFileNameUtf16(const CSzArEx *p, size_t fileIndex, UInt16 *dest) -{ - size_t offs = p->FileNameOffsets[fileIndex]; - size_t len = p->FileNameOffsets[fileIndex + 1] - offs; - if (dest != 0) - { - size_t i; - const Byte *src = p->FileNames + offs * 2; - for (i = 0; i < len; i++) - dest[i] = GetUi16(src + i * 2); - } - return len; -} - -/* -size_t SzArEx_GetFullNameLen(const CSzArEx *p, size_t fileIndex) -{ - size_t len; - if (!p->FileNameOffsets) - return 1; - len = 0; - for (;;) - { - UInt32 parent = (UInt32)(Int32)-1; - len += p->FileNameOffsets[fileIndex + 1] - p->FileNameOffsets[fileIndex]; - if SzBitWithVals_Check(&p->Parents, fileIndex) - parent = p->Parents.Vals[fileIndex]; - if (parent == (UInt32)(Int32)-1) - return len; - fileIndex = parent; - } -} - -UInt16 *SzArEx_GetFullNameUtf16_Back(const CSzArEx *p, size_t fileIndex, UInt16 *dest) -{ - BoolInt needSlash; - if (!p->FileNameOffsets) - { - *(--dest) = 0; - return dest; - } - needSlash = False; - for (;;) - { - UInt32 parent = (UInt32)(Int32)-1; - size_t curLen = p->FileNameOffsets[fileIndex + 1] - p->FileNameOffsets[fileIndex]; - SzArEx_GetFileNameUtf16(p, fileIndex, dest - curLen); - if (needSlash) - *(dest - 1) = '/'; - needSlash = True; - dest -= curLen; - - if SzBitWithVals_Check(&p->Parents, fileIndex) - parent = p->Parents.Vals[fileIndex]; - if (parent == (UInt32)(Int32)-1) - return dest; - fileIndex = parent; - } -} -*/ diff --git a/code/ryzom/client/src/seven_zip/7zBuf.cpp b/code/ryzom/client/src/seven_zip/7zBuf.cpp deleted file mode 100644 index 8865c32a8..000000000 --- a/code/ryzom/client/src/seven_zip/7zBuf.cpp +++ /dev/null @@ -1,36 +0,0 @@ -/* 7zBuf.c -- Byte Buffer -2017-04-03 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "7zBuf.h" - -void Buf_Init(CBuf *p) -{ - p->data = 0; - p->size = 0; -} - -int Buf_Create(CBuf *p, size_t size, ISzAllocPtr alloc) -{ - p->size = 0; - if (size == 0) - { - p->data = 0; - return 1; - } - p->data = (Byte *)ISzAlloc_Alloc(alloc, size); - if (p->data) - { - p->size = size; - return 1; - } - return 0; -} - -void Buf_Free(CBuf *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->data); - p->data = 0; - p->size = 0; -} diff --git a/code/ryzom/client/src/seven_zip/7zBuf.h b/code/ryzom/client/src/seven_zip/7zBuf.h deleted file mode 100644 index 81d1b5b64..000000000 --- a/code/ryzom/client/src/seven_zip/7zBuf.h +++ /dev/null @@ -1,35 +0,0 @@ -/* 7zBuf.h -- Byte Buffer -2017-04-03 : Igor Pavlov : Public domain */ - -#ifndef __7Z_BUF_H -#define __7Z_BUF_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -typedef struct -{ - Byte *data; - size_t size; -} CBuf; - -void Buf_Init(CBuf *p); -int Buf_Create(CBuf *p, size_t size, ISzAllocPtr alloc); -void Buf_Free(CBuf *p, ISzAllocPtr alloc); - -typedef struct -{ - Byte *data; - size_t size; - size_t pos; -} CDynBuf; - -void DynBuf_Construct(CDynBuf *p); -void DynBuf_SeekToBeg(CDynBuf *p); -int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAllocPtr alloc); -void DynBuf_Free(CDynBuf *p, ISzAllocPtr alloc); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/7zBuf2.cpp b/code/ryzom/client/src/seven_zip/7zBuf2.cpp deleted file mode 100644 index 208347416..000000000 --- a/code/ryzom/client/src/seven_zip/7zBuf2.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/* 7zBuf2.c -- Byte Buffer -2017-04-03 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include - -#include "7zBuf.h" - -void DynBuf_Construct(CDynBuf *p) -{ - p->data = 0; - p->size = 0; - p->pos = 0; -} - -void DynBuf_SeekToBeg(CDynBuf *p) -{ - p->pos = 0; -} - -int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAllocPtr alloc) -{ - if (size > p->size - p->pos) - { - size_t newSize = p->pos + size; - Byte *data; - newSize += newSize / 4; - data = (Byte *)ISzAlloc_Alloc(alloc, newSize); - if (!data) - return 0; - p->size = newSize; - if (p->pos != 0) - memcpy(data, p->data, p->pos); - ISzAlloc_Free(alloc, p->data); - p->data = data; - } - if (size != 0) - { - memcpy(p->data + p->pos, buf, size); - p->pos += size; - } - return 1; -} - -void DynBuf_Free(CDynBuf *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->data); - p->data = 0; - p->size = 0; - p->pos = 0; -} diff --git a/code/ryzom/client/src/seven_zip/7zCrcOpt.cpp b/code/ryzom/client/src/seven_zip/7zCrcOpt.cpp deleted file mode 100644 index 73beba298..000000000 --- a/code/ryzom/client/src/seven_zip/7zCrcOpt.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/* 7zCrcOpt.c -- CRC32 calculation -2017-04-03 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "CpuArch.h" - -#ifndef MY_CPU_BE - -#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) - -UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table) -{ - const Byte *p = (const Byte *)data; - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); - for (; size >= 4; size -= 4, p += 4) - { - v ^= *(const UInt32 *)p; - v = - (table + 0x300)[((v ) & 0xFF)] - ^ (table + 0x200)[((v >> 8) & 0xFF)] - ^ (table + 0x100)[((v >> 16) & 0xFF)] - ^ (table + 0x000)[((v >> 24))]; - } - for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); - return v; -} - -UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table) -{ - const Byte *p = (const Byte *)data; - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 7) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); - for (; size >= 8; size -= 8, p += 8) - { - UInt32 d; - v ^= *(const UInt32 *)p; - v = - (table + 0x700)[((v ) & 0xFF)] - ^ (table + 0x600)[((v >> 8) & 0xFF)] - ^ (table + 0x500)[((v >> 16) & 0xFF)] - ^ (table + 0x400)[((v >> 24))]; - d = *((const UInt32 *)p + 1); - v ^= - (table + 0x300)[((d ) & 0xFF)] - ^ (table + 0x200)[((d >> 8) & 0xFF)] - ^ (table + 0x100)[((d >> 16) & 0xFF)] - ^ (table + 0x000)[((d >> 24))]; - } - for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); - return v; -} - -#endif - - -#ifndef MY_CPU_LE - -#define CRC_UINT32_SWAP(v) ((v >> 24) | ((v >> 8) & 0xFF00) | ((v << 8) & 0xFF0000) | (v << 24)) - -#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[(((crc) >> 24) ^ (b))] ^ ((crc) << 8)) - -UInt32 MY_FAST_CALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, const UInt32 *table) -{ - const Byte *p = (const Byte *)data; - table += 0x100; - v = CRC_UINT32_SWAP(v); - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); - for (; size >= 4; size -= 4, p += 4) - { - v ^= *(const UInt32 *)p; - v = - (table + 0x000)[((v ) & 0xFF)] - ^ (table + 0x100)[((v >> 8) & 0xFF)] - ^ (table + 0x200)[((v >> 16) & 0xFF)] - ^ (table + 0x300)[((v >> 24))]; - } - for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); - return CRC_UINT32_SWAP(v); -} - -UInt32 MY_FAST_CALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, const UInt32 *table) -{ - const Byte *p = (const Byte *)data; - table += 0x100; - v = CRC_UINT32_SWAP(v); - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 7) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); - for (; size >= 8; size -= 8, p += 8) - { - UInt32 d; - v ^= *(const UInt32 *)p; - v = - (table + 0x400)[((v ) & 0xFF)] - ^ (table + 0x500)[((v >> 8) & 0xFF)] - ^ (table + 0x600)[((v >> 16) & 0xFF)] - ^ (table + 0x700)[((v >> 24))]; - d = *((const UInt32 *)p + 1); - v ^= - (table + 0x000)[((d ) & 0xFF)] - ^ (table + 0x100)[((d >> 8) & 0xFF)] - ^ (table + 0x200)[((d >> 16) & 0xFF)] - ^ (table + 0x300)[((d >> 24))]; - } - for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); - return CRC_UINT32_SWAP(v); -} - -#endif diff --git a/code/ryzom/client/src/seven_zip/7zDec.cpp b/code/ryzom/client/src/seven_zip/7zDec.cpp deleted file mode 100644 index 7c4635211..000000000 --- a/code/ryzom/client/src/seven_zip/7zDec.cpp +++ /dev/null @@ -1,591 +0,0 @@ -/* 7zDec.c -- Decoding from 7z folder -2019-02-02 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include - -/* #define _7ZIP_PPMD_SUPPPORT */ - -#include "7z.h" -#include "7zCrc.h" - -#include "Bcj2.h" -#include "Bra.h" -#include "CpuArch.h" -#include "Delta.h" -#include "LzmaDec.h" -#include "Lzma2Dec.h" -#ifdef _7ZIP_PPMD_SUPPPORT -#include "Ppmd7.h" -#endif - -#define k_Copy 0 -#define k_Delta 3 -#define k_LZMA2 0x21 -#define k_LZMA 0x30101 -#define k_BCJ 0x3030103 -#define k_BCJ2 0x303011B -#define k_PPC 0x3030205 -#define k_IA64 0x3030401 -#define k_ARM 0x3030501 -#define k_ARMT 0x3030701 -#define k_SPARC 0x3030805 - - -#ifdef _7ZIP_PPMD_SUPPPORT - -#define k_PPMD 0x30401 - -typedef struct -{ - IByteIn vt; - const Byte *cur; - const Byte *end; - const Byte *begin; - UInt64 processed; - BoolInt extra; - SRes res; - const ILookInStream *inStream; -} CByteInToLook; - -static Byte ReadByte(const IByteIn *pp) -{ - CByteInToLook *p = CONTAINER_FROM_VTBL(pp, CByteInToLook, vt); - if (p->cur != p->end) - return *p->cur++; - if (p->res == SZ_OK) - { - size_t size = p->cur - p->begin; - p->processed += size; - p->res = ILookInStream_Skip(p->inStream, size); - size = (1 << 25); - p->res = ILookInStream_Look(p->inStream, (const void **)&p->begin, &size); - p->cur = p->begin; - p->end = p->begin + size; - if (size != 0) - return *p->cur++;; - } - p->extra = True; - return 0; -} - -static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, const ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) -{ - CPpmd7 ppmd; - CByteInToLook s; - SRes res = SZ_OK; - - s.vt.Read = ReadByte; - s.inStream = inStream; - s.begin = s.end = s.cur = NULL; - s.extra = False; - s.res = SZ_OK; - s.processed = 0; - - if (propsSize != 5) - return SZ_ERROR_UNSUPPORTED; - - { - unsigned order = props[0]; - UInt32 memSize = GetUi32(props + 1); - if (order < PPMD7_MIN_ORDER || - order > PPMD7_MAX_ORDER || - memSize < PPMD7_MIN_MEM_SIZE || - memSize > PPMD7_MAX_MEM_SIZE) - return SZ_ERROR_UNSUPPORTED; - Ppmd7_Construct(&ppmd); - if (!Ppmd7_Alloc(&ppmd, memSize, allocMain)) - return SZ_ERROR_MEM; - Ppmd7_Init(&ppmd, order); - } - { - CPpmd7z_RangeDec rc; - Ppmd7z_RangeDec_CreateVTable(&rc); - rc.Stream = &s.vt; - if (!Ppmd7z_RangeDec_Init(&rc)) - res = SZ_ERROR_DATA; - else if (s.extra) - res = (s.res != SZ_OK ? s.res : SZ_ERROR_DATA); - else - { - SizeT i; - for (i = 0; i < outSize; i++) - { - int sym = Ppmd7_DecodeSymbol(&ppmd, &rc.vt); - if (s.extra || sym < 0) - break; - outBuffer[i] = (Byte)sym; - } - if (i != outSize) - res = (s.res != SZ_OK ? s.res : SZ_ERROR_DATA); - else if (s.processed + (s.cur - s.begin) != inSize || !Ppmd7z_RangeDec_IsFinishedOK(&rc)) - res = SZ_ERROR_DATA; - } - } - Ppmd7_Free(&ppmd, allocMain); - return res; -} - -#endif - - -static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) -{ - CLzmaDec state; - SRes res = SZ_OK; - - LzmaDec_Construct(&state); - RINOK(LzmaDec_AllocateProbs(&state, props, propsSize, allocMain)); - state.dic = outBuffer; - state.dicBufSize = outSize; - LzmaDec_Init(&state); - - for (;;) - { - const void *inBuf = NULL; - size_t lookahead = (1 << 18); - if (lookahead > inSize) - lookahead = (size_t)inSize; - res = ILookInStream_Look(inStream, &inBuf, &lookahead); - if (res != SZ_OK) - break; - - { - SizeT inProcessed = (SizeT)lookahead, dicPos = state.dicPos; - ELzmaStatus status; - res = LzmaDec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); - lookahead -= inProcessed; - inSize -= inProcessed; - if (res != SZ_OK) - break; - - if (status == LZMA_STATUS_FINISHED_WITH_MARK) - { - if (outSize != state.dicPos || inSize != 0) - res = SZ_ERROR_DATA; - break; - } - - if (outSize == state.dicPos && inSize == 0 && status == LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) - break; - - if (inProcessed == 0 && dicPos == state.dicPos) - { - res = SZ_ERROR_DATA; - break; - } - - res = ILookInStream_Skip(inStream, inProcessed); - if (res != SZ_OK) - break; - } - } - - LzmaDec_FreeProbs(&state, allocMain); - return res; -} - - -#ifndef _7Z_NO_METHOD_LZMA2 - -static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) -{ - CLzma2Dec state; - SRes res = SZ_OK; - - Lzma2Dec_Construct(&state); - if (propsSize != 1) - return SZ_ERROR_DATA; - RINOK(Lzma2Dec_AllocateProbs(&state, props[0], allocMain)); - state.decoder.dic = outBuffer; - state.decoder.dicBufSize = outSize; - Lzma2Dec_Init(&state); - - for (;;) - { - const void *inBuf = NULL; - size_t lookahead = (1 << 18); - if (lookahead > inSize) - lookahead = (size_t)inSize; - res = ILookInStream_Look(inStream, &inBuf, &lookahead); - if (res != SZ_OK) - break; - - { - SizeT inProcessed = (SizeT)lookahead, dicPos = state.decoder.dicPos; - ELzmaStatus status; - res = Lzma2Dec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); - lookahead -= inProcessed; - inSize -= inProcessed; - if (res != SZ_OK) - break; - - if (status == LZMA_STATUS_FINISHED_WITH_MARK) - { - if (outSize != state.decoder.dicPos || inSize != 0) - res = SZ_ERROR_DATA; - break; - } - - if (inProcessed == 0 && dicPos == state.decoder.dicPos) - { - res = SZ_ERROR_DATA; - break; - } - - res = ILookInStream_Skip(inStream, inProcessed); - if (res != SZ_OK) - break; - } - } - - Lzma2Dec_FreeProbs(&state, allocMain); - return res; -} - -#endif - - -static SRes SzDecodeCopy(UInt64 inSize, ILookInStream *inStream, Byte *outBuffer) -{ - while (inSize > 0) - { - const void *inBuf; - size_t curSize = (1 << 18); - if (curSize > inSize) - curSize = (size_t)inSize; - RINOK(ILookInStream_Look(inStream, &inBuf, &curSize)); - if (curSize == 0) - return SZ_ERROR_INPUT_EOF; - memcpy(outBuffer, inBuf, curSize); - outBuffer += curSize; - inSize -= curSize; - RINOK(ILookInStream_Skip(inStream, curSize)); - } - return SZ_OK; -} - -static BoolInt IS_MAIN_METHOD(UInt32 m) -{ - switch (m) - { - case k_Copy: - case k_LZMA: - #ifndef _7Z_NO_METHOD_LZMA2 - case k_LZMA2: - #endif - #ifdef _7ZIP_PPMD_SUPPPORT - case k_PPMD: - #endif - return True; - } - return False; -} - -static BoolInt IS_SUPPORTED_CODER(const CSzCoderInfo *c) -{ - return - c->NumStreams == 1 - /* && c->MethodID <= (UInt32)0xFFFFFFFF */ - && IS_MAIN_METHOD((UInt32)c->MethodID); -} - -#define IS_BCJ2(c) ((c)->MethodID == k_BCJ2 && (c)->NumStreams == 4) - -static SRes CheckSupportedFolder(const CSzFolder *f) -{ - if (f->NumCoders < 1 || f->NumCoders > 4) - return SZ_ERROR_UNSUPPORTED; - if (!IS_SUPPORTED_CODER(&f->Coders[0])) - return SZ_ERROR_UNSUPPORTED; - if (f->NumCoders == 1) - { - if (f->NumPackStreams != 1 || f->PackStreams[0] != 0 || f->NumBonds != 0) - return SZ_ERROR_UNSUPPORTED; - return SZ_OK; - } - - - #ifndef _7Z_NO_METHODS_FILTERS - - if (f->NumCoders == 2) - { - const CSzCoderInfo *c = &f->Coders[1]; - if ( - /* c->MethodID > (UInt32)0xFFFFFFFF || */ - c->NumStreams != 1 - || f->NumPackStreams != 1 - || f->PackStreams[0] != 0 - || f->NumBonds != 1 - || f->Bonds[0].InIndex != 1 - || f->Bonds[0].OutIndex != 0) - return SZ_ERROR_UNSUPPORTED; - switch ((UInt32)c->MethodID) - { - case k_Delta: - case k_BCJ: - case k_PPC: - case k_IA64: - case k_SPARC: - case k_ARM: - case k_ARMT: - break; - default: - return SZ_ERROR_UNSUPPORTED; - } - return SZ_OK; - } - - #endif - - - if (f->NumCoders == 4) - { - if (!IS_SUPPORTED_CODER(&f->Coders[1]) - || !IS_SUPPORTED_CODER(&f->Coders[2]) - || !IS_BCJ2(&f->Coders[3])) - return SZ_ERROR_UNSUPPORTED; - if (f->NumPackStreams != 4 - || f->PackStreams[0] != 2 - || f->PackStreams[1] != 6 - || f->PackStreams[2] != 1 - || f->PackStreams[3] != 0 - || f->NumBonds != 3 - || f->Bonds[0].InIndex != 5 || f->Bonds[0].OutIndex != 0 - || f->Bonds[1].InIndex != 4 || f->Bonds[1].OutIndex != 1 - || f->Bonds[2].InIndex != 3 || f->Bonds[2].OutIndex != 2) - return SZ_ERROR_UNSUPPORTED; - return SZ_OK; - } - - return SZ_ERROR_UNSUPPORTED; -} - -#define CASE_BRA_CONV(isa) case k_ ## isa: isa ## _Convert(outBuffer, outSize, 0, 0); break; - -static SRes SzFolder_Decode2(const CSzFolder *folder, - const Byte *propsData, - const UInt64 *unpackSizes, - const UInt64 *packPositions, - ILookInStream *inStream, UInt64 startPos, - Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain, - Byte *tempBuf[]) -{ - UInt32 ci; - SizeT tempSizes[3] = { 0, 0, 0}; - SizeT tempSize3 = 0; - Byte *tempBuf3 = 0; - - RINOK(CheckSupportedFolder(folder)); - - for (ci = 0; ci < folder->NumCoders; ci++) - { - const CSzCoderInfo *coder = &folder->Coders[ci]; - - if (IS_MAIN_METHOD((UInt32)coder->MethodID)) - { - UInt32 si = 0; - UInt64 offset; - UInt64 inSize; - Byte *outBufCur = outBuffer; - SizeT outSizeCur = outSize; - if (folder->NumCoders == 4) - { - UInt32 indices[] = { 3, 2, 0 }; - UInt64 unpackSize = unpackSizes[ci]; - si = indices[ci]; - if (ci < 2) - { - Byte *temp; - outSizeCur = (SizeT)unpackSize; - if (outSizeCur != unpackSize) - return SZ_ERROR_MEM; - temp = (Byte *)ISzAlloc_Alloc(allocMain, outSizeCur); - if (!temp && outSizeCur != 0) - return SZ_ERROR_MEM; - outBufCur = tempBuf[1 - ci] = temp; - tempSizes[1 - ci] = outSizeCur; - } - else if (ci == 2) - { - if (unpackSize > outSize) /* check it */ - return SZ_ERROR_PARAM; - tempBuf3 = outBufCur = outBuffer + (outSize - (size_t)unpackSize); - tempSize3 = outSizeCur = (SizeT)unpackSize; - } - else - return SZ_ERROR_UNSUPPORTED; - } - offset = packPositions[si]; - inSize = packPositions[(size_t)si + 1] - offset; - RINOK(LookInStream_SeekTo(inStream, startPos + offset)); - - if (coder->MethodID == k_Copy) - { - if (inSize != outSizeCur) /* check it */ - return SZ_ERROR_DATA; - RINOK(SzDecodeCopy(inSize, inStream, outBufCur)); - } - else if (coder->MethodID == k_LZMA) - { - RINOK(SzDecodeLzma(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain)); - } - #ifndef _7Z_NO_METHOD_LZMA2 - else if (coder->MethodID == k_LZMA2) - { - RINOK(SzDecodeLzma2(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain)); - } - #endif - #ifdef _7ZIP_PPMD_SUPPPORT - else if (coder->MethodID == k_PPMD) - { - RINOK(SzDecodePpmd(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain)); - } - #endif - else - return SZ_ERROR_UNSUPPORTED; - } - else if (coder->MethodID == k_BCJ2) - { - UInt64 offset = packPositions[1]; - UInt64 s3Size = packPositions[2] - offset; - - if (ci != 3) - return SZ_ERROR_UNSUPPORTED; - - tempSizes[2] = (SizeT)s3Size; - if (tempSizes[2] != s3Size) - return SZ_ERROR_MEM; - tempBuf[2] = (Byte *)ISzAlloc_Alloc(allocMain, tempSizes[2]); - if (!tempBuf[2] && tempSizes[2] != 0) - return SZ_ERROR_MEM; - - RINOK(LookInStream_SeekTo(inStream, startPos + offset)); - RINOK(SzDecodeCopy(s3Size, inStream, tempBuf[2])); - - if ((tempSizes[0] & 3) != 0 || - (tempSizes[1] & 3) != 0 || - tempSize3 + tempSizes[0] + tempSizes[1] != outSize) - return SZ_ERROR_DATA; - - { - CBcj2Dec p; - - p.bufs[0] = tempBuf3; p.lims[0] = tempBuf3 + tempSize3; - p.bufs[1] = tempBuf[0]; p.lims[1] = tempBuf[0] + tempSizes[0]; - p.bufs[2] = tempBuf[1]; p.lims[2] = tempBuf[1] + tempSizes[1]; - p.bufs[3] = tempBuf[2]; p.lims[3] = tempBuf[2] + tempSizes[2]; - - p.dest = outBuffer; - p.destLim = outBuffer + outSize; - - Bcj2Dec_Init(&p); - RINOK(Bcj2Dec_Decode(&p)); - - { - unsigned i; - for (i = 0; i < 4; i++) - if (p.bufs[i] != p.lims[i]) - return SZ_ERROR_DATA; - - if (!Bcj2Dec_IsFinished(&p)) - return SZ_ERROR_DATA; - - if (p.dest != p.destLim - || p.state != BCJ2_STREAM_MAIN) - return SZ_ERROR_DATA; - } - } - } - #ifndef _7Z_NO_METHODS_FILTERS - else if (ci == 1) - { - if (coder->MethodID == k_Delta) - { - if (coder->PropsSize != 1) - return SZ_ERROR_UNSUPPORTED; - { - Byte state[DELTA_STATE_SIZE]; - Delta_Init(state); - Delta_Decode(state, (unsigned)(propsData[coder->PropsOffset]) + 1, outBuffer, outSize); - } - } - else - { - if (coder->PropsSize != 0) - return SZ_ERROR_UNSUPPORTED; - switch (coder->MethodID) - { - case k_BCJ: - { - UInt32 state; - x86_Convert_Init(state); - x86_Convert(outBuffer, outSize, 0, &state, 0); - break; - } - CASE_BRA_CONV(PPC) - CASE_BRA_CONV(IA64) - CASE_BRA_CONV(SPARC) - CASE_BRA_CONV(ARM) - CASE_BRA_CONV(ARMT) - default: - return SZ_ERROR_UNSUPPORTED; - } - } - } - #endif - else - return SZ_ERROR_UNSUPPORTED; - } - - return SZ_OK; -} - - -SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, - ILookInStream *inStream, UInt64 startPos, - Byte *outBuffer, size_t outSize, - ISzAllocPtr allocMain) -{ - SRes res; - CSzFolder folder; - CSzData sd; - - const Byte *data = p->CodersData + p->FoCodersOffsets[folderIndex]; - sd.Data = data; - sd.Size = p->FoCodersOffsets[(size_t)folderIndex + 1] - p->FoCodersOffsets[folderIndex]; - - res = SzGetNextFolderItem(&folder, &sd); - - if (res != SZ_OK) - return res; - - if (sd.Size != 0 - || folder.UnpackStream != p->FoToMainUnpackSizeIndex[folderIndex] - || outSize != SzAr_GetFolderUnpackSize(p, folderIndex)) - return SZ_ERROR_FAIL; - { - unsigned i; - Byte *tempBuf[3] = { 0, 0, 0}; - - res = SzFolder_Decode2(&folder, data, - &p->CoderUnpackSizes[p->FoToCoderUnpackSizes[folderIndex]], - p->PackPositions + p->FoStartPackStreamIndex[folderIndex], - inStream, startPos, - outBuffer, (SizeT)outSize, allocMain, tempBuf); - - for (i = 0; i < 3; i++) - ISzAlloc_Free(allocMain, tempBuf[i]); - - if (res == SZ_OK) - if (SzBitWithVals_Check(&p->FolderCRCs, folderIndex)) - if (CrcCalc(outBuffer, outSize) != p->FolderCRCs.Vals[folderIndex]) - res = SZ_ERROR_CRC; - - return res; - } -} diff --git a/code/ryzom/client/src/seven_zip/7zFile.cpp b/code/ryzom/client/src/seven_zip/7zFile.cpp deleted file mode 100644 index 8992fb1c5..000000000 --- a/code/ryzom/client/src/seven_zip/7zFile.cpp +++ /dev/null @@ -1,286 +0,0 @@ -/* 7zFile.c -- File IO -2017-04-03 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "7zFile.h" - -#ifndef USE_WINDOWS_FILE - -#ifndef UNDER_CE -#include -#endif - -#else - -/* - ReadFile and WriteFile functions in Windows have BUG: - If you Read or Write 64MB or more (probably min_failure_size = 64MB - 32KB + 1) - from/to Network file, it returns ERROR_NO_SYSTEM_RESOURCES - (Insufficient system resources exist to complete the requested service). - Probably in some version of Windows there are problems with other sizes: - for 32 MB (maybe also for 16 MB). - And message can be "Network connection was lost" -*/ - -#define kChunkSizeMax (1 << 22) - -#endif - -void File_Construct(CSzFile *p) -{ - #ifdef USE_WINDOWS_FILE - p->handle = INVALID_HANDLE_VALUE; - #else - p->file = NULL; - #endif -} - -#if !defined(UNDER_CE) || !defined(USE_WINDOWS_FILE) -static WRes File_Open(CSzFile *p, const char *name, int writeMode) -{ - #ifdef USE_WINDOWS_FILE - p->handle = CreateFileA(name, - writeMode ? GENERIC_WRITE : GENERIC_READ, - FILE_SHARE_READ, NULL, - writeMode ? CREATE_ALWAYS : OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, NULL); - return (p->handle != INVALID_HANDLE_VALUE) ? 0 : GetLastError(); - #else - p->file = fopen(name, writeMode ? "wb+" : "rb"); - return (p->file != 0) ? 0 : - #ifdef UNDER_CE - 2; /* ENOENT */ - #else - errno; - #endif - #endif -} - -WRes InFile_Open(CSzFile *p, const char *name) { return File_Open(p, name, 0); } -WRes OutFile_Open(CSzFile *p, const char *name) { return File_Open(p, name, 1); } -#endif - -#ifdef USE_WINDOWS_FILE -static WRes File_OpenW(CSzFile *p, const WCHAR *name, int writeMode) -{ - p->handle = CreateFileW(name, - writeMode ? GENERIC_WRITE : GENERIC_READ, - FILE_SHARE_READ, NULL, - writeMode ? CREATE_ALWAYS : OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, NULL); - return (p->handle != INVALID_HANDLE_VALUE) ? 0 : GetLastError(); -} -WRes InFile_OpenW(CSzFile *p, const WCHAR *name) { return File_OpenW(p, name, 0); } -WRes OutFile_OpenW(CSzFile *p, const WCHAR *name) { return File_OpenW(p, name, 1); } -#endif - -WRes File_Close(CSzFile *p) -{ - #ifdef USE_WINDOWS_FILE - if (p->handle != INVALID_HANDLE_VALUE) - { - if (!CloseHandle(p->handle)) - return GetLastError(); - p->handle = INVALID_HANDLE_VALUE; - } - #else - if (p->file != NULL) - { - int res = fclose(p->file); - if (res != 0) - return res; - p->file = NULL; - } - #endif - return 0; -} - -WRes File_Read(CSzFile *p, void *data, size_t *size) -{ - size_t originalSize = *size; - if (originalSize == 0) - return 0; - - #ifdef USE_WINDOWS_FILE - - *size = 0; - do - { - DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; - DWORD processed = 0; - BOOL res = ReadFile(p->handle, data, curSize, &processed, NULL); - data = (void *)((Byte *)data + processed); - originalSize -= processed; - *size += processed; - if (!res) - return GetLastError(); - if (processed == 0) - break; - } - while (originalSize > 0); - return 0; - - #else - - *size = fread(data, 1, originalSize, p->file); - if (*size == originalSize) - return 0; - return ferror(p->file); - - #endif -} - -WRes File_Write(CSzFile *p, const void *data, size_t *size) -{ - size_t originalSize = *size; - if (originalSize == 0) - return 0; - - #ifdef USE_WINDOWS_FILE - - *size = 0; - do - { - DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; - DWORD processed = 0; - BOOL res = WriteFile(p->handle, data, curSize, &processed, NULL); - data = (void *)((Byte *)data + processed); - originalSize -= processed; - *size += processed; - if (!res) - return GetLastError(); - if (processed == 0) - break; - } - while (originalSize > 0); - return 0; - - #else - - *size = fwrite(data, 1, originalSize, p->file); - if (*size == originalSize) - return 0; - return ferror(p->file); - - #endif -} - -WRes File_Seek(CSzFile *p, Int64 *pos, ESzSeek origin) -{ - #ifdef USE_WINDOWS_FILE - - LARGE_INTEGER value; - DWORD moveMethod; - value.LowPart = (DWORD)*pos; - value.HighPart = (LONG)((UInt64)*pos >> 16 >> 16); /* for case when UInt64 is 32-bit only */ - switch (origin) - { - case SZ_SEEK_SET: moveMethod = FILE_BEGIN; break; - case SZ_SEEK_CUR: moveMethod = FILE_CURRENT; break; - case SZ_SEEK_END: moveMethod = FILE_END; break; - default: return ERROR_INVALID_PARAMETER; - } - value.LowPart = SetFilePointer(p->handle, value.LowPart, &value.HighPart, moveMethod); - if (value.LowPart == 0xFFFFFFFF) - { - WRes res = GetLastError(); - if (res != NO_ERROR) - return res; - } - *pos = ((Int64)value.HighPart << 32) | value.LowPart; - return 0; - - #else - - int moveMethod; - int res; - switch (origin) - { - case SZ_SEEK_SET: moveMethod = SEEK_SET; break; - case SZ_SEEK_CUR: moveMethod = SEEK_CUR; break; - case SZ_SEEK_END: moveMethod = SEEK_END; break; - default: return 1; - } - res = fseek(p->file, (long)*pos, moveMethod); - *pos = ftell(p->file); - return res; - - #endif -} - -WRes File_GetLength(CSzFile *p, UInt64 *length) -{ - #ifdef USE_WINDOWS_FILE - - DWORD sizeHigh; - DWORD sizeLow = GetFileSize(p->handle, &sizeHigh); - if (sizeLow == 0xFFFFFFFF) - { - DWORD res = GetLastError(); - if (res != NO_ERROR) - return res; - } - *length = (((UInt64)sizeHigh) << 32) + sizeLow; - return 0; - - #else - - long pos = ftell(p->file); - int res = fseek(p->file, 0, SEEK_END); - *length = ftell(p->file); - fseek(p->file, pos, SEEK_SET); - return res; - - #endif -} - - -/* ---------- FileSeqInStream ---------- */ - -static SRes FileSeqInStream_Read(const ISeqInStream *pp, void *buf, size_t *size) -{ - CFileSeqInStream *p = CONTAINER_FROM_VTBL(pp, CFileSeqInStream, vt); - return File_Read(&p->file, buf, size) == 0 ? SZ_OK : SZ_ERROR_READ; -} - -void FileSeqInStream_CreateVTable(CFileSeqInStream *p) -{ - p->vt.Read = FileSeqInStream_Read; -} - - -/* ---------- FileInStream ---------- */ - -static SRes FileInStream_Read(const ISeekInStream *pp, void *buf, size_t *size) -{ - CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); - return (File_Read(&p->file, buf, size) == 0) ? SZ_OK : SZ_ERROR_READ; -} - -static SRes FileInStream_Seek(const ISeekInStream *pp, Int64 *pos, ESzSeek origin) -{ - CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); - return File_Seek(&p->file, pos, origin); -} - -void FileInStream_CreateVTable(CFileInStream *p) -{ - p->vt.Read = FileInStream_Read; - p->vt.Seek = FileInStream_Seek; -} - - -/* ---------- FileOutStream ---------- */ - -static size_t FileOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) -{ - CFileOutStream *p = CONTAINER_FROM_VTBL(pp, CFileOutStream, vt); - File_Write(&p->file, data, &size); - return size; -} - -void FileOutStream_CreateVTable(CFileOutStream *p) -{ - p->vt.Write = FileOutStream_Write; -} diff --git a/code/ryzom/client/src/seven_zip/7zFile.h b/code/ryzom/client/src/seven_zip/7zFile.h deleted file mode 100644 index 0e7925382..000000000 --- a/code/ryzom/client/src/seven_zip/7zFile.h +++ /dev/null @@ -1,83 +0,0 @@ -/* 7zFile.h -- File IO -2017-04-03 : Igor Pavlov : Public domain */ - -#ifndef __7Z_FILE_H -#define __7Z_FILE_H - -#ifdef _WIN32 -#define USE_WINDOWS_FILE -#endif - -#ifdef USE_WINDOWS_FILE -#include -#else -#include -#endif - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -/* ---------- File ---------- */ - -typedef struct -{ - #ifdef USE_WINDOWS_FILE - HANDLE handle; - #else - FILE *file; - #endif -} CSzFile; - -void File_Construct(CSzFile *p); -#if !defined(UNDER_CE) || !defined(USE_WINDOWS_FILE) -WRes InFile_Open(CSzFile *p, const char *name); -WRes OutFile_Open(CSzFile *p, const char *name); -#endif -#ifdef USE_WINDOWS_FILE -WRes InFile_OpenW(CSzFile *p, const WCHAR *name); -WRes OutFile_OpenW(CSzFile *p, const WCHAR *name); -#endif -WRes File_Close(CSzFile *p); - -/* reads max(*size, remain file's size) bytes */ -WRes File_Read(CSzFile *p, void *data, size_t *size); - -/* writes *size bytes */ -WRes File_Write(CSzFile *p, const void *data, size_t *size); - -WRes File_Seek(CSzFile *p, Int64 *pos, ESzSeek origin); -WRes File_GetLength(CSzFile *p, UInt64 *length); - - -/* ---------- FileInStream ---------- */ - -typedef struct -{ - ISeqInStream vt; - CSzFile file; -} CFileSeqInStream; - -void FileSeqInStream_CreateVTable(CFileSeqInStream *p); - - -typedef struct -{ - ISeekInStream vt; - CSzFile file; -} CFileInStream; - -void FileInStream_CreateVTable(CFileInStream *p); - - -typedef struct -{ - ISeqOutStream vt; - CSzFile file; -} CFileOutStream; - -void FileOutStream_CreateVTable(CFileOutStream *p); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/7zStream.cpp b/code/ryzom/client/src/seven_zip/7zStream.cpp deleted file mode 100644 index 6b5aa1621..000000000 --- a/code/ryzom/client/src/seven_zip/7zStream.cpp +++ /dev/null @@ -1,176 +0,0 @@ -/* 7zStream.c -- 7z Stream functions -2017-04-03 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include - -#include "7zTypes.h" - -SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes errorType) -{ - while (size != 0) - { - size_t processed = size; - RINOK(ISeqInStream_Read(stream, buf, &processed)); - if (processed == 0) - return errorType; - buf = (void *)((Byte *)buf + processed); - size -= processed; - } - return SZ_OK; -} - -SRes SeqInStream_Read(const ISeqInStream *stream, void *buf, size_t size) -{ - return SeqInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); -} - -SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf) -{ - size_t processed = 1; - RINOK(ISeqInStream_Read(stream, buf, &processed)); - return (processed == 1) ? SZ_OK : SZ_ERROR_INPUT_EOF; -} - - - -SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset) -{ - Int64 t = offset; - return ILookInStream_Seek(stream, &t, SZ_SEEK_SET); -} - -SRes LookInStream_LookRead(const ILookInStream *stream, void *buf, size_t *size) -{ - const void *lookBuf; - if (*size == 0) - return SZ_OK; - RINOK(ILookInStream_Look(stream, &lookBuf, size)); - memcpy(buf, lookBuf, *size); - return ILookInStream_Skip(stream, *size); -} - -SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRes errorType) -{ - while (size != 0) - { - size_t processed = size; - RINOK(ILookInStream_Read(stream, buf, &processed)); - if (processed == 0) - return errorType; - buf = (void *)((Byte *)buf + processed); - size -= processed; - } - return SZ_OK; -} - -SRes LookInStream_Read(const ILookInStream *stream, void *buf, size_t size) -{ - return LookInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); -} - - - -#define GET_LookToRead2 CLookToRead2 *p = CONTAINER_FROM_VTBL(pp, CLookToRead2, vt); - -static SRes LookToRead2_Look_Lookahead(const ILookInStream *pp, const void **buf, size_t *size) -{ - SRes res = SZ_OK; - GET_LookToRead2 - size_t size2 = p->size - p->pos; - if (size2 == 0 && *size != 0) - { - p->pos = 0; - p->size = 0; - size2 = p->bufSize; - res = ISeekInStream_Read(p->realStream, p->buf, &size2); - p->size = size2; - } - if (*size > size2) - *size = size2; - *buf = p->buf + p->pos; - return res; -} - -static SRes LookToRead2_Look_Exact(const ILookInStream *pp, const void **buf, size_t *size) -{ - SRes res = SZ_OK; - GET_LookToRead2 - size_t size2 = p->size - p->pos; - if (size2 == 0 && *size != 0) - { - p->pos = 0; - p->size = 0; - if (*size > p->bufSize) - *size = p->bufSize; - res = ISeekInStream_Read(p->realStream, p->buf, size); - size2 = p->size = *size; - } - if (*size > size2) - *size = size2; - *buf = p->buf + p->pos; - return res; -} - -static SRes LookToRead2_Skip(const ILookInStream *pp, size_t offset) -{ - GET_LookToRead2 - p->pos += offset; - return SZ_OK; -} - -static SRes LookToRead2_Read(const ILookInStream *pp, void *buf, size_t *size) -{ - GET_LookToRead2 - size_t rem = p->size - p->pos; - if (rem == 0) - return ISeekInStream_Read(p->realStream, buf, size); - if (rem > *size) - rem = *size; - memcpy(buf, p->buf + p->pos, rem); - p->pos += rem; - *size = rem; - return SZ_OK; -} - -static SRes LookToRead2_Seek(const ILookInStream *pp, Int64 *pos, ESzSeek origin) -{ - GET_LookToRead2 - p->pos = p->size = 0; - return ISeekInStream_Seek(p->realStream, pos, origin); -} - -void LookToRead2_CreateVTable(CLookToRead2 *p, int lookahead) -{ - p->vt.Look = lookahead ? - LookToRead2_Look_Lookahead : - LookToRead2_Look_Exact; - p->vt.Skip = LookToRead2_Skip; - p->vt.Read = LookToRead2_Read; - p->vt.Seek = LookToRead2_Seek; -} - - - -static SRes SecToLook_Read(const ISeqInStream *pp, void *buf, size_t *size) -{ - CSecToLook *p = CONTAINER_FROM_VTBL(pp, CSecToLook, vt); - return LookInStream_LookRead(p->realStream, buf, size); -} - -void SecToLook_CreateVTable(CSecToLook *p) -{ - p->vt.Read = SecToLook_Read; -} - -static SRes SecToRead_Read(const ISeqInStream *pp, void *buf, size_t *size) -{ - CSecToRead *p = CONTAINER_FROM_VTBL(pp, CSecToRead, vt); - return ILookInStream_Read(p->realStream, buf, size); -} - -void SecToRead_CreateVTable(CSecToRead *p) -{ - p->vt.Read = SecToRead_Read; -} diff --git a/code/ryzom/client/src/seven_zip/7zVersion.h b/code/ryzom/client/src/seven_zip/7zVersion.h deleted file mode 100644 index c176823a4..000000000 --- a/code/ryzom/client/src/seven_zip/7zVersion.h +++ /dev/null @@ -1,27 +0,0 @@ -#define MY_VER_MAJOR 19 -#define MY_VER_MINOR 00 -#define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "19.00" -#define MY_VERSION MY_VERSION_NUMBERS - -#ifdef MY_CPU_NAME - #define MY_VERSION_CPU MY_VERSION " (" MY_CPU_NAME ")" -#else - #define MY_VERSION_CPU MY_VERSION -#endif - -#define MY_DATE "2019-02-21" -#undef MY_COPYRIGHT -#undef MY_VERSION_COPYRIGHT_DATE -#define MY_AUTHOR_NAME "Igor Pavlov" -#define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" -#define MY_COPYRIGHT_CR "Copyright (c) 1999-2018 Igor Pavlov" - -#ifdef USE_COPYRIGHT_CR - #define MY_COPYRIGHT MY_COPYRIGHT_CR -#else - #define MY_COPYRIGHT MY_COPYRIGHT_PD -#endif - -#define MY_COPYRIGHT_DATE MY_COPYRIGHT " : " MY_DATE -#define MY_VERSION_COPYRIGHT_DATE MY_VERSION_CPU " : " MY_COPYRIGHT " : " MY_DATE diff --git a/code/ryzom/client/src/seven_zip/7zVersion.rc b/code/ryzom/client/src/seven_zip/7zVersion.rc deleted file mode 100644 index 6ed26de74..000000000 --- a/code/ryzom/client/src/seven_zip/7zVersion.rc +++ /dev/null @@ -1,55 +0,0 @@ -#define MY_VS_FFI_FILEFLAGSMASK 0x0000003FL -#define MY_VOS_NT_WINDOWS32 0x00040004L -#define MY_VOS_CE_WINDOWS32 0x00050004L - -#define MY_VFT_APP 0x00000001L -#define MY_VFT_DLL 0x00000002L - -// #include - -#ifndef MY_VERSION -#include "7zVersion.h" -#endif - -#define MY_VER MY_VER_MAJOR,MY_VER_MINOR,MY_VER_BUILD,0 - -#ifdef DEBUG -#define DBG_FL VS_FF_DEBUG -#else -#define DBG_FL 0 -#endif - -#define MY_VERSION_INFO(fileType, descr, intName, origName) \ -LANGUAGE 9, 1 \ -1 VERSIONINFO \ - FILEVERSION MY_VER \ - PRODUCTVERSION MY_VER \ - FILEFLAGSMASK MY_VS_FFI_FILEFLAGSMASK \ - FILEFLAGS DBG_FL \ - FILEOS MY_VOS_NT_WINDOWS32 \ - FILETYPE fileType \ - FILESUBTYPE 0x0L \ -BEGIN \ - BLOCK "StringFileInfo" \ - BEGIN \ - BLOCK "040904b0" \ - BEGIN \ - VALUE "CompanyName", "Igor Pavlov" \ - VALUE "FileDescription", descr \ - VALUE "FileVersion", MY_VERSION \ - VALUE "InternalName", intName \ - VALUE "LegalCopyright", MY_COPYRIGHT \ - VALUE "OriginalFilename", origName \ - VALUE "ProductName", "7-Zip" \ - VALUE "ProductVersion", MY_VERSION \ - END \ - END \ - BLOCK "VarFileInfo" \ - BEGIN \ - VALUE "Translation", 0x409, 1200 \ - END \ -END - -#define MY_VERSION_INFO_APP(descr, intName) MY_VERSION_INFO(MY_VFT_APP, descr, intName, intName ".exe") - -#define MY_VERSION_INFO_DLL(descr, intName) MY_VERSION_INFO(MY_VFT_DLL, descr, intName, intName ".dll") diff --git a/code/ryzom/client/src/seven_zip/Aes.cpp b/code/ryzom/client/src/seven_zip/Aes.cpp deleted file mode 100644 index 1cdd0e787..000000000 --- a/code/ryzom/client/src/seven_zip/Aes.cpp +++ /dev/null @@ -1,306 +0,0 @@ -/* Aes.c -- AES encryption / decryption -2017-01-24 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "Aes.h" -#include "CpuArch.h" - -static UInt32 T[256 * 4]; -static const Byte Sbox[256] = { - 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, - 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, - 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, - 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, - 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, - 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, - 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, - 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, - 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, - 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, - 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, - 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, - 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, - 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, - 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, - 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16}; - -void MY_FAST_CALL AesCbc_Encode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code(UInt32 *ivAes, Byte *data, size_t numBlocks); - -void MY_FAST_CALL AesCbc_Encode_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); - -AES_CODE_FUNC g_AesCbc_Encode; -AES_CODE_FUNC g_AesCbc_Decode; -AES_CODE_FUNC g_AesCtr_Code; - -static UInt32 D[256 * 4]; -static Byte InvS[256]; - -static const Byte Rcon[11] = { 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36 }; - -#define xtime(x) ((((x) << 1) ^ (((x) & 0x80) != 0 ? 0x1B : 0)) & 0xFF) - -#define Ui32(a0, a1, a2, a3) ((UInt32)(a0) | ((UInt32)(a1) << 8) | ((UInt32)(a2) << 16) | ((UInt32)(a3) << 24)) - -#define gb0(x) ( (x) & 0xFF) -#define gb1(x) (((x) >> ( 8)) & 0xFF) -#define gb2(x) (((x) >> (16)) & 0xFF) -#define gb3(x) (((x) >> (24))) - -#define gb(n, x) gb ## n(x) - -#define TT(x) (T + (x << 8)) -#define DD(x) (D + (x << 8)) - - -void AesGenTables(void) -{ - unsigned i; - for (i = 0; i < 256; i++) - InvS[Sbox[i]] = (Byte)i; - - for (i = 0; i < 256; i++) - { - { - UInt32 a1 = Sbox[i]; - UInt32 a2 = xtime(a1); - UInt32 a3 = a2 ^ a1; - TT(0)[i] = Ui32(a2, a1, a1, a3); - TT(1)[i] = Ui32(a3, a2, a1, a1); - TT(2)[i] = Ui32(a1, a3, a2, a1); - TT(3)[i] = Ui32(a1, a1, a3, a2); - } - { - UInt32 a1 = InvS[i]; - UInt32 a2 = xtime(a1); - UInt32 a4 = xtime(a2); - UInt32 a8 = xtime(a4); - UInt32 a9 = a8 ^ a1; - UInt32 aB = a8 ^ a2 ^ a1; - UInt32 aD = a8 ^ a4 ^ a1; - UInt32 aE = a8 ^ a4 ^ a2; - DD(0)[i] = Ui32(aE, a9, aD, aB); - DD(1)[i] = Ui32(aB, aE, a9, aD); - DD(2)[i] = Ui32(aD, aB, aE, a9); - DD(3)[i] = Ui32(a9, aD, aB, aE); - } - } - - g_AesCbc_Encode = AesCbc_Encode; - g_AesCbc_Decode = AesCbc_Decode; - g_AesCtr_Code = AesCtr_Code; - - #ifdef MY_CPU_X86_OR_AMD64 - if (CPU_Is_Aes_Supported()) - { - g_AesCbc_Encode = AesCbc_Encode_Intel; - g_AesCbc_Decode = AesCbc_Decode_Intel; - g_AesCtr_Code = AesCtr_Code_Intel; - } - #endif -} - - -#define HT(i, x, s) TT(x)[gb(x, s[(i + x) & 3])] - -#define HT4(m, i, s, p) m[i] = \ - HT(i, 0, s) ^ \ - HT(i, 1, s) ^ \ - HT(i, 2, s) ^ \ - HT(i, 3, s) ^ w[p + i] - -#define HT16(m, s, p) \ - HT4(m, 0, s, p); \ - HT4(m, 1, s, p); \ - HT4(m, 2, s, p); \ - HT4(m, 3, s, p); \ - -#define FT(i, x) Sbox[gb(x, m[(i + x) & 3])] -#define FT4(i) dest[i] = Ui32(FT(i, 0), FT(i, 1), FT(i, 2), FT(i, 3)) ^ w[i]; - - -#define HD(i, x, s) DD(x)[gb(x, s[(i - x) & 3])] - -#define HD4(m, i, s, p) m[i] = \ - HD(i, 0, s) ^ \ - HD(i, 1, s) ^ \ - HD(i, 2, s) ^ \ - HD(i, 3, s) ^ w[p + i]; - -#define HD16(m, s, p) \ - HD4(m, 0, s, p); \ - HD4(m, 1, s, p); \ - HD4(m, 2, s, p); \ - HD4(m, 3, s, p); \ - -#define FD(i, x) InvS[gb(x, m[(i - x) & 3])] -#define FD4(i) dest[i] = Ui32(FD(i, 0), FD(i, 1), FD(i, 2), FD(i, 3)) ^ w[i]; - -void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize) -{ - unsigned i, wSize; - wSize = keySize + 28; - keySize /= 4; - w[0] = ((UInt32)keySize / 2) + 3; - w += 4; - - for (i = 0; i < keySize; i++, key += 4) - w[i] = GetUi32(key); - - for (; i < wSize; i++) - { - UInt32 t = w[(size_t)i - 1]; - unsigned rem = i % keySize; - if (rem == 0) - t = Ui32(Sbox[gb1(t)] ^ Rcon[i / keySize], Sbox[gb2(t)], Sbox[gb3(t)], Sbox[gb0(t)]); - else if (keySize > 6 && rem == 4) - t = Ui32(Sbox[gb0(t)], Sbox[gb1(t)], Sbox[gb2(t)], Sbox[gb3(t)]); - w[i] = w[i - keySize] ^ t; - } -} - -void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *w, const Byte *key, unsigned keySize) -{ - unsigned i, num; - Aes_SetKey_Enc(w, key, keySize); - num = keySize + 20; - w += 8; - for (i = 0; i < num; i++) - { - UInt32 r = w[i]; - w[i] = - DD(0)[Sbox[gb0(r)]] ^ - DD(1)[Sbox[gb1(r)]] ^ - DD(2)[Sbox[gb2(r)]] ^ - DD(3)[Sbox[gb3(r)]]; - } -} - -/* Aes_Encode and Aes_Decode functions work with little-endian words. - src and dest are pointers to 4 UInt32 words. - src and dest can point to same block */ - -static void Aes_Encode(const UInt32 *w, UInt32 *dest, const UInt32 *src) -{ - UInt32 s[4]; - UInt32 m[4]; - UInt32 numRounds2 = w[0]; - w += 4; - s[0] = src[0] ^ w[0]; - s[1] = src[1] ^ w[1]; - s[2] = src[2] ^ w[2]; - s[3] = src[3] ^ w[3]; - w += 4; - for (;;) - { - HT16(m, s, 0); - if (--numRounds2 == 0) - break; - HT16(s, m, 4); - w += 8; - } - w += 4; - FT4(0); FT4(1); FT4(2); FT4(3); -} - -static void Aes_Decode(const UInt32 *w, UInt32 *dest, const UInt32 *src) -{ - UInt32 s[4]; - UInt32 m[4]; - UInt32 numRounds2 = w[0]; - w += 4 + numRounds2 * 8; - s[0] = src[0] ^ w[0]; - s[1] = src[1] ^ w[1]; - s[2] = src[2] ^ w[2]; - s[3] = src[3] ^ w[3]; - for (;;) - { - w -= 8; - HD16(m, s, 4); - if (--numRounds2 == 0) - break; - HD16(s, m, 0); - } - FD4(0); FD4(1); FD4(2); FD4(3); -} - -void AesCbc_Init(UInt32 *p, const Byte *iv) -{ - unsigned i; - for (i = 0; i < 4; i++) - p[i] = GetUi32(iv + i * 4); -} - -void MY_FAST_CALL AesCbc_Encode(UInt32 *p, Byte *data, size_t numBlocks) -{ - for (; numBlocks != 0; numBlocks--, data += AES_BLOCK_SIZE) - { - p[0] ^= GetUi32(data); - p[1] ^= GetUi32(data + 4); - p[2] ^= GetUi32(data + 8); - p[3] ^= GetUi32(data + 12); - - Aes_Encode(p + 4, p, p); - - SetUi32(data, p[0]); - SetUi32(data + 4, p[1]); - SetUi32(data + 8, p[2]); - SetUi32(data + 12, p[3]); - } -} - -void MY_FAST_CALL AesCbc_Decode(UInt32 *p, Byte *data, size_t numBlocks) -{ - UInt32 in[4], out[4]; - for (; numBlocks != 0; numBlocks--, data += AES_BLOCK_SIZE) - { - in[0] = GetUi32(data); - in[1] = GetUi32(data + 4); - in[2] = GetUi32(data + 8); - in[3] = GetUi32(data + 12); - - Aes_Decode(p + 4, out, in); - - SetUi32(data, p[0] ^ out[0]); - SetUi32(data + 4, p[1] ^ out[1]); - SetUi32(data + 8, p[2] ^ out[2]); - SetUi32(data + 12, p[3] ^ out[3]); - - p[0] = in[0]; - p[1] = in[1]; - p[2] = in[2]; - p[3] = in[3]; - } -} - -void MY_FAST_CALL AesCtr_Code(UInt32 *p, Byte *data, size_t numBlocks) -{ - for (; numBlocks != 0; numBlocks--) - { - UInt32 temp[4]; - unsigned i; - - if (++p[0] == 0) - p[1]++; - - Aes_Encode(p + 4, temp, p); - - for (i = 0; i < 4; i++, data += 4) - { - UInt32 t = temp[i]; - - #ifdef MY_CPU_LE_UNALIGN - *((UInt32 *)data) ^= t; - #else - data[0] ^= (t & 0xFF); - data[1] ^= ((t >> 8) & 0xFF); - data[2] ^= ((t >> 16) & 0xFF); - data[3] ^= ((t >> 24)); - #endif - } - } -} diff --git a/code/ryzom/client/src/seven_zip/Aes.h b/code/ryzom/client/src/seven_zip/Aes.h deleted file mode 100644 index 64979b5bc..000000000 --- a/code/ryzom/client/src/seven_zip/Aes.h +++ /dev/null @@ -1,38 +0,0 @@ -/* Aes.h -- AES encryption / decryption -2013-01-18 : Igor Pavlov : Public domain */ - -#ifndef __AES_H -#define __AES_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -#define AES_BLOCK_SIZE 16 - -/* Call AesGenTables one time before other AES functions */ -void AesGenTables(void); - -/* UInt32 pointers must be 16-byte aligned */ - -/* 16-byte (4 * 32-bit words) blocks: 1 (IV) + 1 (keyMode) + 15 (AES-256 roundKeys) */ -#define AES_NUM_IVMRK_WORDS ((1 + 1 + 15) * 4) - -/* aes - 16-byte aligned pointer to keyMode+roundKeys sequence */ -/* keySize = 16 or 24 or 32 (bytes) */ -typedef void (MY_FAST_CALL *AES_SET_KEY_FUNC)(UInt32 *aes, const Byte *key, unsigned keySize); -void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *aes, const Byte *key, unsigned keySize); -void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *aes, const Byte *key, unsigned keySize); - -/* ivAes - 16-byte aligned pointer to iv+keyMode+roundKeys sequence: UInt32[AES_NUM_IVMRK_WORDS] */ -void AesCbc_Init(UInt32 *ivAes, const Byte *iv); /* iv size is AES_BLOCK_SIZE */ -/* data - 16-byte aligned pointer to data */ -/* numBlocks - the number of 16-byte blocks in data array */ -typedef void (MY_FAST_CALL *AES_CODE_FUNC)(UInt32 *ivAes, Byte *data, size_t numBlocks); -extern AES_CODE_FUNC g_AesCbc_Encode; -extern AES_CODE_FUNC g_AesCbc_Decode; -extern AES_CODE_FUNC g_AesCtr_Code; - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/AesOpt.cpp b/code/ryzom/client/src/seven_zip/AesOpt.cpp deleted file mode 100644 index 9571c467f..000000000 --- a/code/ryzom/client/src/seven_zip/AesOpt.cpp +++ /dev/null @@ -1,184 +0,0 @@ -/* AesOpt.c -- Intel's AES -2017-06-08 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "CpuArch.h" - -#ifdef MY_CPU_X86_OR_AMD64 -#if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729) -#define USE_INTEL_AES -#endif -#endif - -#ifdef USE_INTEL_AES - -#include - -void MY_FAST_CALL AesCbc_Encode_Intel(__m128i *p, __m128i *data, size_t numBlocks) -{ - __m128i m = *p; - for (; numBlocks != 0; numBlocks--, data++) - { - UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; - const __m128i *w = p + 3; - m = _mm_xor_si128(m, *data); - m = _mm_xor_si128(m, p[2]); - do - { - m = _mm_aesenc_si128(m, w[0]); - m = _mm_aesenc_si128(m, w[1]); - w += 2; - } - while (--numRounds2 != 0); - m = _mm_aesenc_si128(m, w[0]); - m = _mm_aesenclast_si128(m, w[1]); - *data = m; - } - *p = m; -} - -#define NUM_WAYS 3 - -#define AES_OP_W(op, n) { \ - const __m128i t = w[n]; \ - m0 = op(m0, t); \ - m1 = op(m1, t); \ - m2 = op(m2, t); \ - } - -#define AES_DEC(n) AES_OP_W(_mm_aesdec_si128, n) -#define AES_DEC_LAST(n) AES_OP_W(_mm_aesdeclast_si128, n) -#define AES_ENC(n) AES_OP_W(_mm_aesenc_si128, n) -#define AES_ENC_LAST(n) AES_OP_W(_mm_aesenclast_si128, n) - -void MY_FAST_CALL AesCbc_Decode_Intel(__m128i *p, __m128i *data, size_t numBlocks) -{ - __m128i iv = *p; - for (; numBlocks >= NUM_WAYS; numBlocks -= NUM_WAYS, data += NUM_WAYS) - { - UInt32 numRounds2 = *(const UInt32 *)(p + 1); - const __m128i *w = p + numRounds2 * 2; - __m128i m0, m1, m2; - { - const __m128i t = w[2]; - m0 = _mm_xor_si128(t, data[0]); - m1 = _mm_xor_si128(t, data[1]); - m2 = _mm_xor_si128(t, data[2]); - } - numRounds2--; - do - { - AES_DEC(1) - AES_DEC(0) - w -= 2; - } - while (--numRounds2 != 0); - AES_DEC(1) - AES_DEC_LAST(0) - - { - __m128i t; - t = _mm_xor_si128(m0, iv); iv = data[0]; data[0] = t; - t = _mm_xor_si128(m1, iv); iv = data[1]; data[1] = t; - t = _mm_xor_si128(m2, iv); iv = data[2]; data[2] = t; - } - } - for (; numBlocks != 0; numBlocks--, data++) - { - UInt32 numRounds2 = *(const UInt32 *)(p + 1); - const __m128i *w = p + numRounds2 * 2; - __m128i m = _mm_xor_si128(w[2], *data); - numRounds2--; - do - { - m = _mm_aesdec_si128(m, w[1]); - m = _mm_aesdec_si128(m, w[0]); - w -= 2; - } - while (--numRounds2 != 0); - m = _mm_aesdec_si128(m, w[1]); - m = _mm_aesdeclast_si128(m, w[0]); - - m = _mm_xor_si128(m, iv); - iv = *data; - *data = m; - } - *p = iv; -} - -void MY_FAST_CALL AesCtr_Code_Intel(__m128i *p, __m128i *data, size_t numBlocks) -{ - __m128i ctr = *p; - __m128i one; - one.m128i_u64[0] = 1; - one.m128i_u64[1] = 0; - for (; numBlocks >= NUM_WAYS; numBlocks -= NUM_WAYS, data += NUM_WAYS) - { - UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; - const __m128i *w = p; - __m128i m0, m1, m2; - { - const __m128i t = w[2]; - ctr = _mm_add_epi64(ctr, one); m0 = _mm_xor_si128(ctr, t); - ctr = _mm_add_epi64(ctr, one); m1 = _mm_xor_si128(ctr, t); - ctr = _mm_add_epi64(ctr, one); m2 = _mm_xor_si128(ctr, t); - } - w += 3; - do - { - AES_ENC(0) - AES_ENC(1) - w += 2; - } - while (--numRounds2 != 0); - AES_ENC(0) - AES_ENC_LAST(1) - data[0] = _mm_xor_si128(data[0], m0); - data[1] = _mm_xor_si128(data[1], m1); - data[2] = _mm_xor_si128(data[2], m2); - } - for (; numBlocks != 0; numBlocks--, data++) - { - UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; - const __m128i *w = p; - __m128i m; - ctr = _mm_add_epi64(ctr, one); - m = _mm_xor_si128(ctr, p[2]); - w += 3; - do - { - m = _mm_aesenc_si128(m, w[0]); - m = _mm_aesenc_si128(m, w[1]); - w += 2; - } - while (--numRounds2 != 0); - m = _mm_aesenc_si128(m, w[0]); - m = _mm_aesenclast_si128(m, w[1]); - *data = _mm_xor_si128(*data, m); - } - *p = ctr; -} - -#else - -void MY_FAST_CALL AesCbc_Encode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCbc_Decode(UInt32 *ivAes, Byte *data, size_t numBlocks); -void MY_FAST_CALL AesCtr_Code(UInt32 *ivAes, Byte *data, size_t numBlocks); - -void MY_FAST_CALL AesCbc_Encode_Intel(UInt32 *p, Byte *data, size_t numBlocks) -{ - AesCbc_Encode(p, data, numBlocks); -} - -void MY_FAST_CALL AesCbc_Decode_Intel(UInt32 *p, Byte *data, size_t numBlocks) -{ - AesCbc_Decode(p, data, numBlocks); -} - -void MY_FAST_CALL AesCtr_Code_Intel(UInt32 *p, Byte *data, size_t numBlocks) -{ - AesCtr_Code(p, data, numBlocks); -} - -#endif diff --git a/code/ryzom/client/src/seven_zip/Alloc.cpp b/code/ryzom/client/src/seven_zip/Alloc.cpp deleted file mode 100644 index bcede4b85..000000000 --- a/code/ryzom/client/src/seven_zip/Alloc.cpp +++ /dev/null @@ -1,455 +0,0 @@ -/* Alloc.c -- Memory allocation functions -2018-04-27 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include - -#ifdef _WIN32 -#include -#endif -#include - -#include "Alloc.h" - -/* #define _SZ_ALLOC_DEBUG */ - -/* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ -#ifdef _SZ_ALLOC_DEBUG - -#include -int g_allocCount = 0; -int g_allocCountMid = 0; -int g_allocCountBig = 0; - - -#define CONVERT_INT_TO_STR(charType, tempSize) \ - unsigned char temp[tempSize]; unsigned i = 0; \ - while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \ - *s++ = (charType)('0' + (unsigned)val); \ - while (i != 0) { i--; *s++ = temp[i]; } \ - *s = 0; - -static void ConvertUInt64ToString(UInt64 val, char *s) -{ - CONVERT_INT_TO_STR(char, 24); -} - -#define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) - -static void ConvertUInt64ToHex(UInt64 val, char *s) -{ - UInt64 v = val; - unsigned i; - for (i = 1;; i++) - { - v >>= 4; - if (v == 0) - break; - } - s[i] = 0; - do - { - unsigned t = (unsigned)(val & 0xF); - val >>= 4; - s[--i] = GET_HEX_CHAR(t); - } - while (i); -} - -#define DEBUG_OUT_STREAM stderr - -static void Print(const char *s) -{ - fputs(s, DEBUG_OUT_STREAM); -} - -static void PrintAligned(const char *s, size_t align) -{ - size_t len = strlen(s); - for(;;) - { - fputc(' ', DEBUG_OUT_STREAM); - if (len >= align) - break; - ++len; - } - Print(s); -} - -static void PrintLn() -{ - Print("\n"); -} - -static void PrintHex(UInt64 v, size_t align) -{ - char s[32]; - ConvertUInt64ToHex(v, s); - PrintAligned(s, align); -} - -static void PrintDec(UInt64 v, size_t align) -{ - char s[32]; - ConvertUInt64ToString(v, s); - PrintAligned(s, align); -} - -static void PrintAddr(void *p) -{ - PrintHex((UInt64)(size_t)(ptrdiff_t)p, 12); -} - - -#define PRINT_ALLOC(name, cnt, size, ptr) \ - Print(name " "); \ - PrintDec(cnt++, 10); \ - PrintHex(size, 10); \ - PrintAddr(ptr); \ - PrintLn(); - -#define PRINT_FREE(name, cnt, ptr) if (ptr) { \ - Print(name " "); \ - PrintDec(--cnt, 10); \ - PrintAddr(ptr); \ - PrintLn(); } - -#else - -#define PRINT_ALLOC(name, cnt, size, ptr) -#define PRINT_FREE(name, cnt, ptr) -#define Print(s) -#define PrintLn() -#define PrintHex(v, align) -#define PrintDec(v, align) -#define PrintAddr(p) - -#endif - - - -void *MyAlloc(size_t size) -{ - if (size == 0) - return NULL; - #ifdef _SZ_ALLOC_DEBUG - { - void *p = malloc(size); - PRINT_ALLOC("Alloc ", g_allocCount, size, p); - return p; - } - #else - return malloc(size); - #endif -} - -void MyFree(void *address) -{ - PRINT_FREE("Free ", g_allocCount, address); - - free(address); -} - -#ifdef _WIN32 - -void *MidAlloc(size_t size) -{ - if (size == 0) - return NULL; - - PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, NULL); - - return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); -} - -void MidFree(void *address) -{ - PRINT_FREE("Free-Mid", g_allocCountMid, address); - - if (!address) - return; - VirtualFree(address, 0, MEM_RELEASE); -} - -#ifndef MEM_LARGE_PAGES -#undef _7ZIP_LARGE_PAGES -#endif - -#ifdef _7ZIP_LARGE_PAGES -SIZE_T g_LargePageSize = 0; -typedef SIZE_T (WINAPI *GetLargePageMinimumP)(); -#endif - -void SetLargePageSize() -{ - #ifdef _7ZIP_LARGE_PAGES - SIZE_T size; - GetLargePageMinimumP largePageMinimum = (GetLargePageMinimumP) - GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum"); - if (!largePageMinimum) - return; - size = largePageMinimum(); - if (size == 0 || (size & (size - 1)) != 0) - return; - g_LargePageSize = size; - #endif -} - - -void *BigAlloc(size_t size) -{ - if (size == 0) - return NULL; - - PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL); - - #ifdef _7ZIP_LARGE_PAGES - { - SIZE_T ps = g_LargePageSize; - if (ps != 0 && ps <= (1 << 30) && size > (ps / 2)) - { - size_t size2; - ps--; - size2 = (size + ps) & ~ps; - if (size2 >= size) - { - void *res = VirtualAlloc(NULL, size2, MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); - if (res) - return res; - } - } - } - #endif - - return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); -} - -void BigFree(void *address) -{ - PRINT_FREE("Free-Big", g_allocCountBig, address); - - if (!address) - return; - VirtualFree(address, 0, MEM_RELEASE); -} - -#endif - - -static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } -static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MyFree(address); } -const ISzAlloc g_Alloc = { SzAlloc, SzFree }; - -static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MidAlloc(size); } -static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MidFree(address); } -const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; - -static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } -static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); BigFree(address); } -const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; - - -/* - uintptr_t : C99 (optional) - : unsupported in VS6 -*/ - -#ifdef _WIN32 - typedef UINT_PTR UIntPtr; -#else - /* - typedef uintptr_t UIntPtr; - */ - typedef ptrdiff_t UIntPtr; -#endif - - -#define ADJUST_ALLOC_SIZE 0 -/* -#define ADJUST_ALLOC_SIZE (sizeof(void *) - 1) -*/ -/* - Use (ADJUST_ALLOC_SIZE = (sizeof(void *) - 1)), if - MyAlloc() can return address that is NOT multiple of sizeof(void *). -*/ - - -/* -#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((char *)(p) - ((size_t)(UIntPtr)(p) & ((align) - 1)))) -*/ -#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1)))) - -#define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align) - - -#if (_POSIX_C_SOURCE >= 200112L) && !defined(_WIN32) - #define USE_posix_memalign -#endif - -/* - This posix_memalign() is for test purposes only. - We also need special Free() function instead of free(), - if this posix_memalign() is used. -*/ - -/* -static int posix_memalign(void **ptr, size_t align, size_t size) -{ - size_t newSize = size + align; - void *p; - void *pAligned; - *ptr = NULL; - if (newSize < size) - return 12; // ENOMEM - p = MyAlloc(newSize); - if (!p) - return 12; // ENOMEM - pAligned = MY_ALIGN_PTR_UP_PLUS(p, align); - ((void **)pAligned)[-1] = p; - *ptr = pAligned; - return 0; -} -*/ - -/* - ALLOC_ALIGN_SIZE >= sizeof(void *) - ALLOC_ALIGN_SIZE >= cache_line_size -*/ - -#define ALLOC_ALIGN_SIZE ((size_t)1 << 7) - -static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) -{ - #ifndef USE_posix_memalign - - void *p; - void *pAligned; - size_t newSize; - UNUSED_VAR(pp); - - /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned - block to prevent cache line sharing with another allocated blocks */ - - newSize = size + ALLOC_ALIGN_SIZE * 1 + ADJUST_ALLOC_SIZE; - if (newSize < size) - return NULL; - - p = MyAlloc(newSize); - - if (!p) - return NULL; - pAligned = MY_ALIGN_PTR_UP_PLUS(p, ALLOC_ALIGN_SIZE); - - Print(" size="); PrintHex(size, 8); - Print(" a_size="); PrintHex(newSize, 8); - Print(" ptr="); PrintAddr(p); - Print(" a_ptr="); PrintAddr(pAligned); - PrintLn(); - - ((void **)pAligned)[-1] = p; - - return pAligned; - - #else - - void *p; - UNUSED_VAR(pp); - if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size)) - return NULL; - - Print(" posix_memalign="); PrintAddr(p); - PrintLn(); - - return p; - - #endif -} - - -static void SzAlignedFree(ISzAllocPtr pp, void *address) -{ - UNUSED_VAR(pp); - #ifndef USE_posix_memalign - if (address) - MyFree(((void **)address)[-1]); - #else - free(address); - #endif -} - - -const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree }; - - - -#define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) - -/* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */ -#define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] -/* -#define REAL_BLOCK_PTR_VAR(p) ((void **)(p))[-1] -*/ - -static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) -{ - CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); - void *adr; - void *pAligned; - size_t newSize; - size_t extra; - size_t alignSize = (size_t)1 << p->numAlignBits; - - if (alignSize < sizeof(void *)) - alignSize = sizeof(void *); - - if (p->offset >= alignSize) - return NULL; - - /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned - block to prevent cache line sharing with another allocated blocks */ - extra = p->offset & (sizeof(void *) - 1); - newSize = size + alignSize + extra + ADJUST_ALLOC_SIZE; - if (newSize < size) - return NULL; - - adr = ISzAlloc_Alloc(p->baseAlloc, newSize); - - if (!adr) - return NULL; - - pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr + - alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset; - - PrintLn(); - Print("- Aligned: "); - Print(" size="); PrintHex(size, 8); - Print(" a_size="); PrintHex(newSize, 8); - Print(" ptr="); PrintAddr(adr); - Print(" a_ptr="); PrintAddr(pAligned); - PrintLn(); - - REAL_BLOCK_PTR_VAR(pAligned) = adr; - - return pAligned; -} - - -static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address) -{ - if (address) - { - CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); - PrintLn(); - Print("- Aligned Free: "); - PrintLn(); - ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address)); - } -} - - -void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p) -{ - p->vt.Alloc = AlignOffsetAlloc_Alloc; - p->vt.Free = AlignOffsetAlloc_Free; -} diff --git a/code/ryzom/client/src/seven_zip/Alloc.h b/code/ryzom/client/src/seven_zip/Alloc.h deleted file mode 100644 index 648237646..000000000 --- a/code/ryzom/client/src/seven_zip/Alloc.h +++ /dev/null @@ -1,51 +0,0 @@ -/* Alloc.h -- Memory allocation functions -2018-02-19 : Igor Pavlov : Public domain */ - -#ifndef __COMMON_ALLOC_H -#define __COMMON_ALLOC_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -void *MyAlloc(size_t size); -void MyFree(void *address); - -#ifdef _WIN32 - -void SetLargePageSize(); - -void *MidAlloc(size_t size); -void MidFree(void *address); -void *BigAlloc(size_t size); -void BigFree(void *address); - -#else - -#define MidAlloc(size) MyAlloc(size) -#define MidFree(address) MyFree(address) -#define BigAlloc(size) MyAlloc(size) -#define BigFree(address) MyFree(address) - -#endif - -extern const ISzAlloc g_Alloc; -extern const ISzAlloc g_BigAlloc; -extern const ISzAlloc g_MidAlloc; -extern const ISzAlloc g_AlignedAlloc; - - -typedef struct -{ - ISzAlloc vt; - ISzAllocPtr baseAlloc; - unsigned numAlignBits; /* ((1 << numAlignBits) >= sizeof(void *)) */ - size_t offset; /* (offset == (k * sizeof(void *)) && offset < (1 << numAlignBits) */ -} CAlignOffsetAlloc; - -void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p); - - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/Bcj2.cpp b/code/ryzom/client/src/seven_zip/Bcj2.cpp deleted file mode 100644 index 9a0046a65..000000000 --- a/code/ryzom/client/src/seven_zip/Bcj2.cpp +++ /dev/null @@ -1,257 +0,0 @@ -/* Bcj2.c -- BCJ2 Decoder (Converter for x86 code) -2018-04-28 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "Bcj2.h" -#include "CpuArch.h" - -#define CProb UInt16 - -#define kTopValue ((UInt32)1 << 24) -#define kNumModelBits 11 -#define kBitModelTotal (1 << kNumModelBits) -#define kNumMoveBits 5 - -#define _IF_BIT_0 ttt = *prob; bound = (p->range >> kNumModelBits) * ttt; if (p->code < bound) -#define _UPDATE_0 p->range = bound; *prob = (CProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); -#define _UPDATE_1 p->range -= bound; p->code -= bound; *prob = (CProb)(ttt - (ttt >> kNumMoveBits)); - -void Bcj2Dec_Init(CBcj2Dec *p) -{ - unsigned i; - - p->state = BCJ2_DEC_STATE_OK; - p->ip = 0; - p->temp[3] = 0; - p->range = 0; - p->code = 0; - for (i = 0; i < sizeof(p->probs) / sizeof(p->probs[0]); i++) - p->probs[i] = kBitModelTotal >> 1; -} - -SRes Bcj2Dec_Decode(CBcj2Dec *p) -{ - if (p->range <= 5) - { - p->state = BCJ2_DEC_STATE_OK; - for (; p->range != 5; p->range++) - { - if (p->range == 1 && p->code != 0) - return SZ_ERROR_DATA; - - if (p->bufs[BCJ2_STREAM_RC] == p->lims[BCJ2_STREAM_RC]) - { - p->state = BCJ2_STREAM_RC; - return SZ_OK; - } - - p->code = (p->code << 8) | *(p->bufs[BCJ2_STREAM_RC])++; - } - - if (p->code == 0xFFFFFFFF) - return SZ_ERROR_DATA; - - p->range = 0xFFFFFFFF; - } - else if (p->state >= BCJ2_DEC_STATE_ORIG_0) - { - while (p->state <= BCJ2_DEC_STATE_ORIG_3) - { - Byte *dest = p->dest; - if (dest == p->destLim) - return SZ_OK; - *dest = p->temp[(size_t)p->state - BCJ2_DEC_STATE_ORIG_0]; - p->state++; - p->dest = dest + 1; - } - } - - /* - if (BCJ2_IS_32BIT_STREAM(p->state)) - { - const Byte *cur = p->bufs[p->state]; - if (cur == p->lims[p->state]) - return SZ_OK; - p->bufs[p->state] = cur + 4; - - { - UInt32 val; - Byte *dest; - SizeT rem; - - p->ip += 4; - val = GetBe32(cur) - p->ip; - dest = p->dest; - rem = p->destLim - dest; - if (rem < 4) - { - SizeT i; - SetUi32(p->temp, val); - for (i = 0; i < rem; i++) - dest[i] = p->temp[i]; - p->dest = dest + rem; - p->state = BCJ2_DEC_STATE_ORIG_0 + (unsigned)rem; - return SZ_OK; - } - SetUi32(dest, val); - p->temp[3] = (Byte)(val >> 24); - p->dest = dest + 4; - p->state = BCJ2_DEC_STATE_OK; - } - } - */ - - for (;;) - { - if (BCJ2_IS_32BIT_STREAM(p->state)) - p->state = BCJ2_DEC_STATE_OK; - else - { - if (p->range < kTopValue) - { - if (p->bufs[BCJ2_STREAM_RC] == p->lims[BCJ2_STREAM_RC]) - { - p->state = BCJ2_STREAM_RC; - return SZ_OK; - } - p->range <<= 8; - p->code = (p->code << 8) | *(p->bufs[BCJ2_STREAM_RC])++; - } - - { - const Byte *src = p->bufs[BCJ2_STREAM_MAIN]; - const Byte *srcLim; - Byte *dest; - SizeT num = p->lims[BCJ2_STREAM_MAIN] - src; - - if (num == 0) - { - p->state = BCJ2_STREAM_MAIN; - return SZ_OK; - } - - dest = p->dest; - if (num > (SizeT)(p->destLim - dest)) - { - num = p->destLim - dest; - if (num == 0) - { - p->state = BCJ2_DEC_STATE_ORIG; - return SZ_OK; - } - } - - srcLim = src + num; - - if (p->temp[3] == 0x0F && (src[0] & 0xF0) == 0x80) - *dest = src[0]; - else for (;;) - { - Byte b = *src; - *dest = b; - if (b != 0x0F) - { - if ((b & 0xFE) == 0xE8) - break; - dest++; - if (++src != srcLim) - continue; - break; - } - dest++; - if (++src == srcLim) - break; - if ((*src & 0xF0) != 0x80) - continue; - *dest = *src; - break; - } - - num = src - p->bufs[BCJ2_STREAM_MAIN]; - - if (src == srcLim) - { - p->temp[3] = src[-1]; - p->bufs[BCJ2_STREAM_MAIN] = src; - p->ip += (UInt32)num; - p->dest += num; - p->state = - p->bufs[BCJ2_STREAM_MAIN] == - p->lims[BCJ2_STREAM_MAIN] ? - (unsigned)BCJ2_STREAM_MAIN : - (unsigned)BCJ2_DEC_STATE_ORIG; - return SZ_OK; - } - - { - UInt32 bound, ttt; - CProb *prob; - Byte b = src[0]; - Byte prev = (Byte)(num == 0 ? p->temp[3] : src[-1]); - - p->temp[3] = b; - p->bufs[BCJ2_STREAM_MAIN] = src + 1; - num++; - p->ip += (UInt32)num; - p->dest += num; - - prob = p->probs + (unsigned)(b == 0xE8 ? 2 + (unsigned)prev : (b == 0xE9 ? 1 : 0)); - - _IF_BIT_0 - { - _UPDATE_0 - continue; - } - _UPDATE_1 - - } - } - } - - { - UInt32 val; - unsigned cj = (p->temp[3] == 0xE8) ? BCJ2_STREAM_CALL : BCJ2_STREAM_JUMP; - const Byte *cur = p->bufs[cj]; - Byte *dest; - SizeT rem; - - if (cur == p->lims[cj]) - { - p->state = cj; - break; - } - - val = GetBe32(cur); - p->bufs[cj] = cur + 4; - - p->ip += 4; - val -= p->ip; - dest = p->dest; - rem = p->destLim - dest; - - if (rem < 4) - { - p->temp[0] = (Byte)val; if (rem > 0) dest[0] = (Byte)val; val >>= 8; - p->temp[1] = (Byte)val; if (rem > 1) dest[1] = (Byte)val; val >>= 8; - p->temp[2] = (Byte)val; if (rem > 2) dest[2] = (Byte)val; val >>= 8; - p->temp[3] = (Byte)val; - p->dest = dest + rem; - p->state = BCJ2_DEC_STATE_ORIG_0 + (unsigned)rem; - break; - } - - SetUi32(dest, val); - p->temp[3] = (Byte)(val >> 24); - p->dest = dest + 4; - } - } - - if (p->range < kTopValue && p->bufs[BCJ2_STREAM_RC] != p->lims[BCJ2_STREAM_RC]) - { - p->range <<= 8; - p->code = (p->code << 8) | *(p->bufs[BCJ2_STREAM_RC])++; - } - - return SZ_OK; -} diff --git a/code/ryzom/client/src/seven_zip/Bcj2.h b/code/ryzom/client/src/seven_zip/Bcj2.h deleted file mode 100644 index 8824080ac..000000000 --- a/code/ryzom/client/src/seven_zip/Bcj2.h +++ /dev/null @@ -1,146 +0,0 @@ -/* Bcj2.h -- BCJ2 Converter for x86 code -2014-11-10 : Igor Pavlov : Public domain */ - -#ifndef __BCJ2_H -#define __BCJ2_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -#define BCJ2_NUM_STREAMS 4 - -enum -{ - BCJ2_STREAM_MAIN, - BCJ2_STREAM_CALL, - BCJ2_STREAM_JUMP, - BCJ2_STREAM_RC -}; - -enum -{ - BCJ2_DEC_STATE_ORIG_0 = BCJ2_NUM_STREAMS, - BCJ2_DEC_STATE_ORIG_1, - BCJ2_DEC_STATE_ORIG_2, - BCJ2_DEC_STATE_ORIG_3, - - BCJ2_DEC_STATE_ORIG, - BCJ2_DEC_STATE_OK -}; - -enum -{ - BCJ2_ENC_STATE_ORIG = BCJ2_NUM_STREAMS, - BCJ2_ENC_STATE_OK -}; - - -#define BCJ2_IS_32BIT_STREAM(s) ((s) == BCJ2_STREAM_CALL || (s) == BCJ2_STREAM_JUMP) - -/* -CBcj2Dec / CBcj2Enc -bufs sizes: - BUF_SIZE(n) = lims[n] - bufs[n] -bufs sizes for BCJ2_STREAM_CALL and BCJ2_STREAM_JUMP must be mutliply of 4: - (BUF_SIZE(BCJ2_STREAM_CALL) & 3) == 0 - (BUF_SIZE(BCJ2_STREAM_JUMP) & 3) == 0 -*/ - -/* -CBcj2Dec: -dest is allowed to overlap with bufs[BCJ2_STREAM_MAIN], with the following conditions: - bufs[BCJ2_STREAM_MAIN] >= dest && - bufs[BCJ2_STREAM_MAIN] - dest >= tempReserv + - BUF_SIZE(BCJ2_STREAM_CALL) + - BUF_SIZE(BCJ2_STREAM_JUMP) - tempReserv = 0 : for first call of Bcj2Dec_Decode - tempReserv = 4 : for any other calls of Bcj2Dec_Decode - overlap with offset = 1 is not allowed -*/ - -typedef struct -{ - const Byte *bufs[BCJ2_NUM_STREAMS]; - const Byte *lims[BCJ2_NUM_STREAMS]; - Byte *dest; - const Byte *destLim; - - unsigned state; /* BCJ2_STREAM_MAIN has more priority than BCJ2_STATE_ORIG */ - - UInt32 ip; - Byte temp[4]; - UInt32 range; - UInt32 code; - UInt16 probs[2 + 256]; -} CBcj2Dec; - -void Bcj2Dec_Init(CBcj2Dec *p); - -/* Returns: SZ_OK or SZ_ERROR_DATA */ -SRes Bcj2Dec_Decode(CBcj2Dec *p); - -#define Bcj2Dec_IsFinished(_p_) ((_p_)->code == 0) - - - -typedef enum -{ - BCJ2_ENC_FINISH_MODE_CONTINUE, - BCJ2_ENC_FINISH_MODE_END_BLOCK, - BCJ2_ENC_FINISH_MODE_END_STREAM -} EBcj2Enc_FinishMode; - -typedef struct -{ - Byte *bufs[BCJ2_NUM_STREAMS]; - const Byte *lims[BCJ2_NUM_STREAMS]; - const Byte *src; - const Byte *srcLim; - - unsigned state; - EBcj2Enc_FinishMode finishMode; - - Byte prevByte; - - Byte cache; - UInt32 range; - UInt64 low; - UInt64 cacheSize; - - UInt32 ip; - - /* 32-bit ralative offset in JUMP/CALL commands is - - (mod 4 GB) in 32-bit mode - - signed Int32 in 64-bit mode - We use (mod 4 GB) check for fileSize. - Use fileSize up to 2 GB, if you want to support 32-bit and 64-bit code conversion. */ - UInt32 fileIp; - UInt32 fileSize; /* (fileSize <= ((UInt32)1 << 31)), 0 means no_limit */ - UInt32 relatLimit; /* (relatLimit <= ((UInt32)1 << 31)), 0 means desable_conversion */ - - UInt32 tempTarget; - unsigned tempPos; - Byte temp[4 * 2]; - - unsigned flushPos; - - UInt16 probs[2 + 256]; -} CBcj2Enc; - -void Bcj2Enc_Init(CBcj2Enc *p); -void Bcj2Enc_Encode(CBcj2Enc *p); - -#define Bcj2Enc_Get_InputData_Size(p) ((SizeT)((p)->srcLim - (p)->src) + (p)->tempPos) -#define Bcj2Enc_IsFinished(p) ((p)->flushPos == 5) - - -#define BCJ2_RELAT_LIMIT_NUM_BITS 26 -#define BCJ2_RELAT_LIMIT ((UInt32)1 << BCJ2_RELAT_LIMIT_NUM_BITS) - -/* limit for CBcj2Enc::fileSize variable */ -#define BCJ2_FileSize_MAX ((UInt32)1 << 31) - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/Bcj2Enc.cpp b/code/ryzom/client/src/seven_zip/Bcj2Enc.cpp deleted file mode 100644 index bfbeb8e49..000000000 --- a/code/ryzom/client/src/seven_zip/Bcj2Enc.cpp +++ /dev/null @@ -1,311 +0,0 @@ -/* Bcj2Enc.c -- BCJ2 Encoder (Converter for x86 code) -2019-02-02 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -/* #define SHOW_STAT */ - -#ifdef SHOW_STAT -#include -#define PRF(x) x -#else -#define PRF(x) -#endif - -#include - -#include "Bcj2.h" -#include "CpuArch.h" - -#define CProb UInt16 - -#define kTopValue ((UInt32)1 << 24) -#define kNumModelBits 11 -#define kBitModelTotal (1 << kNumModelBits) -#define kNumMoveBits 5 - -void Bcj2Enc_Init(CBcj2Enc *p) -{ - unsigned i; - - p->state = BCJ2_ENC_STATE_OK; - p->finishMode = BCJ2_ENC_FINISH_MODE_CONTINUE; - - p->prevByte = 0; - - p->cache = 0; - p->range = 0xFFFFFFFF; - p->low = 0; - p->cacheSize = 1; - - p->ip = 0; - - p->fileIp = 0; - p->fileSize = 0; - p->relatLimit = BCJ2_RELAT_LIMIT; - - p->tempPos = 0; - - p->flushPos = 0; - - for (i = 0; i < sizeof(p->probs) / sizeof(p->probs[0]); i++) - p->probs[i] = kBitModelTotal >> 1; -} - -static BoolInt MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) -{ - if ((UInt32)p->low < (UInt32)0xFF000000 || (UInt32)(p->low >> 32) != 0) - { - Byte *buf = p->bufs[BCJ2_STREAM_RC]; - do - { - if (buf == p->lims[BCJ2_STREAM_RC]) - { - p->state = BCJ2_STREAM_RC; - p->bufs[BCJ2_STREAM_RC] = buf; - return True; - } - *buf++ = (Byte)(p->cache + (Byte)(p->low >> 32)); - p->cache = 0xFF; - } - while (--p->cacheSize); - p->bufs[BCJ2_STREAM_RC] = buf; - p->cache = (Byte)((UInt32)p->low >> 24); - } - p->cacheSize++; - p->low = (UInt32)p->low << 8; - return False; -} - -static void Bcj2Enc_Encode_2(CBcj2Enc *p) -{ - if (BCJ2_IS_32BIT_STREAM(p->state)) - { - Byte *cur = p->bufs[p->state]; - if (cur == p->lims[p->state]) - return; - SetBe32(cur, p->tempTarget); - p->bufs[p->state] = cur + 4; - } - - p->state = BCJ2_ENC_STATE_ORIG; - - for (;;) - { - if (p->range < kTopValue) - { - if (RangeEnc_ShiftLow(p)) - return; - p->range <<= 8; - } - - { - { - const Byte *src = p->src; - const Byte *srcLim; - Byte *dest; - SizeT num = p->srcLim - src; - - if (p->finishMode == BCJ2_ENC_FINISH_MODE_CONTINUE) - { - if (num <= 4) - return; - num -= 4; - } - else if (num == 0) - break; - - dest = p->bufs[BCJ2_STREAM_MAIN]; - if (num > (SizeT)(p->lims[BCJ2_STREAM_MAIN] - dest)) - { - num = p->lims[BCJ2_STREAM_MAIN] - dest; - if (num == 0) - { - p->state = BCJ2_STREAM_MAIN; - return; - } - } - - srcLim = src + num; - - if (p->prevByte == 0x0F && (src[0] & 0xF0) == 0x80) - *dest = src[0]; - else for (;;) - { - Byte b = *src; - *dest = b; - if (b != 0x0F) - { - if ((b & 0xFE) == 0xE8) - break; - dest++; - if (++src != srcLim) - continue; - break; - } - dest++; - if (++src == srcLim) - break; - if ((*src & 0xF0) != 0x80) - continue; - *dest = *src; - break; - } - - num = src - p->src; - - if (src == srcLim) - { - p->prevByte = src[-1]; - p->bufs[BCJ2_STREAM_MAIN] = dest; - p->src = src; - p->ip += (UInt32)num; - continue; - } - - { - Byte context = (Byte)(num == 0 ? p->prevByte : src[-1]); - BoolInt needConvert; - - p->bufs[BCJ2_STREAM_MAIN] = dest + 1; - p->ip += (UInt32)num + 1; - src++; - - needConvert = False; - - if ((SizeT)(p->srcLim - src) >= 4) - { - UInt32 relatVal = GetUi32(src); - if ((p->fileSize == 0 || (UInt32)(p->ip + 4 + relatVal - p->fileIp) < p->fileSize) - && ((relatVal + p->relatLimit) >> 1) < p->relatLimit) - needConvert = True; - } - - { - UInt32 bound; - unsigned ttt; - Byte b = src[-1]; - CProb *prob = p->probs + (unsigned)(b == 0xE8 ? 2 + (unsigned)context : (b == 0xE9 ? 1 : 0)); - - ttt = *prob; - bound = (p->range >> kNumModelBits) * ttt; - - if (!needConvert) - { - p->range = bound; - *prob = (CProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); - p->src = src; - p->prevByte = b; - continue; - } - - p->low += bound; - p->range -= bound; - *prob = (CProb)(ttt - (ttt >> kNumMoveBits)); - - { - UInt32 relatVal = GetUi32(src); - UInt32 absVal; - p->ip += 4; - absVal = p->ip + relatVal; - p->prevByte = src[3]; - src += 4; - p->src = src; - { - unsigned cj = (b == 0xE8) ? BCJ2_STREAM_CALL : BCJ2_STREAM_JUMP; - Byte *cur = p->bufs[cj]; - if (cur == p->lims[cj]) - { - p->state = cj; - p->tempTarget = absVal; - return; - } - SetBe32(cur, absVal); - p->bufs[cj] = cur + 4; - } - } - } - } - } - } - } - - if (p->finishMode != BCJ2_ENC_FINISH_MODE_END_STREAM) - return; - - for (; p->flushPos < 5; p->flushPos++) - if (RangeEnc_ShiftLow(p)) - return; - p->state = BCJ2_ENC_STATE_OK; -} - - -void Bcj2Enc_Encode(CBcj2Enc *p) -{ - PRF(printf("\n")); - PRF(printf("---- ip = %8d tempPos = %8d src = %8d\n", p->ip, p->tempPos, p->srcLim - p->src)); - - if (p->tempPos != 0) - { - unsigned extra = 0; - - for (;;) - { - const Byte *src = p->src; - const Byte *srcLim = p->srcLim; - EBcj2Enc_FinishMode finishMode = p->finishMode; - - p->src = p->temp; - p->srcLim = p->temp + p->tempPos; - if (src != srcLim) - p->finishMode = BCJ2_ENC_FINISH_MODE_CONTINUE; - - PRF(printf(" ip = %8d tempPos = %8d src = %8d\n", p->ip, p->tempPos, p->srcLim - p->src)); - - Bcj2Enc_Encode_2(p); - - { - unsigned num = (unsigned)(p->src - p->temp); - unsigned tempPos = p->tempPos - num; - unsigned i; - p->tempPos = tempPos; - for (i = 0; i < tempPos; i++) - p->temp[i] = p->temp[(size_t)i + num]; - - p->src = src; - p->srcLim = srcLim; - p->finishMode = finishMode; - - if (p->state != BCJ2_ENC_STATE_ORIG || src == srcLim) - return; - - if (extra >= tempPos) - { - p->src = src - tempPos; - p->tempPos = 0; - break; - } - - p->temp[tempPos] = src[0]; - p->tempPos = tempPos + 1; - p->src = src + 1; - extra++; - } - } - } - - PRF(printf("++++ ip = %8d tempPos = %8d src = %8d\n", p->ip, p->tempPos, p->srcLim - p->src)); - - Bcj2Enc_Encode_2(p); - - if (p->state == BCJ2_ENC_STATE_ORIG) - { - const Byte *src = p->src; - unsigned rem = (unsigned)(p->srcLim - src); - unsigned i; - for (i = 0; i < rem; i++) - p->temp[i] = src[i]; - p->tempPos = rem; - p->src = src + rem; - } -} diff --git a/code/ryzom/client/src/seven_zip/Bra.cpp b/code/ryzom/client/src/seven_zip/Bra.cpp deleted file mode 100644 index aed17e330..000000000 --- a/code/ryzom/client/src/seven_zip/Bra.cpp +++ /dev/null @@ -1,230 +0,0 @@ -/* Bra.c -- Converters for RISC code -2017-04-04 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "CpuArch.h" -#include "Bra.h" - -SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) -{ - Byte *p; - const Byte *lim; - size &= ~(size_t)3; - ip += 4; - p = data; - lim = data + size; - - if (encoding) - - for (;;) - { - for (;;) - { - if (p >= lim) - return p - data; - p += 4; - if (p[-1] == 0xEB) - break; - } - { - UInt32 v = GetUi32(p - 4); - v <<= 2; - v += ip + (UInt32)(p - data); - v >>= 2; - v &= 0x00FFFFFF; - v |= 0xEB000000; - SetUi32(p - 4, v); - } - } - - for (;;) - { - for (;;) - { - if (p >= lim) - return p - data; - p += 4; - if (p[-1] == 0xEB) - break; - } - { - UInt32 v = GetUi32(p - 4); - v <<= 2; - v -= ip + (UInt32)(p - data); - v >>= 2; - v &= 0x00FFFFFF; - v |= 0xEB000000; - SetUi32(p - 4, v); - } - } -} - - -SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) -{ - Byte *p; - const Byte *lim; - size &= ~(size_t)1; - p = data; - lim = data + size - 4; - - if (encoding) - - for (;;) - { - UInt32 b1; - for (;;) - { - UInt32 b3; - if (p > lim) - return p - data; - b1 = p[1]; - b3 = p[3]; - p += 2; - b1 ^= 8; - if ((b3 & b1) >= 0xF8) - break; - } - { - UInt32 v = - ((UInt32)b1 << 19) - + (((UInt32)p[1] & 0x7) << 8) - + (((UInt32)p[-2] << 11)) - + (p[0]); - - p += 2; - { - UInt32 cur = (ip + (UInt32)(p - data)) >> 1; - v += cur; - } - - p[-4] = (Byte)(v >> 11); - p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); - p[-2] = (Byte)v; - p[-1] = (Byte)(0xF8 | (v >> 8)); - } - } - - for (;;) - { - UInt32 b1; - for (;;) - { - UInt32 b3; - if (p > lim) - return p - data; - b1 = p[1]; - b3 = p[3]; - p += 2; - b1 ^= 8; - if ((b3 & b1) >= 0xF8) - break; - } - { - UInt32 v = - ((UInt32)b1 << 19) - + (((UInt32)p[1] & 0x7) << 8) - + (((UInt32)p[-2] << 11)) - + (p[0]); - - p += 2; - { - UInt32 cur = (ip + (UInt32)(p - data)) >> 1; - v -= cur; - } - - /* - SetUi16(p - 4, (UInt16)(((v >> 11) & 0x7FF) | 0xF000)); - SetUi16(p - 2, (UInt16)(v | 0xF800)); - */ - - p[-4] = (Byte)(v >> 11); - p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); - p[-2] = (Byte)v; - p[-1] = (Byte)(0xF8 | (v >> 8)); - } - } -} - - -SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) -{ - Byte *p; - const Byte *lim; - size &= ~(size_t)3; - ip -= 4; - p = data; - lim = data + size; - - for (;;) - { - for (;;) - { - if (p >= lim) - return p - data; - p += 4; - /* if ((v & 0xFC000003) == 0x48000001) */ - if ((p[-4] & 0xFC) == 0x48 && (p[-1] & 3) == 1) - break; - } - { - UInt32 v = GetBe32(p - 4); - if (encoding) - v += ip + (UInt32)(p - data); - else - v -= ip + (UInt32)(p - data); - v &= 0x03FFFFFF; - v |= 0x48000000; - SetBe32(p - 4, v); - } - } -} - - -SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) -{ - Byte *p; - const Byte *lim; - size &= ~(size_t)3; - ip -= 4; - p = data; - lim = data + size; - - for (;;) - { - for (;;) - { - if (p >= lim) - return p - data; - /* - v = GetBe32(p); - p += 4; - m = v + ((UInt32)5 << 29); - m ^= (UInt32)7 << 29; - m += (UInt32)1 << 22; - if ((m & ((UInt32)0x1FF << 23)) == 0) - break; - */ - p += 4; - if ((p[-4] == 0x40 && (p[-3] & 0xC0) == 0) || - (p[-4] == 0x7F && (p[-3] >= 0xC0))) - break; - } - { - UInt32 v = GetBe32(p - 4); - v <<= 2; - if (encoding) - v += ip + (UInt32)(p - data); - else - v -= ip + (UInt32)(p - data); - - v &= 0x01FFFFFF; - v -= (UInt32)1 << 24; - v ^= 0xFF000000; - v >>= 2; - v |= 0x40000000; - SetBe32(p - 4, v); - } - } -} diff --git a/code/ryzom/client/src/seven_zip/Bra.h b/code/ryzom/client/src/seven_zip/Bra.h deleted file mode 100644 index 855e37a6b..000000000 --- a/code/ryzom/client/src/seven_zip/Bra.h +++ /dev/null @@ -1,64 +0,0 @@ -/* Bra.h -- Branch converters for executables -2013-01-18 : Igor Pavlov : Public domain */ - -#ifndef __BRA_H -#define __BRA_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -/* -These functions convert relative addresses to absolute addresses -in CALL instructions to increase the compression ratio. - - In: - data - data buffer - size - size of data - ip - current virtual Instruction Pinter (IP) value - state - state variable for x86 converter - encoding - 0 (for decoding), 1 (for encoding) - - Out: - state - state variable for x86 converter - - Returns: - The number of processed bytes. If you call these functions with multiple calls, - you must start next call with first byte after block of processed bytes. - - Type Endian Alignment LookAhead - - x86 little 1 4 - ARMT little 2 2 - ARM little 4 0 - PPC big 4 0 - SPARC big 4 0 - IA64 little 16 0 - - size must be >= Alignment + LookAhead, if it's not last block. - If (size < Alignment + LookAhead), converter returns 0. - - Example: - - UInt32 ip = 0; - for () - { - ; size must be >= Alignment + LookAhead, if it's not last block - SizeT processed = Convert(data, size, ip, 1); - data += processed; - size -= processed; - ip += processed; - } -*/ - -#define x86_Convert_Init(state) { state = 0; } -SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding); -SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -SizeT IA64_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/Bra86.cpp b/code/ryzom/client/src/seven_zip/Bra86.cpp deleted file mode 100644 index 93ed4d762..000000000 --- a/code/ryzom/client/src/seven_zip/Bra86.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* Bra86.c -- Converter for x86 code (BCJ) -2017-04-03 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "Bra.h" - -#define Test86MSByte(b) ((((b) + 1) & 0xFE) == 0) - -SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding) -{ - SizeT pos = 0; - UInt32 mask = *state & 7; - if (size < 5) - return 0; - size -= 4; - ip += 5; - - for (;;) - { - Byte *p = data + pos; - const Byte *limit = data + size; - for (; p < limit; p++) - if ((*p & 0xFE) == 0xE8) - break; - - { - SizeT d = (SizeT)(p - data - pos); - pos = (SizeT)(p - data); - if (p >= limit) - { - *state = (d > 2 ? 0 : mask >> (unsigned)d); - return pos; - } - if (d > 2) - mask = 0; - else - { - mask >>= (unsigned)d; - if (mask != 0 && (mask > 4 || mask == 3 || Test86MSByte(p[(size_t)(mask >> 1) + 1]))) - { - mask = (mask >> 1) | 4; - pos++; - continue; - } - } - } - - if (Test86MSByte(p[4])) - { - UInt32 v = ((UInt32)p[4] << 24) | ((UInt32)p[3] << 16) | ((UInt32)p[2] << 8) | ((UInt32)p[1]); - UInt32 cur = ip + (UInt32)pos; - pos += 5; - if (encoding) - v += cur; - else - v -= cur; - if (mask != 0) - { - unsigned sh = (mask & 6) << 2; - if (Test86MSByte((Byte)(v >> sh))) - { - v ^= (((UInt32)0x100 << sh) - 1); - if (encoding) - v += cur; - else - v -= cur; - } - mask = 0; - } - p[1] = (Byte)v; - p[2] = (Byte)(v >> 8); - p[3] = (Byte)(v >> 16); - p[4] = (Byte)(0 - ((v >> 24) & 1)); - } - else - { - mask = (mask >> 1) | 4; - pos++; - } - } -} diff --git a/code/ryzom/client/src/seven_zip/BraIA64.cpp b/code/ryzom/client/src/seven_zip/BraIA64.cpp deleted file mode 100644 index d1dbc62c5..000000000 --- a/code/ryzom/client/src/seven_zip/BraIA64.cpp +++ /dev/null @@ -1,53 +0,0 @@ -/* BraIA64.c -- Converter for IA-64 code -2017-01-26 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "CpuArch.h" -#include "Bra.h" - -SizeT IA64_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) -{ - SizeT i; - if (size < 16) - return 0; - size -= 16; - i = 0; - do - { - unsigned m = ((UInt32)0x334B0000 >> (data[i] & 0x1E)) & 3; - if (m) - { - m++; - do - { - Byte *p = data + (i + (size_t)m * 5 - 8); - if (((p[3] >> m) & 15) == 5 - && (((p[-1] | ((UInt32)p[0] << 8)) >> m) & 0x70) == 0) - { - unsigned raw = GetUi32(p); - unsigned v = raw >> m; - v = (v & 0xFFFFF) | ((v & (1 << 23)) >> 3); - - v <<= 4; - if (encoding) - v += ip + (UInt32)i; - else - v -= ip + (UInt32)i; - v >>= 4; - - v &= 0x1FFFFF; - v += 0x700000; - v &= 0x8FFFFF; - raw &= ~((UInt32)0x8FFFFF << m); - raw |= (v << m); - SetUi32(p, raw); - } - } - while (++m <= 4); - } - i += 16; - } - while (i <= size); - return i; -} diff --git a/code/ryzom/client/src/seven_zip/CMakeLists.txt b/code/ryzom/client/src/seven_zip/CMakeLists.txt deleted file mode 100644 index d62750e17..000000000 --- a/code/ryzom/client/src/seven_zip/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) - -FILE(GLOB LIB_SRC *.cpp *.h) - -LIST(REMOVE_ITEM LIB_SRC ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) - -NL_TARGET_LIB(ryzom_sevenzip ${LIB_SRC}) -TARGET_LINK_LIBRARIES(ryzom_sevenzip ${NELMISC_LIBRARIES}) -NL_DEFAULT_PROPS(ryzom_sevenzip "Ryzom, Library: Seven Zip") -NL_ADD_RUNTIME_FLAGS(ryzom_sevenzip) -NL_ADD_LIB_SUFFIX(ryzom_sevenzip) - -ADD_DEFINITIONS(-DUNICODE -D_UNICODE -D_7ZIP_PPMD_SUPPPORT) - -IF(WITH_PCH) - ADD_NATIVE_PRECOMPILED_HEADER(ryzom_sevenzip ${CMAKE_CURRENT_SOURCE_DIR}/Precomp.h ${CMAKE_CURRENT_SOURCE_DIR}/Precomp.cpp) -ENDIF() - -IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) - INSTALL(TARGETS ryzom_sevenzip LIBRARY DESTINATION ${RYZOM_LIB_PREFIX} ARCHIVE DESTINATION ${RYZOM_LIB_PREFIX} COMPONENT libraries) -ENDIF() - -IF(WITH_RYZOM_TOOLS) - ADD_EXECUTABLE(7zDec ${CMAKE_CURRENT_SOURCE_DIR}/7zMain.cpp) - - TARGET_LINK_LIBRARIES(7zDec ryzom_sevenzip) - NL_DEFAULT_PROPS(7zDec "Ryzom, Tools, Misc: Seven Zip Decoder") - NL_ADD_RUNTIME_FLAGS(7zDec) - - INSTALL(TARGETS 7zDec RUNTIME DESTINATION ${RYZOM_BIN_PREFIX} COMPONENT tools) -ENDIF() diff --git a/code/ryzom/client/src/seven_zip/Compiler.h b/code/ryzom/client/src/seven_zip/Compiler.h deleted file mode 100644 index 0cc409d8a..000000000 --- a/code/ryzom/client/src/seven_zip/Compiler.h +++ /dev/null @@ -1,33 +0,0 @@ -/* Compiler.h -2017-04-03 : Igor Pavlov : Public domain */ - -#ifndef __7Z_COMPILER_H -#define __7Z_COMPILER_H - -#ifdef _MSC_VER - - #ifdef UNDER_CE - #define RPC_NO_WINDOWS_H - /* #pragma warning(disable : 4115) // '_RPC_ASYNC_STATE' : named type definition in parentheses */ - #pragma warning(disable : 4201) // nonstandard extension used : nameless struct/union - #pragma warning(disable : 4214) // nonstandard extension used : bit field types other than int - #endif - - #if _MSC_VER >= 1300 - #pragma warning(disable : 4996) // This function or variable may be unsafe - #else - #pragma warning(disable : 4511) // copy constructor could not be generated - #pragma warning(disable : 4512) // assignment operator could not be generated - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #pragma warning(disable : 4702) // unreachable code - #pragma warning(disable : 4710) // not inlined - #pragma warning(disable : 4714) // function marked as __forceinline not inlined - #pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information - #endif - -#endif - -#define UNUSED_VAR(x) (void)x; -/* #define UNUSED_VAR(x) x=x; */ - -#endif diff --git a/code/ryzom/client/src/seven_zip/CpuArch.cpp b/code/ryzom/client/src/seven_zip/CpuArch.cpp deleted file mode 100644 index 02e482e08..000000000 --- a/code/ryzom/client/src/seven_zip/CpuArch.cpp +++ /dev/null @@ -1,218 +0,0 @@ -/* CpuArch.c -- CPU specific code -2018-02-18: Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "CpuArch.h" - -#ifdef MY_CPU_X86_OR_AMD64 - -#if (defined(_MSC_VER) && !defined(MY_CPU_AMD64)) || defined(__GNUC__) -#define USE_ASM -#endif - -#if !defined(USE_ASM) && _MSC_VER >= 1500 -#include -#endif - -#if defined(USE_ASM) && !defined(MY_CPU_AMD64) -static UInt32 CheckFlag(UInt32 flag) -{ - #ifdef _MSC_VER - __asm pushfd; - __asm pop EAX; - __asm mov EDX, EAX; - __asm xor EAX, flag; - __asm push EAX; - __asm popfd; - __asm pushfd; - __asm pop EAX; - __asm xor EAX, EDX; - __asm push EDX; - __asm popfd; - __asm and flag, EAX; - #else - __asm__ __volatile__ ( - "pushf\n\t" - "pop %%EAX\n\t" - "movl %%EAX,%%EDX\n\t" - "xorl %0,%%EAX\n\t" - "push %%EAX\n\t" - "popf\n\t" - "pushf\n\t" - "pop %%EAX\n\t" - "xorl %%EDX,%%EAX\n\t" - "push %%EDX\n\t" - "popf\n\t" - "andl %%EAX, %0\n\t": - "=c" (flag) : "c" (flag) : - "%eax", "%edx"); - #endif - return flag; -} -#define CHECK_CPUID_IS_SUPPORTED if (CheckFlag(1 << 18) == 0 || CheckFlag(1 << 21) == 0) return False; -#else -#define CHECK_CPUID_IS_SUPPORTED -#endif - -void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d) -{ - #ifdef USE_ASM - - #ifdef _MSC_VER - - UInt32 a2, b2, c2, d2; - __asm xor EBX, EBX; - __asm xor ECX, ECX; - __asm xor EDX, EDX; - __asm mov EAX, function; - __asm cpuid; - __asm mov a2, EAX; - __asm mov b2, EBX; - __asm mov c2, ECX; - __asm mov d2, EDX; - - *a = a2; - *b = b2; - *c = c2; - *d = d2; - - #else - - __asm__ __volatile__ ( - #if defined(MY_CPU_AMD64) && defined(__PIC__) - "mov %%rbx, %%rdi;" - "cpuid;" - "xchg %%rbx, %%rdi;" - : "=a" (*a) , - "=D" (*b) , - #elif defined(MY_CPU_X86) && defined(__PIC__) - "mov %%ebx, %%edi;" - "cpuid;" - "xchgl %%ebx, %%edi;" - : "=a" (*a) , - "=D" (*b) , - #else - "cpuid" - : "=a" (*a) , - "=b" (*b) , - #endif - "=c" (*c) , - "=d" (*d) - : "0" (function)) ; - - #endif - - #else - - int CPUInfo[4]; - __cpuid(CPUInfo, function); - *a = CPUInfo[0]; - *b = CPUInfo[1]; - *c = CPUInfo[2]; - *d = CPUInfo[3]; - - #endif -} - -BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p) -{ - CHECK_CPUID_IS_SUPPORTED - MyCPUID(0, &p->maxFunc, &p->vendor[0], &p->vendor[2], &p->vendor[1]); - MyCPUID(1, &p->ver, &p->b, &p->c, &p->d); - return True; -} - -static const UInt32 kVendors[][3] = -{ - { 0x756E6547, 0x49656E69, 0x6C65746E}, - { 0x68747541, 0x69746E65, 0x444D4163}, - { 0x746E6543, 0x48727561, 0x736C7561} -}; - -int x86cpuid_GetFirm(const Cx86cpuid *p) -{ - unsigned i; - for (i = 0; i < sizeof(kVendors) / sizeof(kVendors[i]); i++) - { - const UInt32 *v = kVendors[i]; - if (v[0] == p->vendor[0] && - v[1] == p->vendor[1] && - v[2] == p->vendor[2]) - return (int)i; - } - return -1; -} - -BoolInt CPU_Is_InOrder() -{ - Cx86cpuid p; - int firm; - UInt32 family, model; - if (!x86cpuid_CheckAndRead(&p)) - return True; - - family = x86cpuid_GetFamily(p.ver); - model = x86cpuid_GetModel(p.ver); - - firm = x86cpuid_GetFirm(&p); - - switch (firm) - { - case CPU_FIRM_INTEL: return (family < 6 || (family == 6 && ( - /* In-Order Atom CPU */ - model == 0x1C /* 45 nm, N4xx, D4xx, N5xx, D5xx, 230, 330 */ - || model == 0x26 /* 45 nm, Z6xx */ - || model == 0x27 /* 32 nm, Z2460 */ - || model == 0x35 /* 32 nm, Z2760 */ - || model == 0x36 /* 32 nm, N2xxx, D2xxx */ - ))); - case CPU_FIRM_AMD: return (family < 5 || (family == 5 && (model < 6 || model == 0xA))); - case CPU_FIRM_VIA: return (family < 6 || (family == 6 && model < 0xF)); - } - return True; -} - -#if !defined(MY_CPU_AMD64) && defined(_WIN32) -#include -static BoolInt CPU_Sys_Is_SSE_Supported() -{ - OSVERSIONINFO vi; - vi.dwOSVersionInfoSize = sizeof(vi); - if (!GetVersionEx(&vi)) - return False; - return (vi.dwMajorVersion >= 5); -} -#define CHECK_SYS_SSE_SUPPORT if (!CPU_Sys_Is_SSE_Supported()) return False; -#else -#define CHECK_SYS_SSE_SUPPORT -#endif - -BoolInt CPU_Is_Aes_Supported() -{ - Cx86cpuid p; - CHECK_SYS_SSE_SUPPORT - if (!x86cpuid_CheckAndRead(&p)) - return False; - return (p.c >> 25) & 1; -} - -BoolInt CPU_IsSupported_PageGB() -{ - Cx86cpuid cpuid; - if (!x86cpuid_CheckAndRead(&cpuid)) - return False; - { - UInt32 d[4] = { 0 }; - MyCPUID(0x80000000, &d[0], &d[1], &d[2], &d[3]); - if (d[0] < 0x80000001) - return False; - } - { - UInt32 d[4] = { 0 }; - MyCPUID(0x80000001, &d[0], &d[1], &d[2], &d[3]); - return (d[3] >> 26) & 1; - } -} - -#endif diff --git a/code/ryzom/client/src/seven_zip/CpuArch.h b/code/ryzom/client/src/seven_zip/CpuArch.h deleted file mode 100644 index bd4293880..000000000 --- a/code/ryzom/client/src/seven_zip/CpuArch.h +++ /dev/null @@ -1,336 +0,0 @@ -/* CpuArch.h -- CPU specific code -2018-02-18 : Igor Pavlov : Public domain */ - -#ifndef __CPU_ARCH_H -#define __CPU_ARCH_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -/* -MY_CPU_LE means that CPU is LITTLE ENDIAN. -MY_CPU_BE means that CPU is BIG ENDIAN. -If MY_CPU_LE and MY_CPU_BE are not defined, we don't know about ENDIANNESS of platform. - -MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned memory accesses. -*/ - -#if defined(_M_X64) \ - || defined(_M_AMD64) \ - || defined(__x86_64__) \ - || defined(__AMD64__) \ - || defined(__amd64__) - #define MY_CPU_AMD64 - #ifdef __ILP32__ - #define MY_CPU_NAME "x32" - #else - #define MY_CPU_NAME "x64" - #endif - #define MY_CPU_64BIT -#endif - - -#if defined(_M_IX86) \ - || defined(__i386__) - #define MY_CPU_X86 - #define MY_CPU_NAME "x86" - #define MY_CPU_32BIT -#endif - - -#if defined(_M_ARM64) \ - || defined(__AARCH64EL__) \ - || defined(__AARCH64EB__) \ - || defined(__aarch64__) - #define MY_CPU_ARM64 - #define MY_CPU_NAME "arm64" - #define MY_CPU_64BIT -#endif - - -#if defined(_M_ARM) \ - || defined(_M_ARM_NT) \ - || defined(_M_ARMT) \ - || defined(__arm__) \ - || defined(__thumb__) \ - || defined(__ARMEL__) \ - || defined(__ARMEB__) \ - || defined(__THUMBEL__) \ - || defined(__THUMBEB__) - #define MY_CPU_ARM - #define MY_CPU_NAME "arm" - #define MY_CPU_32BIT -#endif - - -#if defined(_M_IA64) \ - || defined(__ia64__) - #define MY_CPU_IA64 - #define MY_CPU_NAME "ia64" - #define MY_CPU_64BIT -#endif - - -#if defined(__mips64) \ - || defined(__mips64__) \ - || (defined(__mips) && (__mips == 64 || __mips == 4 || __mips == 3)) - #define MY_CPU_NAME "mips64" - #define MY_CPU_64BIT -#elif defined(__mips__) - #define MY_CPU_NAME "mips" - /* #define MY_CPU_32BIT */ -#endif - - -#if defined(__ppc64__) \ - || defined(__powerpc64__) - #ifdef __ILP32__ - #define MY_CPU_NAME "ppc64-32" - #else - #define MY_CPU_NAME "ppc64" - #endif - #define MY_CPU_64BIT -#elif defined(__ppc__) \ - || defined(__powerpc__) - #define MY_CPU_NAME "ppc" - #define MY_CPU_32BIT -#endif - - -#if defined(__sparc64__) - #define MY_CPU_NAME "sparc64" - #define MY_CPU_64BIT -#elif defined(__sparc__) - #define MY_CPU_NAME "sparc" - /* #define MY_CPU_32BIT */ -#endif - - -#if defined(MY_CPU_X86) || defined(MY_CPU_AMD64) -#define MY_CPU_X86_OR_AMD64 -#endif - - -#ifdef _WIN32 - - #ifdef MY_CPU_ARM - #define MY_CPU_ARM_LE - #endif - - #ifdef MY_CPU_ARM64 - #define MY_CPU_ARM64_LE - #endif - - #ifdef _M_IA64 - #define MY_CPU_IA64_LE - #endif - -#endif - - -#if defined(MY_CPU_X86_OR_AMD64) \ - || defined(MY_CPU_ARM_LE) \ - || defined(MY_CPU_ARM64_LE) \ - || defined(MY_CPU_IA64_LE) \ - || defined(__LITTLE_ENDIAN__) \ - || defined(__ARMEL__) \ - || defined(__THUMBEL__) \ - || defined(__AARCH64EL__) \ - || defined(__MIPSEL__) \ - || defined(__MIPSEL) \ - || defined(_MIPSEL) \ - || defined(__BFIN__) \ - || (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) - #define MY_CPU_LE -#endif - -#if defined(__BIG_ENDIAN__) \ - || defined(__ARMEB__) \ - || defined(__THUMBEB__) \ - || defined(__AARCH64EB__) \ - || defined(__MIPSEB__) \ - || defined(__MIPSEB) \ - || defined(_MIPSEB) \ - || defined(__m68k__) \ - || defined(__s390__) \ - || defined(__s390x__) \ - || defined(__zarch__) \ - || (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) - #define MY_CPU_BE -#endif - - -#if defined(MY_CPU_LE) && defined(MY_CPU_BE) - #error Stop_Compiling_Bad_Endian -#endif - - -#if defined(MY_CPU_32BIT) && defined(MY_CPU_64BIT) - #error Stop_Compiling_Bad_32_64_BIT -#endif - - -#ifndef MY_CPU_NAME - #ifdef MY_CPU_LE - #define MY_CPU_NAME "LE" - #elif defined(MY_CPU_BE) - #define MY_CPU_NAME "BE" - #else - /* - #define MY_CPU_NAME "" - */ - #endif -#endif - - - - - -#ifdef MY_CPU_LE - #if defined(MY_CPU_X86_OR_AMD64) \ - || defined(MY_CPU_ARM64) \ - || defined(__ARM_FEATURE_UNALIGNED) - #define MY_CPU_LE_UNALIGN - #endif -#endif - - -#ifdef MY_CPU_LE_UNALIGN - -#define GetUi16(p) (*(const UInt16 *)(const void *)(p)) -#define GetUi32(p) (*(const UInt32 *)(const void *)(p)) -#define GetUi64(p) (*(const UInt64 *)(const void *)(p)) - -#define SetUi16(p, v) { *(UInt16 *)(p) = (v); } -#define SetUi32(p, v) { *(UInt32 *)(p) = (v); } -#define SetUi64(p, v) { *(UInt64 *)(p) = (v); } - -#else - -#define GetUi16(p) ( (UInt16) ( \ - ((const Byte *)(p))[0] | \ - ((UInt16)((const Byte *)(p))[1] << 8) )) - -#define GetUi32(p) ( \ - ((const Byte *)(p))[0] | \ - ((UInt32)((const Byte *)(p))[1] << 8) | \ - ((UInt32)((const Byte *)(p))[2] << 16) | \ - ((UInt32)((const Byte *)(p))[3] << 24)) - -#define GetUi64(p) (GetUi32(p) | ((UInt64)GetUi32(((const Byte *)(p)) + 4) << 32)) - -#define SetUi16(p, v) { Byte *_ppp_ = (Byte *)(p); UInt32 _vvv_ = (v); \ - _ppp_[0] = (Byte)_vvv_; \ - _ppp_[1] = (Byte)(_vvv_ >> 8); } - -#define SetUi32(p, v) { Byte *_ppp_ = (Byte *)(p); UInt32 _vvv_ = (v); \ - _ppp_[0] = (Byte)_vvv_; \ - _ppp_[1] = (Byte)(_vvv_ >> 8); \ - _ppp_[2] = (Byte)(_vvv_ >> 16); \ - _ppp_[3] = (Byte)(_vvv_ >> 24); } - -#define SetUi64(p, v) { Byte *_ppp2_ = (Byte *)(p); UInt64 _vvv2_ = (v); \ - SetUi32(_ppp2_ , (UInt32)_vvv2_); \ - SetUi32(_ppp2_ + 4, (UInt32)(_vvv2_ >> 32)); } - -#endif - -#ifdef __has_builtin - #define MY__has_builtin(x) __has_builtin(x) -#else - #define MY__has_builtin(x) 0 -#endif - -#if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ (_MSC_VER >= 1300) - -/* Note: we use bswap instruction, that is unsupported in 386 cpu */ - -#include - -#pragma intrinsic(_byteswap_ushort) -#pragma intrinsic(_byteswap_ulong) -#pragma intrinsic(_byteswap_uint64) - -/* #define GetBe16(p) _byteswap_ushort(*(const UInt16 *)(const Byte *)(p)) */ -#define GetBe32(p) _byteswap_ulong(*(const UInt32 *)(const Byte *)(p)) -#define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const Byte *)(p)) - -#define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = _byteswap_ulong(v) - -#elif defined(MY_CPU_LE_UNALIGN) && ( \ - (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \ - || (defined(__clang__) && MY__has_builtin(__builtin_bswap16)) ) - -/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const Byte *)(p)) */ -#define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const Byte *)(p)) -#define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const Byte *)(p)) - -#define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = __builtin_bswap32(v) - -#else - -#define GetBe32(p) ( \ - ((UInt32)((const Byte *)(p))[0] << 24) | \ - ((UInt32)((const Byte *)(p))[1] << 16) | \ - ((UInt32)((const Byte *)(p))[2] << 8) | \ - ((const Byte *)(p))[3] ) - -#define GetBe64(p) (((UInt64)GetBe32(p) << 32) | GetBe32(((const Byte *)(p)) + 4)) - -#define SetBe32(p, v) { Byte *_ppp_ = (Byte *)(p); UInt32 _vvv_ = (v); \ - _ppp_[0] = (Byte)(_vvv_ >> 24); \ - _ppp_[1] = (Byte)(_vvv_ >> 16); \ - _ppp_[2] = (Byte)(_vvv_ >> 8); \ - _ppp_[3] = (Byte)_vvv_; } - -#endif - - -#ifndef GetBe16 - -#define GetBe16(p) ( (UInt16) ( \ - ((UInt16)((const Byte *)(p))[0] << 8) | \ - ((const Byte *)(p))[1] )) - -#endif - - - -#ifdef MY_CPU_X86_OR_AMD64 - -typedef struct -{ - UInt32 maxFunc; - UInt32 vendor[3]; - UInt32 ver; - UInt32 b; - UInt32 c; - UInt32 d; -} Cx86cpuid; - -enum -{ - CPU_FIRM_INTEL, - CPU_FIRM_AMD, - CPU_FIRM_VIA -}; - -void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d); - -BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p); -int x86cpuid_GetFirm(const Cx86cpuid *p); - -#define x86cpuid_GetFamily(ver) (((ver >> 16) & 0xFF0) | ((ver >> 8) & 0xF)) -#define x86cpuid_GetModel(ver) (((ver >> 12) & 0xF0) | ((ver >> 4) & 0xF)) -#define x86cpuid_GetStepping(ver) (ver & 0xF) - -BoolInt CPU_Is_InOrder(); -BoolInt CPU_Is_Aes_Supported(); -BoolInt CPU_IsSupported_PageGB(); - -#endif - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/Delta.cpp b/code/ryzom/client/src/seven_zip/Delta.cpp deleted file mode 100644 index e3edd21ed..000000000 --- a/code/ryzom/client/src/seven_zip/Delta.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* Delta.c -- Delta converter -2009-05-26 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "Delta.h" - -void Delta_Init(Byte *state) -{ - unsigned i; - for (i = 0; i < DELTA_STATE_SIZE; i++) - state[i] = 0; -} - -static void MyMemCpy(Byte *dest, const Byte *src, unsigned size) -{ - unsigned i; - for (i = 0; i < size; i++) - dest[i] = src[i]; -} - -void Delta_Encode(Byte *state, unsigned delta, Byte *data, SizeT size) -{ - Byte buf[DELTA_STATE_SIZE]; - unsigned j = 0; - MyMemCpy(buf, state, delta); - { - SizeT i; - for (i = 0; i < size;) - { - for (j = 0; j < delta && i < size; i++, j++) - { - Byte b = data[i]; - data[i] = (Byte)(b - buf[j]); - buf[j] = b; - } - } - } - if (j == delta) - j = 0; - MyMemCpy(state, buf + j, delta - j); - MyMemCpy(state + delta - j, buf, j); -} - -void Delta_Decode(Byte *state, unsigned delta, Byte *data, SizeT size) -{ - Byte buf[DELTA_STATE_SIZE]; - unsigned j = 0; - MyMemCpy(buf, state, delta); - { - SizeT i; - for (i = 0; i < size;) - { - for (j = 0; j < delta && i < size; i++, j++) - { - buf[j] = data[i] = (Byte)(buf[j] + data[i]); - } - } - } - if (j == delta) - j = 0; - MyMemCpy(state, buf + j, delta - j); - MyMemCpy(state + delta - j, buf, j); -} diff --git a/code/ryzom/client/src/seven_zip/Delta.h b/code/ryzom/client/src/seven_zip/Delta.h deleted file mode 100644 index 2fa54ad67..000000000 --- a/code/ryzom/client/src/seven_zip/Delta.h +++ /dev/null @@ -1,19 +0,0 @@ -/* Delta.h -- Delta converter -2013-01-18 : Igor Pavlov : Public domain */ - -#ifndef __DELTA_H -#define __DELTA_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -#define DELTA_STATE_SIZE 256 - -void Delta_Init(Byte *state); -void Delta_Encode(Byte *state, unsigned delta, Byte *data, SizeT size); -void Delta_Decode(Byte *state, unsigned delta, Byte *data, SizeT size); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/LzFind.cpp b/code/ryzom/client/src/seven_zip/LzFind.cpp deleted file mode 100644 index df55e86c1..000000000 --- a/code/ryzom/client/src/seven_zip/LzFind.cpp +++ /dev/null @@ -1,1127 +0,0 @@ -/* LzFind.c -- Match finder for LZ algorithms -2018-07-08 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include - -#include "LzFind.h" -#include "LzHash.h" - -#define kEmptyHashValue 0 -#define kMaxValForNormalize ((UInt32)0xFFFFFFFF) -#define kNormalizeStepMin (1 << 10) /* it must be power of 2 */ -#define kNormalizeMask (~(UInt32)(kNormalizeStepMin - 1)) -#define kMaxHistorySize ((UInt32)7 << 29) - -#define kStartMaxLen 3 - -static void LzInWindow_Free(CMatchFinder *p, ISzAllocPtr alloc) -{ - if (!p->directInput) - { - ISzAlloc_Free(alloc, p->bufferBase); - p->bufferBase = NULL; - } -} - -/* keepSizeBefore + keepSizeAfter + keepSizeReserv must be < 4G) */ - -static int LzInWindow_Create(CMatchFinder *p, UInt32 keepSizeReserv, ISzAllocPtr alloc) -{ - UInt32 blockSize = p->keepSizeBefore + p->keepSizeAfter + keepSizeReserv; - if (p->directInput) - { - p->blockSize = blockSize; - return 1; - } - if (!p->bufferBase || p->blockSize != blockSize) - { - LzInWindow_Free(p, alloc); - p->blockSize = blockSize; - p->bufferBase = (Byte *)ISzAlloc_Alloc(alloc, (size_t)blockSize); - } - return (p->bufferBase != NULL); -} - -Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } - -UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return p->streamPos - p->pos; } - -void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue) -{ - p->posLimit -= subValue; - p->pos -= subValue; - p->streamPos -= subValue; -} - -static void MatchFinder_ReadBlock(CMatchFinder *p) -{ - if (p->streamEndWasReached || p->result != SZ_OK) - return; - - /* We use (p->streamPos - p->pos) value. (p->streamPos < p->pos) is allowed. */ - - if (p->directInput) - { - UInt32 curSize = 0xFFFFFFFF - (p->streamPos - p->pos); - if (curSize > p->directInputRem) - curSize = (UInt32)p->directInputRem; - p->directInputRem -= curSize; - p->streamPos += curSize; - if (p->directInputRem == 0) - p->streamEndWasReached = 1; - return; - } - - for (;;) - { - Byte *dest = p->buffer + (p->streamPos - p->pos); - size_t size = (p->bufferBase + p->blockSize - dest); - if (size == 0) - return; - - p->result = ISeqInStream_Read(p->stream, dest, &size); - if (p->result != SZ_OK) - return; - if (size == 0) - { - p->streamEndWasReached = 1; - return; - } - p->streamPos += (UInt32)size; - if (p->streamPos - p->pos > p->keepSizeAfter) - return; - } -} - -void MatchFinder_MoveBlock(CMatchFinder *p) -{ - memmove(p->bufferBase, - p->buffer - p->keepSizeBefore, - (size_t)(p->streamPos - p->pos) + p->keepSizeBefore); - p->buffer = p->bufferBase + p->keepSizeBefore; -} - -int MatchFinder_NeedMove(CMatchFinder *p) -{ - if (p->directInput) - return 0; - /* if (p->streamEndWasReached) return 0; */ - return ((size_t)(p->bufferBase + p->blockSize - p->buffer) <= p->keepSizeAfter); -} - -void MatchFinder_ReadIfRequired(CMatchFinder *p) -{ - if (p->streamEndWasReached) - return; - if (p->keepSizeAfter >= p->streamPos - p->pos) - MatchFinder_ReadBlock(p); -} - -static void MatchFinder_CheckAndMoveAndRead(CMatchFinder *p) -{ - if (MatchFinder_NeedMove(p)) - MatchFinder_MoveBlock(p); - MatchFinder_ReadBlock(p); -} - -static void MatchFinder_SetDefaultSettings(CMatchFinder *p) -{ - p->cutValue = 32; - p->btMode = 1; - p->numHashBytes = 4; - p->bigHash = 0; -} - -#define kCrcPoly 0xEDB88320 - -void MatchFinder_Construct(CMatchFinder *p) -{ - unsigned i; - p->bufferBase = NULL; - p->directInput = 0; - p->hash = NULL; - p->expectedDataSize = (UInt64)(Int64)-1; - MatchFinder_SetDefaultSettings(p); - - for (i = 0; i < 256; i++) - { - UInt32 r = (UInt32)i; - unsigned j; - for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); - p->crc[i] = r; - } -} - -static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->hash); - p->hash = NULL; -} - -void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc) -{ - MatchFinder_FreeThisClassMemory(p, alloc); - LzInWindow_Free(p, alloc); -} - -static CLzRef* AllocRefs(size_t num, ISzAllocPtr alloc) -{ - size_t sizeInBytes = (size_t)num * sizeof(CLzRef); - if (sizeInBytes / sizeof(CLzRef) != num) - return NULL; - return (CLzRef *)ISzAlloc_Alloc(alloc, sizeInBytes); -} - -int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, - UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAllocPtr alloc) -{ - UInt32 sizeReserv; - - if (historySize > kMaxHistorySize) - { - MatchFinder_Free(p, alloc); - return 0; - } - - sizeReserv = historySize >> 1; - if (historySize >= ((UInt32)3 << 30)) sizeReserv = historySize >> 3; - else if (historySize >= ((UInt32)2 << 30)) sizeReserv = historySize >> 2; - - sizeReserv += (keepAddBufferBefore + matchMaxLen + keepAddBufferAfter) / 2 + (1 << 19); - - p->keepSizeBefore = historySize + keepAddBufferBefore + 1; - p->keepSizeAfter = matchMaxLen + keepAddBufferAfter; - - /* we need one additional byte, since we use MoveBlock after pos++ and before dictionary using */ - - if (LzInWindow_Create(p, sizeReserv, alloc)) - { - UInt32 newCyclicBufferSize = historySize + 1; - UInt32 hs; - p->matchMaxLen = matchMaxLen; - { - p->fixedHashSize = 0; - if (p->numHashBytes == 2) - hs = (1 << 16) - 1; - else - { - hs = historySize; - if (hs > p->expectedDataSize) - hs = (UInt32)p->expectedDataSize; - if (hs != 0) - hs--; - hs |= (hs >> 1); - hs |= (hs >> 2); - hs |= (hs >> 4); - hs |= (hs >> 8); - hs >>= 1; - hs |= 0xFFFF; /* don't change it! It's required for Deflate */ - if (hs > (1 << 24)) - { - if (p->numHashBytes == 3) - hs = (1 << 24) - 1; - else - hs >>= 1; - /* if (bigHash) mode, GetHeads4b() in LzFindMt.c needs (hs >= ((1 << 24) - 1))) */ - } - } - p->hashMask = hs; - hs++; - if (p->numHashBytes > 2) p->fixedHashSize += kHash2Size; - if (p->numHashBytes > 3) p->fixedHashSize += kHash3Size; - if (p->numHashBytes > 4) p->fixedHashSize += kHash4Size; - hs += p->fixedHashSize; - } - - { - size_t newSize; - size_t numSons; - p->historySize = historySize; - p->hashSizeSum = hs; - p->cyclicBufferSize = newCyclicBufferSize; - - numSons = newCyclicBufferSize; - if (p->btMode) - numSons <<= 1; - newSize = hs + numSons; - - if (p->hash && p->numRefs == newSize) - return 1; - - MatchFinder_FreeThisClassMemory(p, alloc); - p->numRefs = newSize; - p->hash = AllocRefs(newSize, alloc); - - if (p->hash) - { - p->son = p->hash + p->hashSizeSum; - return 1; - } - } - } - - MatchFinder_Free(p, alloc); - return 0; -} - -static void MatchFinder_SetLimits(CMatchFinder *p) -{ - UInt32 limit = kMaxValForNormalize - p->pos; - UInt32 limit2 = p->cyclicBufferSize - p->cyclicBufferPos; - - if (limit2 < limit) - limit = limit2; - limit2 = p->streamPos - p->pos; - - if (limit2 <= p->keepSizeAfter) - { - if (limit2 > 0) - limit2 = 1; - } - else - limit2 -= p->keepSizeAfter; - - if (limit2 < limit) - limit = limit2; - - { - UInt32 lenLimit = p->streamPos - p->pos; - if (lenLimit > p->matchMaxLen) - lenLimit = p->matchMaxLen; - p->lenLimit = lenLimit; - } - p->posLimit = p->pos + limit; -} - - -void MatchFinder_Init_LowHash(CMatchFinder *p) -{ - size_t i; - CLzRef *items = p->hash; - size_t numItems = p->fixedHashSize; - for (i = 0; i < numItems; i++) - items[i] = kEmptyHashValue; -} - - -void MatchFinder_Init_HighHash(CMatchFinder *p) -{ - size_t i; - CLzRef *items = p->hash + p->fixedHashSize; - size_t numItems = (size_t)p->hashMask + 1; - for (i = 0; i < numItems; i++) - items[i] = kEmptyHashValue; -} - - -void MatchFinder_Init_3(CMatchFinder *p, int readData) -{ - p->cyclicBufferPos = 0; - p->buffer = p->bufferBase; - p->pos = - p->streamPos = p->cyclicBufferSize; - p->result = SZ_OK; - p->streamEndWasReached = 0; - - if (readData) - MatchFinder_ReadBlock(p); - - MatchFinder_SetLimits(p); -} - - -void MatchFinder_Init(CMatchFinder *p) -{ - MatchFinder_Init_HighHash(p); - MatchFinder_Init_LowHash(p); - MatchFinder_Init_3(p, True); -} - - -static UInt32 MatchFinder_GetSubValue(CMatchFinder *p) -{ - return (p->pos - p->historySize - 1) & kNormalizeMask; -} - -void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems) -{ - size_t i; - for (i = 0; i < numItems; i++) - { - UInt32 value = items[i]; - if (value <= subValue) - value = kEmptyHashValue; - else - value -= subValue; - items[i] = value; - } -} - -static void MatchFinder_Normalize(CMatchFinder *p) -{ - UInt32 subValue = MatchFinder_GetSubValue(p); - MatchFinder_Normalize3(subValue, p->hash, p->numRefs); - MatchFinder_ReduceOffsets(p, subValue); -} - - -MY_NO_INLINE -static void MatchFinder_CheckLimits(CMatchFinder *p) -{ - if (p->pos == kMaxValForNormalize) - MatchFinder_Normalize(p); - if (!p->streamEndWasReached && p->keepSizeAfter == p->streamPos - p->pos) - MatchFinder_CheckAndMoveAndRead(p); - if (p->cyclicBufferPos == p->cyclicBufferSize) - p->cyclicBufferPos = 0; - MatchFinder_SetLimits(p); -} - - -/* - (lenLimit > maxLen) -*/ -MY_FORCE_INLINE -static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, unsigned maxLen) -{ - /* - son[_cyclicBufferPos] = curMatch; - for (;;) - { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) - return distances; - { - const Byte *pb = cur - delta; - curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; - if (pb[maxLen] == cur[maxLen] && *pb == *cur) - { - UInt32 len = 0; - while (++len != lenLimit) - if (pb[len] != cur[len]) - break; - if (maxLen < len) - { - maxLen = len; - *distances++ = len; - *distances++ = delta - 1; - if (len == lenLimit) - return distances; - } - } - } - } - */ - - const Byte *lim = cur + lenLimit; - son[_cyclicBufferPos] = curMatch; - do - { - UInt32 delta = pos - curMatch; - if (delta >= _cyclicBufferSize) - break; - { - ptrdiff_t diff; - curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; - diff = (ptrdiff_t)0 - delta; - if (cur[maxLen] == cur[maxLen + diff]) - { - const Byte *c = cur; - while (*c == c[diff]) - { - if (++c == lim) - { - distances[0] = (UInt32)(lim - cur); - distances[1] = delta - 1; - return distances + 2; - } - } - { - unsigned len = (unsigned)(c - cur); - if (maxLen < len) - { - maxLen = len; - distances[0] = (UInt32)len; - distances[1] = delta - 1; - distances += 2; - } - } - } - } - } - while (--cutValue); - - return distances; -} - - -MY_FORCE_INLINE -UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, UInt32 maxLen) -{ - CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); - unsigned len0 = 0, len1 = 0; - for (;;) - { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) - { - *ptr0 = *ptr1 = kEmptyHashValue; - return distances; - } - { - CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); - const Byte *pb = cur - delta; - unsigned len = (len0 < len1 ? len0 : len1); - UInt32 pair0 = pair[0]; - if (pb[len] == cur[len]) - { - if (++len != lenLimit && pb[len] == cur[len]) - while (++len != lenLimit) - if (pb[len] != cur[len]) - break; - if (maxLen < len) - { - maxLen = (UInt32)len; - *distances++ = (UInt32)len; - *distances++ = delta - 1; - if (len == lenLimit) - { - *ptr1 = pair0; - *ptr0 = pair[1]; - return distances; - } - } - } - if (pb[len] < cur[len]) - { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else - { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; - } - } - } -} - -static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) -{ - CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); - unsigned len0 = 0, len1 = 0; - for (;;) - { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) - { - *ptr0 = *ptr1 = kEmptyHashValue; - return; - } - { - CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); - const Byte *pb = cur - delta; - unsigned len = (len0 < len1 ? len0 : len1); - if (pb[len] == cur[len]) - { - while (++len != lenLimit) - if (pb[len] != cur[len]) - break; - { - if (len == lenLimit) - { - *ptr1 = pair[0]; - *ptr0 = pair[1]; - return; - } - } - } - if (pb[len] < cur[len]) - { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else - { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; - } - } - } -} - -#define MOVE_POS \ - ++p->cyclicBufferPos; \ - p->buffer++; \ - if (++p->pos == p->posLimit) MatchFinder_CheckLimits(p); - -#define MOVE_POS_RET MOVE_POS return (UInt32)offset; - -static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } - -#define GET_MATCHES_HEADER2(minLen, ret_op) \ - unsigned lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ - lenLimit = (unsigned)p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ - cur = p->buffer; - -#define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return 0) -#define SKIP_HEADER(minLen) GET_MATCHES_HEADER2(minLen, continue) - -#define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue - -#define GET_MATCHES_FOOTER(offset, maxLen) \ - offset = (unsigned)(GetMatchesSpec1((UInt32)lenLimit, curMatch, MF_PARAMS(p), \ - distances + offset, (UInt32)maxLen) - distances); MOVE_POS_RET; - -#define SKIP_FOOTER \ - SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; - -#define UPDATE_maxLen { \ - ptrdiff_t diff = (ptrdiff_t)0 - d2; \ - const Byte *c = cur + maxLen; \ - const Byte *lim = cur + lenLimit; \ - for (; c != lim; c++) if (*(c + diff) != *c) break; \ - maxLen = (unsigned)(c - cur); } - -static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - unsigned offset; - GET_MATCHES_HEADER(2) - HASH2_CALC; - curMatch = p->hash[hv]; - p->hash[hv] = p->pos; - offset = 0; - GET_MATCHES_FOOTER(offset, 1) -} - -UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - unsigned offset; - GET_MATCHES_HEADER(3) - HASH_ZIP_CALC; - curMatch = p->hash[hv]; - p->hash[hv] = p->pos; - offset = 0; - GET_MATCHES_FOOTER(offset, 2) -} - -static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - UInt32 h2, d2, pos; - unsigned maxLen, offset; - UInt32 *hash; - GET_MATCHES_HEADER(3) - - HASH3_CALC; - - hash = p->hash; - pos = p->pos; - - d2 = pos - hash[h2]; - - curMatch = (hash + kFix3HashSize)[hv]; - - hash[h2] = pos; - (hash + kFix3HashSize)[hv] = pos; - - maxLen = 2; - offset = 0; - - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) - { - UPDATE_maxLen - distances[0] = (UInt32)maxLen; - distances[1] = d2 - 1; - offset = 2; - if (maxLen == lenLimit) - { - SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); - MOVE_POS_RET; - } - } - - GET_MATCHES_FOOTER(offset, maxLen) -} - -static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - UInt32 h2, h3, d2, d3, pos; - unsigned maxLen, offset; - UInt32 *hash; - GET_MATCHES_HEADER(4) - - HASH4_CALC; - - hash = p->hash; - pos = p->pos; - - d2 = pos - hash [h2]; - d3 = pos - (hash + kFix3HashSize)[h3]; - - curMatch = (hash + kFix4HashSize)[hv]; - - hash [h2] = pos; - (hash + kFix3HashSize)[h3] = pos; - (hash + kFix4HashSize)[hv] = pos; - - maxLen = 0; - offset = 0; - - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) - { - maxLen = 2; - distances[0] = 2; - distances[1] = d2 - 1; - offset = 2; - } - - if (d2 != d3 && d3 < p->cyclicBufferSize && *(cur - d3) == *cur) - { - maxLen = 3; - distances[(size_t)offset + 1] = d3 - 1; - offset += 2; - d2 = d3; - } - - if (offset != 0) - { - UPDATE_maxLen - distances[(size_t)offset - 2] = (UInt32)maxLen; - if (maxLen == lenLimit) - { - SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); - MOVE_POS_RET; - } - } - - if (maxLen < 3) - maxLen = 3; - - GET_MATCHES_FOOTER(offset, maxLen) -} - -/* -static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - UInt32 h2, h3, h4, d2, d3, d4, maxLen, offset, pos; - UInt32 *hash; - GET_MATCHES_HEADER(5) - - HASH5_CALC; - - hash = p->hash; - pos = p->pos; - - d2 = pos - hash [h2]; - d3 = pos - (hash + kFix3HashSize)[h3]; - d4 = pos - (hash + kFix4HashSize)[h4]; - - curMatch = (hash + kFix5HashSize)[hv]; - - hash [h2] = pos; - (hash + kFix3HashSize)[h3] = pos; - (hash + kFix4HashSize)[h4] = pos; - (hash + kFix5HashSize)[hv] = pos; - - maxLen = 0; - offset = 0; - - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) - { - distances[0] = maxLen = 2; - distances[1] = d2 - 1; - offset = 2; - if (*(cur - d2 + 2) == cur[2]) - distances[0] = maxLen = 3; - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) - { - distances[2] = maxLen = 3; - distances[3] = d3 - 1; - offset = 4; - d2 = d3; - } - } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) - { - distances[0] = maxLen = 3; - distances[1] = d3 - 1; - offset = 2; - d2 = d3; - } - - if (d2 != d4 && d4 < p->cyclicBufferSize - && *(cur - d4) == *cur - && *(cur - d4 + 3) == *(cur + 3)) - { - maxLen = 4; - distances[(size_t)offset + 1] = d4 - 1; - offset += 2; - d2 = d4; - } - - if (offset != 0) - { - UPDATE_maxLen - distances[(size_t)offset - 2] = maxLen; - if (maxLen == lenLimit) - { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); - MOVE_POS_RET; - } - } - - if (maxLen < 4) - maxLen = 4; - - GET_MATCHES_FOOTER(offset, maxLen) -} -*/ - -static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - UInt32 h2, h3, d2, d3, pos; - unsigned maxLen, offset; - UInt32 *hash; - GET_MATCHES_HEADER(4) - - HASH4_CALC; - - hash = p->hash; - pos = p->pos; - - d2 = pos - hash [h2]; - d3 = pos - (hash + kFix3HashSize)[h3]; - curMatch = (hash + kFix4HashSize)[hv]; - - hash [h2] = pos; - (hash + kFix3HashSize)[h3] = pos; - (hash + kFix4HashSize)[hv] = pos; - - maxLen = 0; - offset = 0; - - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) - { - maxLen = 2; - distances[0] = 2; - distances[1] = d2 - 1; - offset = 2; - } - - if (d2 != d3 && d3 < p->cyclicBufferSize && *(cur - d3) == *cur) - { - maxLen = 3; - distances[(size_t)offset + 1] = d3 - 1; - offset += 2; - d2 = d3; - } - - if (offset != 0) - { - UPDATE_maxLen - distances[(size_t)offset - 2] = (UInt32)maxLen; - if (maxLen == lenLimit) - { - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS_RET; - } - } - - if (maxLen < 3) - maxLen = 3; - - offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances + offset, maxLen) - (distances)); - MOVE_POS_RET -} - -/* -static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - UInt32 h2, h3, h4, d2, d3, d4, maxLen, offset, pos - UInt32 *hash; - GET_MATCHES_HEADER(5) - - HASH5_CALC; - - hash = p->hash; - pos = p->pos; - - d2 = pos - hash [h2]; - d3 = pos - (hash + kFix3HashSize)[h3]; - d4 = pos - (hash + kFix4HashSize)[h4]; - - curMatch = (hash + kFix5HashSize)[hv]; - - hash [h2] = pos; - (hash + kFix3HashSize)[h3] = pos; - (hash + kFix4HashSize)[h4] = pos; - (hash + kFix5HashSize)[hv] = pos; - - maxLen = 0; - offset = 0; - - if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) - { - distances[0] = maxLen = 2; - distances[1] = d2 - 1; - offset = 2; - if (*(cur - d2 + 2) == cur[2]) - distances[0] = maxLen = 3; - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) - { - distances[2] = maxLen = 3; - distances[3] = d3 - 1; - offset = 4; - d2 = d3; - } - } - else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) - { - distances[0] = maxLen = 3; - distances[1] = d3 - 1; - offset = 2; - d2 = d3; - } - - if (d2 != d4 && d4 < p->cyclicBufferSize - && *(cur - d4) == *cur - && *(cur - d4 + 3) == *(cur + 3)) - { - maxLen = 4; - distances[(size_t)offset + 1] = d4 - 1; - offset += 2; - d2 = d4; - } - - if (offset != 0) - { - UPDATE_maxLen - distances[(size_t)offset - 2] = maxLen; - if (maxLen == lenLimit) - { - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS_RET; - } - } - - if (maxLen < 4) - maxLen = 4; - - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances + offset, maxLen) - (distances)); - MOVE_POS_RET -} -*/ - -UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - unsigned offset; - GET_MATCHES_HEADER(3) - HASH_ZIP_CALC; - curMatch = p->hash[hv]; - p->hash[hv] = p->pos; - offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances, 2) - (distances)); - MOVE_POS_RET -} - -static void Bt2_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - SKIP_HEADER(2) - HASH2_CALC; - curMatch = p->hash[hv]; - p->hash[hv] = p->pos; - SKIP_FOOTER - } - while (--num != 0); -} - -void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - SKIP_HEADER(3) - HASH_ZIP_CALC; - curMatch = p->hash[hv]; - p->hash[hv] = p->pos; - SKIP_FOOTER - } - while (--num != 0); -} - -static void Bt3_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - UInt32 h2; - UInt32 *hash; - SKIP_HEADER(3) - HASH3_CALC; - hash = p->hash; - curMatch = (hash + kFix3HashSize)[hv]; - hash[h2] = - (hash + kFix3HashSize)[hv] = p->pos; - SKIP_FOOTER - } - while (--num != 0); -} - -static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - UInt32 h2, h3; - UInt32 *hash; - SKIP_HEADER(4) - HASH4_CALC; - hash = p->hash; - curMatch = (hash + kFix4HashSize)[hv]; - hash [h2] = - (hash + kFix3HashSize)[h3] = - (hash + kFix4HashSize)[hv] = p->pos; - SKIP_FOOTER - } - while (--num != 0); -} - -/* -static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - UInt32 h2, h3, h4; - UInt32 *hash; - SKIP_HEADER(5) - HASH5_CALC; - hash = p->hash; - curMatch = (hash + kFix5HashSize)[hv]; - hash [h2] = - (hash + kFix3HashSize)[h3] = - (hash + kFix4HashSize)[h4] = - (hash + kFix5HashSize)[hv] = p->pos; - SKIP_FOOTER - } - while (--num != 0); -} -*/ - -static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - UInt32 h2, h3; - UInt32 *hash; - SKIP_HEADER(4) - HASH4_CALC; - hash = p->hash; - curMatch = (hash + kFix4HashSize)[hv]; - hash [h2] = - (hash + kFix3HashSize)[h3] = - (hash + kFix4HashSize)[hv] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } - while (--num != 0); -} - -/* -static void Hc5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - UInt32 h2, h3, h4; - UInt32 *hash; - SKIP_HEADER(5) - HASH5_CALC; - hash = p->hash; - curMatch = hash + kFix5HashSize)[hv]; - hash [h2] = - (hash + kFix3HashSize)[h3] = - (hash + kFix4HashSize)[h4] = - (hash + kFix5HashSize)[hv] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } - while (--num != 0); -} -*/ - -void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - SKIP_HEADER(3) - HASH_ZIP_CALC; - curMatch = p->hash[hv]; - p->hash[hv] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } - while (--num != 0); -} - -void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable) -{ - vTable->Init = (Mf_Init_Func)MatchFinder_Init; - vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinder_GetNumAvailableBytes; - vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinder_GetPointerToCurrentPos; - if (!p->btMode) - { - /* if (p->numHashBytes <= 4) */ - { - vTable->GetMatches = (Mf_GetMatches_Func)Hc4_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Hc4_MatchFinder_Skip; - } - /* - else - { - vTable->GetMatches = (Mf_GetMatches_Func)Hc5_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Hc5_MatchFinder_Skip; - } - */ - } - else if (p->numHashBytes == 2) - { - vTable->GetMatches = (Mf_GetMatches_Func)Bt2_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt2_MatchFinder_Skip; - } - else if (p->numHashBytes == 3) - { - vTable->GetMatches = (Mf_GetMatches_Func)Bt3_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt3_MatchFinder_Skip; - } - else /* if (p->numHashBytes == 4) */ - { - vTable->GetMatches = (Mf_GetMatches_Func)Bt4_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt4_MatchFinder_Skip; - } - /* - else - { - vTable->GetMatches = (Mf_GetMatches_Func)Bt5_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt5_MatchFinder_Skip; - } - */ -} diff --git a/code/ryzom/client/src/seven_zip/LzFind.h b/code/ryzom/client/src/seven_zip/LzFind.h deleted file mode 100644 index 42c13be15..000000000 --- a/code/ryzom/client/src/seven_zip/LzFind.h +++ /dev/null @@ -1,121 +0,0 @@ -/* LzFind.h -- Match finder for LZ algorithms -2017-06-10 : Igor Pavlov : Public domain */ - -#ifndef __LZ_FIND_H -#define __LZ_FIND_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -typedef UInt32 CLzRef; - -typedef struct _CMatchFinder -{ - Byte *buffer; - UInt32 pos; - UInt32 posLimit; - UInt32 streamPos; - UInt32 lenLimit; - - UInt32 cyclicBufferPos; - UInt32 cyclicBufferSize; /* it must be = (historySize + 1) */ - - Byte streamEndWasReached; - Byte btMode; - Byte bigHash; - Byte directInput; - - UInt32 matchMaxLen; - CLzRef *hash; - CLzRef *son; - UInt32 hashMask; - UInt32 cutValue; - - Byte *bufferBase; - ISeqInStream *stream; - - UInt32 blockSize; - UInt32 keepSizeBefore; - UInt32 keepSizeAfter; - - UInt32 numHashBytes; - size_t directInputRem; - UInt32 historySize; - UInt32 fixedHashSize; - UInt32 hashSizeSum; - SRes result; - UInt32 crc[256]; - size_t numRefs; - - UInt64 expectedDataSize; -} CMatchFinder; - -#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer) - -#define Inline_MatchFinder_GetNumAvailableBytes(p) ((p)->streamPos - (p)->pos) - -#define Inline_MatchFinder_IsFinishedOK(p) \ - ((p)->streamEndWasReached \ - && (p)->streamPos == (p)->pos \ - && (!(p)->directInput || (p)->directInputRem == 0)) - -int MatchFinder_NeedMove(CMatchFinder *p); -Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p); -void MatchFinder_MoveBlock(CMatchFinder *p); -void MatchFinder_ReadIfRequired(CMatchFinder *p); - -void MatchFinder_Construct(CMatchFinder *p); - -/* Conditions: - historySize <= 3 GB - keepAddBufferBefore + matchMaxLen + keepAddBufferAfter < 511MB -*/ -int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, - UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAllocPtr alloc); -void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc); -void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems); -void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue); - -UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *buffer, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, - UInt32 *distances, UInt32 maxLen); - -/* -Conditions: - Mf_GetNumAvailableBytes_Func must be called before each Mf_GetMatchLen_Func. - Mf_GetPointerToCurrentPos_Func's result must be used only before any other function -*/ - -typedef void (*Mf_Init_Func)(void *object); -typedef UInt32 (*Mf_GetNumAvailableBytes_Func)(void *object); -typedef const Byte * (*Mf_GetPointerToCurrentPos_Func)(void *object); -typedef UInt32 (*Mf_GetMatches_Func)(void *object, UInt32 *distances); -typedef void (*Mf_Skip_Func)(void *object, UInt32); - -typedef struct _IMatchFinder -{ - Mf_Init_Func Init; - Mf_GetNumAvailableBytes_Func GetNumAvailableBytes; - Mf_GetPointerToCurrentPos_Func GetPointerToCurrentPos; - Mf_GetMatches_Func GetMatches; - Mf_Skip_Func Skip; -} IMatchFinder; - -void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable); - -void MatchFinder_Init_LowHash(CMatchFinder *p); -void MatchFinder_Init_HighHash(CMatchFinder *p); -void MatchFinder_Init_3(CMatchFinder *p, int readData); -void MatchFinder_Init(CMatchFinder *p); - -UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); -UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); - -void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num); -void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/LzHash.h b/code/ryzom/client/src/seven_zip/LzHash.h deleted file mode 100644 index e7c942303..000000000 --- a/code/ryzom/client/src/seven_zip/LzHash.h +++ /dev/null @@ -1,57 +0,0 @@ -/* LzHash.h -- HASH functions for LZ algorithms -2015-04-12 : Igor Pavlov : Public domain */ - -#ifndef __LZ_HASH_H -#define __LZ_HASH_H - -#define kHash2Size (1 << 10) -#define kHash3Size (1 << 16) -#define kHash4Size (1 << 20) - -#define kFix3HashSize (kHash2Size) -#define kFix4HashSize (kHash2Size + kHash3Size) -#define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) - -#define HASH2_CALC hv = cur[0] | ((UInt32)cur[1] << 8); - -#define HASH3_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - hv = (temp ^ ((UInt32)cur[2] << 8)) & p->hashMask; } - -#define HASH4_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - temp ^= ((UInt32)cur[2] << 8); \ - h3 = temp & (kHash3Size - 1); \ - hv = (temp ^ (p->crc[cur[3]] << 5)) & p->hashMask; } - -#define HASH5_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - temp ^= ((UInt32)cur[2] << 8); \ - h3 = temp & (kHash3Size - 1); \ - temp ^= (p->crc[cur[3]] << 5); \ - h4 = temp & (kHash4Size - 1); \ - hv = (temp ^ (p->crc[cur[4]] << 3)) & p->hashMask; } - -/* #define HASH_ZIP_CALC hv = ((cur[0] | ((UInt32)cur[1] << 8)) ^ p->crc[cur[2]]) & 0xFFFF; */ -#define HASH_ZIP_CALC hv = ((cur[2] | ((UInt32)cur[0] << 8)) ^ p->crc[cur[1]]) & 0xFFFF; - - -#define MT_HASH2_CALC \ - h2 = (p->crc[cur[0]] ^ cur[1]) & (kHash2Size - 1); - -#define MT_HASH3_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - h3 = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); } - -#define MT_HASH4_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - h2 = temp & (kHash2Size - 1); \ - temp ^= ((UInt32)cur[2] << 8); \ - h3 = temp & (kHash3Size - 1); \ - h4 = (temp ^ (p->crc[cur[3]] << 5)) & (kHash4Size - 1); } - -#endif diff --git a/code/ryzom/client/src/seven_zip/Lzma2Dec.cpp b/code/ryzom/client/src/seven_zip/Lzma2Dec.cpp deleted file mode 100644 index 4e138a4ae..000000000 --- a/code/ryzom/client/src/seven_zip/Lzma2Dec.cpp +++ /dev/null @@ -1,488 +0,0 @@ -/* Lzma2Dec.c -- LZMA2 Decoder -2019-02-02 : Igor Pavlov : Public domain */ - -/* #define SHOW_DEBUG_INFO */ - -#include "Precomp.h" - -#ifdef SHOW_DEBUG_INFO -#include -#endif - -#include - -#include "Lzma2Dec.h" - -/* -00000000 - End of data -00000001 U U - Uncompressed, reset dic, need reset state and set new prop -00000010 U U - Uncompressed, no reset -100uuuuu U U P P - LZMA, no reset -101uuuuu U U P P - LZMA, reset state -110uuuuu U U P P S - LZMA, reset state + set new prop -111uuuuu U U P P S - LZMA, reset state + set new prop, reset dic - - u, U - Unpack Size - P - Pack Size - S - Props -*/ - -#define LZMA2_CONTROL_COPY_RESET_DIC 1 - -#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & (1 << 7)) == 0) - -#define LZMA2_LCLP_MAX 4 -#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) - -#ifdef SHOW_DEBUG_INFO -#define PRF(x) x -#else -#define PRF(x) -#endif - -typedef enum -{ - LZMA2_STATE_CONTROL, - LZMA2_STATE_UNPACK0, - LZMA2_STATE_UNPACK1, - LZMA2_STATE_PACK0, - LZMA2_STATE_PACK1, - LZMA2_STATE_PROP, - LZMA2_STATE_DATA, - LZMA2_STATE_DATA_CONT, - LZMA2_STATE_FINISHED, - LZMA2_STATE_ERROR -} ELzma2State; - -static SRes Lzma2Dec_GetOldProps(Byte prop, Byte *props) -{ - UInt32 dicSize; - if (prop > 40) - return SZ_ERROR_UNSUPPORTED; - dicSize = (prop == 40) ? 0xFFFFFFFF : LZMA2_DIC_SIZE_FROM_PROP(prop); - props[0] = (Byte)LZMA2_LCLP_MAX; - props[1] = (Byte)(dicSize); - props[2] = (Byte)(dicSize >> 8); - props[3] = (Byte)(dicSize >> 16); - props[4] = (Byte)(dicSize >> 24); - return SZ_OK; -} - -SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) -{ - Byte props[LZMA_PROPS_SIZE]; - RINOK(Lzma2Dec_GetOldProps(prop, props)); - return LzmaDec_AllocateProbs(&p->decoder, props, LZMA_PROPS_SIZE, alloc); -} - -SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) -{ - Byte props[LZMA_PROPS_SIZE]; - RINOK(Lzma2Dec_GetOldProps(prop, props)); - return LzmaDec_Allocate(&p->decoder, props, LZMA_PROPS_SIZE, alloc); -} - -void Lzma2Dec_Init(CLzma2Dec *p) -{ - p->state = LZMA2_STATE_CONTROL; - p->needInitLevel = 0xE0; - p->isExtraMode = False; - p->unpackSize = 0; - - // p->decoder.dicPos = 0; // we can use it instead of full init - LzmaDec_Init(&p->decoder); -} - -static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) -{ - switch (p->state) - { - case LZMA2_STATE_CONTROL: - p->isExtraMode = False; - p->control = b; - PRF(printf("\n %8X", (unsigned)p->decoder.dicPos)); - PRF(printf(" %02X", (unsigned)b)); - if (b == 0) - return LZMA2_STATE_FINISHED; - if (LZMA2_IS_UNCOMPRESSED_STATE(p)) - { - if (b == LZMA2_CONTROL_COPY_RESET_DIC) - p->needInitLevel = 0xC0; - else if (b > 2 || p->needInitLevel == 0xE0) - return LZMA2_STATE_ERROR; - } - else - { - if (b < p->needInitLevel) - return LZMA2_STATE_ERROR; - p->needInitLevel = 0; - p->unpackSize = (UInt32)(b & 0x1F) << 16; - } - return LZMA2_STATE_UNPACK0; - - case LZMA2_STATE_UNPACK0: - p->unpackSize |= (UInt32)b << 8; - return LZMA2_STATE_UNPACK1; - - case LZMA2_STATE_UNPACK1: - p->unpackSize |= (UInt32)b; - p->unpackSize++; - PRF(printf(" %7u", (unsigned)p->unpackSize)); - return LZMA2_IS_UNCOMPRESSED_STATE(p) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; - - case LZMA2_STATE_PACK0: - p->packSize = (UInt32)b << 8; - return LZMA2_STATE_PACK1; - - case LZMA2_STATE_PACK1: - p->packSize |= (UInt32)b; - p->packSize++; - // if (p->packSize < 5) return LZMA2_STATE_ERROR; - PRF(printf(" %5u", (unsigned)p->packSize)); - return (p->control & 0x40) ? LZMA2_STATE_PROP : LZMA2_STATE_DATA; - - case LZMA2_STATE_PROP: - { - unsigned lc, lp; - if (b >= (9 * 5 * 5)) - return LZMA2_STATE_ERROR; - lc = b % 9; - b /= 9; - p->decoder.prop.pb = (Byte)(b / 5); - lp = b % 5; - if (lc + lp > LZMA2_LCLP_MAX) - return LZMA2_STATE_ERROR; - p->decoder.prop.lc = (Byte)lc; - p->decoder.prop.lp = (Byte)lp; - return LZMA2_STATE_DATA; - } - } - return LZMA2_STATE_ERROR; -} - -static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, const Byte *src, SizeT size) -{ - memcpy(p->dic + p->dicPos, src, size); - p->dicPos += size; - if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= size) - p->checkDicSize = p->prop.dicSize; - p->processedPos += (UInt32)size; -} - -void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState); - - -SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, - const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) -{ - SizeT inSize = *srcLen; - *srcLen = 0; - *status = LZMA_STATUS_NOT_SPECIFIED; - - while (p->state != LZMA2_STATE_ERROR) - { - SizeT dicPos; - - if (p->state == LZMA2_STATE_FINISHED) - { - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return SZ_OK; - } - - dicPos = p->decoder.dicPos; - - if (dicPos == dicLimit && finishMode == LZMA_FINISH_ANY) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_OK; - } - - if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT) - { - if (*srcLen == inSize) - { - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - (*srcLen)++; - p->state = Lzma2Dec_UpdateState(p, *src++); - if (dicPos == dicLimit && p->state != LZMA2_STATE_FINISHED) - break; - continue; - } - - { - SizeT inCur = inSize - *srcLen; - SizeT outCur = dicLimit - dicPos; - ELzmaFinishMode curFinishMode = LZMA_FINISH_ANY; - - if (outCur >= p->unpackSize) - { - outCur = (SizeT)p->unpackSize; - curFinishMode = LZMA_FINISH_END; - } - - if (LZMA2_IS_UNCOMPRESSED_STATE(p)) - { - if (inCur == 0) - { - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - - if (p->state == LZMA2_STATE_DATA) - { - BoolInt initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); - LzmaDec_InitDicAndState(&p->decoder, initDic, False); - } - - if (inCur > outCur) - inCur = outCur; - if (inCur == 0) - break; - - LzmaDec_UpdateWithUncompressed(&p->decoder, src, inCur); - - src += inCur; - *srcLen += inCur; - p->unpackSize -= (UInt32)inCur; - p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; - } - else - { - SRes res; - - if (p->state == LZMA2_STATE_DATA) - { - BoolInt initDic = (p->control >= 0xE0); - BoolInt initState = (p->control >= 0xA0); - LzmaDec_InitDicAndState(&p->decoder, initDic, initState); - p->state = LZMA2_STATE_DATA_CONT; - } - - if (inCur > p->packSize) - inCur = (SizeT)p->packSize; - - res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status); - - src += inCur; - *srcLen += inCur; - p->packSize -= (UInt32)inCur; - outCur = p->decoder.dicPos - dicPos; - p->unpackSize -= (UInt32)outCur; - - if (res != 0) - break; - - if (*status == LZMA_STATUS_NEEDS_MORE_INPUT) - { - if (p->packSize == 0) - break; - return SZ_OK; - } - - if (inCur == 0 && outCur == 0) - { - if (*status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK - || p->unpackSize != 0 - || p->packSize != 0) - break; - p->state = LZMA2_STATE_CONTROL; - } - - *status = LZMA_STATUS_NOT_SPECIFIED; - } - } - } - - *status = LZMA_STATUS_NOT_SPECIFIED; - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; -} - - - - -ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, - SizeT outSize, - const Byte *src, SizeT *srcLen, - int checkFinishBlock) -{ - SizeT inSize = *srcLen; - *srcLen = 0; - - while (p->state != LZMA2_STATE_ERROR) - { - if (p->state == LZMA2_STATE_FINISHED) - return (ELzma2ParseStatus)LZMA_STATUS_FINISHED_WITH_MARK; - - if (outSize == 0 && !checkFinishBlock) - return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; - - if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT) - { - if (*srcLen == inSize) - return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; - (*srcLen)++; - - p->state = Lzma2Dec_UpdateState(p, *src++); - - if (p->state == LZMA2_STATE_UNPACK0) - { - // if (p->decoder.dicPos != 0) - if (p->control == LZMA2_CONTROL_COPY_RESET_DIC || p->control >= 0xE0) - return LZMA2_PARSE_STATUS_NEW_BLOCK; - // if (outSize == 0) return LZMA_STATUS_NOT_FINISHED; - } - - // The following code can be commented. - // It's not big problem, if we read additional input bytes. - // It will be stopped later in LZMA2_STATE_DATA / LZMA2_STATE_DATA_CONT state. - - if (outSize == 0 && p->state != LZMA2_STATE_FINISHED) - { - // checkFinishBlock is true. So we expect that block must be finished, - // We can return LZMA_STATUS_NOT_SPECIFIED or LZMA_STATUS_NOT_FINISHED here - // break; - return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; - } - - if (p->state == LZMA2_STATE_DATA) - return LZMA2_PARSE_STATUS_NEW_CHUNK; - - continue; - } - - if (outSize == 0) - return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; - - { - SizeT inCur = inSize - *srcLen; - - if (LZMA2_IS_UNCOMPRESSED_STATE(p)) - { - if (inCur == 0) - return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; - if (inCur > p->unpackSize) - inCur = p->unpackSize; - if (inCur > outSize) - inCur = outSize; - p->decoder.dicPos += inCur; - src += inCur; - *srcLen += inCur; - outSize -= inCur; - p->unpackSize -= (UInt32)inCur; - p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; - } - else - { - p->isExtraMode = True; - - if (inCur == 0) - { - if (p->packSize != 0) - return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; - } - else if (p->state == LZMA2_STATE_DATA) - { - p->state = LZMA2_STATE_DATA_CONT; - if (*src != 0) - { - // first byte of lzma chunk must be Zero - *srcLen += 1; - p->packSize--; - break; - } - } - - if (inCur > p->packSize) - inCur = (SizeT)p->packSize; - - src += inCur; - *srcLen += inCur; - p->packSize -= (UInt32)inCur; - - if (p->packSize == 0) - { - SizeT rem = outSize; - if (rem > p->unpackSize) - rem = p->unpackSize; - p->decoder.dicPos += rem; - p->unpackSize -= (UInt32)rem; - outSize -= rem; - if (p->unpackSize == 0) - p->state = LZMA2_STATE_CONTROL; - } - } - } - } - - p->state = LZMA2_STATE_ERROR; - return (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; -} - - - - -SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) -{ - SizeT outSize = *destLen, inSize = *srcLen; - *srcLen = *destLen = 0; - - for (;;) - { - SizeT inCur = inSize, outCur, dicPos; - ELzmaFinishMode curFinishMode; - SRes res; - - if (p->decoder.dicPos == p->decoder.dicBufSize) - p->decoder.dicPos = 0; - dicPos = p->decoder.dicPos; - curFinishMode = LZMA_FINISH_ANY; - outCur = p->decoder.dicBufSize - dicPos; - - if (outCur >= outSize) - { - outCur = outSize; - curFinishMode = finishMode; - } - - res = Lzma2Dec_DecodeToDic(p, dicPos + outCur, src, &inCur, curFinishMode, status); - - src += inCur; - inSize -= inCur; - *srcLen += inCur; - outCur = p->decoder.dicPos - dicPos; - memcpy(dest, p->decoder.dic + dicPos, outCur); - dest += outCur; - outSize -= outCur; - *destLen += outCur; - if (res != 0) - return res; - if (outCur == 0 || outSize == 0) - return SZ_OK; - } -} - - -SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAllocPtr alloc) -{ - CLzma2Dec p; - SRes res; - SizeT outSize = *destLen, inSize = *srcLen; - *destLen = *srcLen = 0; - *status = LZMA_STATUS_NOT_SPECIFIED; - Lzma2Dec_Construct(&p); - RINOK(Lzma2Dec_AllocateProbs(&p, prop, alloc)); - p.decoder.dic = dest; - p.decoder.dicBufSize = outSize; - Lzma2Dec_Init(&p); - *srcLen = inSize; - res = Lzma2Dec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status); - *destLen = p.decoder.dicPos; - if (res == SZ_OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT) - res = SZ_ERROR_INPUT_EOF; - Lzma2Dec_FreeProbs(&p, alloc); - return res; -} diff --git a/code/ryzom/client/src/seven_zip/Lzma2Dec.h b/code/ryzom/client/src/seven_zip/Lzma2Dec.h deleted file mode 100644 index b8ddeac89..000000000 --- a/code/ryzom/client/src/seven_zip/Lzma2Dec.h +++ /dev/null @@ -1,120 +0,0 @@ -/* Lzma2Dec.h -- LZMA2 Decoder -2018-02-19 : Igor Pavlov : Public domain */ - -#ifndef __LZMA2_DEC_H -#define __LZMA2_DEC_H - -#include "LzmaDec.h" - -EXTERN_C_BEGIN - -/* ---------- State Interface ---------- */ - -typedef struct -{ - unsigned state; - Byte control; - Byte needInitLevel; - Byte isExtraMode; - Byte _pad_; - UInt32 packSize; - UInt32 unpackSize; - CLzmaDec decoder; -} CLzma2Dec; - -#define Lzma2Dec_Construct(p) LzmaDec_Construct(&(p)->decoder) -#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc) -#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc) - -SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); -SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); -void Lzma2Dec_Init(CLzma2Dec *p); - -/* -finishMode: - It has meaning only if the decoding reaches output limit (*destLen or dicLimit). - LZMA_FINISH_ANY - use smallest number of input bytes - LZMA_FINISH_END - read EndOfStream marker after decoding - -Returns: - SZ_OK - status: - LZMA_STATUS_FINISHED_WITH_MARK - LZMA_STATUS_NOT_FINISHED - LZMA_STATUS_NEEDS_MORE_INPUT - SZ_ERROR_DATA - Data error -*/ - -SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, - const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); - -SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); - - -/* ---------- LZMA2 block and chunk parsing ---------- */ - -/* -Lzma2Dec_Parse() parses compressed data stream up to next independent block or next chunk data. -It can return LZMA_STATUS_* code or LZMA2_PARSE_STATUS_* code: - - LZMA2_PARSE_STATUS_NEW_BLOCK - there is new block, and 1 additional byte (control byte of next block header) was read from input. - - LZMA2_PARSE_STATUS_NEW_CHUNK - there is new chunk, and only lzma2 header of new chunk was read. - CLzma2Dec::unpackSize contains unpack size of that chunk -*/ - -typedef enum -{ -/* - LZMA_STATUS_NOT_SPECIFIED // data error - LZMA_STATUS_FINISHED_WITH_MARK - LZMA_STATUS_NOT_FINISHED // - LZMA_STATUS_NEEDS_MORE_INPUT - LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK // unused -*/ - LZMA2_PARSE_STATUS_NEW_BLOCK = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + 1, - LZMA2_PARSE_STATUS_NEW_CHUNK -} ELzma2ParseStatus; - -ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, - SizeT outSize, // output size - const Byte *src, SizeT *srcLen, - int checkFinishBlock // set (checkFinishBlock = 1), if it must read full input data, if decoder.dicPos reaches blockMax position. - ); - -/* -LZMA2 parser doesn't decode LZMA chunks, so we must read - full input LZMA chunk to decode some part of LZMA chunk. - -Lzma2Dec_GetUnpackExtra() returns the value that shows - max possible number of output bytes that can be output by decoder - at current input positon. -*/ - -#define Lzma2Dec_GetUnpackExtra(p) ((p)->isExtraMode ? (p)->unpackSize : 0); - - -/* ---------- One Call Interface ---------- */ - -/* -finishMode: - It has meaning only if the decoding reaches output limit (*destLen). - LZMA_FINISH_ANY - use smallest number of input bytes - LZMA_FINISH_END - read EndOfStream marker after decoding - -Returns: - SZ_OK - status: - LZMA_STATUS_FINISHED_WITH_MARK - LZMA_STATUS_NOT_FINISHED - SZ_ERROR_DATA - Data error - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_UNSUPPORTED - Unsupported properties - SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src). -*/ - -SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAllocPtr alloc); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/Lzma2Enc.cpp b/code/ryzom/client/src/seven_zip/Lzma2Enc.cpp deleted file mode 100644 index 9af0622b0..000000000 --- a/code/ryzom/client/src/seven_zip/Lzma2Enc.cpp +++ /dev/null @@ -1,803 +0,0 @@ -/* Lzma2Enc.c -- LZMA2 Encoder -2018-07-04 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include - -#define _7ZIP_ST - -#include "Lzma2Enc.h" - -#ifndef _7ZIP_ST -#include "MtCoder.h" -#else -#define MTCODER__THREADS_MAX 1 -#endif - -#define LZMA2_CONTROL_LZMA (1 << 7) -#define LZMA2_CONTROL_COPY_NO_RESET 2 -#define LZMA2_CONTROL_COPY_RESET_DIC 1 -#define LZMA2_CONTROL_EOF 0 - -#define LZMA2_LCLP_MAX 4 - -#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) - -#define LZMA2_PACK_SIZE_MAX (1 << 16) -#define LZMA2_COPY_CHUNK_SIZE LZMA2_PACK_SIZE_MAX -#define LZMA2_UNPACK_SIZE_MAX (1 << 21) -#define LZMA2_KEEP_WINDOW_SIZE LZMA2_UNPACK_SIZE_MAX - -#define LZMA2_CHUNK_SIZE_COMPRESSED_MAX ((1 << 16) + 16) - - -#define PRF(x) /* x */ - - -/* ---------- CLimitedSeqInStream ---------- */ - -typedef struct -{ - ISeqInStream vt; - ISeqInStream *realStream; - UInt64 limit; - UInt64 processed; - int finished; -} CLimitedSeqInStream; - -static void LimitedSeqInStream_Init(CLimitedSeqInStream *p) -{ - p->limit = (UInt64)(Int64)-1; - p->processed = 0; - p->finished = 0; -} - -static SRes LimitedSeqInStream_Read(const ISeqInStream *pp, void *data, size_t *size) -{ - CLimitedSeqInStream *p = CONTAINER_FROM_VTBL(pp, CLimitedSeqInStream, vt); - size_t size2 = *size; - SRes res = SZ_OK; - - if (p->limit != (UInt64)(Int64)-1) - { - UInt64 rem = p->limit - p->processed; - if (size2 > rem) - size2 = (size_t)rem; - } - if (size2 != 0) - { - res = ISeqInStream_Read(p->realStream, data, &size2); - p->finished = (size2 == 0 ? 1 : 0); - p->processed += size2; - } - *size = size2; - return res; -} - - -/* ---------- CLzma2EncInt ---------- */ - -typedef struct -{ - CLzmaEncHandle enc; - Byte propsAreSet; - Byte propsByte; - Byte needInitState; - Byte needInitProp; - UInt64 srcPos; -} CLzma2EncInt; - - -static SRes Lzma2EncInt_InitStream(CLzma2EncInt *p, const CLzma2EncProps *props) -{ - if (!p->propsAreSet) - { - SizeT propsSize = LZMA_PROPS_SIZE; - Byte propsEncoded[LZMA_PROPS_SIZE]; - RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); - RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); - p->propsByte = propsEncoded[0]; - p->propsAreSet = True; - } - return SZ_OK; -} - -static void Lzma2EncInt_InitBlock(CLzma2EncInt *p) -{ - p->srcPos = 0; - p->needInitState = True; - p->needInitProp = True; -} - - -SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, - ISzAllocPtr alloc, ISzAllocPtr allocBig); -SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, - UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, - Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); -const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); -void LzmaEnc_Finish(CLzmaEncHandle pp); -void LzmaEnc_SaveState(CLzmaEncHandle pp); -void LzmaEnc_RestoreState(CLzmaEncHandle pp); - -/* -UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp); -*/ - -static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, - size_t *packSizeRes, ISeqOutStream *outStream) -{ - size_t packSizeLimit = *packSizeRes; - size_t packSize = packSizeLimit; - UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX; - unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0); - BoolInt useCopyBlock; - SRes res; - - *packSizeRes = 0; - if (packSize < lzHeaderSize) - return SZ_ERROR_OUTPUT_EOF; - packSize -= lzHeaderSize; - - LzmaEnc_SaveState(p->enc); - res = LzmaEnc_CodeOneMemBlock(p->enc, p->needInitState, - outBuf + lzHeaderSize, &packSize, LZMA2_PACK_SIZE_MAX, &unpackSize); - - PRF(printf("\npackSize = %7d unpackSize = %7d ", packSize, unpackSize)); - - if (unpackSize == 0) - return res; - - if (res == SZ_OK) - useCopyBlock = (packSize + 2 >= unpackSize || packSize > (1 << 16)); - else - { - if (res != SZ_ERROR_OUTPUT_EOF) - return res; - res = SZ_OK; - useCopyBlock = True; - } - - if (useCopyBlock) - { - size_t destPos = 0; - PRF(printf("################# COPY ")); - - while (unpackSize > 0) - { - UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE; - if (packSizeLimit - destPos < u + 3) - return SZ_ERROR_OUTPUT_EOF; - outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET); - outBuf[destPos++] = (Byte)((u - 1) >> 8); - outBuf[destPos++] = (Byte)(u - 1); - memcpy(outBuf + destPos, LzmaEnc_GetCurBuf(p->enc) - unpackSize, u); - unpackSize -= u; - destPos += u; - p->srcPos += u; - - if (outStream) - { - *packSizeRes += destPos; - if (ISeqOutStream_Write(outStream, outBuf, destPos) != destPos) - return SZ_ERROR_WRITE; - destPos = 0; - } - else - *packSizeRes = destPos; - /* needInitState = True; */ - } - - LzmaEnc_RestoreState(p->enc); - return SZ_OK; - } - - { - size_t destPos = 0; - UInt32 u = unpackSize - 1; - UInt32 pm = (UInt32)(packSize - 1); - unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0); - - PRF(printf(" ")); - - outBuf[destPos++] = (Byte)(LZMA2_CONTROL_LZMA | (mode << 5) | ((u >> 16) & 0x1F)); - outBuf[destPos++] = (Byte)(u >> 8); - outBuf[destPos++] = (Byte)u; - outBuf[destPos++] = (Byte)(pm >> 8); - outBuf[destPos++] = (Byte)pm; - - if (p->needInitProp) - outBuf[destPos++] = p->propsByte; - - p->needInitProp = False; - p->needInitState = False; - destPos += packSize; - p->srcPos += unpackSize; - - if (outStream) - if (ISeqOutStream_Write(outStream, outBuf, destPos) != destPos) - return SZ_ERROR_WRITE; - - *packSizeRes = destPos; - return SZ_OK; - } -} - - -/* ---------- Lzma2 Props ---------- */ - -void Lzma2EncProps_Init(CLzma2EncProps *p) -{ - LzmaEncProps_Init(&p->lzmaProps); - p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO; - p->numBlockThreads_Reduced = -1; - p->numBlockThreads_Max = -1; - p->numTotalThreads = -1; -} - -void Lzma2EncProps_Normalize(CLzma2EncProps *p) -{ - UInt64 fileSize; - int t1, t1n, t2, t2r, t3; - { - CLzmaEncProps lzmaProps = p->lzmaProps; - LzmaEncProps_Normalize(&lzmaProps); - t1n = lzmaProps.numThreads; - } - - t1 = p->lzmaProps.numThreads; - t2 = p->numBlockThreads_Max; - t3 = p->numTotalThreads; - - if (t2 > MTCODER__THREADS_MAX) - t2 = MTCODER__THREADS_MAX; - - if (t3 <= 0) - { - if (t2 <= 0) - t2 = 1; - t3 = t1n * t2; - } - else if (t2 <= 0) - { - t2 = t3 / t1n; - if (t2 == 0) - { - t1 = 1; - t2 = t3; - } - if (t2 > MTCODER__THREADS_MAX) - t2 = MTCODER__THREADS_MAX; - } - else if (t1 <= 0) - { - t1 = t3 / t2; - if (t1 == 0) - t1 = 1; - } - else - t3 = t1n * t2; - - p->lzmaProps.numThreads = t1; - - t2r = t2; - - fileSize = p->lzmaProps.reduceSize; - - if ( p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID - && p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO - && (p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) - p->lzmaProps.reduceSize = p->blockSize; - - LzmaEncProps_Normalize(&p->lzmaProps); - - p->lzmaProps.reduceSize = fileSize; - - t1 = p->lzmaProps.numThreads; - - if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) - { - t2r = t2 = 1; - t3 = t1; - } - else if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO && t2 <= 1) - { - /* if there is no block multi-threading, we use SOLID block */ - p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; - } - else - { - if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) - { - const UInt32 kMinSize = (UInt32)1 << 20; - const UInt32 kMaxSize = (UInt32)1 << 28; - const UInt32 dictSize = p->lzmaProps.dictSize; - UInt64 blockSize = (UInt64)dictSize << 2; - if (blockSize < kMinSize) blockSize = kMinSize; - if (blockSize > kMaxSize) blockSize = kMaxSize; - if (blockSize < dictSize) blockSize = dictSize; - blockSize += (kMinSize - 1); - blockSize &= ~(UInt64)(kMinSize - 1); - p->blockSize = blockSize; - } - - if (t2 > 1 && fileSize != (UInt64)(Int64)-1) - { - UInt64 numBlocks = fileSize / p->blockSize; - if (numBlocks * p->blockSize != fileSize) - numBlocks++; - if (numBlocks < (unsigned)t2) - { - t2r = (unsigned)numBlocks; - if (t2r == 0) - t2r = 1; - t3 = t1 * t2r; - } - } - } - - p->numBlockThreads_Max = t2; - p->numBlockThreads_Reduced = t2r; - p->numTotalThreads = t3; -} - - -static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) -{ - return (p && ICompressProgress_Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; -} - - -/* ---------- Lzma2 ---------- */ - -typedef struct -{ - Byte propEncoded; - CLzma2EncProps props; - UInt64 expectedDataSize; - - Byte *tempBufLzma; - - ISzAllocPtr alloc; - ISzAllocPtr allocBig; - - CLzma2EncInt coders[MTCODER__THREADS_MAX]; - - #ifndef _7ZIP_ST - - ISeqOutStream *outStream; - Byte *outBuf; - size_t outBuf_Rem; /* remainder in outBuf */ - - size_t outBufSize; /* size of allocated outBufs[i] */ - size_t outBufsDataSizes[MTCODER__BLOCKS_MAX]; - BoolInt mtCoder_WasConstructed; - CMtCoder mtCoder; - Byte *outBufs[MTCODER__BLOCKS_MAX]; - - #endif - -} CLzma2Enc; - - - -CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - CLzma2Enc *p = (CLzma2Enc *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Enc)); - if (!p) - return NULL; - Lzma2EncProps_Init(&p->props); - Lzma2EncProps_Normalize(&p->props); - p->expectedDataSize = (UInt64)(Int64)-1; - p->tempBufLzma = NULL; - p->alloc = alloc; - p->allocBig = allocBig; - { - unsigned i; - for (i = 0; i < MTCODER__THREADS_MAX; i++) - p->coders[i].enc = NULL; - } - - #ifndef _7ZIP_ST - p->mtCoder_WasConstructed = False; - { - unsigned i; - for (i = 0; i < MTCODER__BLOCKS_MAX; i++) - p->outBufs[i] = NULL; - p->outBufSize = 0; - } - #endif - - return p; -} - - -#ifndef _7ZIP_ST - -static void Lzma2Enc_FreeOutBufs(CLzma2Enc *p) -{ - unsigned i; - for (i = 0; i < MTCODER__BLOCKS_MAX; i++) - if (p->outBufs[i]) - { - ISzAlloc_Free(p->alloc, p->outBufs[i]); - p->outBufs[i] = NULL; - } - p->outBufSize = 0; -} - -#endif - - -void Lzma2Enc_Destroy(CLzma2EncHandle pp) -{ - CLzma2Enc *p = (CLzma2Enc *)pp; - unsigned i; - for (i = 0; i < MTCODER__THREADS_MAX; i++) - { - CLzma2EncInt *t = &p->coders[i]; - if (t->enc) - { - LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig); - t->enc = NULL; - } - } - - - #ifndef _7ZIP_ST - if (p->mtCoder_WasConstructed) - { - MtCoder_Destruct(&p->mtCoder); - p->mtCoder_WasConstructed = False; - } - Lzma2Enc_FreeOutBufs(p); - #endif - - ISzAlloc_Free(p->alloc, p->tempBufLzma); - p->tempBufLzma = NULL; - - ISzAlloc_Free(p->alloc, pp); -} - - -SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) -{ - CLzma2Enc *p = (CLzma2Enc *)pp; - CLzmaEncProps lzmaProps = props->lzmaProps; - LzmaEncProps_Normalize(&lzmaProps); - if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX) - return SZ_ERROR_PARAM; - p->props = *props; - Lzma2EncProps_Normalize(&p->props); - return SZ_OK; -} - - -void Lzma2Enc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) -{ - CLzma2Enc *p = (CLzma2Enc *)pp; - p->expectedDataSize = expectedDataSiize; -} - - -Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) -{ - CLzma2Enc *p = (CLzma2Enc *)pp; - unsigned i; - UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps); - for (i = 0; i < 40; i++) - if (dicSize <= LZMA2_DIC_SIZE_FROM_PROP(i)) - break; - return (Byte)i; -} - - -static SRes Lzma2Enc_EncodeMt1( - CLzma2Enc *me, - CLzma2EncInt *p, - ISeqOutStream *outStream, - Byte *outBuf, size_t *outBufSize, - ISeqInStream *inStream, - const Byte *inData, size_t inDataSize, - int finished, - ICompressProgress *progress) -{ - UInt64 unpackTotal = 0; - UInt64 packTotal = 0; - size_t outLim = 0; - CLimitedSeqInStream limitedInStream; - - if (outBuf) - { - outLim = *outBufSize; - *outBufSize = 0; - } - - if (!p->enc) - { - p->propsAreSet = False; - p->enc = LzmaEnc_Create(me->alloc); - if (!p->enc) - return SZ_ERROR_MEM; - } - - limitedInStream.realStream = inStream; - if (inStream) - { - limitedInStream.vt.Read = LimitedSeqInStream_Read; - } - - if (!outBuf) - { - // outStream version works only in one thread. So we use CLzma2Enc::tempBufLzma - if (!me->tempBufLzma) - { - me->tempBufLzma = (Byte *)ISzAlloc_Alloc(me->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); - if (!me->tempBufLzma) - return SZ_ERROR_MEM; - } - } - - RINOK(Lzma2EncInt_InitStream(p, &me->props)); - - for (;;) - { - SRes res = SZ_OK; - size_t inSizeCur = 0; - - Lzma2EncInt_InitBlock(p); - - LimitedSeqInStream_Init(&limitedInStream); - limitedInStream.limit = me->props.blockSize; - - if (inStream) - { - UInt64 expected = (UInt64)(Int64)-1; - // inStream version works only in one thread. So we use CLzma2Enc::expectedDataSize - if (me->expectedDataSize != (UInt64)(Int64)-1 - && me->expectedDataSize >= unpackTotal) - expected = me->expectedDataSize - unpackTotal; - if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID - && expected > me->props.blockSize) - expected = (size_t)me->props.blockSize; - - LzmaEnc_SetDataSize(p->enc, expected); - - RINOK(LzmaEnc_PrepareForLzma2(p->enc, - &limitedInStream.vt, - LZMA2_KEEP_WINDOW_SIZE, - me->alloc, - me->allocBig)); - } - else - { - inSizeCur = inDataSize - (size_t)unpackTotal; - if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID - && inSizeCur > me->props.blockSize) - inSizeCur = (size_t)me->props.blockSize; - - // LzmaEnc_SetDataSize(p->enc, inSizeCur); - - RINOK(LzmaEnc_MemPrepare(p->enc, - inData + (size_t)unpackTotal, inSizeCur, - LZMA2_KEEP_WINDOW_SIZE, - me->alloc, - me->allocBig)); - } - - for (;;) - { - size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; - if (outBuf) - packSize = outLim - (size_t)packTotal; - - res = Lzma2EncInt_EncodeSubblock(p, - outBuf ? outBuf + (size_t)packTotal : me->tempBufLzma, &packSize, - outBuf ? NULL : outStream); - - if (res != SZ_OK) - break; - - packTotal += packSize; - if (outBuf) - *outBufSize = (size_t)packTotal; - - res = Progress(progress, unpackTotal + p->srcPos, packTotal); - if (res != SZ_OK) - break; - - /* - if (LzmaEnc_GetNumAvailableBytes(p->enc) == 0) - break; - */ - - if (packSize == 0) - break; - } - - LzmaEnc_Finish(p->enc); - - unpackTotal += p->srcPos; - - RINOK(res); - - if (p->srcPos != (inStream ? limitedInStream.processed : inSizeCur)) - return SZ_ERROR_FAIL; - - if (inStream ? limitedInStream.finished : (unpackTotal == inDataSize)) - { - if (finished) - { - if (outBuf) - { - size_t destPos = *outBufSize; - if (destPos >= outLim) - return SZ_ERROR_OUTPUT_EOF; - outBuf[destPos] = 0; - *outBufSize = destPos + 1; - } - else - { - Byte b = 0; - if (ISeqOutStream_Write(outStream, &b, 1) != 1) - return SZ_ERROR_WRITE; - } - } - return SZ_OK; - } - } -} - - - -#ifndef _7ZIP_ST - -static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, - const Byte *src, size_t srcSize, int finished) -{ - CLzma2Enc *me = (CLzma2Enc *)pp; - size_t destSize = me->outBufSize; - SRes res; - CMtProgressThunk progressThunk; - - Byte *dest = me->outBufs[outBufIndex]; - - me->outBufsDataSizes[outBufIndex] = 0; - - if (!dest) - { - dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); - if (!dest) - return SZ_ERROR_MEM; - me->outBufs[outBufIndex] = dest; - } - - MtProgressThunk_CreateVTable(&progressThunk); - progressThunk.mtProgress = &me->mtCoder.mtProgress; - progressThunk.inSize = 0; - progressThunk.outSize = 0; - - res = Lzma2Enc_EncodeMt1(me, - &me->coders[coderIndex], - NULL, dest, &destSize, - NULL, src, srcSize, - finished, - &progressThunk.vt); - - me->outBufsDataSizes[outBufIndex] = destSize; - - return res; -} - - -static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex) -{ - CLzma2Enc *me = (CLzma2Enc *)pp; - size_t size = me->outBufsDataSizes[outBufIndex]; - const Byte *data = me->outBufs[outBufIndex]; - - if (me->outStream) - return ISeqOutStream_Write(me->outStream, data, size) == size ? SZ_OK : SZ_ERROR_WRITE; - - if (size > me->outBuf_Rem) - return SZ_ERROR_OUTPUT_EOF; - memcpy(me->outBuf, data, size); - me->outBuf_Rem -= size; - me->outBuf += size; - return SZ_OK; -} - -#endif - - - -SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, - ISeqOutStream *outStream, - Byte *outBuf, size_t *outBufSize, - ISeqInStream *inStream, - const Byte *inData, size_t inDataSize, - ICompressProgress *progress) -{ - CLzma2Enc *p = (CLzma2Enc *)pp; - - if (inStream && inData) - return SZ_ERROR_PARAM; - - if (outStream && outBuf) - return SZ_ERROR_PARAM; - - { - unsigned i; - for (i = 0; i < MTCODER__THREADS_MAX; i++) - p->coders[i].propsAreSet = False; - } - - #ifndef _7ZIP_ST - - if (p->props.numBlockThreads_Reduced > 1) - { - IMtCoderCallback2 vt; - - if (!p->mtCoder_WasConstructed) - { - p->mtCoder_WasConstructed = True; - MtCoder_Construct(&p->mtCoder); - } - - vt.Code = Lzma2Enc_MtCallback_Code; - vt.Write = Lzma2Enc_MtCallback_Write; - - p->outStream = outStream; - p->outBuf = NULL; - p->outBuf_Rem = 0; - if (!outStream) - { - p->outBuf = outBuf; - p->outBuf_Rem = *outBufSize; - *outBufSize = 0; - } - - p->mtCoder.allocBig = p->allocBig; - p->mtCoder.progress = progress; - p->mtCoder.inStream = inStream; - p->mtCoder.inData = inData; - p->mtCoder.inDataSize = inDataSize; - p->mtCoder.mtCallback = &vt; - p->mtCoder.mtCallbackObject = p; - - p->mtCoder.blockSize = (size_t)p->props.blockSize; - if (p->mtCoder.blockSize != p->props.blockSize) - return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ - - { - size_t destBlockSize = p->mtCoder.blockSize + (p->mtCoder.blockSize >> 10) + 16; - if (destBlockSize < p->mtCoder.blockSize) - return SZ_ERROR_PARAM; - if (p->outBufSize != destBlockSize) - Lzma2Enc_FreeOutBufs(p); - p->outBufSize = destBlockSize; - } - - p->mtCoder.numThreadsMax = p->props.numBlockThreads_Max; - p->mtCoder.expectedDataSize = p->expectedDataSize; - - { - SRes res = MtCoder_Code(&p->mtCoder); - if (!outStream) - *outBufSize = p->outBuf - outBuf; - return res; - } - } - - #endif - - - return Lzma2Enc_EncodeMt1(p, - &p->coders[0], - outStream, outBuf, outBufSize, - inStream, inData, inDataSize, - True, /* finished */ - progress); -} diff --git a/code/ryzom/client/src/seven_zip/Lzma2Enc.h b/code/ryzom/client/src/seven_zip/Lzma2Enc.h deleted file mode 100644 index 6a6110ff7..000000000 --- a/code/ryzom/client/src/seven_zip/Lzma2Enc.h +++ /dev/null @@ -1,55 +0,0 @@ -/* Lzma2Enc.h -- LZMA2 Encoder -2017-07-27 : Igor Pavlov : Public domain */ - -#ifndef __LZMA2_ENC_H -#define __LZMA2_ENC_H - -#include "LzmaEnc.h" - -EXTERN_C_BEGIN - -#define LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO 0 -#define LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID ((UInt64)(Int64)-1) - -typedef struct -{ - CLzmaEncProps lzmaProps; - UInt64 blockSize; - int numBlockThreads_Reduced; - int numBlockThreads_Max; - int numTotalThreads; -} CLzma2EncProps; - -void Lzma2EncProps_Init(CLzma2EncProps *p); -void Lzma2EncProps_Normalize(CLzma2EncProps *p); - -/* ---------- CLzmaEnc2Handle Interface ---------- */ - -/* Lzma2Enc_* functions can return the following exit codes: -SRes: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater in props - SZ_ERROR_WRITE - ISeqOutStream write callback error - SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output - SZ_ERROR_PROGRESS - some break from progress callback - SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) -*/ - -typedef void * CLzma2EncHandle; - -CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); -void Lzma2Enc_Destroy(CLzma2EncHandle p); -SRes Lzma2Enc_SetProps(CLzma2EncHandle p, const CLzma2EncProps *props); -void Lzma2Enc_SetDataSize(CLzma2EncHandle p, UInt64 expectedDataSiize); -Byte Lzma2Enc_WriteProperties(CLzma2EncHandle p); -SRes Lzma2Enc_Encode2(CLzma2EncHandle p, - ISeqOutStream *outStream, - Byte *outBuf, size_t *outBufSize, - ISeqInStream *inStream, - const Byte *inData, size_t inDataSize, - ICompressProgress *progress); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/Lzma86.h b/code/ryzom/client/src/seven_zip/Lzma86.h deleted file mode 100644 index bebed5cb7..000000000 --- a/code/ryzom/client/src/seven_zip/Lzma86.h +++ /dev/null @@ -1,111 +0,0 @@ -/* Lzma86.h -- LZMA + x86 (BCJ) Filter -2013-01-18 : Igor Pavlov : Public domain */ - -#ifndef __LZMA86_H -#define __LZMA86_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -#define LZMA86_SIZE_OFFSET (1 + 5) -#define LZMA86_HEADER_SIZE (LZMA86_SIZE_OFFSET + 8) - -/* -It's an example for LZMA + x86 Filter use. -You can use .lzma86 extension, if you write that stream to file. -.lzma86 header adds one additional byte to standard .lzma header. -.lzma86 header (14 bytes): - Offset Size Description - 0 1 = 0 - no filter, pure LZMA - = 1 - x86 filter + LZMA - 1 1 lc, lp and pb in encoded form - 2 4 dictSize (little endian) - 6 8 uncompressed size (little endian) - - -Lzma86_Encode -------------- -level - compression level: 0 <= level <= 9, the default value for "level" is 5. - -dictSize - The dictionary size in bytes. The maximum value is - 128 MB = (1 << 27) bytes for 32-bit version - 1 GB = (1 << 30) bytes for 64-bit version - The default value is 16 MB = (1 << 24) bytes, for level = 5. - It's recommended to use the dictionary that is larger than 4 KB and - that can be calculated as (1 << N) or (3 << N) sizes. - For better compression ratio dictSize must be >= inSize. - -filterMode: - SZ_FILTER_NO - no Filter - SZ_FILTER_YES - x86 Filter - SZ_FILTER_AUTO - it tries both alternatives to select best. - Encoder will use 2 or 3 passes: - 2 passes when FILTER_NO provides better compression. - 3 passes when FILTER_YES provides better compression. - -Lzma86Encode allocates Data with MyAlloc functions. -RAM Requirements for compressing: - RamSize = dictionarySize * 11.5 + 6MB + FilterBlockSize - filterMode FilterBlockSize - SZ_FILTER_NO 0 - SZ_FILTER_YES inSize - SZ_FILTER_AUTO inSize - - -Return code: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ - -enum ESzFilterMode -{ - SZ_FILTER_NO, - SZ_FILTER_YES, - SZ_FILTER_AUTO -}; - -SRes Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, - int level, UInt32 dictSize, int filterMode); - - -/* -Lzma86_GetUnpackSize: - In: - src - input data - srcLen - input data size - Out: - unpackSize - size of uncompressed stream - Return code: - SZ_OK - OK - SZ_ERROR_INPUT_EOF - Error in headers -*/ - -SRes Lzma86_GetUnpackSize(const Byte *src, SizeT srcLen, UInt64 *unpackSize); - -/* -Lzma86_Decode: - In: - dest - output data - destLen - output data size - src - input data - srcLen - input data size - Out: - destLen - processed output size - srcLen - processed input size - Return code: - SZ_OK - OK - SZ_ERROR_DATA - Data error - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_UNSUPPORTED - unsupported file - SZ_ERROR_INPUT_EOF - it needs more bytes in input buffer -*/ - -SRes Lzma86_Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/Lzma86Dec.cpp b/code/ryzom/client/src/seven_zip/Lzma86Dec.cpp deleted file mode 100644 index 21031745c..000000000 --- a/code/ryzom/client/src/seven_zip/Lzma86Dec.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* Lzma86Dec.c -- LZMA + x86 (BCJ) Filter Decoder -2016-05-16 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "Lzma86.h" - -#include "Alloc.h" -#include "Bra.h" -#include "LzmaDec.h" - -SRes Lzma86_GetUnpackSize(const Byte *src, SizeT srcLen, UInt64 *unpackSize) -{ - unsigned i; - if (srcLen < LZMA86_HEADER_SIZE) - return SZ_ERROR_INPUT_EOF; - *unpackSize = 0; - for (i = 0; i < sizeof(UInt64); i++) - *unpackSize += ((UInt64)src[LZMA86_SIZE_OFFSET + i]) << (8 * i); - return SZ_OK; -} - -SRes Lzma86_Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen) -{ - SRes res; - int useFilter; - SizeT inSizePure; - ELzmaStatus status; - - if (*srcLen < LZMA86_HEADER_SIZE) - return SZ_ERROR_INPUT_EOF; - - useFilter = src[0]; - - if (useFilter > 1) - { - *destLen = 0; - return SZ_ERROR_UNSUPPORTED; - } - - inSizePure = *srcLen - LZMA86_HEADER_SIZE; - res = LzmaDecode(dest, destLen, src + LZMA86_HEADER_SIZE, &inSizePure, - src + 1, LZMA_PROPS_SIZE, LZMA_FINISH_ANY, &status, &g_Alloc); - *srcLen = inSizePure + LZMA86_HEADER_SIZE; - if (res != SZ_OK) - return res; - if (useFilter == 1) - { - UInt32 x86State; - x86_Convert_Init(x86State); - x86_Convert(dest, *destLen, 0, &x86State, 0); - } - return SZ_OK; -} diff --git a/code/ryzom/client/src/seven_zip/Lzma86Enc.cpp b/code/ryzom/client/src/seven_zip/Lzma86Enc.cpp deleted file mode 100644 index 2617bab8e..000000000 --- a/code/ryzom/client/src/seven_zip/Lzma86Enc.cpp +++ /dev/null @@ -1,106 +0,0 @@ -/* Lzma86Enc.c -- LZMA + x86 (BCJ) Filter Encoder -2018-07-04 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include - -#include "Lzma86.h" - -#include "Alloc.h" -#include "Bra.h" -#include "LzmaEnc.h" - -#define SZE_OUT_OVERFLOW SZE_DATA_ERROR - -int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, - int level, UInt32 dictSize, int filterMode) -{ - size_t outSize2 = *destLen; - Byte *filteredStream; - BoolInt useFilter; - int mainResult = SZ_ERROR_OUTPUT_EOF; - CLzmaEncProps props; - LzmaEncProps_Init(&props); - props.level = level; - props.dictSize = dictSize; - - *destLen = 0; - if (outSize2 < LZMA86_HEADER_SIZE) - return SZ_ERROR_OUTPUT_EOF; - - { - int i; - UInt64 t = srcLen; - for (i = 0; i < 8; i++, t >>= 8) - dest[LZMA86_SIZE_OFFSET + i] = (Byte)t; - } - - filteredStream = 0; - useFilter = (filterMode != SZ_FILTER_NO); - if (useFilter) - { - if (srcLen != 0) - { - filteredStream = (Byte *)MyAlloc(srcLen); - if (filteredStream == 0) - return SZ_ERROR_MEM; - memcpy(filteredStream, src, srcLen); - } - { - UInt32 x86State; - x86_Convert_Init(x86State); - x86_Convert(filteredStream, srcLen, 0, &x86State, 1); - } - } - - { - size_t minSize = 0; - BoolInt bestIsFiltered = False; - - /* passes for SZ_FILTER_AUTO: - 0 - BCJ + LZMA - 1 - LZMA - 2 - BCJ + LZMA agaian, if pass 0 (BCJ + LZMA) is better. - */ - int numPasses = (filterMode == SZ_FILTER_AUTO) ? 3 : 1; - - int i; - for (i = 0; i < numPasses; i++) - { - size_t outSizeProcessed = outSize2 - LZMA86_HEADER_SIZE; - size_t outPropsSize = 5; - SRes curRes; - BoolInt curModeIsFiltered = (numPasses > 1 && i == numPasses - 1); - if (curModeIsFiltered && !bestIsFiltered) - break; - if (useFilter && i == 0) - curModeIsFiltered = True; - - curRes = LzmaEncode(dest + LZMA86_HEADER_SIZE, &outSizeProcessed, - curModeIsFiltered ? filteredStream : src, srcLen, - &props, dest + 1, &outPropsSize, 0, - NULL, &g_Alloc, &g_Alloc); - - if (curRes != SZ_ERROR_OUTPUT_EOF) - { - if (curRes != SZ_OK) - { - mainResult = curRes; - break; - } - if (outSizeProcessed <= minSize || mainResult != SZ_OK) - { - minSize = outSizeProcessed; - bestIsFiltered = curModeIsFiltered; - mainResult = SZ_OK; - } - } - } - dest[0] = (Byte)(bestIsFiltered ? 1 : 0); - *destLen = LZMA86_HEADER_SIZE + minSize; - } - if (useFilter) - MyFree(filteredStream); - return mainResult; -} diff --git a/code/ryzom/client/src/seven_zip/LzmaDec.cpp b/code/ryzom/client/src/seven_zip/LzmaDec.cpp deleted file mode 100644 index ba3e1dd50..000000000 --- a/code/ryzom/client/src/seven_zip/LzmaDec.cpp +++ /dev/null @@ -1,1185 +0,0 @@ -/* LzmaDec.c -- LZMA Decoder -2018-07-04 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include - -/* #include "CpuArch.h" */ -#include "LzmaDec.h" - -#define kNumTopBits 24 -#define kTopValue ((UInt32)1 << kNumTopBits) - -#define kNumBitModelTotalBits 11 -#define kBitModelTotal (1 << kNumBitModelTotalBits) -#define kNumMoveBits 5 - -#define RC_INIT_SIZE 5 - -#define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } - -#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) -#define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); -#define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); -#define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ - { UPDATE_0(p); i = (i + i); A0; } else \ - { UPDATE_1(p); i = (i + i) + 1; A1; } - -#define TREE_GET_BIT(probs, i) { GET_BIT2(probs + i, i, ;, ;); } - -#define REV_BIT(p, i, A0, A1) IF_BIT_0(p + i) \ - { UPDATE_0(p + i); A0; } else \ - { UPDATE_1(p + i); A1; } -#define REV_BIT_VAR( p, i, m) REV_BIT(p, i, i += m; m += m, m += m; i += m; ) -#define REV_BIT_CONST(p, i, m) REV_BIT(p, i, i += m; , i += m * 2; ) -#define REV_BIT_LAST( p, i, m) REV_BIT(p, i, i -= m , ; ) - -#define TREE_DECODE(probs, limit, i) \ - { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } - -/* #define _LZMA_SIZE_OPT */ - -#ifdef _LZMA_SIZE_OPT -#define TREE_6_DECODE(probs, i) TREE_DECODE(probs, (1 << 6), i) -#else -#define TREE_6_DECODE(probs, i) \ - { i = 1; \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - i -= 0x40; } -#endif - -#define NORMAL_LITER_DEC TREE_GET_BIT(prob, symbol) -#define MATCHED_LITER_DEC \ - matchByte += matchByte; \ - bit = offs; \ - offs &= matchByte; \ - probLit = prob + (offs + bit + symbol); \ - GET_BIT2(probLit, symbol, offs ^= bit; , ;) - - - -#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } - -#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) -#define UPDATE_0_CHECK range = bound; -#define UPDATE_1_CHECK range -= bound; code -= bound; -#define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \ - { UPDATE_0_CHECK; i = (i + i); A0; } else \ - { UPDATE_1_CHECK; i = (i + i) + 1; A1; } -#define GET_BIT_CHECK(p, i) GET_BIT2_CHECK(p, i, ; , ;) -#define TREE_DECODE_CHECK(probs, limit, i) \ - { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } - - -#define REV_BIT_CHECK(p, i, m) IF_BIT_0_CHECK(p + i) \ - { UPDATE_0_CHECK; i += m; m += m; } else \ - { UPDATE_1_CHECK; m += m; i += m; } - - -#define kNumPosBitsMax 4 -#define kNumPosStatesMax (1 << kNumPosBitsMax) - -#define kLenNumLowBits 3 -#define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumHighBits 8 -#define kLenNumHighSymbols (1 << kLenNumHighBits) - -#define LenLow 0 -#define LenHigh (LenLow + 2 * (kNumPosStatesMax << kLenNumLowBits)) -#define kNumLenProbs (LenHigh + kLenNumHighSymbols) - -#define LenChoice LenLow -#define LenChoice2 (LenLow + (1 << kLenNumLowBits)) - -#define kNumStates 12 -#define kNumStates2 16 -#define kNumLitStates 7 - -#define kStartPosModelIndex 4 -#define kEndPosModelIndex 14 -#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) - -#define kNumPosSlotBits 6 -#define kNumLenToPosStates 4 - -#define kNumAlignBits 4 -#define kAlignTableSize (1 << kNumAlignBits) - -#define kMatchMinLen 2 -#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) - -/* External ASM code needs same CLzmaProb array layout. So don't change it. */ - -/* (probs_1664) is faster and better for code size at some platforms */ -/* -#ifdef MY_CPU_X86_OR_AMD64 -*/ -#define kStartOffset 1664 -#define GET_PROBS p->probs_1664 -/* -#define GET_PROBS p->probs + kStartOffset -#else -#define kStartOffset 0 -#define GET_PROBS p->probs -#endif -*/ - -#define SpecPos (-kStartOffset) -#define IsRep0Long (SpecPos + kNumFullDistances) -#define RepLenCoder (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) -#define LenCoder (RepLenCoder + kNumLenProbs) -#define IsMatch (LenCoder + kNumLenProbs) -#define Align (IsMatch + (kNumStates2 << kNumPosBitsMax)) -#define IsRep (Align + kAlignTableSize) -#define IsRepG0 (IsRep + kNumStates) -#define IsRepG1 (IsRepG0 + kNumStates) -#define IsRepG2 (IsRepG1 + kNumStates) -#define PosSlot (IsRepG2 + kNumStates) -#define Literal (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) -#define NUM_BASE_PROBS (Literal + kStartOffset) - -#if Align != 0 && kStartOffset != 0 - #error Stop_Compiling_Bad_LZMA_kAlign -#endif - -#if NUM_BASE_PROBS != 1984 - #error Stop_Compiling_Bad_LZMA_PROBS -#endif - - -#define LZMA_LIT_SIZE 0x300 - -#define LzmaProps_GetNumProbs(p) (NUM_BASE_PROBS + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) - - -#define CALC_POS_STATE(processedPos, pbMask) (((processedPos) & (pbMask)) << 4) -#define COMBINED_PS_STATE (posState + state) -#define GET_LEN_STATE (posState) - -#define LZMA_DIC_MIN (1 << 12) - -/* -p->remainLen : shows status of LZMA decoder: - < kMatchSpecLenStart : normal remain - = kMatchSpecLenStart : finished - = kMatchSpecLenStart + 1 : need init range coder - = kMatchSpecLenStart + 2 : need init range coder and state -*/ - -/* ---------- LZMA_DECODE_REAL ---------- */ -/* -LzmaDec_DecodeReal_3() can be implemented in external ASM file. -3 - is the code compatibility version of that function for check at link time. -*/ - -#define LZMA_DECODE_REAL LzmaDec_DecodeReal_3 - -/* -LZMA_DECODE_REAL() -In: - RangeCoder is normalized - if (p->dicPos == limit) - { - LzmaDec_TryDummy() was called before to exclude LITERAL and MATCH-REP cases. - So first symbol can be only MATCH-NON-REP. And if that MATCH-NON-REP symbol - is not END_OF_PAYALOAD_MARKER, then function returns error code. - } - -Processing: - first LZMA symbol will be decoded in any case - All checks for limits are at the end of main loop, - It will decode new LZMA-symbols while (p->buf < bufLimit && dicPos < limit), - RangeCoder is still without last normalization when (p->buf < bufLimit) is being checked. - -Out: - RangeCoder is normalized - Result: - SZ_OK - OK - SZ_ERROR_DATA - Error - p->remainLen: - < kMatchSpecLenStart : normal remain - = kMatchSpecLenStart : finished -*/ - - -#ifdef _LZMA_DEC_OPT - -int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit); - -#else - -static -int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit) -{ - CLzmaProb *probs = GET_PROBS; - unsigned state = (unsigned)p->state; - UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; - unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; - unsigned lc = p->prop.lc; - unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); - - Byte *dic = p->dic; - SizeT dicBufSize = p->dicBufSize; - SizeT dicPos = p->dicPos; - - UInt32 processedPos = p->processedPos; - UInt32 checkDicSize = p->checkDicSize; - unsigned len = 0; - - const Byte *buf = p->buf; - UInt32 range = p->range; - UInt32 code = p->code; - - do - { - CLzmaProb *prob; - UInt32 bound; - unsigned ttt; - unsigned posState = CALC_POS_STATE(processedPos, pbMask); - - prob = probs + IsMatch + COMBINED_PS_STATE; - IF_BIT_0(prob) - { - unsigned symbol; - UPDATE_0(prob); - prob = probs + Literal; - if (processedPos != 0 || checkDicSize != 0) - prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); - processedPos++; - - if (state < kNumLitStates) - { - state -= (state < 4) ? state : 3; - symbol = 1; - #ifdef _LZMA_SIZE_OPT - do { NORMAL_LITER_DEC } while (symbol < 0x100); - #else - NORMAL_LITER_DEC - NORMAL_LITER_DEC - NORMAL_LITER_DEC - NORMAL_LITER_DEC - NORMAL_LITER_DEC - NORMAL_LITER_DEC - NORMAL_LITER_DEC - NORMAL_LITER_DEC - #endif - } - else - { - unsigned matchByte = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; - unsigned offs = 0x100; - state -= (state < 10) ? 3 : 6; - symbol = 1; - #ifdef _LZMA_SIZE_OPT - do - { - unsigned bit; - CLzmaProb *probLit; - MATCHED_LITER_DEC - } - while (symbol < 0x100); - #else - { - unsigned bit; - CLzmaProb *probLit; - MATCHED_LITER_DEC - MATCHED_LITER_DEC - MATCHED_LITER_DEC - MATCHED_LITER_DEC - MATCHED_LITER_DEC - MATCHED_LITER_DEC - MATCHED_LITER_DEC - MATCHED_LITER_DEC - } - #endif - } - - dic[dicPos++] = (Byte)symbol; - continue; - } - - { - UPDATE_1(prob); - prob = probs + IsRep + state; - IF_BIT_0(prob) - { - UPDATE_0(prob); - state += kNumStates; - prob = probs + LenCoder; - } - else - { - UPDATE_1(prob); - /* - // that case was checked before with kBadRepCode - if (checkDicSize == 0 && processedPos == 0) - return SZ_ERROR_DATA; - */ - prob = probs + IsRepG0 + state; - IF_BIT_0(prob) - { - UPDATE_0(prob); - prob = probs + IsRep0Long + COMBINED_PS_STATE; - IF_BIT_0(prob) - { - UPDATE_0(prob); - dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; - dicPos++; - processedPos++; - state = state < kNumLitStates ? 9 : 11; - continue; - } - UPDATE_1(prob); - } - else - { - UInt32 distance; - UPDATE_1(prob); - prob = probs + IsRepG1 + state; - IF_BIT_0(prob) - { - UPDATE_0(prob); - distance = rep1; - } - else - { - UPDATE_1(prob); - prob = probs + IsRepG2 + state; - IF_BIT_0(prob) - { - UPDATE_0(prob); - distance = rep2; - } - else - { - UPDATE_1(prob); - distance = rep3; - rep3 = rep2; - } - rep2 = rep1; - } - rep1 = rep0; - rep0 = distance; - } - state = state < kNumLitStates ? 8 : 11; - prob = probs + RepLenCoder; - } - - #ifdef _LZMA_SIZE_OPT - { - unsigned lim, offset; - CLzmaProb *probLen = prob + LenChoice; - IF_BIT_0(probLen) - { - UPDATE_0(probLen); - probLen = prob + LenLow + GET_LEN_STATE; - offset = 0; - lim = (1 << kLenNumLowBits); - } - else - { - UPDATE_1(probLen); - probLen = prob + LenChoice2; - IF_BIT_0(probLen) - { - UPDATE_0(probLen); - probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); - offset = kLenNumLowSymbols; - lim = (1 << kLenNumLowBits); - } - else - { - UPDATE_1(probLen); - probLen = prob + LenHigh; - offset = kLenNumLowSymbols * 2; - lim = (1 << kLenNumHighBits); - } - } - TREE_DECODE(probLen, lim, len); - len += offset; - } - #else - { - CLzmaProb *probLen = prob + LenChoice; - IF_BIT_0(probLen) - { - UPDATE_0(probLen); - probLen = prob + LenLow + GET_LEN_STATE; - len = 1; - TREE_GET_BIT(probLen, len); - TREE_GET_BIT(probLen, len); - TREE_GET_BIT(probLen, len); - len -= 8; - } - else - { - UPDATE_1(probLen); - probLen = prob + LenChoice2; - IF_BIT_0(probLen) - { - UPDATE_0(probLen); - probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); - len = 1; - TREE_GET_BIT(probLen, len); - TREE_GET_BIT(probLen, len); - TREE_GET_BIT(probLen, len); - } - else - { - UPDATE_1(probLen); - probLen = prob + LenHigh; - TREE_DECODE(probLen, (1 << kLenNumHighBits), len); - len += kLenNumLowSymbols * 2; - } - } - } - #endif - - if (state >= kNumStates) - { - UInt32 distance; - prob = probs + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); - TREE_6_DECODE(prob, distance); - if (distance >= kStartPosModelIndex) - { - unsigned posSlot = (unsigned)distance; - unsigned numDirectBits = (unsigned)(((distance >> 1) - 1)); - distance = (2 | (distance & 1)); - if (posSlot < kEndPosModelIndex) - { - distance <<= numDirectBits; - prob = probs + SpecPos; - { - UInt32 m = 1; - distance++; - do - { - REV_BIT_VAR(prob, distance, m); - } - while (--numDirectBits); - distance -= m; - } - } - else - { - numDirectBits -= kNumAlignBits; - do - { - NORMALIZE - range >>= 1; - - { - UInt32 t; - code -= range; - t = (0 - ((UInt32)code >> 31)); /* (UInt32)((Int32)code >> 31) */ - distance = (distance << 1) + (t + 1); - code += range & t; - } - /* - distance <<= 1; - if (code >= range) - { - code -= range; - distance |= 1; - } - */ - } - while (--numDirectBits); - prob = probs + Align; - distance <<= kNumAlignBits; - { - unsigned i = 1; - REV_BIT_CONST(prob, i, 1); - REV_BIT_CONST(prob, i, 2); - REV_BIT_CONST(prob, i, 4); - REV_BIT_LAST (prob, i, 8); - distance |= i; - } - if (distance == (UInt32)0xFFFFFFFF) - { - len = kMatchSpecLenStart; - state -= kNumStates; - break; - } - } - } - - rep3 = rep2; - rep2 = rep1; - rep1 = rep0; - rep0 = distance + 1; - state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; - if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) - { - p->dicPos = dicPos; - return SZ_ERROR_DATA; - } - } - - len += kMatchMinLen; - - { - SizeT rem; - unsigned curLen; - SizeT pos; - - if ((rem = limit - dicPos) == 0) - { - p->dicPos = dicPos; - return SZ_ERROR_DATA; - } - - curLen = ((rem < len) ? (unsigned)rem : len); - pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); - - processedPos += (UInt32)curLen; - - len -= curLen; - if (curLen <= dicBufSize - pos) - { - Byte *dest = dic + dicPos; - ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; - const Byte *lim = dest + curLen; - dicPos += (SizeT)curLen; - do - *(dest) = (Byte)*(dest + src); - while (++dest != lim); - } - else - { - do - { - dic[dicPos++] = dic[pos]; - if (++pos == dicBufSize) - pos = 0; - } - while (--curLen != 0); - } - } - } - } - while (dicPos < limit && buf < bufLimit); - - NORMALIZE; - - p->buf = buf; - p->range = range; - p->code = code; - p->remainLen = (UInt32)len; - p->dicPos = dicPos; - p->processedPos = processedPos; - p->reps[0] = rep0; - p->reps[1] = rep1; - p->reps[2] = rep2; - p->reps[3] = rep3; - p->state = (UInt32)state; - - return SZ_OK; -} -#endif - -static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) -{ - if (p->remainLen != 0 && p->remainLen < kMatchSpecLenStart) - { - Byte *dic = p->dic; - SizeT dicPos = p->dicPos; - SizeT dicBufSize = p->dicBufSize; - unsigned len = (unsigned)p->remainLen; - SizeT rep0 = p->reps[0]; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ - SizeT rem = limit - dicPos; - if (rem < len) - len = (unsigned)(rem); - - if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) - p->checkDicSize = p->prop.dicSize; - - p->processedPos += (UInt32)len; - p->remainLen -= (UInt32)len; - while (len != 0) - { - len--; - dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; - dicPos++; - } - p->dicPos = dicPos; - } -} - - -#define kRange0 0xFFFFFFFF -#define kBound0 ((kRange0 >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1)) -#define kBadRepCode (kBound0 + (((kRange0 - kBound0) >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1))) -#if kBadRepCode != (0xC0000000 - 0x400) - #error Stop_Compiling_Bad_LZMA_Check -#endif - -static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) -{ - do - { - SizeT limit2 = limit; - if (p->checkDicSize == 0) - { - UInt32 rem = p->prop.dicSize - p->processedPos; - if (limit - p->dicPos > rem) - limit2 = p->dicPos + rem; - - if (p->processedPos == 0) - if (p->code >= kBadRepCode) - return SZ_ERROR_DATA; - } - - RINOK(LZMA_DECODE_REAL(p, limit2, bufLimit)); - - if (p->checkDicSize == 0 && p->processedPos >= p->prop.dicSize) - p->checkDicSize = p->prop.dicSize; - - LzmaDec_WriteRem(p, limit); - } - while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); - - return 0; -} - -typedef enum -{ - DUMMY_ERROR, /* unexpected end of input stream */ - DUMMY_LIT, - DUMMY_MATCH, - DUMMY_REP -} ELzmaDummy; - -static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inSize) -{ - UInt32 range = p->range; - UInt32 code = p->code; - const Byte *bufLimit = buf + inSize; - const CLzmaProb *probs = GET_PROBS; - unsigned state = (unsigned)p->state; - ELzmaDummy res; - - { - const CLzmaProb *prob; - UInt32 bound; - unsigned ttt; - unsigned posState = CALC_POS_STATE(p->processedPos, (1 << p->prop.pb) - 1); - - prob = probs + IsMatch + COMBINED_PS_STATE; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK - - /* if (bufLimit - buf >= 7) return DUMMY_LIT; */ - - prob = probs + Literal; - if (p->checkDicSize != 0 || p->processedPos != 0) - prob += ((UInt32)LZMA_LIT_SIZE * - ((((p->processedPos) & ((1 << (p->prop.lp)) - 1)) << p->prop.lc) + - (p->dic[(p->dicPos == 0 ? p->dicBufSize : p->dicPos) - 1] >> (8 - p->prop.lc)))); - - if (state < kNumLitStates) - { - unsigned symbol = 1; - do { GET_BIT_CHECK(prob + symbol, symbol) } while (symbol < 0x100); - } - else - { - unsigned matchByte = p->dic[p->dicPos - p->reps[0] + - (p->dicPos < p->reps[0] ? p->dicBufSize : 0)]; - unsigned offs = 0x100; - unsigned symbol = 1; - do - { - unsigned bit; - const CLzmaProb *probLit; - matchByte += matchByte; - bit = offs; - offs &= matchByte; - probLit = prob + (offs + bit + symbol); - GET_BIT2_CHECK(probLit, symbol, offs ^= bit; , ; ) - } - while (symbol < 0x100); - } - res = DUMMY_LIT; - } - else - { - unsigned len; - UPDATE_1_CHECK; - - prob = probs + IsRep + state; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - state = 0; - prob = probs + LenCoder; - res = DUMMY_MATCH; - } - else - { - UPDATE_1_CHECK; - res = DUMMY_REP; - prob = probs + IsRepG0 + state; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - prob = probs + IsRep0Long + COMBINED_PS_STATE; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - NORMALIZE_CHECK; - return DUMMY_REP; - } - else - { - UPDATE_1_CHECK; - } - } - else - { - UPDATE_1_CHECK; - prob = probs + IsRepG1 + state; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - } - else - { - UPDATE_1_CHECK; - prob = probs + IsRepG2 + state; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - } - else - { - UPDATE_1_CHECK; - } - } - } - state = kNumStates; - prob = probs + RepLenCoder; - } - { - unsigned limit, offset; - const CLzmaProb *probLen = prob + LenChoice; - IF_BIT_0_CHECK(probLen) - { - UPDATE_0_CHECK; - probLen = prob + LenLow + GET_LEN_STATE; - offset = 0; - limit = 1 << kLenNumLowBits; - } - else - { - UPDATE_1_CHECK; - probLen = prob + LenChoice2; - IF_BIT_0_CHECK(probLen) - { - UPDATE_0_CHECK; - probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); - offset = kLenNumLowSymbols; - limit = 1 << kLenNumLowBits; - } - else - { - UPDATE_1_CHECK; - probLen = prob + LenHigh; - offset = kLenNumLowSymbols * 2; - limit = 1 << kLenNumHighBits; - } - } - TREE_DECODE_CHECK(probLen, limit, len); - len += offset; - } - - if (state < 4) - { - unsigned posSlot; - prob = probs + PosSlot + - ((len < kNumLenToPosStates - 1 ? len : kNumLenToPosStates - 1) << - kNumPosSlotBits); - TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); - if (posSlot >= kStartPosModelIndex) - { - unsigned numDirectBits = ((posSlot >> 1) - 1); - - /* if (bufLimit - buf >= 8) return DUMMY_MATCH; */ - - if (posSlot < kEndPosModelIndex) - { - prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits); - } - else - { - numDirectBits -= kNumAlignBits; - do - { - NORMALIZE_CHECK - range >>= 1; - code -= range & (((code - range) >> 31) - 1); - /* if (code >= range) code -= range; */ - } - while (--numDirectBits); - prob = probs + Align; - numDirectBits = kNumAlignBits; - } - { - unsigned i = 1; - unsigned m = 1; - do - { - REV_BIT_CHECK(prob, i, m); - } - while (--numDirectBits); - } - } - } - } - } - NORMALIZE_CHECK; - return res; -} - - -void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState) -{ - p->remainLen = kMatchSpecLenStart + 1; - p->tempBufSize = 0; - - if (initDic) - { - p->processedPos = 0; - p->checkDicSize = 0; - p->remainLen = kMatchSpecLenStart + 2; - } - if (initState) - p->remainLen = kMatchSpecLenStart + 2; -} - -void LzmaDec_Init(CLzmaDec *p) -{ - p->dicPos = 0; - LzmaDec_InitDicAndState(p, True, True); -} - - -SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, - ELzmaFinishMode finishMode, ELzmaStatus *status) -{ - SizeT inSize = *srcLen; - (*srcLen) = 0; - - *status = LZMA_STATUS_NOT_SPECIFIED; - - if (p->remainLen > kMatchSpecLenStart) - { - for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) - p->tempBuf[p->tempBufSize++] = *src++; - if (p->tempBufSize != 0 && p->tempBuf[0] != 0) - return SZ_ERROR_DATA; - if (p->tempBufSize < RC_INIT_SIZE) - { - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - p->code = - ((UInt32)p->tempBuf[1] << 24) - | ((UInt32)p->tempBuf[2] << 16) - | ((UInt32)p->tempBuf[3] << 8) - | ((UInt32)p->tempBuf[4]); - p->range = 0xFFFFFFFF; - p->tempBufSize = 0; - - if (p->remainLen > kMatchSpecLenStart + 1) - { - SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); - SizeT i; - CLzmaProb *probs = p->probs; - for (i = 0; i < numProbs; i++) - probs[i] = kBitModelTotal >> 1; - p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; - p->state = 0; - } - - p->remainLen = 0; - } - - LzmaDec_WriteRem(p, dicLimit); - - while (p->remainLen != kMatchSpecLenStart) - { - int checkEndMarkNow = 0; - - if (p->dicPos >= dicLimit) - { - if (p->remainLen == 0 && p->code == 0) - { - *status = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK; - return SZ_OK; - } - if (finishMode == LZMA_FINISH_ANY) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_OK; - } - if (p->remainLen != 0) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; - } - checkEndMarkNow = 1; - } - - if (p->tempBufSize == 0) - { - SizeT processed; - const Byte *bufLimit; - if (inSize < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) - { - int dummyRes = LzmaDec_TryDummy(p, src, inSize); - if (dummyRes == DUMMY_ERROR) - { - memcpy(p->tempBuf, src, inSize); - p->tempBufSize = (unsigned)inSize; - (*srcLen) += inSize; - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (checkEndMarkNow && dummyRes != DUMMY_MATCH) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; - } - bufLimit = src; - } - else - bufLimit = src + inSize - LZMA_REQUIRED_INPUT_MAX; - p->buf = src; - if (LzmaDec_DecodeReal2(p, dicLimit, bufLimit) != 0) - return SZ_ERROR_DATA; - processed = (SizeT)(p->buf - src); - (*srcLen) += processed; - src += processed; - inSize -= processed; - } - else - { - unsigned rem = p->tempBufSize, lookAhead = 0; - while (rem < LZMA_REQUIRED_INPUT_MAX && lookAhead < inSize) - p->tempBuf[rem++] = src[lookAhead++]; - p->tempBufSize = rem; - if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) - { - int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, (SizeT)rem); - if (dummyRes == DUMMY_ERROR) - { - (*srcLen) += (SizeT)lookAhead; - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (checkEndMarkNow && dummyRes != DUMMY_MATCH) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; - } - } - p->buf = p->tempBuf; - if (LzmaDec_DecodeReal2(p, dicLimit, p->buf) != 0) - return SZ_ERROR_DATA; - - { - unsigned kkk = (unsigned)(p->buf - p->tempBuf); - if (rem < kkk) - return SZ_ERROR_FAIL; /* some internal error */ - rem -= kkk; - if (lookAhead < rem) - return SZ_ERROR_FAIL; /* some internal error */ - lookAhead -= rem; - } - (*srcLen) += (SizeT)lookAhead; - src += lookAhead; - inSize -= (SizeT)lookAhead; - p->tempBufSize = 0; - } - } - - if (p->code != 0) - return SZ_ERROR_DATA; - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return SZ_OK; -} - - -SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) -{ - SizeT outSize = *destLen; - SizeT inSize = *srcLen; - *srcLen = *destLen = 0; - for (;;) - { - SizeT inSizeCur = inSize, outSizeCur, dicPos; - ELzmaFinishMode curFinishMode; - SRes res; - if (p->dicPos == p->dicBufSize) - p->dicPos = 0; - dicPos = p->dicPos; - if (outSize > p->dicBufSize - dicPos) - { - outSizeCur = p->dicBufSize; - curFinishMode = LZMA_FINISH_ANY; - } - else - { - outSizeCur = dicPos + outSize; - curFinishMode = finishMode; - } - - res = LzmaDec_DecodeToDic(p, outSizeCur, src, &inSizeCur, curFinishMode, status); - src += inSizeCur; - inSize -= inSizeCur; - *srcLen += inSizeCur; - outSizeCur = p->dicPos - dicPos; - memcpy(dest, p->dic + dicPos, outSizeCur); - dest += outSizeCur; - outSize -= outSizeCur; - *destLen += outSizeCur; - if (res != 0) - return res; - if (outSizeCur == 0 || outSize == 0) - return SZ_OK; - } -} - -void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->probs); - p->probs = NULL; -} - -static void LzmaDec_FreeDict(CLzmaDec *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->dic); - p->dic = NULL; -} - -void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc) -{ - LzmaDec_FreeProbs(p, alloc); - LzmaDec_FreeDict(p, alloc); -} - -SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size) -{ - UInt32 dicSize; - Byte d; - - if (size < LZMA_PROPS_SIZE) - return SZ_ERROR_UNSUPPORTED; - else - dicSize = data[1] | ((UInt32)data[2] << 8) | ((UInt32)data[3] << 16) | ((UInt32)data[4] << 24); - - if (dicSize < LZMA_DIC_MIN) - dicSize = LZMA_DIC_MIN; - p->dicSize = dicSize; - - d = data[0]; - if (d >= (9 * 5 * 5)) - return SZ_ERROR_UNSUPPORTED; - - p->lc = (Byte)(d % 9); - d /= 9; - p->pb = (Byte)(d / 5); - p->lp = (Byte)(d % 5); - - return SZ_OK; -} - -static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAllocPtr alloc) -{ - UInt32 numProbs = LzmaProps_GetNumProbs(propNew); - if (!p->probs || numProbs != p->numProbs) - { - LzmaDec_FreeProbs(p, alloc); - p->probs = (CLzmaProb *)ISzAlloc_Alloc(alloc, numProbs * sizeof(CLzmaProb)); - if (!p->probs) - return SZ_ERROR_MEM; - p->probs_1664 = p->probs + 1664; - p->numProbs = numProbs; - } - return SZ_OK; -} - -SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) -{ - CLzmaProps propNew; - RINOK(LzmaProps_Decode(&propNew, props, propsSize)); - RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); - p->prop = propNew; - return SZ_OK; -} - -SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) -{ - CLzmaProps propNew; - SizeT dicBufSize; - RINOK(LzmaProps_Decode(&propNew, props, propsSize)); - RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); - - { - UInt32 dictSize = propNew.dicSize; - SizeT mask = ((UInt32)1 << 12) - 1; - if (dictSize >= ((UInt32)1 << 30)) mask = ((UInt32)1 << 22) - 1; - else if (dictSize >= ((UInt32)1 << 22)) mask = ((UInt32)1 << 20) - 1;; - dicBufSize = ((SizeT)dictSize + mask) & ~mask; - if (dicBufSize < dictSize) - dicBufSize = dictSize; - } - - if (!p->dic || dicBufSize != p->dicBufSize) - { - LzmaDec_FreeDict(p, alloc); - p->dic = (Byte *)ISzAlloc_Alloc(alloc, dicBufSize); - if (!p->dic) - { - LzmaDec_FreeProbs(p, alloc); - return SZ_ERROR_MEM; - } - } - p->dicBufSize = dicBufSize; - p->prop = propNew; - return SZ_OK; -} - -SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, - ELzmaStatus *status, ISzAllocPtr alloc) -{ - CLzmaDec p; - SRes res; - SizeT outSize = *destLen, inSize = *srcLen; - *destLen = *srcLen = 0; - *status = LZMA_STATUS_NOT_SPECIFIED; - if (inSize < RC_INIT_SIZE) - return SZ_ERROR_INPUT_EOF; - LzmaDec_Construct(&p); - RINOK(LzmaDec_AllocateProbs(&p, propData, propSize, alloc)); - p.dic = dest; - p.dicBufSize = outSize; - LzmaDec_Init(&p); - *srcLen = inSize; - res = LzmaDec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status); - *destLen = p.dicPos; - if (res == SZ_OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT) - res = SZ_ERROR_INPUT_EOF; - LzmaDec_FreeProbs(&p, alloc); - return res; -} diff --git a/code/ryzom/client/src/seven_zip/LzmaDec.h b/code/ryzom/client/src/seven_zip/LzmaDec.h deleted file mode 100644 index 1f0927ab1..000000000 --- a/code/ryzom/client/src/seven_zip/LzmaDec.h +++ /dev/null @@ -1,234 +0,0 @@ -/* LzmaDec.h -- LZMA Decoder -2018-04-21 : Igor Pavlov : Public domain */ - -#ifndef __LZMA_DEC_H -#define __LZMA_DEC_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -/* #define _LZMA_PROB32 */ -/* _LZMA_PROB32 can increase the speed on some CPUs, - but memory usage for CLzmaDec::probs will be doubled in that case */ - -typedef -#ifdef _LZMA_PROB32 - UInt32 -#else - UInt16 -#endif - CLzmaProb; - - -/* ---------- LZMA Properties ---------- */ - -#define LZMA_PROPS_SIZE 5 - -typedef struct _CLzmaProps -{ - Byte lc; - Byte lp; - Byte pb; - Byte _pad_; - UInt32 dicSize; -} CLzmaProps; - -/* LzmaProps_Decode - decodes properties -Returns: - SZ_OK - SZ_ERROR_UNSUPPORTED - Unsupported properties -*/ - -SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size); - - -/* ---------- LZMA Decoder state ---------- */ - -/* LZMA_REQUIRED_INPUT_MAX = number of required input bytes for worst case. - Num bits = log2((2^11 / 31) ^ 22) + 26 < 134 + 26 = 160; */ - -#define LZMA_REQUIRED_INPUT_MAX 20 - -typedef struct -{ - /* Don't change this structure. ASM code can use it. */ - CLzmaProps prop; - CLzmaProb *probs; - CLzmaProb *probs_1664; - Byte *dic; - SizeT dicBufSize; - SizeT dicPos; - const Byte *buf; - UInt32 range; - UInt32 code; - UInt32 processedPos; - UInt32 checkDicSize; - UInt32 reps[4]; - UInt32 state; - UInt32 remainLen; - - UInt32 numProbs; - unsigned tempBufSize; - Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; -} CLzmaDec; - -#define LzmaDec_Construct(p) { (p)->dic = NULL; (p)->probs = NULL; } - -void LzmaDec_Init(CLzmaDec *p); - -/* There are two types of LZMA streams: - - Stream with end mark. That end mark adds about 6 bytes to compressed size. - - Stream without end mark. You must know exact uncompressed size to decompress such stream. */ - -typedef enum -{ - LZMA_FINISH_ANY, /* finish at any point */ - LZMA_FINISH_END /* block must be finished at the end */ -} ELzmaFinishMode; - -/* ELzmaFinishMode has meaning only if the decoding reaches output limit !!! - - You must use LZMA_FINISH_END, when you know that current output buffer - covers last bytes of block. In other cases you must use LZMA_FINISH_ANY. - - If LZMA decoder sees end marker before reaching output limit, it returns SZ_OK, - and output value of destLen will be less than output buffer size limit. - You can check status result also. - - You can use multiple checks to test data integrity after full decompression: - 1) Check Result and "status" variable. - 2) Check that output(destLen) = uncompressedSize, if you know real uncompressedSize. - 3) Check that output(srcLen) = compressedSize, if you know real compressedSize. - You must use correct finish mode in that case. */ - -typedef enum -{ - LZMA_STATUS_NOT_SPECIFIED, /* use main error code instead */ - LZMA_STATUS_FINISHED_WITH_MARK, /* stream was finished with end mark. */ - LZMA_STATUS_NOT_FINISHED, /* stream was not finished */ - LZMA_STATUS_NEEDS_MORE_INPUT, /* you must provide more input bytes */ - LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK /* there is probability that stream was finished without end mark */ -} ELzmaStatus; - -/* ELzmaStatus is used only as output value for function call */ - - -/* ---------- Interfaces ---------- */ - -/* There are 3 levels of interfaces: - 1) Dictionary Interface - 2) Buffer Interface - 3) One Call Interface - You can select any of these interfaces, but don't mix functions from different - groups for same object. */ - - -/* There are two variants to allocate state for Dictionary Interface: - 1) LzmaDec_Allocate / LzmaDec_Free - 2) LzmaDec_AllocateProbs / LzmaDec_FreeProbs - You can use variant 2, if you set dictionary buffer manually. - For Buffer Interface you must always use variant 1. - -LzmaDec_Allocate* can return: - SZ_OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_UNSUPPORTED - Unsupported properties -*/ - -SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); -void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc); - -SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); -void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc); - -/* ---------- Dictionary Interface ---------- */ - -/* You can use it, if you want to eliminate the overhead for data copying from - dictionary to some other external buffer. - You must work with CLzmaDec variables directly in this interface. - - STEPS: - LzmaDec_Construct() - LzmaDec_Allocate() - for (each new stream) - { - LzmaDec_Init() - while (it needs more decompression) - { - LzmaDec_DecodeToDic() - use data from CLzmaDec::dic and update CLzmaDec::dicPos - } - } - LzmaDec_Free() -*/ - -/* LzmaDec_DecodeToDic - - The decoding to internal dictionary buffer (CLzmaDec::dic). - You must manually update CLzmaDec::dicPos, if it reaches CLzmaDec::dicBufSize !!! - -finishMode: - It has meaning only if the decoding reaches output limit (dicLimit). - LZMA_FINISH_ANY - Decode just dicLimit bytes. - LZMA_FINISH_END - Stream must be finished after dicLimit. - -Returns: - SZ_OK - status: - LZMA_STATUS_FINISHED_WITH_MARK - LZMA_STATUS_NOT_FINISHED - LZMA_STATUS_NEEDS_MORE_INPUT - LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK - SZ_ERROR_DATA - Data error -*/ - -SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, - const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); - - -/* ---------- Buffer Interface ---------- */ - -/* It's zlib-like interface. - See LzmaDec_DecodeToDic description for information about STEPS and return results, - but you must use LzmaDec_DecodeToBuf instead of LzmaDec_DecodeToDic and you don't need - to work with CLzmaDec variables manually. - -finishMode: - It has meaning only if the decoding reaches output limit (*destLen). - LZMA_FINISH_ANY - Decode just destLen bytes. - LZMA_FINISH_END - Stream must be finished after (*destLen). -*/ - -SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); - - -/* ---------- One Call Interface ---------- */ - -/* LzmaDecode - -finishMode: - It has meaning only if the decoding reaches output limit (*destLen). - LZMA_FINISH_ANY - Decode just destLen bytes. - LZMA_FINISH_END - Stream must be finished after (*destLen). - -Returns: - SZ_OK - status: - LZMA_STATUS_FINISHED_WITH_MARK - LZMA_STATUS_NOT_FINISHED - LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK - SZ_ERROR_DATA - Data error - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_UNSUPPORTED - Unsupported properties - SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src). -*/ - -SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, - ELzmaStatus *status, ISzAllocPtr alloc); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/LzmaEnc.cpp b/code/ryzom/client/src/seven_zip/LzmaEnc.cpp deleted file mode 100644 index 54d0d0507..000000000 --- a/code/ryzom/client/src/seven_zip/LzmaEnc.cpp +++ /dev/null @@ -1,2978 +0,0 @@ -/* LzmaEnc.c -- LZMA Encoder -2019-01-10: Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include - -#define _7ZIP_ST - -/* #define SHOW_STAT */ -/* #define SHOW_STAT2 */ - -#if defined(SHOW_STAT) || defined(SHOW_STAT2) -#include -#endif - -#include "LzmaEnc.h" - -#include "LzFind.h" -#ifndef _7ZIP_ST -#include "LzFindMt.h" -#endif - -#ifdef SHOW_STAT -static unsigned g_STAT_OFFSET = 0; -#endif - -#define kLzmaMaxHistorySize ((UInt32)3 << 29) -/* #define kLzmaMaxHistorySize ((UInt32)7 << 29) */ - -#define kNumTopBits 24 -#define kTopValue ((UInt32)1 << kNumTopBits) - -#define kNumBitModelTotalBits 11 -#define kBitModelTotal (1 << kNumBitModelTotalBits) -#define kNumMoveBits 5 -#define kProbInitValue (kBitModelTotal >> 1) - -#define kNumMoveReducingBits 4 -#define kNumBitPriceShiftBits 4 -#define kBitPrice (1 << kNumBitPriceShiftBits) - -#define REP_LEN_COUNT 64 - -void LzmaEncProps_Init(CLzmaEncProps *p) -{ - p->level = 5; - p->dictSize = p->mc = 0; - p->reduceSize = (UInt64)(Int64)-1; - p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1; - p->writeEndMark = 0; -} - -void LzmaEncProps_Normalize(CLzmaEncProps *p) -{ - int level = p->level; - if (level < 0) level = 5; - p->level = level; - - if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level <= 7 ? (1 << 25) : (1 << 26))); - if (p->dictSize > p->reduceSize) - { - unsigned i; - UInt32 reduceSize = (UInt32)p->reduceSize; - for (i = 11; i <= 30; i++) - { - if (reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; } - if (reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; } - } - } - - if (p->lc < 0) p->lc = 3; - if (p->lp < 0) p->lp = 0; - if (p->pb < 0) p->pb = 2; - - if (p->algo < 0) p->algo = (level < 5 ? 0 : 1); - if (p->fb < 0) p->fb = (level < 7 ? 32 : 64); - if (p->btMode < 0) p->btMode = (p->algo == 0 ? 0 : 1); - if (p->numHashBytes < 0) p->numHashBytes = 4; - if (p->mc == 0) p->mc = (16 + (p->fb >> 1)) >> (p->btMode ? 0 : 1); - - if (p->numThreads < 0) - p->numThreads = - #ifndef _7ZIP_ST - ((p->btMode && p->algo) ? 2 : 1); - #else - 1; - #endif -} - -UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2) -{ - CLzmaEncProps props = *props2; - LzmaEncProps_Normalize(&props); - return props.dictSize; -} - -#if (_MSC_VER >= 1400) -/* BSR code is fast for some new CPUs */ -/* #define LZMA_LOG_BSR */ -#endif - -#ifdef LZMA_LOG_BSR - -#define kDicLogSizeMaxCompress 32 - -#define BSR2_RET(pos, res) { unsigned long zz; _BitScanReverse(&zz, (pos)); res = (zz + zz) + ((pos >> (zz - 1)) & 1); } - -static unsigned GetPosSlot1(UInt32 pos) -{ - unsigned res; - BSR2_RET(pos, res); - return res; -} -#define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } -#define GetPosSlot(pos, res) { if (pos < 2) res = pos; else BSR2_RET(pos, res); } - -#else - -#define kNumLogBits (9 + sizeof(size_t) / 2) -/* #define kNumLogBits (11 + sizeof(size_t) / 8 * 3) */ - -#define kDicLogSizeMaxCompress ((kNumLogBits - 1) * 2 + 7) - -static void LzmaEnc_FastPosInit(Byte *g_FastPos) -{ - unsigned slot; - g_FastPos[0] = 0; - g_FastPos[1] = 1; - g_FastPos += 2; - - for (slot = 2; slot < kNumLogBits * 2; slot++) - { - size_t k = ((size_t)1 << ((slot >> 1) - 1)); - size_t j; - for (j = 0; j < k; j++) - g_FastPos[j] = (Byte)slot; - g_FastPos += k; - } -} - -/* we can use ((limit - pos) >> 31) only if (pos < ((UInt32)1 << 31)) */ -/* -#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ - (0 - (((((UInt32)1 << (kNumLogBits + 6)) - 1) - pos) >> 31))); \ - res = p->g_FastPos[pos >> zz] + (zz * 2); } -*/ - -/* -#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ - (0 - (((((UInt32)1 << (kNumLogBits)) - 1) - (pos >> 6)) >> 31))); \ - res = p->g_FastPos[pos >> zz] + (zz * 2); } -*/ - -#define BSR2_RET(pos, res) { unsigned zz = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ - res = p->g_FastPos[pos >> zz] + (zz * 2); } - -/* -#define BSR2_RET(pos, res) { res = (pos < (1 << (kNumLogBits + 6))) ? \ - p->g_FastPos[pos >> 6] + 12 : \ - p->g_FastPos[pos >> (6 + kNumLogBits - 1)] + (6 + (kNumLogBits - 1)) * 2; } -*/ - -#define GetPosSlot1(pos) p->g_FastPos[pos] -#define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } -#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos & (kNumFullDistances - 1)]; else BSR2_RET(pos, res); } - -#endif - - -#define LZMA_NUM_REPS 4 - -typedef UInt16 CState; -typedef UInt16 CExtra; - -typedef struct -{ - UInt32 price; - CState state; - CExtra extra; - // 0 : normal - // 1 : LIT : MATCH - // > 1 : MATCH (extra-1) : LIT : REP0 (len) - UInt32 len; - UInt32 dist; - UInt32 reps[LZMA_NUM_REPS]; -} COptimal; - - -// 18.06 -#define kNumOpts (1 << 11) -#define kPackReserve (kNumOpts * 8) -// #define kNumOpts (1 << 12) -// #define kPackReserve (1 + kNumOpts * 2) - -#define kNumLenToPosStates 4 -#define kNumPosSlotBits 6 -#define kDicLogSizeMin 0 -#define kDicLogSizeMax 32 -#define kDistTableSizeMax (kDicLogSizeMax * 2) - -#define kNumAlignBits 4 -#define kAlignTableSize (1 << kNumAlignBits) -#define kAlignMask (kAlignTableSize - 1) - -#define kStartPosModelIndex 4 -#define kEndPosModelIndex 14 -#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) - -typedef -#ifdef _LZMA_PROB32 - UInt32 -#else - UInt16 -#endif - CLzmaProb; - -#define LZMA_PB_MAX 4 -#define LZMA_LC_MAX 8 -#define LZMA_LP_MAX 4 - -#define LZMA_NUM_PB_STATES_MAX (1 << LZMA_PB_MAX) - -#define kLenNumLowBits 3 -#define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumHighBits 8 -#define kLenNumHighSymbols (1 << kLenNumHighBits) -#define kLenNumSymbolsTotal (kLenNumLowSymbols * 2 + kLenNumHighSymbols) - -#define LZMA_MATCH_LEN_MIN 2 -#define LZMA_MATCH_LEN_MAX (LZMA_MATCH_LEN_MIN + kLenNumSymbolsTotal - 1) - -#define kNumStates 12 - - -typedef struct -{ - CLzmaProb low[LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)]; - CLzmaProb high[kLenNumHighSymbols]; -} CLenEnc; - - -typedef struct -{ - unsigned tableSize; - UInt32 prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal]; - // UInt32 prices1[LZMA_NUM_PB_STATES_MAX][kLenNumLowSymbols * 2]; - // UInt32 prices2[kLenNumSymbolsTotal]; -} CLenPriceEnc; - -#define GET_PRICE_LEN(p, posState, len) \ - ((p)->prices[posState][(size_t)(len) - LZMA_MATCH_LEN_MIN]) - -/* -#define GET_PRICE_LEN(p, posState, len) \ - ((p)->prices2[(size_t)(len) - 2] + ((p)->prices1[posState][((len) - 2) & (kLenNumLowSymbols * 2 - 1)] & (((len) - 2 - kLenNumLowSymbols * 2) >> 9))) -*/ - -typedef struct -{ - UInt32 range; - unsigned cache; - UInt64 low; - UInt64 cacheSize; - Byte *buf; - Byte *bufLim; - Byte *bufBase; - ISeqOutStream *outStream; - UInt64 processed; - SRes res; -} CRangeEnc; - - -typedef struct -{ - CLzmaProb *litProbs; - - unsigned state; - UInt32 reps[LZMA_NUM_REPS]; - - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; - CLzmaProb isRep[kNumStates]; - CLzmaProb isRepG0[kNumStates]; - CLzmaProb isRepG1[kNumStates]; - CLzmaProb isRepG2[kNumStates]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; - - CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances]; - - CLenEnc lenProbs; - CLenEnc repLenProbs; - -} CSaveState; - - -typedef UInt32 CProbPrice; - - -typedef struct -{ - void *matchFinderObj; - IMatchFinder matchFinder; - - unsigned optCur; - unsigned optEnd; - - unsigned longestMatchLen; - unsigned numPairs; - UInt32 numAvail; - - unsigned state; - unsigned numFastBytes; - unsigned additionalOffset; - UInt32 reps[LZMA_NUM_REPS]; - unsigned lpMask, pbMask; - CLzmaProb *litProbs; - CRangeEnc rc; - - UInt32 backRes; - - unsigned lc, lp, pb; - unsigned lclp; - - BoolInt fastMode; - BoolInt writeEndMark; - BoolInt finished; - BoolInt multiThread; - BoolInt needInit; - // BoolInt _maxMode; - - UInt64 nowPos64; - - unsigned matchPriceCount; - // unsigned alignPriceCount; - int repLenEncCounter; - - unsigned distTableSize; - - UInt32 dictSize; - SRes result; - - #ifndef _7ZIP_ST - BoolInt mtMode; - // begin of CMatchFinderMt is used in LZ thread - CMatchFinderMt matchFinderMt; - // end of CMatchFinderMt is used in BT and HASH threads - #endif - - CMatchFinder matchFinderBase; - - #ifndef _7ZIP_ST - Byte pad[128]; - #endif - - // LZ thread - CProbPrice ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; - - UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2 + 1]; - - UInt32 alignPrices[kAlignTableSize]; - UInt32 posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; - UInt32 distancesPrices[kNumLenToPosStates][kNumFullDistances]; - - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; - CLzmaProb isRep[kNumStates]; - CLzmaProb isRepG0[kNumStates]; - CLzmaProb isRepG1[kNumStates]; - CLzmaProb isRepG2[kNumStates]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances]; - - CLenEnc lenProbs; - CLenEnc repLenProbs; - - #ifndef LZMA_LOG_BSR - Byte g_FastPos[1 << kNumLogBits]; - #endif - - CLenPriceEnc lenEnc; - CLenPriceEnc repLenEnc; - - COptimal opt[kNumOpts]; - - CSaveState saveState; - - #ifndef _7ZIP_ST - Byte pad2[128]; - #endif -} CLzmaEnc; - - - -#define COPY_ARR(dest, src, arr) memcpy(dest->arr, src->arr, sizeof(src->arr)); - -void LzmaEnc_SaveState(CLzmaEncHandle pp) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - CSaveState *dest = &p->saveState; - - dest->state = p->state; - - dest->lenProbs = p->lenProbs; - dest->repLenProbs = p->repLenProbs; - - COPY_ARR(dest, p, reps); - - COPY_ARR(dest, p, posAlignEncoder); - COPY_ARR(dest, p, isRep); - COPY_ARR(dest, p, isRepG0); - COPY_ARR(dest, p, isRepG1); - COPY_ARR(dest, p, isRepG2); - COPY_ARR(dest, p, isMatch); - COPY_ARR(dest, p, isRep0Long); - COPY_ARR(dest, p, posSlotEncoder); - COPY_ARR(dest, p, posEncoders); - - memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << p->lclp) * sizeof(CLzmaProb)); -} - - -void LzmaEnc_RestoreState(CLzmaEncHandle pp) -{ - CLzmaEnc *dest = (CLzmaEnc *)pp; - const CSaveState *p = &dest->saveState; - - dest->state = p->state; - - dest->lenProbs = p->lenProbs; - dest->repLenProbs = p->repLenProbs; - - COPY_ARR(dest, p, reps); - - COPY_ARR(dest, p, posAlignEncoder); - COPY_ARR(dest, p, isRep); - COPY_ARR(dest, p, isRepG0); - COPY_ARR(dest, p, isRepG1); - COPY_ARR(dest, p, isRepG2); - COPY_ARR(dest, p, isMatch); - COPY_ARR(dest, p, isRep0Long); - COPY_ARR(dest, p, posSlotEncoder); - COPY_ARR(dest, p, posEncoders); - - memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << dest->lclp) * sizeof(CLzmaProb)); -} - - - -SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - CLzmaEncProps props = *props2; - LzmaEncProps_Normalize(&props); - - if (props.lc > LZMA_LC_MAX - || props.lp > LZMA_LP_MAX - || props.pb > LZMA_PB_MAX - || props.dictSize > ((UInt64)1 << kDicLogSizeMaxCompress) - || props.dictSize > kLzmaMaxHistorySize) - return SZ_ERROR_PARAM; - - p->dictSize = props.dictSize; - { - unsigned fb = props.fb; - if (fb < 5) - fb = 5; - if (fb > LZMA_MATCH_LEN_MAX) - fb = LZMA_MATCH_LEN_MAX; - p->numFastBytes = fb; - } - p->lc = props.lc; - p->lp = props.lp; - p->pb = props.pb; - p->fastMode = (props.algo == 0); - // p->_maxMode = True; - p->matchFinderBase.btMode = (Byte)(props.btMode ? 1 : 0); - { - unsigned numHashBytes = 4; - if (props.btMode) - { - if (props.numHashBytes < 2) - numHashBytes = 2; - else if (props.numHashBytes < 4) - numHashBytes = props.numHashBytes; - } - p->matchFinderBase.numHashBytes = numHashBytes; - } - - p->matchFinderBase.cutValue = props.mc; - - p->writeEndMark = props.writeEndMark; - - #ifndef _7ZIP_ST - /* - if (newMultiThread != _multiThread) - { - ReleaseMatchFinder(); - _multiThread = newMultiThread; - } - */ - p->multiThread = (props.numThreads > 1); - #endif - - return SZ_OK; -} - - -void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - p->matchFinderBase.expectedDataSize = expectedDataSiize; -} - - -#define kState_Start 0 -#define kState_LitAfterMatch 4 -#define kState_LitAfterRep 5 -#define kState_MatchAfterLit 7 -#define kState_RepAfterLit 8 - -static const Byte kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; -static const Byte kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; -static const Byte kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; -static const Byte kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; - -#define IsLitState(s) ((s) < 7) -#define GetLenToPosState2(len) (((len) < kNumLenToPosStates - 1) ? (len) : kNumLenToPosStates - 1) -#define GetLenToPosState(len) (((len) < kNumLenToPosStates + 1) ? (len) - 2 : kNumLenToPosStates - 1) - -#define kInfinityPrice (1 << 30) - -static void RangeEnc_Construct(CRangeEnc *p) -{ - p->outStream = NULL; - p->bufBase = NULL; -} - -#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) -#define RangeEnc_GetProcessed_sizet(p) ((size_t)(p)->processed + ((p)->buf - (p)->bufBase) + (size_t)(p)->cacheSize) - -#define RC_BUF_SIZE (1 << 16) - -static int RangeEnc_Alloc(CRangeEnc *p, ISzAllocPtr alloc) -{ - if (!p->bufBase) - { - p->bufBase = (Byte *)ISzAlloc_Alloc(alloc, RC_BUF_SIZE); - if (!p->bufBase) - return 0; - p->bufLim = p->bufBase + RC_BUF_SIZE; - } - return 1; -} - -static void RangeEnc_Free(CRangeEnc *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->bufBase); - p->bufBase = 0; -} - -static void RangeEnc_Init(CRangeEnc *p) -{ - /* Stream.Init(); */ - p->range = 0xFFFFFFFF; - p->cache = 0; - p->low = 0; - p->cacheSize = 0; - - p->buf = p->bufBase; - - p->processed = 0; - p->res = SZ_OK; -} - -MY_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) -{ - size_t num; - if (p->res != SZ_OK) - return; - num = p->buf - p->bufBase; - if (num != ISeqOutStream_Write(p->outStream, p->bufBase, num)) - p->res = SZ_ERROR_WRITE; - p->processed += num; - p->buf = p->bufBase; -} - -MY_NO_INLINE static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) -{ - UInt32 low = (UInt32)p->low; - unsigned high = (unsigned)(p->low >> 32); - p->low = (UInt32)(low << 8); - if (low < (UInt32)0xFF000000 || high != 0) - { - { - Byte *buf = p->buf; - *buf++ = (Byte)(p->cache + high); - p->cache = (unsigned)(low >> 24); - p->buf = buf; - if (buf == p->bufLim) - RangeEnc_FlushStream(p); - if (p->cacheSize == 0) - return; - } - high += 0xFF; - for (;;) - { - Byte *buf = p->buf; - *buf++ = (Byte)(high); - p->buf = buf; - if (buf == p->bufLim) - RangeEnc_FlushStream(p); - if (--p->cacheSize == 0) - return; - } - } - p->cacheSize++; -} - -static void RangeEnc_FlushData(CRangeEnc *p) -{ - int i; - for (i = 0; i < 5; i++) - RangeEnc_ShiftLow(p); -} - -#define RC_NORM(p) if (range < kTopValue) { range <<= 8; RangeEnc_ShiftLow(p); } - -#define RC_BIT_PRE(p, prob) \ - ttt = *(prob); \ - newBound = (range >> kNumBitModelTotalBits) * ttt; - -// #define _LZMA_ENC_USE_BRANCH - -#ifdef _LZMA_ENC_USE_BRANCH - -#define RC_BIT(p, prob, bit) { \ - RC_BIT_PRE(p, prob) \ - if (bit == 0) { range = newBound; ttt += (kBitModelTotal - ttt) >> kNumMoveBits; } \ - else { (p)->low += newBound; range -= newBound; ttt -= ttt >> kNumMoveBits; } \ - *(prob) = (CLzmaProb)ttt; \ - RC_NORM(p) \ - } - -#else - -#define RC_BIT(p, prob, bit) { \ - UInt32 mask; \ - RC_BIT_PRE(p, prob) \ - mask = 0 - (UInt32)bit; \ - range &= mask; \ - mask &= newBound; \ - range -= mask; \ - (p)->low += mask; \ - mask = (UInt32)bit - 1; \ - range += newBound & mask; \ - mask &= (kBitModelTotal - ((1 << kNumMoveBits) - 1)); \ - mask += ((1 << kNumMoveBits) - 1); \ - ttt += (Int32)(mask - ttt) >> kNumMoveBits; \ - *(prob) = (CLzmaProb)ttt; \ - RC_NORM(p) \ - } - -#endif - - - - -#define RC_BIT_0_BASE(p, prob) \ - range = newBound; *(prob) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); - -#define RC_BIT_1_BASE(p, prob) \ - range -= newBound; (p)->low += newBound; *(prob) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); \ - -#define RC_BIT_0(p, prob) \ - RC_BIT_0_BASE(p, prob) \ - RC_NORM(p) - -#define RC_BIT_1(p, prob) \ - RC_BIT_1_BASE(p, prob) \ - RC_NORM(p) - -static void RangeEnc_EncodeBit_0(CRangeEnc *p, CLzmaProb *prob) -{ - UInt32 range, ttt, newBound; - range = p->range; - RC_BIT_PRE(p, prob) - RC_BIT_0(p, prob) - p->range = range; -} - -static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 sym) -{ - UInt32 range = p->range; - sym |= 0x100; - do - { - UInt32 ttt, newBound; - // RangeEnc_EncodeBit(p, probs + (sym >> 8), (sym >> 7) & 1); - CLzmaProb *prob = probs + (sym >> 8); - UInt32 bit = (sym >> 7) & 1; - sym <<= 1; - RC_BIT(p, prob, bit); - } - while (sym < 0x10000); - p->range = range; -} - -static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 sym, UInt32 matchByte) -{ - UInt32 range = p->range; - UInt32 offs = 0x100; - sym |= 0x100; - do - { - UInt32 ttt, newBound; - CLzmaProb *prob; - UInt32 bit; - matchByte <<= 1; - // RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (sym >> 8)), (sym >> 7) & 1); - prob = probs + (offs + (matchByte & offs) + (sym >> 8)); - bit = (sym >> 7) & 1; - sym <<= 1; - offs &= ~(matchByte ^ sym); - RC_BIT(p, prob, bit); - } - while (sym < 0x10000); - p->range = range; -} - - - -static void LzmaEnc_InitPriceTables(CProbPrice *ProbPrices) -{ - UInt32 i; - for (i = 0; i < (kBitModelTotal >> kNumMoveReducingBits); i++) - { - const unsigned kCyclesBits = kNumBitPriceShiftBits; - UInt32 w = (i << kNumMoveReducingBits) + (1 << (kNumMoveReducingBits - 1)); - unsigned bitCount = 0; - unsigned j; - for (j = 0; j < kCyclesBits; j++) - { - w = w * w; - bitCount <<= 1; - while (w >= ((UInt32)1 << 16)) - { - w >>= 1; - bitCount++; - } - } - ProbPrices[i] = (CProbPrice)((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); - // printf("\n%3d: %5d", i, ProbPrices[i]); - } -} - - -#define GET_PRICE(prob, bit) \ - p->ProbPrices[((prob) ^ (unsigned)(((-(int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; - -#define GET_PRICEa(prob, bit) \ - ProbPrices[((prob) ^ (unsigned)((-((int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; - -#define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits] -#define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] - -#define GET_PRICEa_0(prob) ProbPrices[(prob) >> kNumMoveReducingBits] -#define GET_PRICEa_1(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] - - -static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 sym, const CProbPrice *ProbPrices) -{ - UInt32 price = 0; - sym |= 0x100; - do - { - unsigned bit = sym & 1; - sym >>= 1; - price += GET_PRICEa(probs[sym], bit); - } - while (sym >= 2); - return price; -} - - -static UInt32 LitEnc_Matched_GetPrice(const CLzmaProb *probs, UInt32 sym, UInt32 matchByte, const CProbPrice *ProbPrices) -{ - UInt32 price = 0; - UInt32 offs = 0x100; - sym |= 0x100; - do - { - matchByte <<= 1; - price += GET_PRICEa(probs[offs + (matchByte & offs) + (sym >> 8)], (sym >> 7) & 1); - sym <<= 1; - offs &= ~(matchByte ^ sym); - } - while (sym < 0x10000); - return price; -} - - -static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, unsigned numBits, unsigned sym) -{ - UInt32 range = rc->range; - unsigned m = 1; - do - { - UInt32 ttt, newBound; - unsigned bit = sym & 1; - // RangeEnc_EncodeBit(rc, probs + m, bit); - sym >>= 1; - RC_BIT(rc, probs + m, bit); - m = (m << 1) | bit; - } - while (--numBits); - rc->range = range; -} - - - -static void LenEnc_Init(CLenEnc *p) -{ - unsigned i; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)); i++) - p->low[i] = kProbInitValue; - for (i = 0; i < kLenNumHighSymbols; i++) - p->high[i] = kProbInitValue; -} - -static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, unsigned sym, unsigned posState) -{ - UInt32 range, ttt, newBound; - CLzmaProb *probs = p->low; - range = rc->range; - RC_BIT_PRE(rc, probs); - if (sym >= kLenNumLowSymbols) - { - RC_BIT_1(rc, probs); - probs += kLenNumLowSymbols; - RC_BIT_PRE(rc, probs); - if (sym >= kLenNumLowSymbols * 2) - { - RC_BIT_1(rc, probs); - rc->range = range; - // RcTree_Encode(rc, p->high, kLenNumHighBits, sym - kLenNumLowSymbols * 2); - LitEnc_Encode(rc, p->high, sym - kLenNumLowSymbols * 2); - return; - } - sym -= kLenNumLowSymbols; - } - - // RcTree_Encode(rc, probs + (posState << kLenNumLowBits), kLenNumLowBits, sym); - { - unsigned m; - unsigned bit; - RC_BIT_0(rc, probs); - probs += (posState << (1 + kLenNumLowBits)); - bit = (sym >> 2) ; RC_BIT(rc, probs + 1, bit); m = (1 << 1) + bit; - bit = (sym >> 1) & 1; RC_BIT(rc, probs + m, bit); m = (m << 1) + bit; - bit = sym & 1; RC_BIT(rc, probs + m, bit); - rc->range = range; - } -} - -static void SetPrices_3(const CLzmaProb *probs, UInt32 startPrice, UInt32 *prices, const CProbPrice *ProbPrices) -{ - unsigned i; - for (i = 0; i < 8; i += 2) - { - UInt32 price = startPrice; - UInt32 prob; - price += GET_PRICEa(probs[1 ], (i >> 2)); - price += GET_PRICEa(probs[2 + (i >> 2)], (i >> 1) & 1); - prob = probs[4 + (i >> 1)]; - prices[i ] = price + GET_PRICEa_0(prob); - prices[i + 1] = price + GET_PRICEa_1(prob); - } -} - - -MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTables( - CLenPriceEnc *p, - unsigned numPosStates, - const CLenEnc *enc, - const CProbPrice *ProbPrices) -{ - UInt32 b; - - { - unsigned prob = enc->low[0]; - UInt32 a, c; - unsigned posState; - b = GET_PRICEa_1(prob); - a = GET_PRICEa_0(prob); - c = b + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); - for (posState = 0; posState < numPosStates; posState++) - { - UInt32 *prices = p->prices[posState]; - const CLzmaProb *probs = enc->low + (posState << (1 + kLenNumLowBits)); - SetPrices_3(probs, a, prices, ProbPrices); - SetPrices_3(probs + kLenNumLowSymbols, c, prices + kLenNumLowSymbols, ProbPrices); - } - } - - /* - { - unsigned i; - UInt32 b; - a = GET_PRICEa_0(enc->low[0]); - for (i = 0; i < kLenNumLowSymbols; i++) - p->prices2[i] = a; - a = GET_PRICEa_1(enc->low[0]); - b = a + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); - for (i = kLenNumLowSymbols; i < kLenNumLowSymbols * 2; i++) - p->prices2[i] = b; - a += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); - } - */ - - // p->counter = numSymbols; - // p->counter = 64; - - { - unsigned i = p->tableSize; - - if (i > kLenNumLowSymbols * 2) - { - const CLzmaProb *probs = enc->high; - UInt32 *prices = p->prices[0] + kLenNumLowSymbols * 2; - i -= kLenNumLowSymbols * 2 - 1; - i >>= 1; - b += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); - do - { - /* - p->prices2[i] = a + - // RcTree_GetPrice(enc->high, kLenNumHighBits, i - kLenNumLowSymbols * 2, ProbPrices); - LitEnc_GetPrice(probs, i - kLenNumLowSymbols * 2, ProbPrices); - */ - // UInt32 price = a + RcTree_GetPrice(probs, kLenNumHighBits - 1, sym, ProbPrices); - unsigned sym = --i + (1 << (kLenNumHighBits - 1)); - UInt32 price = b; - do - { - unsigned bit = sym & 1; - sym >>= 1; - price += GET_PRICEa(probs[sym], bit); - } - while (sym >= 2); - - { - unsigned prob = probs[(size_t)i + (1 << (kLenNumHighBits - 1))]; - prices[(size_t)i * 2 ] = price + GET_PRICEa_0(prob); - prices[(size_t)i * 2 + 1] = price + GET_PRICEa_1(prob); - } - } - while (i); - - { - unsigned posState; - size_t num = (p->tableSize - kLenNumLowSymbols * 2) * sizeof(p->prices[0][0]); - for (posState = 1; posState < numPosStates; posState++) - memcpy(p->prices[posState] + kLenNumLowSymbols * 2, p->prices[0] + kLenNumLowSymbols * 2, num); - } - } - } -} - -/* - #ifdef SHOW_STAT - g_STAT_OFFSET += num; - printf("\n MovePos %u", num); - #endif -*/ - -#define MOVE_POS(p, num) { \ - p->additionalOffset += (num); \ - p->matchFinder.Skip(p->matchFinderObj, (UInt32)(num)); } - - -static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) -{ - unsigned numPairs; - - p->additionalOffset++; - p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); - numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); - *numPairsRes = numPairs; - - #ifdef SHOW_STAT - printf("\n i = %u numPairs = %u ", g_STAT_OFFSET, numPairs / 2); - g_STAT_OFFSET++; - { - unsigned i; - for (i = 0; i < numPairs; i += 2) - printf("%2u %6u | ", p->matches[i], p->matches[i + 1]); - } - #endif - - if (numPairs == 0) - return 0; - { - unsigned len = p->matches[(size_t)numPairs - 2]; - if (len != p->numFastBytes) - return len; - { - UInt32 numAvail = p->numAvail; - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; - { - const Byte *p1 = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - const Byte *p2 = p1 + len; - ptrdiff_t dif = (ptrdiff_t)-1 - p->matches[(size_t)numPairs - 1]; - const Byte *lim = p1 + numAvail; - for (; p2 != lim && *p2 == p2[dif]; p2++) - {} - return (unsigned)(p2 - p1); - } - } - } -} - -#define MARK_LIT ((UInt32)(Int32)-1) - -#define MakeAs_Lit(p) { (p)->dist = MARK_LIT; (p)->extra = 0; } -#define MakeAs_ShortRep(p) { (p)->dist = 0; (p)->extra = 0; } -#define IsShortRep(p) ((p)->dist == 0) - - -#define GetPrice_ShortRep(p, state, posState) \ - ( GET_PRICE_0(p->isRepG0[state]) + GET_PRICE_0(p->isRep0Long[state][posState])) - -#define GetPrice_Rep_0(p, state, posState) ( \ - GET_PRICE_1(p->isMatch[state][posState]) \ - + GET_PRICE_1(p->isRep0Long[state][posState])) \ - + GET_PRICE_1(p->isRep[state]) \ - + GET_PRICE_0(p->isRepG0[state]) - -MY_FORCE_INLINE -static UInt32 GetPrice_PureRep(const CLzmaEnc *p, unsigned repIndex, size_t state, size_t posState) -{ - UInt32 price; - UInt32 prob = p->isRepG0[state]; - if (repIndex == 0) - { - price = GET_PRICE_0(prob); - price += GET_PRICE_1(p->isRep0Long[state][posState]); - } - else - { - price = GET_PRICE_1(prob); - prob = p->isRepG1[state]; - if (repIndex == 1) - price += GET_PRICE_0(prob); - else - { - price += GET_PRICE_1(prob); - price += GET_PRICE(p->isRepG2[state], repIndex - 2); - } - } - return price; -} - - -static unsigned Backward(CLzmaEnc *p, unsigned cur) -{ - unsigned wr = cur + 1; - p->optEnd = wr; - - for (;;) - { - UInt32 dist = p->opt[cur].dist; - unsigned len = (unsigned)p->opt[cur].len; - unsigned extra = (unsigned)p->opt[cur].extra; - cur -= len; - - if (extra) - { - wr--; - p->opt[wr].len = (UInt32)len; - cur -= extra; - len = extra; - if (extra == 1) - { - p->opt[wr].dist = dist; - dist = MARK_LIT; - } - else - { - p->opt[wr].dist = 0; - len--; - wr--; - p->opt[wr].dist = MARK_LIT; - p->opt[wr].len = 1; - } - } - - if (cur == 0) - { - p->backRes = dist; - p->optCur = wr; - return len; - } - - wr--; - p->opt[wr].dist = dist; - p->opt[wr].len = (UInt32)len; - } -} - - - -#define LIT_PROBS(pos, prevByte) \ - (p->litProbs + (UInt32)3 * (((((pos) << 8) + (prevByte)) & p->lpMask) << p->lc)) - - -static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) -{ - unsigned last, cur; - UInt32 reps[LZMA_NUM_REPS]; - unsigned repLens[LZMA_NUM_REPS]; - UInt32 *matches; - - { - UInt32 numAvail; - unsigned numPairs, mainLen, repMaxIndex, i, posState; - UInt32 matchPrice, repMatchPrice; - const Byte *data; - Byte curByte, matchByte; - - p->optCur = p->optEnd = 0; - - if (p->additionalOffset == 0) - mainLen = ReadMatchDistances(p, &numPairs); - else - { - mainLen = p->longestMatchLen; - numPairs = p->numPairs; - } - - numAvail = p->numAvail; - if (numAvail < 2) - { - p->backRes = MARK_LIT; - return 1; - } - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; - - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repMaxIndex = 0; - - for (i = 0; i < LZMA_NUM_REPS; i++) - { - unsigned len; - const Byte *data2; - reps[i] = p->reps[i]; - data2 = data - reps[i]; - if (data[0] != data2[0] || data[1] != data2[1]) - { - repLens[i] = 0; - continue; - } - for (len = 2; len < numAvail && data[len] == data2[len]; len++) - {} - repLens[i] = len; - if (len > repLens[repMaxIndex]) - repMaxIndex = i; - } - - if (repLens[repMaxIndex] >= p->numFastBytes) - { - unsigned len; - p->backRes = (UInt32)repMaxIndex; - len = repLens[repMaxIndex]; - MOVE_POS(p, len - 1) - return len; - } - - matches = p->matches; - - if (mainLen >= p->numFastBytes) - { - p->backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; - MOVE_POS(p, mainLen - 1) - return mainLen; - } - - curByte = *data; - matchByte = *(data - reps[0]); - - last = repLens[repMaxIndex]; - if (last <= mainLen) - last = mainLen; - - if (last < 2 && curByte != matchByte) - { - p->backRes = MARK_LIT; - return 1; - } - - p->opt[0].state = (CState)p->state; - - posState = (position & p->pbMask); - - { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + - (!IsLitState(p->state) ? - LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); - } - - MakeAs_Lit(&p->opt[1]); - - matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); - repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); - - // 18.06 - if (matchByte == curByte && repLens[0] == 0) - { - UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, p->state, posState); - if (shortRepPrice < p->opt[1].price) - { - p->opt[1].price = shortRepPrice; - MakeAs_ShortRep(&p->opt[1]); - } - if (last < 2) - { - p->backRes = p->opt[1].dist; - return 1; - } - } - - p->opt[1].len = 1; - - p->opt[0].reps[0] = reps[0]; - p->opt[0].reps[1] = reps[1]; - p->opt[0].reps[2] = reps[2]; - p->opt[0].reps[3] = reps[3]; - - // ---------- REP ---------- - - for (i = 0; i < LZMA_NUM_REPS; i++) - { - unsigned repLen = repLens[i]; - UInt32 price; - if (repLen < 2) - continue; - price = repMatchPrice + GetPrice_PureRep(p, i, p->state, posState); - do - { - UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, repLen); - COptimal *opt = &p->opt[repLen]; - if (price2 < opt->price) - { - opt->price = price2; - opt->len = (UInt32)repLen; - opt->dist = (UInt32)i; - opt->extra = 0; - } - } - while (--repLen >= 2); - } - - - // ---------- MATCH ---------- - { - unsigned len = repLens[0] + 1; - if (len <= mainLen) - { - unsigned offs = 0; - UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); - - if (len < 2) - len = 2; - else - while (len > matches[offs]) - offs += 2; - - for (; ; len++) - { - COptimal *opt; - UInt32 dist = matches[(size_t)offs + 1]; - UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); - unsigned lenToPosState = GetLenToPosState(len); - - if (dist < kNumFullDistances) - price += p->distancesPrices[lenToPosState][dist & (kNumFullDistances - 1)]; - else - { - unsigned slot; - GetPosSlot2(dist, slot); - price += p->alignPrices[dist & kAlignMask]; - price += p->posSlotPrices[lenToPosState][slot]; - } - - opt = &p->opt[len]; - - if (price < opt->price) - { - opt->price = price; - opt->len = (UInt32)len; - opt->dist = dist + LZMA_NUM_REPS; - opt->extra = 0; - } - - if (len == matches[offs]) - { - offs += 2; - if (offs == numPairs) - break; - } - } - } - } - - - cur = 0; - - #ifdef SHOW_STAT2 - /* if (position >= 0) */ - { - unsigned i; - printf("\n pos = %4X", position); - for (i = cur; i <= last; i++) - printf("\nprice[%4X] = %u", position - cur + i, p->opt[i].price); - } - #endif - } - - - - // ---------- Optimal Parsing ---------- - - for (;;) - { - unsigned numAvail; - UInt32 numAvailFull; - unsigned newLen, numPairs, prev, state, posState, startLen; - UInt32 litPrice, matchPrice, repMatchPrice; - BoolInt nextIsLit; - Byte curByte, matchByte; - const Byte *data; - COptimal *curOpt, *nextOpt; - - if (++cur == last) - break; - - // 18.06 - if (cur >= kNumOpts - 64) - { - unsigned j, best; - UInt32 price = p->opt[cur].price; - best = cur; - for (j = cur + 1; j <= last; j++) - { - UInt32 price2 = p->opt[j].price; - if (price >= price2) - { - price = price2; - best = j; - } - } - { - unsigned delta = best - cur; - if (delta != 0) - { - MOVE_POS(p, delta); - } - } - cur = best; - break; - } - - newLen = ReadMatchDistances(p, &numPairs); - - if (newLen >= p->numFastBytes) - { - p->numPairs = numPairs; - p->longestMatchLen = newLen; - break; - } - - curOpt = &p->opt[cur]; - - position++; - - // we need that check here, if skip_items in p->opt are possible - /* - if (curOpt->price >= kInfinityPrice) - continue; - */ - - prev = cur - curOpt->len; - - if (curOpt->len == 1) - { - state = (unsigned)p->opt[prev].state; - if (IsShortRep(curOpt)) - state = kShortRepNextStates[state]; - else - state = kLiteralNextStates[state]; - } - else - { - const COptimal *prevOpt; - UInt32 b0; - UInt32 dist = curOpt->dist; - - if (curOpt->extra) - { - prev -= (unsigned)curOpt->extra; - state = kState_RepAfterLit; - if (curOpt->extra == 1) - state = (dist < LZMA_NUM_REPS ? kState_RepAfterLit : kState_MatchAfterLit); - } - else - { - state = (unsigned)p->opt[prev].state; - if (dist < LZMA_NUM_REPS) - state = kRepNextStates[state]; - else - state = kMatchNextStates[state]; - } - - prevOpt = &p->opt[prev]; - b0 = prevOpt->reps[0]; - - if (dist < LZMA_NUM_REPS) - { - if (dist == 0) - { - reps[0] = b0; - reps[1] = prevOpt->reps[1]; - reps[2] = prevOpt->reps[2]; - reps[3] = prevOpt->reps[3]; - } - else - { - reps[1] = b0; - b0 = prevOpt->reps[1]; - if (dist == 1) - { - reps[0] = b0; - reps[2] = prevOpt->reps[2]; - reps[3] = prevOpt->reps[3]; - } - else - { - reps[2] = b0; - reps[0] = prevOpt->reps[dist]; - reps[3] = prevOpt->reps[dist ^ 1]; - } - } - } - else - { - reps[0] = (dist - LZMA_NUM_REPS + 1); - reps[1] = b0; - reps[2] = prevOpt->reps[1]; - reps[3] = prevOpt->reps[2]; - } - } - - curOpt->state = (CState)state; - curOpt->reps[0] = reps[0]; - curOpt->reps[1] = reps[1]; - curOpt->reps[2] = reps[2]; - curOpt->reps[3] = reps[3]; - - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - curByte = *data; - matchByte = *(data - reps[0]); - - posState = (position & p->pbMask); - - /* - The order of Price checks: - < LIT - <= SHORT_REP - < LIT : REP_0 - < REP [ : LIT : REP_0 ] - < MATCH [ : LIT : REP_0 ] - */ - - { - UInt32 curPrice = curOpt->price; - unsigned prob = p->isMatch[state][posState]; - matchPrice = curPrice + GET_PRICE_1(prob); - litPrice = curPrice + GET_PRICE_0(prob); - } - - nextOpt = &p->opt[(size_t)cur + 1]; - nextIsLit = False; - - // here we can allow skip_items in p->opt, if we don't check (nextOpt->price < kInfinityPrice) - // 18.new.06 - if ((nextOpt->price < kInfinityPrice - // && !IsLitState(state) - && matchByte == curByte) - || litPrice > nextOpt->price - ) - litPrice = 0; - else - { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - litPrice += (!IsLitState(state) ? - LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); - - if (litPrice < nextOpt->price) - { - nextOpt->price = litPrice; - nextOpt->len = 1; - MakeAs_Lit(nextOpt); - nextIsLit = True; - } - } - - repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); - - numAvailFull = p->numAvail; - { - unsigned temp = kNumOpts - 1 - cur; - if (numAvailFull > temp) - numAvailFull = (UInt32)temp; - } - - // 18.06 - // ---------- SHORT_REP ---------- - if (IsLitState(state)) // 18.new - if (matchByte == curByte) - if (repMatchPrice < nextOpt->price) // 18.new - // if (numAvailFull < 2 || data[1] != *(data - reps[0] + 1)) - if ( - // nextOpt->price >= kInfinityPrice || - nextOpt->len < 2 // we can check nextOpt->len, if skip items are not allowed in p->opt - || (nextOpt->dist != 0 - // && nextOpt->extra <= 1 // 17.old - ) - ) - { - UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, state, posState); - // if (shortRepPrice <= nextOpt->price) // 17.old - if (shortRepPrice < nextOpt->price) // 18.new - { - nextOpt->price = shortRepPrice; - nextOpt->len = 1; - MakeAs_ShortRep(nextOpt); - nextIsLit = False; - } - } - - if (numAvailFull < 2) - continue; - numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes); - - // numAvail <= p->numFastBytes - - // ---------- LIT : REP_0 ---------- - - if (!nextIsLit - && litPrice != 0 // 18.new - && matchByte != curByte - && numAvailFull > 2) - { - const Byte *data2 = data - reps[0]; - if (data[1] == data2[1] && data[2] == data2[2]) - { - unsigned len; - unsigned limit = p->numFastBytes + 1; - if (limit > numAvailFull) - limit = numAvailFull; - for (len = 3; len < limit && data[len] == data2[len]; len++) - {} - - { - unsigned state2 = kLiteralNextStates[state]; - unsigned posState2 = (position + 1) & p->pbMask; - UInt32 price = litPrice + GetPrice_Rep_0(p, state2, posState2); - { - unsigned offset = cur + len; - - if (last < offset) - last = offset; - - // do - { - UInt32 price2; - COptimal *opt; - len--; - // price2 = price + GetPrice_Len_Rep_0(p, len, state2, posState2); - price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len); - - opt = &p->opt[offset]; - // offset--; - if (price2 < opt->price) - { - opt->price = price2; - opt->len = (UInt32)len; - opt->dist = 0; - opt->extra = 1; - } - } - // while (len >= 3); - } - } - } - } - - startLen = 2; /* speed optimization */ - - { - // ---------- REP ---------- - unsigned repIndex = 0; // 17.old - // unsigned repIndex = IsLitState(state) ? 0 : 1; // 18.notused - for (; repIndex < LZMA_NUM_REPS; repIndex++) - { - unsigned len; - UInt32 price; - const Byte *data2 = data - reps[repIndex]; - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - - for (len = 2; len < numAvail && data[len] == data2[len]; len++) - {} - - // if (len < startLen) continue; // 18.new: speed optimization - - { - unsigned offset = cur + len; - if (last < offset) - last = offset; - } - { - unsigned len2 = len; - price = repMatchPrice + GetPrice_PureRep(p, repIndex, state, posState); - do - { - UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, len2); - COptimal *opt = &p->opt[cur + len2]; - if (price2 < opt->price) - { - opt->price = price2; - opt->len = (UInt32)len2; - opt->dist = (UInt32)repIndex; - opt->extra = 0; - } - } - while (--len2 >= 2); - } - - if (repIndex == 0) startLen = len + 1; // 17.old - // startLen = len + 1; // 18.new - - /* if (_maxMode) */ - { - // ---------- REP : LIT : REP_0 ---------- - // numFastBytes + 1 + numFastBytes - - unsigned len2 = len + 1; - unsigned limit = len2 + p->numFastBytes; - if (limit > numAvailFull) - limit = numAvailFull; - - len2 += 2; - if (len2 <= limit) - if (data[len2 - 2] == data2[len2 - 2]) - if (data[len2 - 1] == data2[len2 - 1]) - { - unsigned state2 = kRepNextStates[state]; - unsigned posState2 = (position + len) & p->pbMask; - price += GET_PRICE_LEN(&p->repLenEnc, posState, len) - + GET_PRICE_0(p->isMatch[state2][posState2]) - + LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), - data[len], data2[len], p->ProbPrices); - - // state2 = kLiteralNextStates[state2]; - state2 = kState_LitAfterRep; - posState2 = (posState2 + 1) & p->pbMask; - - - price += GetPrice_Rep_0(p, state2, posState2); - - for (; len2 < limit && data[len2] == data2[len2]; len2++) - {} - - len2 -= len; - // if (len2 >= 3) - { - { - unsigned offset = cur + len + len2; - - if (last < offset) - last = offset; - // do - { - UInt32 price2; - COptimal *opt; - len2--; - // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); - price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); - - opt = &p->opt[offset]; - // offset--; - if (price2 < opt->price) - { - opt->price = price2; - opt->len = (UInt32)len2; - opt->extra = (CExtra)(len + 1); - opt->dist = (UInt32)repIndex; - } - } - // while (len2 >= 3); - } - } - } - } - } - } - - - // ---------- MATCH ---------- - /* for (unsigned len = 2; len <= newLen; len++) */ - if (newLen > numAvail) - { - newLen = numAvail; - for (numPairs = 0; newLen > matches[numPairs]; numPairs += 2); - matches[numPairs] = (UInt32)newLen; - numPairs += 2; - } - - // startLen = 2; /* speed optimization */ - - if (newLen >= startLen) - { - UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]); - UInt32 dist; - unsigned offs, posSlot, len; - - { - unsigned offset = cur + newLen; - if (last < offset) - last = offset; - } - - offs = 0; - while (startLen > matches[offs]) - offs += 2; - dist = matches[(size_t)offs + 1]; - - // if (dist >= kNumFullDistances) - GetPosSlot2(dist, posSlot); - - for (len = /*2*/ startLen; ; len++) - { - UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); - { - COptimal *opt; - unsigned lenNorm = len - 2; - lenNorm = GetLenToPosState2(lenNorm); - if (dist < kNumFullDistances) - price += p->distancesPrices[lenNorm][dist & (kNumFullDistances - 1)]; - else - price += p->posSlotPrices[lenNorm][posSlot] + p->alignPrices[dist & kAlignMask]; - - opt = &p->opt[cur + len]; - if (price < opt->price) - { - opt->price = price; - opt->len = (UInt32)len; - opt->dist = dist + LZMA_NUM_REPS; - opt->extra = 0; - } - } - - if (len == matches[offs]) - { - // if (p->_maxMode) { - // MATCH : LIT : REP_0 - - const Byte *data2 = data - dist - 1; - unsigned len2 = len + 1; - unsigned limit = len2 + p->numFastBytes; - if (limit > numAvailFull) - limit = numAvailFull; - - len2 += 2; - if (len2 <= limit) - if (data[len2 - 2] == data2[len2 - 2]) - if (data[len2 - 1] == data2[len2 - 1]) - { - for (; len2 < limit && data[len2] == data2[len2]; len2++) - {} - - len2 -= len; - - // if (len2 >= 3) - { - unsigned state2 = kMatchNextStates[state]; - unsigned posState2 = (position + len) & p->pbMask; - unsigned offset; - price += GET_PRICE_0(p->isMatch[state2][posState2]); - price += LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), - data[len], data2[len], p->ProbPrices); - - // state2 = kLiteralNextStates[state2]; - state2 = kState_LitAfterMatch; - - posState2 = (posState2 + 1) & p->pbMask; - price += GetPrice_Rep_0(p, state2, posState2); - - offset = cur + len + len2; - - if (last < offset) - last = offset; - // do - { - UInt32 price2; - COptimal *opt; - len2--; - // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); - price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); - opt = &p->opt[offset]; - // offset--; - if (price2 < opt->price) - { - opt->price = price2; - opt->len = (UInt32)len2; - opt->extra = (CExtra)(len + 1); - opt->dist = dist + LZMA_NUM_REPS; - } - } - // while (len2 >= 3); - } - - } - - offs += 2; - if (offs == numPairs) - break; - dist = matches[(size_t)offs + 1]; - // if (dist >= kNumFullDistances) - GetPosSlot2(dist, posSlot); - } - } - } - } - - do - p->opt[last].price = kInfinityPrice; - while (--last); - - return Backward(p, cur); -} - - - -#define ChangePair(smallDist, bigDist) (((bigDist) >> 7) > (smallDist)) - - - -static unsigned GetOptimumFast(CLzmaEnc *p) -{ - UInt32 numAvail, mainDist; - unsigned mainLen, numPairs, repIndex, repLen, i; - const Byte *data; - - if (p->additionalOffset == 0) - mainLen = ReadMatchDistances(p, &numPairs); - else - { - mainLen = p->longestMatchLen; - numPairs = p->numPairs; - } - - numAvail = p->numAvail; - p->backRes = MARK_LIT; - if (numAvail < 2) - return 1; - // if (mainLen < 2 && p->state == 0) return 1; // 18.06.notused - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repLen = repIndex = 0; - - for (i = 0; i < LZMA_NUM_REPS; i++) - { - unsigned len; - const Byte *data2 = data - p->reps[i]; - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - for (len = 2; len < numAvail && data[len] == data2[len]; len++) - {} - if (len >= p->numFastBytes) - { - p->backRes = (UInt32)i; - MOVE_POS(p, len - 1) - return len; - } - if (len > repLen) - { - repIndex = i; - repLen = len; - } - } - - if (mainLen >= p->numFastBytes) - { - p->backRes = p->matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; - MOVE_POS(p, mainLen - 1) - return mainLen; - } - - mainDist = 0; /* for GCC */ - - if (mainLen >= 2) - { - mainDist = p->matches[(size_t)numPairs - 1]; - while (numPairs > 2) - { - UInt32 dist2; - if (mainLen != p->matches[(size_t)numPairs - 4] + 1) - break; - dist2 = p->matches[(size_t)numPairs - 3]; - if (!ChangePair(dist2, mainDist)) - break; - numPairs -= 2; - mainLen--; - mainDist = dist2; - } - if (mainLen == 2 && mainDist >= 0x80) - mainLen = 1; - } - - if (repLen >= 2) - if ( repLen + 1 >= mainLen - || (repLen + 2 >= mainLen && mainDist >= (1 << 9)) - || (repLen + 3 >= mainLen && mainDist >= (1 << 15))) - { - p->backRes = (UInt32)repIndex; - MOVE_POS(p, repLen - 1) - return repLen; - } - - if (mainLen < 2 || numAvail <= 2) - return 1; - - { - unsigned len1 = ReadMatchDistances(p, &p->numPairs); - p->longestMatchLen = len1; - - if (len1 >= 2) - { - UInt32 newDist = p->matches[(size_t)p->numPairs - 1]; - if ( (len1 >= mainLen && newDist < mainDist) - || (len1 == mainLen + 1 && !ChangePair(mainDist, newDist)) - || (len1 > mainLen + 1) - || (len1 + 1 >= mainLen && mainLen >= 3 && ChangePair(newDist, mainDist))) - return 1; - } - } - - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - - for (i = 0; i < LZMA_NUM_REPS; i++) - { - unsigned len, limit; - const Byte *data2 = data - p->reps[i]; - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - limit = mainLen - 1; - for (len = 2;; len++) - { - if (len >= limit) - return 1; - if (data[len] != data2[len]) - break; - } - } - - p->backRes = mainDist + LZMA_NUM_REPS; - if (mainLen != 2) - { - MOVE_POS(p, mainLen - 2) - } - return mainLen; -} - - - - -static void WriteEndMarker(CLzmaEnc *p, unsigned posState) -{ - UInt32 range; - range = p->rc.range; - { - UInt32 ttt, newBound; - CLzmaProb *prob = &p->isMatch[p->state][posState]; - RC_BIT_PRE(&p->rc, prob) - RC_BIT_1(&p->rc, prob) - prob = &p->isRep[p->state]; - RC_BIT_PRE(&p->rc, prob) - RC_BIT_0(&p->rc, prob) - } - p->state = kMatchNextStates[p->state]; - - p->rc.range = range; - LenEnc_Encode(&p->lenProbs, &p->rc, 0, posState); - range = p->rc.range; - - { - // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[0], (1 << kNumPosSlotBits) - 1); - CLzmaProb *probs = p->posSlotEncoder[0]; - unsigned m = 1; - do - { - UInt32 ttt, newBound; - RC_BIT_PRE(p, probs + m) - RC_BIT_1(&p->rc, probs + m); - m = (m << 1) + 1; - } - while (m < (1 << kNumPosSlotBits)); - } - { - // RangeEnc_EncodeDirectBits(&p->rc, ((UInt32)1 << (30 - kNumAlignBits)) - 1, 30 - kNumAlignBits); UInt32 range = p->range; - unsigned numBits = 30 - kNumAlignBits; - do - { - range >>= 1; - p->rc.low += range; - RC_NORM(&p->rc) - } - while (--numBits); - } - - { - // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); - CLzmaProb *probs = p->posAlignEncoder; - unsigned m = 1; - do - { - UInt32 ttt, newBound; - RC_BIT_PRE(p, probs + m) - RC_BIT_1(&p->rc, probs + m); - m = (m << 1) + 1; - } - while (m < kAlignTableSize); - } - p->rc.range = range; -} - - -static SRes CheckErrors(CLzmaEnc *p) -{ - if (p->result != SZ_OK) - return p->result; - if (p->rc.res != SZ_OK) - p->result = SZ_ERROR_WRITE; - if (p->matchFinderBase.result != SZ_OK) - p->result = SZ_ERROR_READ; - if (p->result != SZ_OK) - p->finished = True; - return p->result; -} - - -MY_NO_INLINE static SRes Flush(CLzmaEnc *p, UInt32 nowPos) -{ - /* ReleaseMFStream(); */ - p->finished = True; - if (p->writeEndMark) - WriteEndMarker(p, nowPos & p->pbMask); - RangeEnc_FlushData(&p->rc); - RangeEnc_FlushStream(&p->rc); - return CheckErrors(p); -} - - -MY_NO_INLINE static void FillAlignPrices(CLzmaEnc *p) -{ - unsigned i; - const CProbPrice *ProbPrices = p->ProbPrices; - const CLzmaProb *probs = p->posAlignEncoder; - // p->alignPriceCount = 0; - for (i = 0; i < kAlignTableSize / 2; i++) - { - UInt32 price = 0; - unsigned sym = i; - unsigned m = 1; - unsigned bit; - UInt32 prob; - bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; - bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; - bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; - prob = probs[m]; - p->alignPrices[i ] = price + GET_PRICEa_0(prob); - p->alignPrices[i + 8] = price + GET_PRICEa_1(prob); - // p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); - } -} - - -MY_NO_INLINE static void FillDistancesPrices(CLzmaEnc *p) -{ - // int y; for (y = 0; y < 100; y++) { - - UInt32 tempPrices[kNumFullDistances]; - unsigned i, lps; - - const CProbPrice *ProbPrices = p->ProbPrices; - p->matchPriceCount = 0; - - for (i = kStartPosModelIndex / 2; i < kNumFullDistances / 2; i++) - { - unsigned posSlot = GetPosSlot1(i); - unsigned footerBits = (posSlot >> 1) - 1; - unsigned base = ((2 | (posSlot & 1)) << footerBits); - const CLzmaProb *probs = p->posEncoders + (size_t)base * 2; - // tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base, footerBits, i - base, p->ProbPrices); - UInt32 price = 0; - unsigned m = 1; - unsigned sym = i; - unsigned offset = (unsigned)1 << footerBits; - base += i; - - if (footerBits) - do - { - unsigned bit = sym & 1; - sym >>= 1; - price += GET_PRICEa(probs[m], bit); - m = (m << 1) + bit; - } - while (--footerBits); - - { - unsigned prob = probs[m]; - tempPrices[base ] = price + GET_PRICEa_0(prob); - tempPrices[base + offset] = price + GET_PRICEa_1(prob); - } - } - - for (lps = 0; lps < kNumLenToPosStates; lps++) - { - unsigned slot; - unsigned distTableSize2 = (p->distTableSize + 1) >> 1; - UInt32 *posSlotPrices = p->posSlotPrices[lps]; - const CLzmaProb *probs = p->posSlotEncoder[lps]; - - for (slot = 0; slot < distTableSize2; slot++) - { - // posSlotPrices[slot] = RcTree_GetPrice(encoder, kNumPosSlotBits, slot, p->ProbPrices); - UInt32 price; - unsigned bit; - unsigned sym = slot + (1 << (kNumPosSlotBits - 1)); - unsigned prob; - bit = sym & 1; sym >>= 1; price = GET_PRICEa(probs[sym], bit); - bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); - bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); - bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); - bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); - prob = probs[(size_t)slot + (1 << (kNumPosSlotBits - 1))]; - posSlotPrices[(size_t)slot * 2 ] = price + GET_PRICEa_0(prob); - posSlotPrices[(size_t)slot * 2 + 1] = price + GET_PRICEa_1(prob); - } - - { - UInt32 delta = ((UInt32)((kEndPosModelIndex / 2 - 1) - kNumAlignBits) << kNumBitPriceShiftBits); - for (slot = kEndPosModelIndex / 2; slot < distTableSize2; slot++) - { - posSlotPrices[(size_t)slot * 2 ] += delta; - posSlotPrices[(size_t)slot * 2 + 1] += delta; - delta += ((UInt32)1 << kNumBitPriceShiftBits); - } - } - - { - UInt32 *dp = p->distancesPrices[lps]; - - dp[0] = posSlotPrices[0]; - dp[1] = posSlotPrices[1]; - dp[2] = posSlotPrices[2]; - dp[3] = posSlotPrices[3]; - - for (i = 4; i < kNumFullDistances; i += 2) - { - UInt32 slotPrice = posSlotPrices[GetPosSlot1(i)]; - dp[i ] = slotPrice + tempPrices[i]; - dp[i + 1] = slotPrice + tempPrices[i + 1]; - } - } - } - // } -} - - - -void LzmaEnc_Construct(CLzmaEnc *p) -{ - RangeEnc_Construct(&p->rc); - MatchFinder_Construct(&p->matchFinderBase); - - #ifndef _7ZIP_ST - MatchFinderMt_Construct(&p->matchFinderMt); - p->matchFinderMt.MatchFinder = &p->matchFinderBase; - #endif - - { - CLzmaEncProps props; - LzmaEncProps_Init(&props); - LzmaEnc_SetProps(p, &props); - } - - #ifndef LZMA_LOG_BSR - LzmaEnc_FastPosInit(p->g_FastPos); - #endif - - LzmaEnc_InitPriceTables(p->ProbPrices); - p->litProbs = NULL; - p->saveState.litProbs = NULL; - -} - -CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc) -{ - void *p; - p = ISzAlloc_Alloc(alloc, sizeof(CLzmaEnc)); - if (p) - LzmaEnc_Construct((CLzmaEnc *)p); - return p; -} - -void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->litProbs); - ISzAlloc_Free(alloc, p->saveState.litProbs); - p->litProbs = NULL; - p->saveState.litProbs = NULL; -} - -void LzmaEnc_Destruct(CLzmaEnc *p, ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - #ifndef _7ZIP_ST - MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); - #endif - - MatchFinder_Free(&p->matchFinderBase, allocBig); - LzmaEnc_FreeLits(p, alloc); - RangeEnc_Free(&p->rc, alloc); -} - -void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - LzmaEnc_Destruct((CLzmaEnc *)p, alloc, allocBig); - ISzAlloc_Free(alloc, p); -} - - -static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpackSize) -{ - UInt32 nowPos32, startPos32; - if (p->needInit) - { - p->matchFinder.Init(p->matchFinderObj); - p->needInit = 0; - } - - if (p->finished) - return p->result; - RINOK(CheckErrors(p)); - - nowPos32 = (UInt32)p->nowPos64; - startPos32 = nowPos32; - - if (p->nowPos64 == 0) - { - unsigned numPairs; - Byte curByte; - if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) - return Flush(p, nowPos32); - ReadMatchDistances(p, &numPairs); - RangeEnc_EncodeBit_0(&p->rc, &p->isMatch[kState_Start][0]); - // p->state = kLiteralNextStates[p->state]; - curByte = *(p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset); - LitEnc_Encode(&p->rc, p->litProbs, curByte); - p->additionalOffset--; - nowPos32++; - } - - if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) != 0) - - for (;;) - { - UInt32 dist; - unsigned len, posState; - UInt32 range, ttt, newBound; - CLzmaProb *probs; - - if (p->fastMode) - len = GetOptimumFast(p); - else - { - unsigned oci = p->optCur; - if (p->optEnd == oci) - len = GetOptimum(p, nowPos32); - else - { - const COptimal *opt = &p->opt[oci]; - len = opt->len; - p->backRes = opt->dist; - p->optCur = oci + 1; - } - } - - posState = (unsigned)nowPos32 & p->pbMask; - range = p->rc.range; - probs = &p->isMatch[p->state][posState]; - - RC_BIT_PRE(&p->rc, probs) - - dist = p->backRes; - - #ifdef SHOW_STAT2 - printf("\n pos = %6X, len = %3u pos = %6u", nowPos32, len, dist); - #endif - - if (dist == MARK_LIT) - { - Byte curByte; - const Byte *data; - unsigned state; - - RC_BIT_0(&p->rc, probs); - p->rc.range = range; - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; - probs = LIT_PROBS(nowPos32, *(data - 1)); - curByte = *data; - state = p->state; - p->state = kLiteralNextStates[state]; - if (IsLitState(state)) - LitEnc_Encode(&p->rc, probs, curByte); - else - LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0])); - } - else - { - RC_BIT_1(&p->rc, probs); - probs = &p->isRep[p->state]; - RC_BIT_PRE(&p->rc, probs) - - if (dist < LZMA_NUM_REPS) - { - RC_BIT_1(&p->rc, probs); - probs = &p->isRepG0[p->state]; - RC_BIT_PRE(&p->rc, probs) - if (dist == 0) - { - RC_BIT_0(&p->rc, probs); - probs = &p->isRep0Long[p->state][posState]; - RC_BIT_PRE(&p->rc, probs) - if (len != 1) - { - RC_BIT_1_BASE(&p->rc, probs); - } - else - { - RC_BIT_0_BASE(&p->rc, probs); - p->state = kShortRepNextStates[p->state]; - } - } - else - { - RC_BIT_1(&p->rc, probs); - probs = &p->isRepG1[p->state]; - RC_BIT_PRE(&p->rc, probs) - if (dist == 1) - { - RC_BIT_0_BASE(&p->rc, probs); - dist = p->reps[1]; - } - else - { - RC_BIT_1(&p->rc, probs); - probs = &p->isRepG2[p->state]; - RC_BIT_PRE(&p->rc, probs) - if (dist == 2) - { - RC_BIT_0_BASE(&p->rc, probs); - dist = p->reps[2]; - } - else - { - RC_BIT_1_BASE(&p->rc, probs); - dist = p->reps[3]; - p->reps[3] = p->reps[2]; - } - p->reps[2] = p->reps[1]; - } - p->reps[1] = p->reps[0]; - p->reps[0] = dist; - } - - RC_NORM(&p->rc) - - p->rc.range = range; - - if (len != 1) - { - LenEnc_Encode(&p->repLenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); - --p->repLenEncCounter; - p->state = kRepNextStates[p->state]; - } - } - else - { - unsigned posSlot; - RC_BIT_0(&p->rc, probs); - p->rc.range = range; - p->state = kMatchNextStates[p->state]; - - LenEnc_Encode(&p->lenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); - // --p->lenEnc.counter; - - dist -= LZMA_NUM_REPS; - p->reps[3] = p->reps[2]; - p->reps[2] = p->reps[1]; - p->reps[1] = p->reps[0]; - p->reps[0] = dist + 1; - - p->matchPriceCount++; - GetPosSlot(dist, posSlot); - // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], posSlot); - { - UInt32 sym = (UInt32)posSlot + (1 << kNumPosSlotBits); - range = p->rc.range; - probs = p->posSlotEncoder[GetLenToPosState(len)]; - do - { - CLzmaProb *prob = probs + (sym >> kNumPosSlotBits); - UInt32 bit = (sym >> (kNumPosSlotBits - 1)) & 1; - sym <<= 1; - RC_BIT(&p->rc, prob, bit); - } - while (sym < (1 << kNumPosSlotBits * 2)); - p->rc.range = range; - } - - if (dist >= kStartPosModelIndex) - { - unsigned footerBits = ((posSlot >> 1) - 1); - - if (dist < kNumFullDistances) - { - unsigned base = ((2 | (posSlot & 1)) << footerBits); - RcTree_ReverseEncode(&p->rc, p->posEncoders + base, footerBits, (unsigned)(dist /* - base */)); - } - else - { - UInt32 pos2 = (dist | 0xF) << (32 - footerBits); - range = p->rc.range; - // RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); - /* - do - { - range >>= 1; - p->rc.low += range & (0 - ((dist >> --footerBits) & 1)); - RC_NORM(&p->rc) - } - while (footerBits > kNumAlignBits); - */ - do - { - range >>= 1; - p->rc.low += range & (0 - (pos2 >> 31)); - pos2 += pos2; - RC_NORM(&p->rc) - } - while (pos2 != 0xF0000000); - - - // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); - - { - unsigned m = 1; - unsigned bit; - bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; - bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; - bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; - bit = dist & 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); - p->rc.range = range; - // p->alignPriceCount++; - } - } - } - } - } - - nowPos32 += (UInt32)len; - p->additionalOffset -= len; - - if (p->additionalOffset == 0) - { - UInt32 processed; - - if (!p->fastMode) - { - /* - if (p->alignPriceCount >= 16) // kAlignTableSize - FillAlignPrices(p); - if (p->matchPriceCount >= 128) - FillDistancesPrices(p); - if (p->lenEnc.counter <= 0) - LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); - */ - if (p->matchPriceCount >= 64) - { - FillAlignPrices(p); - // { int y; for (y = 0; y < 100; y++) { - FillDistancesPrices(p); - // }} - LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); - } - if (p->repLenEncCounter <= 0) - { - p->repLenEncCounter = REP_LEN_COUNT; - LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); - } - } - - if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) - break; - processed = nowPos32 - startPos32; - - if (maxPackSize) - { - if (processed + kNumOpts + 300 >= maxUnpackSize - || RangeEnc_GetProcessed_sizet(&p->rc) + kPackReserve >= maxPackSize) - break; - } - else if (processed >= (1 << 17)) - { - p->nowPos64 += nowPos32 - startPos32; - return CheckErrors(p); - } - } - } - - p->nowPos64 += nowPos32 - startPos32; - return Flush(p, nowPos32); -} - - - -#define kBigHashDicLimit ((UInt32)1 << 24) - -static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - UInt32 beforeSize = kNumOpts; - if (!RangeEnc_Alloc(&p->rc, alloc)) - return SZ_ERROR_MEM; - - #ifndef _7ZIP_ST - p->mtMode = (p->multiThread && !p->fastMode && (p->matchFinderBase.btMode != 0)); - #endif - - { - unsigned lclp = p->lc + p->lp; - if (!p->litProbs || !p->saveState.litProbs || p->lclp != lclp) - { - LzmaEnc_FreeLits(p, alloc); - p->litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); - p->saveState.litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); - if (!p->litProbs || !p->saveState.litProbs) - { - LzmaEnc_FreeLits(p, alloc); - return SZ_ERROR_MEM; - } - p->lclp = lclp; - } - } - - p->matchFinderBase.bigHash = (Byte)(p->dictSize > kBigHashDicLimit ? 1 : 0); - - if (beforeSize + p->dictSize < keepWindowSize) - beforeSize = keepWindowSize - p->dictSize; - - #ifndef _7ZIP_ST - if (p->mtMode) - { - RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, - LZMA_MATCH_LEN_MAX - + 1 /* 18.04 */ - , allocBig)); - p->matchFinderObj = &p->matchFinderMt; - p->matchFinderBase.bigHash = (Byte)( - (p->dictSize > kBigHashDicLimit && p->matchFinderBase.hashMask >= 0xFFFFFF) ? 1 : 0); - MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder); - } - else - #endif - { - if (!MatchFinder_Create(&p->matchFinderBase, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)) - return SZ_ERROR_MEM; - p->matchFinderObj = &p->matchFinderBase; - MatchFinder_CreateVTable(&p->matchFinderBase, &p->matchFinder); - } - - return SZ_OK; -} - -void LzmaEnc_Init(CLzmaEnc *p) -{ - unsigned i; - p->state = 0; - p->reps[0] = - p->reps[1] = - p->reps[2] = - p->reps[3] = 1; - - RangeEnc_Init(&p->rc); - - for (i = 0; i < (1 << kNumAlignBits); i++) - p->posAlignEncoder[i] = kProbInitValue; - - for (i = 0; i < kNumStates; i++) - { - unsigned j; - for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++) - { - p->isMatch[i][j] = kProbInitValue; - p->isRep0Long[i][j] = kProbInitValue; - } - p->isRep[i] = kProbInitValue; - p->isRepG0[i] = kProbInitValue; - p->isRepG1[i] = kProbInitValue; - p->isRepG2[i] = kProbInitValue; - } - - { - for (i = 0; i < kNumLenToPosStates; i++) - { - CLzmaProb *probs = p->posSlotEncoder[i]; - unsigned j; - for (j = 0; j < (1 << kNumPosSlotBits); j++) - probs[j] = kProbInitValue; - } - } - { - for (i = 0; i < kNumFullDistances; i++) - p->posEncoders[i] = kProbInitValue; - } - - { - UInt32 num = (UInt32)0x300 << (p->lp + p->lc); - UInt32 k; - CLzmaProb *probs = p->litProbs; - for (k = 0; k < num; k++) - probs[k] = kProbInitValue; - } - - - LenEnc_Init(&p->lenProbs); - LenEnc_Init(&p->repLenProbs); - - p->optEnd = 0; - p->optCur = 0; - - { - for (i = 0; i < kNumOpts; i++) - p->opt[i].price = kInfinityPrice; - } - - p->additionalOffset = 0; - - p->pbMask = (1 << p->pb) - 1; - p->lpMask = ((UInt32)0x100 << p->lp) - ((unsigned)0x100 >> p->lc); -} - - -void LzmaEnc_InitPrices(CLzmaEnc *p) -{ - if (!p->fastMode) - { - FillDistancesPrices(p); - FillAlignPrices(p); - } - - p->lenEnc.tableSize = - p->repLenEnc.tableSize = - p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN; - - p->repLenEncCounter = REP_LEN_COUNT; - - LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); - LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); -} - -static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - unsigned i; - for (i = kEndPosModelIndex / 2; i < kDicLogSizeMax; i++) - if (p->dictSize <= ((UInt32)1 << i)) - break; - p->distTableSize = i * 2; - - p->finished = False; - p->result = SZ_OK; - RINOK(LzmaEnc_Alloc(p, keepWindowSize, alloc, allocBig)); - LzmaEnc_Init(p); - LzmaEnc_InitPrices(p); - p->nowPos64 = 0; - return SZ_OK; -} - -static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, - ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - p->matchFinderBase.stream = inStream; - p->needInit = 1; - p->rc.outStream = outStream; - return LzmaEnc_AllocAndInit(p, 0, alloc, allocBig); -} - -SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, - ISeqInStream *inStream, UInt32 keepWindowSize, - ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - p->matchFinderBase.stream = inStream; - p->needInit = 1; - return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); -} - -static void LzmaEnc_SetInputBuf(CLzmaEnc *p, const Byte *src, SizeT srcLen) -{ - p->matchFinderBase.directInput = 1; - p->matchFinderBase.bufferBase = (Byte *)src; - p->matchFinderBase.directInputRem = srcLen; -} - -SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, - UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - LzmaEnc_SetInputBuf(p, src, srcLen); - p->needInit = 1; - - LzmaEnc_SetDataSize(pp, srcLen); - return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); -} - -void LzmaEnc_Finish(CLzmaEncHandle pp) -{ - #ifndef _7ZIP_ST - CLzmaEnc *p = (CLzmaEnc *)pp; - if (p->mtMode) - MatchFinderMt_ReleaseStream(&p->matchFinderMt); - #else - UNUSED_VAR(pp); - #endif -} - - -typedef struct -{ - ISeqOutStream vt; - Byte *data; - SizeT rem; - BoolInt overflow; -} CLzmaEnc_SeqOutStreamBuf; - -static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size) -{ - CLzmaEnc_SeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CLzmaEnc_SeqOutStreamBuf, vt); - if (p->rem < size) - { - size = p->rem; - p->overflow = True; - } - memcpy(p->data, data, size); - p->rem -= size; - p->data += size; - return size; -} - - -UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp) -{ - const CLzmaEnc *p = (CLzmaEnc *)pp; - return p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); -} - - -const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp) -{ - const CLzmaEnc *p = (CLzmaEnc *)pp; - return p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; -} - - -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, - Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - UInt64 nowPos64; - SRes res; - CLzmaEnc_SeqOutStreamBuf outStream; - - outStream.vt.Write = SeqOutStreamBuf_Write; - outStream.data = dest; - outStream.rem = *destLen; - outStream.overflow = False; - - p->writeEndMark = False; - p->finished = False; - p->result = SZ_OK; - - if (reInit) - LzmaEnc_Init(p); - LzmaEnc_InitPrices(p); - - nowPos64 = p->nowPos64; - RangeEnc_Init(&p->rc); - p->rc.outStream = &outStream.vt; - - if (desiredPackSize == 0) - return SZ_ERROR_OUTPUT_EOF; - - res = LzmaEnc_CodeOneBlock(p, desiredPackSize, *unpackSize); - - *unpackSize = (UInt32)(p->nowPos64 - nowPos64); - *destLen -= outStream.rem; - if (outStream.overflow) - return SZ_ERROR_OUTPUT_EOF; - - return res; -} - - -static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) -{ - SRes res = SZ_OK; - - #ifndef _7ZIP_ST - Byte allocaDummy[0x300]; - allocaDummy[0] = 0; - allocaDummy[1] = allocaDummy[0]; - #endif - - for (;;) - { - res = LzmaEnc_CodeOneBlock(p, 0, 0); - if (res != SZ_OK || p->finished) - break; - if (progress) - { - res = ICompressProgress_Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc)); - if (res != SZ_OK) - { - res = SZ_ERROR_PROGRESS; - break; - } - } - } - - LzmaEnc_Finish(p); - - /* - if (res == SZ_OK && !Inline_MatchFinder_IsFinishedOK(&p->matchFinderBase)) - res = SZ_ERROR_FAIL; - } - */ - - return res; -} - - -SRes LzmaEnc_Encode(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress, - ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - RINOK(LzmaEnc_Prepare(pp, outStream, inStream, alloc, allocBig)); - return LzmaEnc_Encode2((CLzmaEnc *)pp, progress); -} - - -SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, Byte *props, SizeT *size) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - unsigned i; - UInt32 dictSize = p->dictSize; - if (*size < LZMA_PROPS_SIZE) - return SZ_ERROR_PARAM; - *size = LZMA_PROPS_SIZE; - props[0] = (Byte)((p->pb * 5 + p->lp) * 9 + p->lc); - - if (dictSize >= ((UInt32)1 << 22)) - { - UInt32 kDictMask = ((UInt32)1 << 20) - 1; - if (dictSize < (UInt32)0xFFFFFFFF - kDictMask) - dictSize = (dictSize + kDictMask) & ~kDictMask; - } - else for (i = 11; i <= 30; i++) - { - if (dictSize <= ((UInt32)2 << i)) { dictSize = (2 << i); break; } - if (dictSize <= ((UInt32)3 << i)) { dictSize = (3 << i); break; } - } - - for (i = 0; i < 4; i++) - props[1 + i] = (Byte)(dictSize >> (8 * i)); - return SZ_OK; -} - - -unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle pp) -{ - return ((CLzmaEnc *)pp)->writeEndMark; -} - - -SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - SRes res; - CLzmaEnc *p = (CLzmaEnc *)pp; - - CLzmaEnc_SeqOutStreamBuf outStream; - - outStream.vt.Write = SeqOutStreamBuf_Write; - outStream.data = dest; - outStream.rem = *destLen; - outStream.overflow = False; - - p->writeEndMark = writeEndMark; - p->rc.outStream = &outStream.vt; - - res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig); - - if (res == SZ_OK) - { - res = LzmaEnc_Encode2(p, progress); - if (res == SZ_OK && p->nowPos64 != srcLen) - res = SZ_ERROR_FAIL; - } - - *destLen -= outStream.rem; - if (outStream.overflow) - return SZ_ERROR_OUTPUT_EOF; - return res; -} - - -SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - CLzmaEnc *p = (CLzmaEnc *)LzmaEnc_Create(alloc); - SRes res; - if (!p) - return SZ_ERROR_MEM; - - res = LzmaEnc_SetProps(p, props); - if (res == SZ_OK) - { - res = LzmaEnc_WriteProperties(p, propsEncoded, propsSize); - if (res == SZ_OK) - res = LzmaEnc_MemEncode(p, dest, destLen, src, srcLen, - writeEndMark, progress, alloc, allocBig); - } - - LzmaEnc_Destroy(p, alloc, allocBig); - return res; -} diff --git a/code/ryzom/client/src/seven_zip/LzmaEnc.h b/code/ryzom/client/src/seven_zip/LzmaEnc.h deleted file mode 100644 index 9194ee576..000000000 --- a/code/ryzom/client/src/seven_zip/LzmaEnc.h +++ /dev/null @@ -1,76 +0,0 @@ -/* LzmaEnc.h -- LZMA Encoder -2017-07-27 : Igor Pavlov : Public domain */ - -#ifndef __LZMA_ENC_H -#define __LZMA_ENC_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -#define LZMA_PROPS_SIZE 5 - -typedef struct _CLzmaEncProps -{ - int level; /* 0 <= level <= 9 */ - UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version - (1 << 12) <= dictSize <= (3 << 29) for 64-bit version - default = (1 << 24) */ - int lc; /* 0 <= lc <= 8, default = 3 */ - int lp; /* 0 <= lp <= 4, default = 0 */ - int pb; /* 0 <= pb <= 4, default = 2 */ - int algo; /* 0 - fast, 1 - normal, default = 1 */ - int fb; /* 5 <= fb <= 273, default = 32 */ - int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */ - int numHashBytes; /* 2, 3 or 4, default = 4 */ - UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ - unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */ - int numThreads; /* 1 or 2, default = 2 */ - - UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1. - Encoder uses this value to reduce dictionary size */ -} CLzmaEncProps; - -void LzmaEncProps_Init(CLzmaEncProps *p); -void LzmaEncProps_Normalize(CLzmaEncProps *p); -UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2); - - -/* ---------- CLzmaEncHandle Interface ---------- */ - -/* LzmaEnc* functions can return the following exit codes: -SRes: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater in props - SZ_ERROR_WRITE - ISeqOutStream write callback error - SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output - SZ_ERROR_PROGRESS - some break from progress callback - SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) -*/ - -typedef void * CLzmaEncHandle; - -CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc); -void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig); - -SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props); -void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize); -SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size); -unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p); - -SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, - ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); -SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); - - -/* ---------- One Call Interface ---------- */ - -SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/LzmaLib.cpp b/code/ryzom/client/src/seven_zip/LzmaLib.cpp deleted file mode 100644 index 706e9e58c..000000000 --- a/code/ryzom/client/src/seven_zip/LzmaLib.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/* LzmaLib.c -- LZMA library wrapper -2015-06-13 : Igor Pavlov : Public domain */ - -#include "Alloc.h" -#include "LzmaDec.h" -#include "LzmaEnc.h" -#include "LzmaLib.h" - -MY_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t srcLen, - unsigned char *outProps, size_t *outPropsSize, - int level, /* 0 <= level <= 9, default = 5 */ - unsigned dictSize, /* use (1 << N) or (3 << N). 4 KB < dictSize <= 128 MB */ - int lc, /* 0 <= lc <= 8, default = 3 */ - int lp, /* 0 <= lp <= 4, default = 0 */ - int pb, /* 0 <= pb <= 4, default = 2 */ - int fb, /* 5 <= fb <= 273, default = 32 */ - int numThreads /* 1 or 2, default = 2 */ -) -{ - CLzmaEncProps props; - LzmaEncProps_Init(&props); - props.level = level; - props.dictSize = dictSize; - props.lc = lc; - props.lp = lp; - props.pb = pb; - props.fb = fb; - props.numThreads = numThreads; - - return LzmaEncode(dest, destLen, src, srcLen, &props, outProps, outPropsSize, 0, - NULL, &g_Alloc, &g_Alloc); -} - - -MY_STDAPI LzmaUncompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t *srcLen, - const unsigned char *props, size_t propsSize) -{ - ELzmaStatus status; - return LzmaDecode(dest, destLen, src, srcLen, props, (unsigned)propsSize, LZMA_FINISH_ANY, &status, &g_Alloc); -} diff --git a/code/ryzom/client/src/seven_zip/LzmaLib.h b/code/ryzom/client/src/seven_zip/LzmaLib.h deleted file mode 100644 index 88fa87d35..000000000 --- a/code/ryzom/client/src/seven_zip/LzmaLib.h +++ /dev/null @@ -1,131 +0,0 @@ -/* LzmaLib.h -- LZMA library interface -2013-01-18 : Igor Pavlov : Public domain */ - -#ifndef __LZMA_LIB_H -#define __LZMA_LIB_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -#define MY_STDAPI int MY_STD_CALL - -#define LZMA_PROPS_SIZE 5 - -/* -RAM requirements for LZMA: - for compression: (dictSize * 11.5 + 6 MB) + state_size - for decompression: dictSize + state_size - state_size = (4 + (1.5 << (lc + lp))) KB - by default (lc=3, lp=0), state_size = 16 KB. - -LZMA properties (5 bytes) format - Offset Size Description - 0 1 lc, lp and pb in encoded form. - 1 4 dictSize (little endian). -*/ - -/* -LzmaCompress ------------- - -outPropsSize - - In: the pointer to the size of outProps buffer; *outPropsSize = LZMA_PROPS_SIZE = 5. - Out: the pointer to the size of written properties in outProps buffer; *outPropsSize = LZMA_PROPS_SIZE = 5. - - LZMA Encoder will use defult values for any parameter, if it is - -1 for any from: level, loc, lp, pb, fb, numThreads - 0 for dictSize - -level - compression level: 0 <= level <= 9; - - level dictSize algo fb - 0: 16 KB 0 32 - 1: 64 KB 0 32 - 2: 256 KB 0 32 - 3: 1 MB 0 32 - 4: 4 MB 0 32 - 5: 16 MB 1 32 - 6: 32 MB 1 32 - 7+: 64 MB 1 64 - - The default value for "level" is 5. - - algo = 0 means fast method - algo = 1 means normal method - -dictSize - The dictionary size in bytes. The maximum value is - 128 MB = (1 << 27) bytes for 32-bit version - 1 GB = (1 << 30) bytes for 64-bit version - The default value is 16 MB = (1 << 24) bytes. - It's recommended to use the dictionary that is larger than 4 KB and - that can be calculated as (1 << N) or (3 << N) sizes. - -lc - The number of literal context bits (high bits of previous literal). - It can be in the range from 0 to 8. The default value is 3. - Sometimes lc=4 gives the gain for big files. - -lp - The number of literal pos bits (low bits of current position for literals). - It can be in the range from 0 to 4. The default value is 0. - The lp switch is intended for periodical data when the period is equal to 2^lp. - For example, for 32-bit (4 bytes) periodical data you can use lp=2. Often it's - better to set lc=0, if you change lp switch. - -pb - The number of pos bits (low bits of current position). - It can be in the range from 0 to 4. The default value is 2. - The pb switch is intended for periodical data when the period is equal 2^pb. - -fb - Word size (the number of fast bytes). - It can be in the range from 5 to 273. The default value is 32. - Usually, a big number gives a little bit better compression ratio and - slower compression process. - -numThreads - The number of thereads. 1 or 2. The default value is 2. - Fast mode (algo = 0) can use only 1 thread. - -Out: - destLen - processed output size -Returns: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ - -MY_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t srcLen, - unsigned char *outProps, size_t *outPropsSize, /* *outPropsSize must be = 5 */ - int level, /* 0 <= level <= 9, default = 5 */ - unsigned dictSize, /* default = (1 << 24) */ - int lc, /* 0 <= lc <= 8, default = 3 */ - int lp, /* 0 <= lp <= 4, default = 0 */ - int pb, /* 0 <= pb <= 4, default = 2 */ - int fb, /* 5 <= fb <= 273, default = 32 */ - int numThreads /* 1 or 2, default = 2 */ - ); - -/* -LzmaUncompress --------------- -In: - dest - output data - destLen - output data size - src - input data - srcLen - input data size -Out: - destLen - processed output size - srcLen - processed input size -Returns: - SZ_OK - OK - SZ_ERROR_DATA - Data error - SZ_ERROR_MEM - Memory allocation arror - SZ_ERROR_UNSUPPORTED - Unsupported properties - SZ_ERROR_INPUT_EOF - it needs more bytes in input buffer (src) -*/ - -MY_STDAPI LzmaUncompress(unsigned char *dest, size_t *destLen, const unsigned char *src, SizeT *srcLen, - const unsigned char *props, size_t propsSize); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/Ppmd.h b/code/ryzom/client/src/seven_zip/Ppmd.h deleted file mode 100644 index a5c1e3ef2..000000000 --- a/code/ryzom/client/src/seven_zip/Ppmd.h +++ /dev/null @@ -1,85 +0,0 @@ -/* Ppmd.h -- PPMD codec common code -2017-04-03 : Igor Pavlov : Public domain -This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ - -#ifndef __PPMD_H -#define __PPMD_H - -#include "CpuArch.h" - -EXTERN_C_BEGIN - -#ifdef MY_CPU_32BIT - #define PPMD_32BIT -#endif - -#define PPMD_INT_BITS 7 -#define PPMD_PERIOD_BITS 7 -#define PPMD_BIN_SCALE (1 << (PPMD_INT_BITS + PPMD_PERIOD_BITS)) - -#define PPMD_GET_MEAN_SPEC(summ, shift, round) (((summ) + (1 << ((shift) - (round)))) >> (shift)) -#define PPMD_GET_MEAN(summ) PPMD_GET_MEAN_SPEC((summ), PPMD_PERIOD_BITS, 2) -#define PPMD_UPDATE_PROB_0(prob) ((prob) + (1 << PPMD_INT_BITS) - PPMD_GET_MEAN(prob)) -#define PPMD_UPDATE_PROB_1(prob) ((prob) - PPMD_GET_MEAN(prob)) - -#define PPMD_N1 4 -#define PPMD_N2 4 -#define PPMD_N3 4 -#define PPMD_N4 ((128 + 3 - 1 * PPMD_N1 - 2 * PPMD_N2 - 3 * PPMD_N3) / 4) -#define PPMD_NUM_INDEXES (PPMD_N1 + PPMD_N2 + PPMD_N3 + PPMD_N4) - -#pragma pack(push, 1) -/* Most compilers works OK here even without #pragma pack(push, 1), but some GCC compilers need it. */ - -/* SEE-contexts for PPM-contexts with masked symbols */ -typedef struct -{ - UInt16 Summ; /* Freq */ - Byte Shift; /* Speed of Freq change; low Shift is for fast change */ - Byte Count; /* Count to next change of Shift */ -} CPpmd_See; - -#define Ppmd_See_Update(p) if ((p)->Shift < PPMD_PERIOD_BITS && --(p)->Count == 0) \ - { (p)->Summ <<= 1; (p)->Count = (Byte)(3 << (p)->Shift++); } - -typedef struct -{ - Byte Symbol; - Byte Freq; - UInt16 SuccessorLow; - UInt16 SuccessorHigh; -} CPpmd_State; - -#pragma pack(pop) - -typedef - #ifdef PPMD_32BIT - CPpmd_State * - #else - UInt32 - #endif - CPpmd_State_Ref; - -typedef - #ifdef PPMD_32BIT - void * - #else - UInt32 - #endif - CPpmd_Void_Ref; - -typedef - #ifdef PPMD_32BIT - Byte * - #else - UInt32 - #endif - CPpmd_Byte_Ref; - -#define PPMD_SetAllBitsIn256Bytes(p) \ - { size_t z; for (z = 0; z < 256 / sizeof(p[0]); z += 8) { \ - p[z+7] = p[z+6] = p[z+5] = p[z+4] = p[z+3] = p[z+2] = p[z+1] = p[z+0] = ~(size_t)0; }} - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/Ppmd7.cpp b/code/ryzom/client/src/seven_zip/Ppmd7.cpp deleted file mode 100644 index 470aadccf..000000000 --- a/code/ryzom/client/src/seven_zip/Ppmd7.cpp +++ /dev/null @@ -1,712 +0,0 @@ -/* Ppmd7.c -- PPMdH codec -2018-07-04 : Igor Pavlov : Public domain -This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ - -#include "Precomp.h" - -#include - -#include "Ppmd7.h" - -const Byte PPMD7_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }; -static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051}; - -#define MAX_FREQ 124 -#define UNIT_SIZE 12 - -#define U2B(nu) ((UInt32)(nu) * UNIT_SIZE) -#define U2I(nu) (p->Units2Indx[(size_t)(nu) - 1]) -#define I2U(indx) (p->Indx2Units[indx]) - -#ifdef PPMD_32BIT - #define REF(ptr) (ptr) -#else - #define REF(ptr) ((UInt32)((Byte *)(ptr) - (p)->Base)) -#endif - -#define STATS_REF(ptr) ((CPpmd_State_Ref)REF(ptr)) - -#define CTX(ref) ((CPpmd7_Context *)Ppmd7_GetContext(p, ref)) -#define STATS(ctx) Ppmd7_GetStats(p, ctx) -#define ONE_STATE(ctx) Ppmd7Context_OneState(ctx) -#define SUFFIX(ctx) CTX((ctx)->Suffix) - -typedef CPpmd7_Context * CTX_PTR; - -struct CPpmd7_Node_; - -typedef - #ifdef PPMD_32BIT - struct CPpmd7_Node_ * - #else - UInt32 - #endif - CPpmd7_Node_Ref; - -typedef struct CPpmd7_Node_ -{ - UInt16 Stamp; /* must be at offset 0 as CPpmd7_Context::NumStats. Stamp=0 means free */ - UInt16 NU; - CPpmd7_Node_Ref Next; /* must be at offset >= 4 */ - CPpmd7_Node_Ref Prev; -} CPpmd7_Node; - -#ifdef PPMD_32BIT - #define NODE(ptr) (ptr) -#else - #define NODE(offs) ((CPpmd7_Node *)(p->Base + (offs))) -#endif - -void Ppmd7_Construct(CPpmd7 *p) -{ - unsigned i, k, m; - - p->Base = 0; - - for (i = 0, k = 0; i < PPMD_NUM_INDEXES; i++) - { - unsigned step = (i >= 12 ? 4 : (i >> 2) + 1); - do { p->Units2Indx[k++] = (Byte)i; } while (--step); - p->Indx2Units[i] = (Byte)k; - } - - p->NS2BSIndx[0] = (0 << 1); - p->NS2BSIndx[1] = (1 << 1); - memset(p->NS2BSIndx + 2, (2 << 1), 9); - memset(p->NS2BSIndx + 11, (3 << 1), 256 - 11); - - for (i = 0; i < 3; i++) - p->NS2Indx[i] = (Byte)i; - for (m = i, k = 1; i < 256; i++) - { - p->NS2Indx[i] = (Byte)m; - if (--k == 0) - k = (++m) - 2; - } - - memset(p->HB2Flag, 0, 0x40); - memset(p->HB2Flag + 0x40, 8, 0x100 - 0x40); -} - -void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->Base); - p->Size = 0; - p->Base = 0; -} - -BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc) -{ - if (!p->Base || p->Size != size) - { - size_t size2; - Ppmd7_Free(p, alloc); - size2 = 0 - #ifndef PPMD_32BIT - + UNIT_SIZE - #endif - ; - p->AlignOffset = - #ifdef PPMD_32BIT - (4 - size) & 3; - #else - 4 - (size & 3); - #endif - if ((p->Base = (Byte *)ISzAlloc_Alloc(alloc, p->AlignOffset + size + size2)) == 0) - return False; - p->Size = size; - } - return True; -} - -static void InsertNode(CPpmd7 *p, void *node, unsigned indx) -{ - *((CPpmd_Void_Ref *)node) = p->FreeList[indx]; - p->FreeList[indx] = REF(node); -} - -static void *RemoveNode(CPpmd7 *p, unsigned indx) -{ - CPpmd_Void_Ref *node = (CPpmd_Void_Ref *)Ppmd7_GetPtr(p, p->FreeList[indx]); - p->FreeList[indx] = *node; - return node; -} - -static void SplitBlock(CPpmd7 *p, void *ptr, unsigned oldIndx, unsigned newIndx) -{ - unsigned i, nu = I2U(oldIndx) - I2U(newIndx); - ptr = (Byte *)ptr + U2B(I2U(newIndx)); - if (I2U(i = U2I(nu)) != nu) - { - unsigned k = I2U(--i); - InsertNode(p, ((Byte *)ptr) + U2B(k), nu - k - 1); - } - InsertNode(p, ptr, i); -} - -static void GlueFreeBlocks(CPpmd7 *p) -{ - #ifdef PPMD_32BIT - CPpmd7_Node headItem; - CPpmd7_Node_Ref head = &headItem; - #else - CPpmd7_Node_Ref head = p->AlignOffset + p->Size; - #endif - - CPpmd7_Node_Ref n = head; - unsigned i; - - p->GlueCount = 255; - - /* create doubly-linked list of free blocks */ - for (i = 0; i < PPMD_NUM_INDEXES; i++) - { - UInt16 nu = I2U(i); - CPpmd7_Node_Ref next = (CPpmd7_Node_Ref)p->FreeList[i]; - p->FreeList[i] = 0; - while (next != 0) - { - CPpmd7_Node *node = NODE(next); - node->Next = n; - n = NODE(n)->Prev = next; - next = *(const CPpmd7_Node_Ref *)node; - node->Stamp = 0; - node->NU = (UInt16)nu; - } - } - NODE(head)->Stamp = 1; - NODE(head)->Next = n; - NODE(n)->Prev = head; - if (p->LoUnit != p->HiUnit) - ((CPpmd7_Node *)p->LoUnit)->Stamp = 1; - - /* Glue free blocks */ - while (n != head) - { - CPpmd7_Node *node = NODE(n); - UInt32 nu = (UInt32)node->NU; - for (;;) - { - CPpmd7_Node *node2 = NODE(n) + nu; - nu += node2->NU; - if (node2->Stamp != 0 || nu >= 0x10000) - break; - NODE(node2->Prev)->Next = node2->Next; - NODE(node2->Next)->Prev = node2->Prev; - node->NU = (UInt16)nu; - } - n = node->Next; - } - - /* Fill lists of free blocks */ - for (n = NODE(head)->Next; n != head;) - { - CPpmd7_Node *node = NODE(n); - unsigned nu; - CPpmd7_Node_Ref next = node->Next; - for (nu = node->NU; nu > 128; nu -= 128, node += 128) - InsertNode(p, node, PPMD_NUM_INDEXES - 1); - if (I2U(i = U2I(nu)) != nu) - { - unsigned k = I2U(--i); - InsertNode(p, node + k, nu - k - 1); - } - InsertNode(p, node, i); - n = next; - } -} - -static void *AllocUnitsRare(CPpmd7 *p, unsigned indx) -{ - unsigned i; - void *retVal; - if (p->GlueCount == 0) - { - GlueFreeBlocks(p); - if (p->FreeList[indx] != 0) - return RemoveNode(p, indx); - } - i = indx; - do - { - if (++i == PPMD_NUM_INDEXES) - { - UInt32 numBytes = U2B(I2U(indx)); - p->GlueCount--; - return ((UInt32)(p->UnitsStart - p->Text) > numBytes) ? (p->UnitsStart -= numBytes) : (NULL); - } - } - while (p->FreeList[i] == 0); - retVal = RemoveNode(p, i); - SplitBlock(p, retVal, i, indx); - return retVal; -} - -static void *AllocUnits(CPpmd7 *p, unsigned indx) -{ - UInt32 numBytes; - if (p->FreeList[indx] != 0) - return RemoveNode(p, indx); - numBytes = U2B(I2U(indx)); - if (numBytes <= (UInt32)(p->HiUnit - p->LoUnit)) - { - void *retVal = p->LoUnit; - p->LoUnit += numBytes; - return retVal; - } - return AllocUnitsRare(p, indx); -} - -#define MyMem12Cpy(dest, src, num) \ - { UInt32 *d = (UInt32 *)dest; const UInt32 *s = (const UInt32 *)src; UInt32 n = num; \ - do { d[0] = s[0]; d[1] = s[1]; d[2] = s[2]; s += 3; d += 3; } while (--n); } - -static void *ShrinkUnits(CPpmd7 *p, void *oldPtr, unsigned oldNU, unsigned newNU) -{ - unsigned i0 = U2I(oldNU); - unsigned i1 = U2I(newNU); - if (i0 == i1) - return oldPtr; - if (p->FreeList[i1] != 0) - { - void *ptr = RemoveNode(p, i1); - MyMem12Cpy(ptr, oldPtr, newNU); - InsertNode(p, oldPtr, i0); - return ptr; - } - SplitBlock(p, oldPtr, i0, i1); - return oldPtr; -} - -#define SUCCESSOR(p) ((CPpmd_Void_Ref)((p)->SuccessorLow | ((UInt32)(p)->SuccessorHigh << 16))) - -static void SetSuccessor(CPpmd_State *p, CPpmd_Void_Ref v) -{ - (p)->SuccessorLow = (UInt16)((UInt32)(v) & 0xFFFF); - (p)->SuccessorHigh = (UInt16)(((UInt32)(v) >> 16) & 0xFFFF); -} - -static void RestartModel(CPpmd7 *p) -{ - unsigned i, k, m; - - memset(p->FreeList, 0, sizeof(p->FreeList)); - p->Text = p->Base + p->AlignOffset; - p->HiUnit = p->Text + p->Size; - p->LoUnit = p->UnitsStart = p->HiUnit - p->Size / 8 / UNIT_SIZE * 7 * UNIT_SIZE; - p->GlueCount = 0; - - p->OrderFall = p->MaxOrder; - p->RunLength = p->InitRL = -(Int32)((p->MaxOrder < 12) ? p->MaxOrder : 12) - 1; - p->PrevSuccess = 0; - - p->MinContext = p->MaxContext = (CTX_PTR)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ - p->MinContext->Suffix = 0; - p->MinContext->NumStats = 256; - p->MinContext->SummFreq = 256 + 1; - p->FoundState = (CPpmd_State *)p->LoUnit; /* AllocUnits(p, PPMD_NUM_INDEXES - 1); */ - p->LoUnit += U2B(256 / 2); - p->MinContext->Stats = REF(p->FoundState); - for (i = 0; i < 256; i++) - { - CPpmd_State *s = &p->FoundState[i]; - s->Symbol = (Byte)i; - s->Freq = 1; - SetSuccessor(s, 0); - } - - for (i = 0; i < 128; i++) - for (k = 0; k < 8; k++) - { - UInt16 *dest = p->BinSumm[i] + k; - UInt16 val = (UInt16)(PPMD_BIN_SCALE - kInitBinEsc[k] / (i + 2)); - for (m = 0; m < 64; m += 8) - dest[m] = val; - } - - for (i = 0; i < 25; i++) - for (k = 0; k < 16; k++) - { - CPpmd_See *s = &p->See[i][k]; - s->Summ = (UInt16)((5 * i + 10) << (s->Shift = PPMD_PERIOD_BITS - 4)); - s->Count = 4; - } -} - -void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder) -{ - p->MaxOrder = maxOrder; - RestartModel(p); - p->DummySee.Shift = PPMD_PERIOD_BITS; - p->DummySee.Summ = 0; /* unused */ - p->DummySee.Count = 64; /* unused */ -} - -static CTX_PTR CreateSuccessors(CPpmd7 *p, BoolInt skip) -{ - CPpmd_State upState; - CTX_PTR c = p->MinContext; - CPpmd_Byte_Ref upBranch = (CPpmd_Byte_Ref)SUCCESSOR(p->FoundState); - CPpmd_State *ps[PPMD7_MAX_ORDER]; - unsigned numPs = 0; - - if (!skip) - ps[numPs++] = p->FoundState; - - while (c->Suffix) - { - CPpmd_Void_Ref successor; - CPpmd_State *s; - c = SUFFIX(c); - if (c->NumStats != 1) - { - for (s = STATS(c); s->Symbol != p->FoundState->Symbol; s++); - } - else - s = ONE_STATE(c); - successor = SUCCESSOR(s); - if (successor != upBranch) - { - c = CTX(successor); - if (numPs == 0) - return c; - break; - } - ps[numPs++] = s; - } - - upState.Symbol = *(const Byte *)Ppmd7_GetPtr(p, upBranch); - SetSuccessor(&upState, upBranch + 1); - - if (c->NumStats == 1) - upState.Freq = ONE_STATE(c)->Freq; - else - { - UInt32 cf, s0; - CPpmd_State *s; - for (s = STATS(c); s->Symbol != upState.Symbol; s++); - cf = s->Freq - 1; - s0 = c->SummFreq - c->NumStats - cf; - upState.Freq = (Byte)(1 + ((2 * cf <= s0) ? (5 * cf > s0) : ((2 * cf + 3 * s0 - 1) / (2 * s0)))); - } - - do - { - /* Create Child */ - CTX_PTR c1; /* = AllocContext(p); */ - if (p->HiUnit != p->LoUnit) - c1 = (CTX_PTR)(p->HiUnit -= UNIT_SIZE); - else if (p->FreeList[0] != 0) - c1 = (CTX_PTR)RemoveNode(p, 0); - else - { - c1 = (CTX_PTR)AllocUnitsRare(p, 0); - if (!c1) - return NULL; - } - c1->NumStats = 1; - *ONE_STATE(c1) = upState; - c1->Suffix = REF(c); - SetSuccessor(ps[--numPs], REF(c1)); - c = c1; - } - while (numPs != 0); - - return c; -} - -static void SwapStates(CPpmd_State *t1, CPpmd_State *t2) -{ - CPpmd_State tmp = *t1; - *t1 = *t2; - *t2 = tmp; -} - -static void UpdateModel(CPpmd7 *p) -{ - CPpmd_Void_Ref successor, fSuccessor = SUCCESSOR(p->FoundState); - CTX_PTR c; - unsigned s0, ns; - - if (p->FoundState->Freq < MAX_FREQ / 4 && p->MinContext->Suffix != 0) - { - c = SUFFIX(p->MinContext); - - if (c->NumStats == 1) - { - CPpmd_State *s = ONE_STATE(c); - if (s->Freq < 32) - s->Freq++; - } - else - { - CPpmd_State *s = STATS(c); - if (s->Symbol != p->FoundState->Symbol) - { - do { s++; } while (s->Symbol != p->FoundState->Symbol); - if (s[0].Freq >= s[-1].Freq) - { - SwapStates(&s[0], &s[-1]); - s--; - } - } - if (s->Freq < MAX_FREQ - 9) - { - s->Freq += 2; - c->SummFreq += 2; - } - } - } - - if (p->OrderFall == 0) - { - p->MinContext = p->MaxContext = CreateSuccessors(p, True); - if (p->MinContext == 0) - { - RestartModel(p); - return; - } - SetSuccessor(p->FoundState, REF(p->MinContext)); - return; - } - - *p->Text++ = p->FoundState->Symbol; - successor = REF(p->Text); - if (p->Text >= p->UnitsStart) - { - RestartModel(p); - return; - } - - if (fSuccessor) - { - if (fSuccessor <= successor) - { - CTX_PTR cs = CreateSuccessors(p, False); - if (cs == NULL) - { - RestartModel(p); - return; - } - fSuccessor = REF(cs); - } - if (--p->OrderFall == 0) - { - successor = fSuccessor; - p->Text -= (p->MaxContext != p->MinContext); - } - } - else - { - SetSuccessor(p->FoundState, successor); - fSuccessor = REF(p->MinContext); - } - - s0 = p->MinContext->SummFreq - (ns = p->MinContext->NumStats) - (p->FoundState->Freq - 1); - - for (c = p->MaxContext; c != p->MinContext; c = SUFFIX(c)) - { - unsigned ns1; - UInt32 cf, sf; - if ((ns1 = c->NumStats) != 1) - { - if ((ns1 & 1) == 0) - { - /* Expand for one UNIT */ - unsigned oldNU = ns1 >> 1; - unsigned i = U2I(oldNU); - if (i != U2I((size_t)oldNU + 1)) - { - void *ptr = AllocUnits(p, i + 1); - void *oldPtr; - if (!ptr) - { - RestartModel(p); - return; - } - oldPtr = STATS(c); - MyMem12Cpy(ptr, oldPtr, oldNU); - InsertNode(p, oldPtr, i); - c->Stats = STATS_REF(ptr); - } - } - c->SummFreq = (UInt16)(c->SummFreq + (2 * ns1 < ns) + 2 * ((4 * ns1 <= ns) & (c->SummFreq <= 8 * ns1))); - } - else - { - CPpmd_State *s = (CPpmd_State*)AllocUnits(p, 0); - if (!s) - { - RestartModel(p); - return; - } - *s = *ONE_STATE(c); - c->Stats = REF(s); - if (s->Freq < MAX_FREQ / 4 - 1) - s->Freq <<= 1; - else - s->Freq = MAX_FREQ - 4; - c->SummFreq = (UInt16)(s->Freq + p->InitEsc + (ns > 3)); - } - cf = 2 * (UInt32)p->FoundState->Freq * (c->SummFreq + 6); - sf = (UInt32)s0 + c->SummFreq; - if (cf < 6 * sf) - { - cf = 1 + (cf > sf) + (cf >= 4 * sf); - c->SummFreq += 3; - } - else - { - cf = 4 + (cf >= 9 * sf) + (cf >= 12 * sf) + (cf >= 15 * sf); - c->SummFreq = (UInt16)(c->SummFreq + cf); - } - { - CPpmd_State *s = STATS(c) + ns1; - SetSuccessor(s, successor); - s->Symbol = p->FoundState->Symbol; - s->Freq = (Byte)cf; - c->NumStats = (UInt16)(ns1 + 1); - } - } - p->MaxContext = p->MinContext = CTX(fSuccessor); -} - -static void Rescale(CPpmd7 *p) -{ - unsigned i, adder, sumFreq, escFreq; - CPpmd_State *stats = STATS(p->MinContext); - CPpmd_State *s = p->FoundState; - { - CPpmd_State tmp = *s; - for (; s != stats; s--) - s[0] = s[-1]; - *s = tmp; - } - escFreq = p->MinContext->SummFreq - s->Freq; - s->Freq += 4; - adder = (p->OrderFall != 0); - s->Freq = (Byte)((s->Freq + adder) >> 1); - sumFreq = s->Freq; - - i = p->MinContext->NumStats - 1; - do - { - escFreq -= (++s)->Freq; - s->Freq = (Byte)((s->Freq + adder) >> 1); - sumFreq += s->Freq; - if (s[0].Freq > s[-1].Freq) - { - CPpmd_State *s1 = s; - CPpmd_State tmp = *s1; - do - s1[0] = s1[-1]; - while (--s1 != stats && tmp.Freq > s1[-1].Freq); - *s1 = tmp; - } - } - while (--i); - - if (s->Freq == 0) - { - unsigned numStats = p->MinContext->NumStats; - unsigned n0, n1; - do { i++; } while ((--s)->Freq == 0); - escFreq += i; - p->MinContext->NumStats = (UInt16)(p->MinContext->NumStats - i); - if (p->MinContext->NumStats == 1) - { - CPpmd_State tmp = *stats; - do - { - tmp.Freq = (Byte)(tmp.Freq - (tmp.Freq >> 1)); - escFreq >>= 1; - } - while (escFreq > 1); - InsertNode(p, stats, U2I(((numStats + 1) >> 1))); - *(p->FoundState = ONE_STATE(p->MinContext)) = tmp; - return; - } - n0 = (numStats + 1) >> 1; - n1 = (p->MinContext->NumStats + 1) >> 1; - if (n0 != n1) - p->MinContext->Stats = STATS_REF(ShrinkUnits(p, stats, n0, n1)); - } - p->MinContext->SummFreq = (UInt16)(sumFreq + escFreq - (escFreq >> 1)); - p->FoundState = STATS(p->MinContext); -} - -CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *escFreq) -{ - CPpmd_See *see; - unsigned nonMasked = p->MinContext->NumStats - numMasked; - if (p->MinContext->NumStats != 256) - { - see = p->See[(unsigned)p->NS2Indx[(size_t)nonMasked - 1]] + - (nonMasked < (unsigned)SUFFIX(p->MinContext)->NumStats - p->MinContext->NumStats) + - 2 * (unsigned)(p->MinContext->SummFreq < 11 * p->MinContext->NumStats) + - 4 * (unsigned)(numMasked > nonMasked) + - p->HiBitsFlag; - { - unsigned r = (see->Summ >> see->Shift); - see->Summ = (UInt16)(see->Summ - r); - *escFreq = r + (r == 0); - } - } - else - { - see = &p->DummySee; - *escFreq = 1; - } - return see; -} - -static void NextContext(CPpmd7 *p) -{ - CTX_PTR c = CTX(SUCCESSOR(p->FoundState)); - if (p->OrderFall == 0 && (Byte *)c > p->Text) - p->MinContext = p->MaxContext = c; - else - UpdateModel(p); -} - -void Ppmd7_Update1(CPpmd7 *p) -{ - CPpmd_State *s = p->FoundState; - s->Freq += 4; - p->MinContext->SummFreq += 4; - if (s[0].Freq > s[-1].Freq) - { - SwapStates(&s[0], &s[-1]); - p->FoundState = --s; - if (s->Freq > MAX_FREQ) - Rescale(p); - } - NextContext(p); -} - -void Ppmd7_Update1_0(CPpmd7 *p) -{ - p->PrevSuccess = (2 * p->FoundState->Freq > p->MinContext->SummFreq); - p->RunLength += p->PrevSuccess; - p->MinContext->SummFreq += 4; - if ((p->FoundState->Freq += 4) > MAX_FREQ) - Rescale(p); - NextContext(p); -} - -void Ppmd7_UpdateBin(CPpmd7 *p) -{ - p->FoundState->Freq = (Byte)(p->FoundState->Freq + (p->FoundState->Freq < 128 ? 1: 0)); - p->PrevSuccess = 1; - p->RunLength++; - NextContext(p); -} - -void Ppmd7_Update2(CPpmd7 *p) -{ - p->MinContext->SummFreq += 4; - if ((p->FoundState->Freq += 4) > MAX_FREQ) - Rescale(p); - p->RunLength = p->InitRL; - UpdateModel(p); -} diff --git a/code/ryzom/client/src/seven_zip/Ppmd7.h b/code/ryzom/client/src/seven_zip/Ppmd7.h deleted file mode 100644 index 610539a04..000000000 --- a/code/ryzom/client/src/seven_zip/Ppmd7.h +++ /dev/null @@ -1,142 +0,0 @@ -/* Ppmd7.h -- PPMdH compression codec -2018-07-04 : Igor Pavlov : Public domain -This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ - -/* This code supports virtual RangeDecoder and includes the implementation -of RangeCoder from 7z, instead of RangeCoder from original PPMd var.H. -If you need the compatibility with original PPMd var.H, you can use external RangeDecoder */ - -#ifndef __PPMD7_H -#define __PPMD7_H - -#include "Ppmd.h" - -EXTERN_C_BEGIN - -#define PPMD7_MIN_ORDER 2 -#define PPMD7_MAX_ORDER 64 - -#define PPMD7_MIN_MEM_SIZE (1 << 11) -#define PPMD7_MAX_MEM_SIZE (0xFFFFFFFF - 12 * 3) - -struct CPpmd7_Context_; - -typedef - #ifdef PPMD_32BIT - struct CPpmd7_Context_ * - #else - UInt32 - #endif - CPpmd7_Context_Ref; - -typedef struct CPpmd7_Context_ -{ - UInt16 NumStats; - UInt16 SummFreq; - CPpmd_State_Ref Stats; - CPpmd7_Context_Ref Suffix; -} CPpmd7_Context; - -#define Ppmd7Context_OneState(p) ((CPpmd_State *)&(p)->SummFreq) - -typedef struct -{ - CPpmd7_Context *MinContext, *MaxContext; - CPpmd_State *FoundState; - unsigned OrderFall, InitEsc, PrevSuccess, MaxOrder, HiBitsFlag; - Int32 RunLength, InitRL; /* must be 32-bit at least */ - - UInt32 Size; - UInt32 GlueCount; - Byte *Base, *LoUnit, *HiUnit, *Text, *UnitsStart; - UInt32 AlignOffset; - - Byte Indx2Units[PPMD_NUM_INDEXES]; - Byte Units2Indx[128]; - CPpmd_Void_Ref FreeList[PPMD_NUM_INDEXES]; - Byte NS2Indx[256], NS2BSIndx[256], HB2Flag[256]; - CPpmd_See DummySee, See[25][16]; - UInt16 BinSumm[128][64]; -} CPpmd7; - -void Ppmd7_Construct(CPpmd7 *p); -BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc); -void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc); -void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder); -#define Ppmd7_WasAllocated(p) ((p)->Base != NULL) - - -/* ---------- Internal Functions ---------- */ - -extern const Byte PPMD7_kExpEscape[16]; - -#ifdef PPMD_32BIT - #define Ppmd7_GetPtr(p, ptr) (ptr) - #define Ppmd7_GetContext(p, ptr) (ptr) - #define Ppmd7_GetStats(p, ctx) ((ctx)->Stats) -#else - #define Ppmd7_GetPtr(p, offs) ((void *)((p)->Base + (offs))) - #define Ppmd7_GetContext(p, offs) ((CPpmd7_Context *)Ppmd7_GetPtr((p), (offs))) - #define Ppmd7_GetStats(p, ctx) ((CPpmd_State *)Ppmd7_GetPtr((p), ((ctx)->Stats))) -#endif - -void Ppmd7_Update1(CPpmd7 *p); -void Ppmd7_Update1_0(CPpmd7 *p); -void Ppmd7_Update2(CPpmd7 *p); -void Ppmd7_UpdateBin(CPpmd7 *p); - -#define Ppmd7_GetBinSumm(p) \ - &p->BinSumm[(size_t)(unsigned)Ppmd7Context_OneState(p->MinContext)->Freq - 1][p->PrevSuccess + \ - p->NS2BSIndx[(size_t)Ppmd7_GetContext(p, p->MinContext->Suffix)->NumStats - 1] + \ - (p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]) + \ - 2 * p->HB2Flag[(unsigned)Ppmd7Context_OneState(p->MinContext)->Symbol] + \ - ((p->RunLength >> 26) & 0x20)] - -CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *scale); - - -/* ---------- Decode ---------- */ - -typedef struct IPpmd7_RangeDec IPpmd7_RangeDec; - -struct IPpmd7_RangeDec -{ - UInt32 (*GetThreshold)(const IPpmd7_RangeDec *p, UInt32 total); - void (*Decode)(const IPpmd7_RangeDec *p, UInt32 start, UInt32 size); - UInt32 (*DecodeBit)(const IPpmd7_RangeDec *p, UInt32 size0); -}; - -typedef struct -{ - IPpmd7_RangeDec vt; - UInt32 Range; - UInt32 Code; - IByteIn *Stream; -} CPpmd7z_RangeDec; - -void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p); -BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p); -#define Ppmd7z_RangeDec_IsFinishedOK(p) ((p)->Code == 0) - -int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc); - - -/* ---------- Encode ---------- */ - -typedef struct -{ - UInt64 Low; - UInt32 Range; - Byte Cache; - UInt64 CacheSize; - IByteOut *Stream; -} CPpmd7z_RangeEnc; - -void Ppmd7z_RangeEnc_Init(CPpmd7z_RangeEnc *p); -void Ppmd7z_RangeEnc_FlushData(CPpmd7z_RangeEnc *p); - -void Ppmd7_EncodeSymbol(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/Ppmd7Dec.cpp b/code/ryzom/client/src/seven_zip/Ppmd7Dec.cpp deleted file mode 100644 index 311e9f9dd..000000000 --- a/code/ryzom/client/src/seven_zip/Ppmd7Dec.cpp +++ /dev/null @@ -1,191 +0,0 @@ -/* Ppmd7Dec.c -- PPMdH Decoder -2018-07-04 : Igor Pavlov : Public domain -This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ - -#include "Precomp.h" - -#include "Ppmd7.h" - -#define kTopValue (1 << 24) - -BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p) -{ - unsigned i; - p->Code = 0; - p->Range = 0xFFFFFFFF; - if (IByteIn_Read(p->Stream) != 0) - return False; - for (i = 0; i < 4; i++) - p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); - return (p->Code < 0xFFFFFFFF); -} - -#define GET_Ppmd7z_RangeDec CPpmd7z_RangeDec *p = CONTAINER_FROM_VTBL(pp, CPpmd7z_RangeDec, vt); - -static UInt32 Range_GetThreshold(const IPpmd7_RangeDec *pp, UInt32 total) -{ - GET_Ppmd7z_RangeDec - return p->Code / (p->Range /= total); -} - -static void Range_Normalize(CPpmd7z_RangeDec *p) -{ - if (p->Range < kTopValue) - { - p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); - p->Range <<= 8; - if (p->Range < kTopValue) - { - p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); - p->Range <<= 8; - } - } -} - -static void Range_Decode(const IPpmd7_RangeDec *pp, UInt32 start, UInt32 size) -{ - GET_Ppmd7z_RangeDec - p->Code -= start * p->Range; - p->Range *= size; - Range_Normalize(p); -} - -static UInt32 Range_DecodeBit(const IPpmd7_RangeDec *pp, UInt32 size0) -{ - GET_Ppmd7z_RangeDec - UInt32 newBound = (p->Range >> 14) * size0; - UInt32 symbol; - if (p->Code < newBound) - { - symbol = 0; - p->Range = newBound; - } - else - { - symbol = 1; - p->Code -= newBound; - p->Range -= newBound; - } - Range_Normalize(p); - return symbol; -} - -void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p) -{ - p->vt.GetThreshold = Range_GetThreshold; - p->vt.Decode = Range_Decode; - p->vt.DecodeBit = Range_DecodeBit; -} - - -#define MASK(sym) ((signed char *)charMask)[sym] - -int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc) -{ - size_t charMask[256 / sizeof(size_t)]; - if (p->MinContext->NumStats != 1) - { - CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext); - unsigned i; - UInt32 count, hiCnt; - if ((count = rc->GetThreshold(rc, p->MinContext->SummFreq)) < (hiCnt = s->Freq)) - { - Byte symbol; - rc->Decode(rc, 0, s->Freq); - p->FoundState = s; - symbol = s->Symbol; - Ppmd7_Update1_0(p); - return symbol; - } - p->PrevSuccess = 0; - i = p->MinContext->NumStats - 1; - do - { - if ((hiCnt += (++s)->Freq) > count) - { - Byte symbol; - rc->Decode(rc, hiCnt - s->Freq, s->Freq); - p->FoundState = s; - symbol = s->Symbol; - Ppmd7_Update1(p); - return symbol; - } - } - while (--i); - if (count >= p->MinContext->SummFreq) - return -2; - p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]; - rc->Decode(rc, hiCnt, p->MinContext->SummFreq - hiCnt); - PPMD_SetAllBitsIn256Bytes(charMask); - MASK(s->Symbol) = 0; - i = p->MinContext->NumStats - 1; - do { MASK((--s)->Symbol) = 0; } while (--i); - } - else - { - UInt16 *prob = Ppmd7_GetBinSumm(p); - if (rc->DecodeBit(rc, *prob) == 0) - { - Byte symbol; - *prob = (UInt16)PPMD_UPDATE_PROB_0(*prob); - symbol = (p->FoundState = Ppmd7Context_OneState(p->MinContext))->Symbol; - Ppmd7_UpdateBin(p); - return symbol; - } - *prob = (UInt16)PPMD_UPDATE_PROB_1(*prob); - p->InitEsc = PPMD7_kExpEscape[*prob >> 10]; - PPMD_SetAllBitsIn256Bytes(charMask); - MASK(Ppmd7Context_OneState(p->MinContext)->Symbol) = 0; - p->PrevSuccess = 0; - } - for (;;) - { - CPpmd_State *ps[256], *s; - UInt32 freqSum, count, hiCnt; - CPpmd_See *see; - unsigned i, num, numMasked = p->MinContext->NumStats; - do - { - p->OrderFall++; - if (!p->MinContext->Suffix) - return -1; - p->MinContext = Ppmd7_GetContext(p, p->MinContext->Suffix); - } - while (p->MinContext->NumStats == numMasked); - hiCnt = 0; - s = Ppmd7_GetStats(p, p->MinContext); - i = 0; - num = p->MinContext->NumStats - numMasked; - do - { - int k = (int)(MASK(s->Symbol)); - hiCnt += (s->Freq & k); - ps[i] = s++; - i -= k; - } - while (i != num); - - see = Ppmd7_MakeEscFreq(p, numMasked, &freqSum); - freqSum += hiCnt; - count = rc->GetThreshold(rc, freqSum); - - if (count < hiCnt) - { - Byte symbol; - CPpmd_State **pps = ps; - for (hiCnt = 0; (hiCnt += (*pps)->Freq) <= count; pps++); - s = *pps; - rc->Decode(rc, hiCnt - s->Freq, s->Freq); - Ppmd_See_Update(see); - p->FoundState = s; - symbol = s->Symbol; - Ppmd7_Update2(p); - return symbol; - } - if (count >= freqSum) - return -2; - rc->Decode(rc, hiCnt, freqSum - hiCnt); - see->Summ = (UInt16)(see->Summ + freqSum); - do { MASK(ps[--i]->Symbol) = 0; } while (i != 0); - } -} diff --git a/code/ryzom/client/src/seven_zip/Ppmd7Enc.cpp b/code/ryzom/client/src/seven_zip/Ppmd7Enc.cpp deleted file mode 100644 index 286b8712c..000000000 --- a/code/ryzom/client/src/seven_zip/Ppmd7Enc.cpp +++ /dev/null @@ -1,187 +0,0 @@ -/* Ppmd7Enc.c -- PPMdH Encoder -2017-04-03 : Igor Pavlov : Public domain -This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ - -#include "Precomp.h" - -#include "Ppmd7.h" - -#define kTopValue (1 << 24) - -void Ppmd7z_RangeEnc_Init(CPpmd7z_RangeEnc *p) -{ - p->Low = 0; - p->Range = 0xFFFFFFFF; - p->Cache = 0; - p->CacheSize = 1; -} - -static void RangeEnc_ShiftLow(CPpmd7z_RangeEnc *p) -{ - if ((UInt32)p->Low < (UInt32)0xFF000000 || (unsigned)(p->Low >> 32) != 0) - { - Byte temp = p->Cache; - do - { - IByteOut_Write(p->Stream, (Byte)(temp + (Byte)(p->Low >> 32))); - temp = 0xFF; - } - while (--p->CacheSize != 0); - p->Cache = (Byte)((UInt32)p->Low >> 24); - } - p->CacheSize++; - p->Low = (UInt32)p->Low << 8; -} - -static void RangeEnc_Encode(CPpmd7z_RangeEnc *p, UInt32 start, UInt32 size, UInt32 total) -{ - p->Low += start * (p->Range /= total); - p->Range *= size; - while (p->Range < kTopValue) - { - p->Range <<= 8; - RangeEnc_ShiftLow(p); - } -} - -static void RangeEnc_EncodeBit_0(CPpmd7z_RangeEnc *p, UInt32 size0) -{ - p->Range = (p->Range >> 14) * size0; - while (p->Range < kTopValue) - { - p->Range <<= 8; - RangeEnc_ShiftLow(p); - } -} - -static void RangeEnc_EncodeBit_1(CPpmd7z_RangeEnc *p, UInt32 size0) -{ - UInt32 newBound = (p->Range >> 14) * size0; - p->Low += newBound; - p->Range -= newBound; - while (p->Range < kTopValue) - { - p->Range <<= 8; - RangeEnc_ShiftLow(p); - } -} - -void Ppmd7z_RangeEnc_FlushData(CPpmd7z_RangeEnc *p) -{ - unsigned i; - for (i = 0; i < 5; i++) - RangeEnc_ShiftLow(p); -} - - -#define MASK(sym) ((signed char *)charMask)[sym] - -void Ppmd7_EncodeSymbol(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol) -{ - size_t charMask[256 / sizeof(size_t)]; - if (p->MinContext->NumStats != 1) - { - CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext); - UInt32 sum; - unsigned i; - if (s->Symbol == symbol) - { - RangeEnc_Encode(rc, 0, s->Freq, p->MinContext->SummFreq); - p->FoundState = s; - Ppmd7_Update1_0(p); - return; - } - p->PrevSuccess = 0; - sum = s->Freq; - i = p->MinContext->NumStats - 1; - do - { - if ((++s)->Symbol == symbol) - { - RangeEnc_Encode(rc, sum, s->Freq, p->MinContext->SummFreq); - p->FoundState = s; - Ppmd7_Update1(p); - return; - } - sum += s->Freq; - } - while (--i); - - p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]; - PPMD_SetAllBitsIn256Bytes(charMask); - MASK(s->Symbol) = 0; - i = p->MinContext->NumStats - 1; - do { MASK((--s)->Symbol) = 0; } while (--i); - RangeEnc_Encode(rc, sum, p->MinContext->SummFreq - sum, p->MinContext->SummFreq); - } - else - { - UInt16 *prob = Ppmd7_GetBinSumm(p); - CPpmd_State *s = Ppmd7Context_OneState(p->MinContext); - if (s->Symbol == symbol) - { - RangeEnc_EncodeBit_0(rc, *prob); - *prob = (UInt16)PPMD_UPDATE_PROB_0(*prob); - p->FoundState = s; - Ppmd7_UpdateBin(p); - return; - } - else - { - RangeEnc_EncodeBit_1(rc, *prob); - *prob = (UInt16)PPMD_UPDATE_PROB_1(*prob); - p->InitEsc = PPMD7_kExpEscape[*prob >> 10]; - PPMD_SetAllBitsIn256Bytes(charMask); - MASK(s->Symbol) = 0; - p->PrevSuccess = 0; - } - } - for (;;) - { - UInt32 escFreq; - CPpmd_See *see; - CPpmd_State *s; - UInt32 sum; - unsigned i, numMasked = p->MinContext->NumStats; - do - { - p->OrderFall++; - if (!p->MinContext->Suffix) - return; /* EndMarker (symbol = -1) */ - p->MinContext = Ppmd7_GetContext(p, p->MinContext->Suffix); - } - while (p->MinContext->NumStats == numMasked); - - see = Ppmd7_MakeEscFreq(p, numMasked, &escFreq); - s = Ppmd7_GetStats(p, p->MinContext); - sum = 0; - i = p->MinContext->NumStats; - do - { - int cur = s->Symbol; - if (cur == symbol) - { - UInt32 low = sum; - CPpmd_State *s1 = s; - do - { - sum += (s->Freq & (int)(MASK(s->Symbol))); - s++; - } - while (--i); - RangeEnc_Encode(rc, low, s1->Freq, sum + escFreq); - Ppmd_See_Update(see); - p->FoundState = s1; - Ppmd7_Update2(p); - return; - } - sum += (s->Freq & (int)(MASK(cur))); - MASK(cur) = 0; - s++; - } - while (--i); - - RangeEnc_Encode(rc, sum, escFreq, sum + escFreq); - see->Summ = (UInt16)(see->Summ + sum + escFreq); - } -} diff --git a/code/ryzom/client/src/seven_zip/Precomp.cpp b/code/ryzom/client/src/seven_zip/Precomp.cpp deleted file mode 100644 index 01605e3c2..000000000 --- a/code/ryzom/client/src/seven_zip/Precomp.cpp +++ /dev/null @@ -1,4 +0,0 @@ -/* Precomp.c -- StdAfx -2013-01-21 : Igor Pavlov : Public domain */ - -#include "Precomp.h" diff --git a/code/ryzom/client/src/seven_zip/Precomp.h b/code/ryzom/client/src/seven_zip/Precomp.h deleted file mode 100644 index dcaf49475..000000000 --- a/code/ryzom/client/src/seven_zip/Precomp.h +++ /dev/null @@ -1,19 +0,0 @@ -/* Precomp.h -- StdAfx -2013-11-12 : Igor Pavlov : Public domain */ - -#ifndef __7Z_PRECOMP_H -#define __7Z_PRECOMP_H - -#if defined(_MSC_VER) && defined(_DEBUG) - #define _CRTDBG_MAP_ALLOC - #include - #include - #define DEBUG_NEW new(_NORMAL_BLOCK, __FILE__, __LINE__) -#endif - -#define _7ZIP_ST - -#include "Compiler.h" -/* #include "7zTypes.h" */ - -#endif diff --git a/code/ryzom/client/src/seven_zip/RotateDefs.h b/code/ryzom/client/src/seven_zip/RotateDefs.h deleted file mode 100644 index 8f01d1a6c..000000000 --- a/code/ryzom/client/src/seven_zip/RotateDefs.h +++ /dev/null @@ -1,30 +0,0 @@ -/* RotateDefs.h -- Rotate functions -2015-03-25 : Igor Pavlov : Public domain */ - -#ifndef __ROTATE_DEFS_H -#define __ROTATE_DEFS_H - -#ifdef _MSC_VER - -#include - -/* don't use _rotl with MINGW. It can insert slow call to function. */ - -/* #if (_MSC_VER >= 1200) */ -#pragma intrinsic(_rotl) -#pragma intrinsic(_rotr) -/* #endif */ - -#define rotlFixed(x, n) _rotl((x), (n)) -#define rotrFixed(x, n) _rotr((x), (n)) - -#else - -/* new compilers can translate these macros to fast commands. */ - -#define rotlFixed(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) -#define rotrFixed(x, n) (((x) >> (n)) | ((x) << (32 - (n)))) - -#endif - -#endif diff --git a/code/ryzom/client/src/seven_zip/Sha256.cpp b/code/ryzom/client/src/seven_zip/Sha256.cpp deleted file mode 100644 index 04b688c6b..000000000 --- a/code/ryzom/client/src/seven_zip/Sha256.cpp +++ /dev/null @@ -1,248 +0,0 @@ -/* Crypto/Sha256.c -- SHA-256 Hash -2017-04-03 : Igor Pavlov : Public domain -This code is based on public domain code from Wei Dai's Crypto++ library. */ - -#include "Precomp.h" - -#include - -#include "CpuArch.h" -#include "RotateDefs.h" -#include "Sha256.h" - -/* define it for speed optimization */ -#ifndef _SFX -#define _SHA256_UNROLL -#define _SHA256_UNROLL2 -#endif - -/* #define _SHA256_UNROLL2 */ - -void Sha256_Init(CSha256 *p) -{ - p->state[0] = 0x6a09e667; - p->state[1] = 0xbb67ae85; - p->state[2] = 0x3c6ef372; - p->state[3] = 0xa54ff53a; - p->state[4] = 0x510e527f; - p->state[5] = 0x9b05688c; - p->state[6] = 0x1f83d9ab; - p->state[7] = 0x5be0cd19; - p->count = 0; -} - -#define S0(x) (rotrFixed(x, 2) ^ rotrFixed(x,13) ^ rotrFixed(x, 22)) -#define S1(x) (rotrFixed(x, 6) ^ rotrFixed(x,11) ^ rotrFixed(x, 25)) -#define s0(x) (rotrFixed(x, 7) ^ rotrFixed(x,18) ^ (x >> 3)) -#define s1(x) (rotrFixed(x,17) ^ rotrFixed(x,19) ^ (x >> 10)) - -#define blk0(i) (W[i]) -#define blk2(i) (W[i] += s1(W[((i)-2)&15]) + W[((i)-7)&15] + s0(W[((i)-15)&15])) - -#define Ch(x,y,z) (z^(x&(y^z))) -#define Maj(x,y,z) ((x&y)|(z&(x|y))) - -#ifdef _SHA256_UNROLL2 - -#define R(a,b,c,d,e,f,g,h, i) \ - h += S1(e) + Ch(e,f,g) + K[(i)+(size_t)(j)] + (j ? blk2(i) : blk0(i)); \ - d += h; \ - h += S0(a) + Maj(a, b, c) - -#define RX_8(i) \ - R(a,b,c,d,e,f,g,h, i); \ - R(h,a,b,c,d,e,f,g, i+1); \ - R(g,h,a,b,c,d,e,f, i+2); \ - R(f,g,h,a,b,c,d,e, i+3); \ - R(e,f,g,h,a,b,c,d, i+4); \ - R(d,e,f,g,h,a,b,c, i+5); \ - R(c,d,e,f,g,h,a,b, i+6); \ - R(b,c,d,e,f,g,h,a, i+7) - -#define RX_16 RX_8(0); RX_8(8); - -#else - -#define a(i) T[(0-(i))&7] -#define b(i) T[(1-(i))&7] -#define c(i) T[(2-(i))&7] -#define d(i) T[(3-(i))&7] -#define e(i) T[(4-(i))&7] -#define f(i) T[(5-(i))&7] -#define g(i) T[(6-(i))&7] -#define h(i) T[(7-(i))&7] - -#define R(i) \ - h(i) += S1(e(i)) + Ch(e(i),f(i),g(i)) + K[(i)+(size_t)(j)] + (j ? blk2(i) : blk0(i)); \ - d(i) += h(i); \ - h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) \ - -#ifdef _SHA256_UNROLL - -#define RX_8(i) R(i+0); R(i+1); R(i+2); R(i+3); R(i+4); R(i+5); R(i+6); R(i+7); -#define RX_16 RX_8(0); RX_8(8); - -#else - -#define RX_16 unsigned i; for (i = 0; i < 16; i++) { R(i); } - -#endif - -#endif - -static const UInt32 K[64] = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, - 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, - 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, - 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, - 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, - 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, - 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, - 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, - 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, - 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, - 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, - 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, - 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 -}; - -static void Sha256_WriteByteBlock(CSha256 *p) -{ - UInt32 W[16]; - unsigned j; - UInt32 *state; - - #ifdef _SHA256_UNROLL2 - UInt32 a,b,c,d,e,f,g,h; - #else - UInt32 T[8]; - #endif - - for (j = 0; j < 16; j += 4) - { - const Byte *ccc = p->buffer + j * 4; - W[j ] = GetBe32(ccc); - W[j + 1] = GetBe32(ccc + 4); - W[j + 2] = GetBe32(ccc + 8); - W[j + 3] = GetBe32(ccc + 12); - } - - state = p->state; - - #ifdef _SHA256_UNROLL2 - a = state[0]; - b = state[1]; - c = state[2]; - d = state[3]; - e = state[4]; - f = state[5]; - g = state[6]; - h = state[7]; - #else - for (j = 0; j < 8; j++) - T[j] = state[j]; - #endif - - for (j = 0; j < 64; j += 16) - { - RX_16 - } - - #ifdef _SHA256_UNROLL2 - state[0] += a; - state[1] += b; - state[2] += c; - state[3] += d; - state[4] += e; - state[5] += f; - state[6] += g; - state[7] += h; - #else - for (j = 0; j < 8; j++) - state[j] += T[j]; - #endif - - /* Wipe variables */ - /* memset(W, 0, sizeof(W)); */ - /* memset(T, 0, sizeof(T)); */ -} - -#undef S0 -#undef S1 -#undef s0 -#undef s1 - -void Sha256_Update(CSha256 *p, const Byte *data, size_t size) -{ - if (size == 0) - return; - - { - unsigned pos = (unsigned)p->count & 0x3F; - unsigned num; - - p->count += size; - - num = 64 - pos; - if (num > size) - { - memcpy(p->buffer + pos, data, size); - return; - } - - size -= num; - memcpy(p->buffer + pos, data, num); - data += num; - } - - for (;;) - { - Sha256_WriteByteBlock(p); - if (size < 64) - break; - size -= 64; - memcpy(p->buffer, data, 64); - data += 64; - } - - if (size != 0) - memcpy(p->buffer, data, size); -} - -void Sha256_Final(CSha256 *p, Byte *digest) -{ - unsigned pos = (unsigned)p->count & 0x3F; - unsigned i; - - p->buffer[pos++] = 0x80; - - while (pos != (64 - 8)) - { - pos &= 0x3F; - if (pos == 0) - Sha256_WriteByteBlock(p); - p->buffer[pos++] = 0; - } - - { - UInt64 numBits = (p->count << 3); - SetBe32(p->buffer + 64 - 8, (UInt32)(numBits >> 32)); - SetBe32(p->buffer + 64 - 4, (UInt32)(numBits)); - } - - Sha256_WriteByteBlock(p); - - for (i = 0; i < 8; i += 2) - { - UInt32 v0 = p->state[i]; - UInt32 v1 = p->state[i + 1]; - SetBe32(digest , v0); - SetBe32(digest + 4, v1); - digest += 8; - } - - Sha256_Init(p); -} diff --git a/code/ryzom/client/src/seven_zip/Sha256.h b/code/ryzom/client/src/seven_zip/Sha256.h deleted file mode 100644 index 3f455dbc0..000000000 --- a/code/ryzom/client/src/seven_zip/Sha256.h +++ /dev/null @@ -1,26 +0,0 @@ -/* Sha256.h -- SHA-256 Hash -2013-01-18 : Igor Pavlov : Public domain */ - -#ifndef __CRYPTO_SHA256_H -#define __CRYPTO_SHA256_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -#define SHA256_DIGEST_SIZE 32 - -typedef struct -{ - UInt32 state[8]; - UInt64 count; - Byte buffer[64]; -} CSha256; - -void Sha256_Init(CSha256 *p); -void Sha256_Update(CSha256 *p, const Byte *data, size_t size); -void Sha256_Final(CSha256 *p, Byte *digest); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/Sort.cpp b/code/ryzom/client/src/seven_zip/Sort.cpp deleted file mode 100644 index e1097e380..000000000 --- a/code/ryzom/client/src/seven_zip/Sort.cpp +++ /dev/null @@ -1,141 +0,0 @@ -/* Sort.c -- Sort functions -2014-04-05 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "Sort.h" - -#define HeapSortDown(p, k, size, temp) \ - { for (;;) { \ - size_t s = (k << 1); \ - if (s > size) break; \ - if (s < size && p[s + 1] > p[s]) s++; \ - if (temp >= p[s]) break; \ - p[k] = p[s]; k = s; \ - } p[k] = temp; } - -void HeapSort(UInt32 *p, size_t size) -{ - if (size <= 1) - return; - p--; - { - size_t i = size / 2; - do - { - UInt32 temp = p[i]; - size_t k = i; - HeapSortDown(p, k, size, temp) - } - while (--i != 0); - } - /* - do - { - size_t k = 1; - UInt32 temp = p[size]; - p[size--] = p[1]; - HeapSortDown(p, k, size, temp) - } - while (size > 1); - */ - while (size > 3) - { - UInt32 temp = p[size]; - size_t k = (p[3] > p[2]) ? 3 : 2; - p[size--] = p[1]; - p[1] = p[k]; - HeapSortDown(p, k, size, temp) - } - { - UInt32 temp = p[size]; - p[size] = p[1]; - if (size > 2 && p[2] < temp) - { - p[1] = p[2]; - p[2] = temp; - } - else - p[1] = temp; - } -} - -void HeapSort64(UInt64 *p, size_t size) -{ - if (size <= 1) - return; - p--; - { - size_t i = size / 2; - do - { - UInt64 temp = p[i]; - size_t k = i; - HeapSortDown(p, k, size, temp) - } - while (--i != 0); - } - /* - do - { - size_t k = 1; - UInt64 temp = p[size]; - p[size--] = p[1]; - HeapSortDown(p, k, size, temp) - } - while (size > 1); - */ - while (size > 3) - { - UInt64 temp = p[size]; - size_t k = (p[3] > p[2]) ? 3 : 2; - p[size--] = p[1]; - p[1] = p[k]; - HeapSortDown(p, k, size, temp) - } - { - UInt64 temp = p[size]; - p[size] = p[1]; - if (size > 2 && p[2] < temp) - { - p[1] = p[2]; - p[2] = temp; - } - else - p[1] = temp; - } -} - -/* -#define HeapSortRefDown(p, vals, n, size, temp) \ - { size_t k = n; UInt32 val = vals[temp]; for (;;) { \ - size_t s = (k << 1); \ - if (s > size) break; \ - if (s < size && vals[p[s + 1]] > vals[p[s]]) s++; \ - if (val >= vals[p[s]]) break; \ - p[k] = p[s]; k = s; \ - } p[k] = temp; } - -void HeapSortRef(UInt32 *p, UInt32 *vals, size_t size) -{ - if (size <= 1) - return; - p--; - { - size_t i = size / 2; - do - { - UInt32 temp = p[i]; - HeapSortRefDown(p, vals, i, size, temp); - } - while (--i != 0); - } - do - { - UInt32 temp = p[size]; - p[size--] = p[1]; - HeapSortRefDown(p, vals, 1, size, temp); - } - while (size > 1); -} -*/ diff --git a/code/ryzom/client/src/seven_zip/Sort.h b/code/ryzom/client/src/seven_zip/Sort.h deleted file mode 100644 index 2e2963a23..000000000 --- a/code/ryzom/client/src/seven_zip/Sort.h +++ /dev/null @@ -1,18 +0,0 @@ -/* Sort.h -- Sort functions -2014-04-05 : Igor Pavlov : Public domain */ - -#ifndef __7Z_SORT_H -#define __7Z_SORT_H - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -void HeapSort(UInt32 *p, size_t size); -void HeapSort64(UInt64 *p, size_t size); - -/* void HeapSortRef(UInt32 *p, UInt32 *vals, size_t size); */ - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/Xz.cpp b/code/ryzom/client/src/seven_zip/Xz.cpp deleted file mode 100644 index d9f83df16..000000000 --- a/code/ryzom/client/src/seven_zip/Xz.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* Xz.c - Xz -2017-05-12 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "7zCrc.h" -#include "CpuArch.h" -#include "Xz.h" -#include "XzCrc64.h" - -const Byte XZ_SIG[XZ_SIG_SIZE] = { 0xFD, '7', 'z', 'X', 'Z', 0 }; -/* const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; */ - -unsigned Xz_WriteVarInt(Byte *buf, UInt64 v) -{ - unsigned i = 0; - do - { - buf[i++] = (Byte)((v & 0x7F) | 0x80); - v >>= 7; - } - while (v != 0); - buf[(size_t)i - 1] &= 0x7F; - return i; -} - -void Xz_Construct(CXzStream *p) -{ - p->numBlocks = 0; - p->blocks = NULL; - p->flags = 0; -} - -void Xz_Free(CXzStream *p, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, p->blocks); - p->numBlocks = 0; - p->blocks = NULL; -} - -unsigned XzFlags_GetCheckSize(CXzStreamFlags f) -{ - unsigned t = XzFlags_GetCheckType(f); - return (t == 0) ? 0 : (4 << ((t - 1) / 3)); -} - -void XzCheck_Init(CXzCheck *p, unsigned mode) -{ - p->mode = mode; - switch (mode) - { - case XZ_CHECK_CRC32: p->crc = CRC_INIT_VAL; break; - case XZ_CHECK_CRC64: p->crc64 = CRC64_INIT_VAL; break; - case XZ_CHECK_SHA256: Sha256_Init(&p->sha); break; - } -} - -void XzCheck_Update(CXzCheck *p, const void *data, size_t size) -{ - switch (p->mode) - { - case XZ_CHECK_CRC32: p->crc = CrcUpdate(p->crc, data, size); break; - case XZ_CHECK_CRC64: p->crc64 = Crc64Update(p->crc64, data, size); break; - case XZ_CHECK_SHA256: Sha256_Update(&p->sha, (const Byte *)data, size); break; - } -} - -int XzCheck_Final(CXzCheck *p, Byte *digest) -{ - switch (p->mode) - { - case XZ_CHECK_CRC32: - SetUi32(digest, CRC_GET_DIGEST(p->crc)); - break; - case XZ_CHECK_CRC64: - { - int i; - UInt64 v = CRC64_GET_DIGEST(p->crc64); - for (i = 0; i < 8; i++, v >>= 8) - digest[i] = (Byte)(v & 0xFF); - break; - } - case XZ_CHECK_SHA256: - Sha256_Final(&p->sha, digest); - break; - default: - return 0; - } - return 1; -} diff --git a/code/ryzom/client/src/seven_zip/Xz.h b/code/ryzom/client/src/seven_zip/Xz.h deleted file mode 100644 index 544ee18f2..000000000 --- a/code/ryzom/client/src/seven_zip/Xz.h +++ /dev/null @@ -1,460 +0,0 @@ -/* Xz.h - Xz interface -2018-07-04 : Igor Pavlov : Public domain */ - -#ifndef __XZ_H -#define __XZ_H - -#include "Sha256.h" - -EXTERN_C_BEGIN - -#define XZ_ID_Subblock 1 -#define XZ_ID_Delta 3 -#define XZ_ID_X86 4 -#define XZ_ID_PPC 5 -#define XZ_ID_IA64 6 -#define XZ_ID_ARM 7 -#define XZ_ID_ARMT 8 -#define XZ_ID_SPARC 9 -#define XZ_ID_LZMA2 0x21 - -unsigned Xz_ReadVarInt(const Byte *p, size_t maxSize, UInt64 *value); -unsigned Xz_WriteVarInt(Byte *buf, UInt64 v); - -/* ---------- xz block ---------- */ - -#define XZ_BLOCK_HEADER_SIZE_MAX 1024 - -#define XZ_NUM_FILTERS_MAX 4 -#define XZ_BF_NUM_FILTERS_MASK 3 -#define XZ_BF_PACK_SIZE (1 << 6) -#define XZ_BF_UNPACK_SIZE (1 << 7) - -#define XZ_FILTER_PROPS_SIZE_MAX 20 - -typedef struct -{ - UInt64 id; - UInt32 propsSize; - Byte props[XZ_FILTER_PROPS_SIZE_MAX]; -} CXzFilter; - -typedef struct -{ - UInt64 packSize; - UInt64 unpackSize; - Byte flags; - CXzFilter filters[XZ_NUM_FILTERS_MAX]; -} CXzBlock; - -#define XzBlock_GetNumFilters(p) (((p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1) -#define XzBlock_HasPackSize(p) (((p)->flags & XZ_BF_PACK_SIZE) != 0) -#define XzBlock_HasUnpackSize(p) (((p)->flags & XZ_BF_UNPACK_SIZE) != 0) -#define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0) - -SRes XzBlock_Parse(CXzBlock *p, const Byte *header); -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes); - -/* ---------- xz stream ---------- */ - -#define XZ_SIG_SIZE 6 -#define XZ_FOOTER_SIG_SIZE 2 - -extern const Byte XZ_SIG[XZ_SIG_SIZE]; - -/* -extern const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE]; -*/ - -#define XZ_FOOTER_SIG_0 'Y' -#define XZ_FOOTER_SIG_1 'Z' - -#define XZ_STREAM_FLAGS_SIZE 2 -#define XZ_STREAM_CRC_SIZE 4 - -#define XZ_STREAM_HEADER_SIZE (XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE + XZ_STREAM_CRC_SIZE) -#define XZ_STREAM_FOOTER_SIZE (XZ_FOOTER_SIG_SIZE + XZ_STREAM_FLAGS_SIZE + XZ_STREAM_CRC_SIZE + 4) - -#define XZ_CHECK_MASK 0xF -#define XZ_CHECK_NO 0 -#define XZ_CHECK_CRC32 1 -#define XZ_CHECK_CRC64 4 -#define XZ_CHECK_SHA256 10 - -typedef struct -{ - unsigned mode; - UInt32 crc; - UInt64 crc64; - CSha256 sha; -} CXzCheck; - -void XzCheck_Init(CXzCheck *p, unsigned mode); -void XzCheck_Update(CXzCheck *p, const void *data, size_t size); -int XzCheck_Final(CXzCheck *p, Byte *digest); - -typedef UInt16 CXzStreamFlags; - -#define XzFlags_IsSupported(f) ((f) <= XZ_CHECK_MASK) -#define XzFlags_GetCheckType(f) ((f) & XZ_CHECK_MASK) -#define XzFlags_HasDataCrc32(f) (Xz_GetCheckType(f) == XZ_CHECK_CRC32) -unsigned XzFlags_GetCheckSize(CXzStreamFlags f); - -SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf); -SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream); - -typedef struct -{ - UInt64 unpackSize; - UInt64 totalSize; -} CXzBlockSizes; - -typedef struct -{ - CXzStreamFlags flags; - size_t numBlocks; - CXzBlockSizes *blocks; - UInt64 startOffset; -} CXzStream; - -void Xz_Construct(CXzStream *p); -void Xz_Free(CXzStream *p, ISzAllocPtr alloc); - -#define XZ_SIZE_OVERFLOW ((UInt64)(Int64)-1) - -UInt64 Xz_GetUnpackSize(const CXzStream *p); -UInt64 Xz_GetPackSize(const CXzStream *p); - -typedef struct -{ - size_t num; - size_t numAllocated; - CXzStream *streams; -} CXzs; - -void Xzs_Construct(CXzs *p); -void Xzs_Free(CXzs *p, ISzAllocPtr alloc); -SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc); - -UInt64 Xzs_GetNumBlocks(const CXzs *p); -UInt64 Xzs_GetUnpackSize(const CXzs *p); - - -// ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder - -typedef enum -{ - CODER_STATUS_NOT_SPECIFIED, /* use main error code instead */ - CODER_STATUS_FINISHED_WITH_MARK, /* stream was finished with end mark. */ - CODER_STATUS_NOT_FINISHED, /* stream was not finished */ - CODER_STATUS_NEEDS_MORE_INPUT /* you must provide more input bytes */ -} ECoderStatus; - - -// ECoderFinishMode values are identical to ELzmaFinishMode - -typedef enum -{ - CODER_FINISH_ANY, /* finish at any point */ - CODER_FINISH_END /* block must be finished at the end */ -} ECoderFinishMode; - - -typedef struct _IStateCoder -{ - void *p; - void (*Free)(void *p, ISzAllocPtr alloc); - SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAllocPtr alloc); - void (*Init)(void *p); - SRes (*Code2)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, - // int *wasFinished, - ECoderStatus *status); - SizeT (*Filter)(void *p, Byte *data, SizeT size); -} IStateCoder; - - - -#define MIXCODER_NUM_FILTERS_MAX 4 - -typedef struct -{ - ISzAllocPtr alloc; - Byte *buf; - unsigned numCoders; - - Byte *outBuf; - size_t outBufSize; - size_t outWritten; // is equal to lzmaDecoder.dicPos (in outBuf mode) - BoolInt wasFinished; - SRes res; - ECoderStatus status; - // BoolInt SingleBufMode; - - int finished[MIXCODER_NUM_FILTERS_MAX - 1]; - size_t pos[MIXCODER_NUM_FILTERS_MAX - 1]; - size_t size[MIXCODER_NUM_FILTERS_MAX - 1]; - UInt64 ids[MIXCODER_NUM_FILTERS_MAX]; - SRes results[MIXCODER_NUM_FILTERS_MAX]; - IStateCoder coders[MIXCODER_NUM_FILTERS_MAX]; -} CMixCoder; - - -typedef enum -{ - XZ_STATE_STREAM_HEADER, - XZ_STATE_STREAM_INDEX, - XZ_STATE_STREAM_INDEX_CRC, - XZ_STATE_STREAM_FOOTER, - XZ_STATE_STREAM_PADDING, - XZ_STATE_BLOCK_HEADER, - XZ_STATE_BLOCK, - XZ_STATE_BLOCK_FOOTER -} EXzState; - - -typedef struct -{ - EXzState state; - UInt32 pos; - unsigned alignPos; - unsigned indexPreSize; - - CXzStreamFlags streamFlags; - - UInt32 blockHeaderSize; - UInt64 packSize; - UInt64 unpackSize; - - UInt64 numBlocks; // number of finished blocks in current stream - UInt64 indexSize; - UInt64 indexPos; - UInt64 padSize; - - UInt64 numStartedStreams; - UInt64 numFinishedStreams; - UInt64 numTotalBlocks; - - UInt32 crc; - CMixCoder decoder; - CXzBlock block; - CXzCheck check; - CSha256 sha; - - BoolInt parseMode; - BoolInt headerParsedOk; - BoolInt decodeToStreamSignature; - unsigned decodeOnlyOneBlock; - - Byte *outBuf; - size_t outBufSize; - size_t outDataWritten; // the size of data in (outBuf) that were fully unpacked - - Byte shaDigest[SHA256_DIGEST_SIZE]; - Byte buf[XZ_BLOCK_HEADER_SIZE_MAX]; -} CXzUnpacker; - -/* alloc : aligned for cache line allocation is better */ -void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc); -void XzUnpacker_Init(CXzUnpacker *p); -void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize); -void XzUnpacker_Free(CXzUnpacker *p); - -/* - XzUnpacker - The sequence for decoding functions: - { - XzUnpacker_Construct() - [Decoding_Calls] - XzUnpacker_Free() - } - - [Decoding_Calls] - - There are 3 types of interfaces for [Decoding_Calls] calls: - - Interface-1 : Partial output buffers: - { - XzUnpacker_Init() - for() - XzUnpacker_Code(); - } - - Interface-2 : Direct output buffer: - Use it, if you know exact size of decoded data, and you need - whole xz unpacked data in one output buffer. - xz unpacker doesn't allocate additional buffer for lzma2 dictionary in that mode. - { - XzUnpacker_Init() - XzUnpacker_SetOutBufMode(); // to set output buffer and size - for() - XzUnpacker_Code(); // (dest = NULL) in XzUnpacker_Code() - } - - Interface-3 : Direct output buffer : One call full decoding - It unpacks whole input buffer to output buffer in one call. - It uses Interface-2 internally. - { - XzUnpacker_CodeFull() - } -*/ - -/* -finishMode: - It has meaning only if the decoding reaches output limit (*destLen). - CODER_FINISH_ANY - use smallest number of input bytes - CODER_FINISH_END - read EndOfStream marker after decoding - -Returns: - SZ_OK - status: - CODER_STATUS_NOT_FINISHED, - CODER_STATUS_NEEDS_MORE_INPUT - maybe there are more xz streams, - call XzUnpacker_IsStreamWasFinished to check that current stream was finished - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_DATA - Data error - SZ_ERROR_UNSUPPORTED - Unsupported method or method properties - SZ_ERROR_CRC - CRC error - // SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src). - - SZ_ERROR_NO_ARCHIVE - the error with xz Stream Header with one of the following reasons: - - xz Stream Signature failure - - CRC32 of xz Stream Header is failed - - The size of Stream padding is not multiple of four bytes. - It's possible to get that error, if xz stream was finished and the stream - contains some another data. In that case you can call XzUnpacker_GetExtraSize() - function to get real size of xz stream. -*/ - - -SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, int srcFinished, - ECoderFinishMode finishMode, ECoderStatus *status); - -SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, - ECoderFinishMode finishMode, ECoderStatus *status); - -BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); - -/* -XzUnpacker_GetExtraSize() returns then number of uncofirmed bytes, - if it's in (XZ_STATE_STREAM_HEADER) state or in (XZ_STATE_STREAM_PADDING) state. -These bytes can be some bytes after xz archive, or -it can be start of new xz stream. - -Call XzUnpacker_GetExtraSize() after XzUnpacker_Code() function to detect real size of -xz stream in two cases, if XzUnpacker_Code() returns: - res == SZ_OK && status == CODER_STATUS_NEEDS_MORE_INPUT - res == SZ_ERROR_NO_ARCHIVE -*/ - -UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p); - - -/* - for random block decoding: - XzUnpacker_Init(); - set CXzUnpacker::streamFlags - XzUnpacker_PrepareToRandomBlockDecoding() - loop - { - XzUnpacker_Code() - XzUnpacker_IsBlockFinished() - } -*/ - -void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p); -BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p); - -#define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags)) - - - -/* ---------- Multi Threading Decoding ---------- */ - - -typedef struct -{ - size_t inBufSize_ST; - size_t outStep_ST; - BoolInt ignoreErrors; - - #ifndef _7ZIP_ST - unsigned numThreads; - size_t inBufSize_MT; - size_t memUseMax; - #endif -} CXzDecMtProps; - -void XzDecMtProps_Init(CXzDecMtProps *p); - - -typedef void * CXzDecMtHandle; - -/* - alloc : XzDecMt uses CAlignOffsetAlloc for addresses allocated by (alloc). - allocMid : for big allocations, aligned allocation is better -*/ - -CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); -void XzDecMt_Destroy(CXzDecMtHandle p); - - -typedef struct -{ - Byte UnpackSize_Defined; - Byte NumStreams_Defined; - Byte NumBlocks_Defined; - - Byte DataAfterEnd; - Byte DecodingTruncated; // Decoding was Truncated, we need only partial output data - - UInt64 InSize; // pack size processed - UInt64 OutSize; - - UInt64 NumStreams; - UInt64 NumBlocks; - - SRes DecodeRes; - SRes ReadRes; - SRes ProgressRes; - SRes CombinedRes; - SRes CombinedRes_Type; - -} CXzStatInfo; - -void XzStatInfo_Clear(CXzStatInfo *p); - -/* -XzDecMt_Decode() -SRes: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_NO_ARCHIVE - is not xz archive - SZ_ERROR_ARCHIVE - Headers error - SZ_ERROR_DATA - Data Error - SZ_ERROR_CRC - CRC Error - SZ_ERROR_INPUT_EOF - it needs more input data - SZ_ERROR_WRITE - ISeqOutStream error - (SZ_ERROR_READ) - ISeqInStream errors - (SZ_ERROR_PROGRESS) - ICompressProgress errors - // SZ_ERROR_THREAD - error in multi-threading functions - MY_SRes_HRESULT_FROM_WRes(WRes_error) - error in multi-threading function -*/ - -SRes XzDecMt_Decode(CXzDecMtHandle p, - const CXzDecMtProps *props, - const UInt64 *outDataSize, // NULL means undefined - int finishMode, // 0 - partial unpacking is allowed, 1 - xz stream(s) must be finished - ISeqOutStream *outStream, - // Byte *outBuf, size_t *outBufSize, - ISeqInStream *inStream, - // const Byte *inData, size_t inDataSize, - CXzStatInfo *stat, - int *isMT, // 0 means that ST (Single-Thread) version was used - ICompressProgress *progress); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/XzCrc64.cpp b/code/ryzom/client/src/seven_zip/XzCrc64.cpp deleted file mode 100644 index b6d02cbeb..000000000 --- a/code/ryzom/client/src/seven_zip/XzCrc64.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/* XzCrc64.c -- CRC64 calculation -2017-05-23 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "XzCrc64.h" -#include "CpuArch.h" - -#define kCrc64Poly UINT64_CONST(0xC96C5795D7870F42) - -#ifdef MY_CPU_LE - #define CRC64_NUM_TABLES 4 -#else - #define CRC64_NUM_TABLES 5 - #define CRC_UINT64_SWAP(v) \ - ((v >> 56) \ - | ((v >> 40) & ((UInt64)0xFF << 8)) \ - | ((v >> 24) & ((UInt64)0xFF << 16)) \ - | ((v >> 8) & ((UInt64)0xFF << 24)) \ - | ((v << 8) & ((UInt64)0xFF << 32)) \ - | ((v << 24) & ((UInt64)0xFF << 40)) \ - | ((v << 40) & ((UInt64)0xFF << 48)) \ - | ((v << 56))) - - UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table); -#endif - -#ifndef MY_CPU_BE - UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table); -#endif - -typedef UInt64 (MY_FAST_CALL *CRC64_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table); - -static CRC64_FUNC g_Crc64Update; -UInt64 g_Crc64Table[256 * CRC64_NUM_TABLES]; - -UInt64 MY_FAST_CALL Crc64Update(UInt64 v, const void *data, size_t size) -{ - return g_Crc64Update(v, data, size, g_Crc64Table); -} - -UInt64 MY_FAST_CALL Crc64Calc(const void *data, size_t size) -{ - return g_Crc64Update(CRC64_INIT_VAL, data, size, g_Crc64Table) ^ CRC64_INIT_VAL; -} - -void MY_FAST_CALL Crc64GenerateTable() -{ - UInt32 i; - for (i = 0; i < 256; i++) - { - UInt64 r = i; - unsigned j; - for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrc64Poly & ((UInt64)0 - (r & 1))); - g_Crc64Table[i] = r; - } - for (i = 256; i < 256 * CRC64_NUM_TABLES; i++) - { - UInt64 r = g_Crc64Table[(size_t)i - 256]; - g_Crc64Table[i] = g_Crc64Table[r & 0xFF] ^ (r >> 8); - } - - #ifdef MY_CPU_LE - - g_Crc64Update = XzCrc64UpdateT4; - - #else - { - #ifndef MY_CPU_BE - UInt32 k = 1; - if (*(const Byte *)&k == 1) - g_Crc64Update = XzCrc64UpdateT4; - else - #endif - { - for (i = 256 * CRC64_NUM_TABLES - 1; i >= 256; i--) - { - UInt64 x = g_Crc64Table[(size_t)i - 256]; - g_Crc64Table[i] = CRC_UINT64_SWAP(x); - } - g_Crc64Update = XzCrc64UpdateT1_BeT4; - } - } - #endif -} diff --git a/code/ryzom/client/src/seven_zip/XzCrc64.h b/code/ryzom/client/src/seven_zip/XzCrc64.h deleted file mode 100644 index 08dbc330c..000000000 --- a/code/ryzom/client/src/seven_zip/XzCrc64.h +++ /dev/null @@ -1,26 +0,0 @@ -/* XzCrc64.h -- CRC64 calculation -2013-01-18 : Igor Pavlov : Public domain */ - -#ifndef __XZ_CRC64_H -#define __XZ_CRC64_H - -#include - -#include "7zTypes.h" - -EXTERN_C_BEGIN - -extern UInt64 g_Crc64Table[]; - -void MY_FAST_CALL Crc64GenerateTable(void); - -#define CRC64_INIT_VAL UINT64_CONST(0xFFFFFFFFFFFFFFFF) -#define CRC64_GET_DIGEST(crc) ((crc) ^ CRC64_INIT_VAL) -#define CRC64_UPDATE_BYTE(crc, b) (g_Crc64Table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) - -UInt64 MY_FAST_CALL Crc64Update(UInt64 crc, const void *data, size_t size); -UInt64 MY_FAST_CALL Crc64Calc(const void *data, size_t size); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/XzCrc64Opt.cpp b/code/ryzom/client/src/seven_zip/XzCrc64Opt.cpp deleted file mode 100644 index b2852de4d..000000000 --- a/code/ryzom/client/src/seven_zip/XzCrc64Opt.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/* XzCrc64Opt.c -- CRC64 calculation -2017-06-30 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include "CpuArch.h" - -#ifndef MY_CPU_BE - -#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) - -UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table) -{ - const Byte *p = (const Byte *)data; - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC64_UPDATE_BYTE_2(v, *p); - for (; size >= 4; size -= 4, p += 4) - { - UInt32 d = (UInt32)v ^ *(const UInt32 *)p; - v = (v >> 32) - ^ (table + 0x300)[((d ) & 0xFF)] - ^ (table + 0x200)[((d >> 8) & 0xFF)] - ^ (table + 0x100)[((d >> 16) & 0xFF)] - ^ (table + 0x000)[((d >> 24))]; - } - for (; size > 0; size--, p++) - v = CRC64_UPDATE_BYTE_2(v, *p); - return v; -} - -#endif - - -#ifndef MY_CPU_LE - -#define CRC_UINT64_SWAP(v) \ - ((v >> 56) \ - | ((v >> 40) & ((UInt64)0xFF << 8)) \ - | ((v >> 24) & ((UInt64)0xFF << 16)) \ - | ((v >> 8) & ((UInt64)0xFF << 24)) \ - | ((v << 8) & ((UInt64)0xFF << 32)) \ - | ((v << 24) & ((UInt64)0xFF << 40)) \ - | ((v << 40) & ((UInt64)0xFF << 48)) \ - | ((v << 56))) - -#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) - -UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table) -{ - const Byte *p = (const Byte *)data; - table += 0x100; - v = CRC_UINT64_SWAP(v); - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC64_UPDATE_BYTE_2_BE(v, *p); - for (; size >= 4; size -= 4, p += 4) - { - UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)p; - v = (v << 32) - ^ (table + 0x000)[((d ) & 0xFF)] - ^ (table + 0x100)[((d >> 8) & 0xFF)] - ^ (table + 0x200)[((d >> 16) & 0xFF)] - ^ (table + 0x300)[((d >> 24))]; - } - for (; size > 0; size--, p++) - v = CRC64_UPDATE_BYTE_2_BE(v, *p); - return CRC_UINT64_SWAP(v); -} - -#endif diff --git a/code/ryzom/client/src/seven_zip/XzDec.cpp b/code/ryzom/client/src/seven_zip/XzDec.cpp deleted file mode 100644 index 395e83f67..000000000 --- a/code/ryzom/client/src/seven_zip/XzDec.cpp +++ /dev/null @@ -1,2766 +0,0 @@ -/* XzDec.c -- Xz Decode -2019-02-02 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -// #include - -// #define XZ_DUMP - -/* #define XZ_DUMP */ - -#ifdef XZ_DUMP -#include -#endif - -// #define SHOW_DEBUG_INFO - -#ifdef SHOW_DEBUG_INFO -#include -#endif - -#ifdef SHOW_DEBUG_INFO -#define PRF(x) x -#else -#define PRF(x) -#endif - -#define PRF_STR(s) PRF(printf("\n" s "\n")) -#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) - -#include -#include - -#include "7zCrc.h" -#include "Alloc.h" -#include "Bra.h" -#include "CpuArch.h" -#include "Delta.h" -#include "Lzma2Dec.h" - -// #define USE_SUBBLOCK - -#ifdef USE_SUBBLOCK -#include "Bcj3Dec.c" -#include "SbDec.h" -#endif - -#include "Xz.h" - -#define XZ_CHECK_SIZE_MAX 64 - -#define CODER_BUF_SIZE ((size_t)1 << 17) - -unsigned Xz_ReadVarInt(const Byte *p, size_t maxSize, UInt64 *value) -{ - unsigned i, limit; - *value = 0; - limit = (maxSize > 9) ? 9 : (unsigned)maxSize; - - for (i = 0; i < limit;) - { - Byte b = p[i]; - *value |= (UInt64)(b & 0x7F) << (7 * i++); - if ((b & 0x80) == 0) - return (b == 0 && i != 1) ? 0 : i; - } - return 0; -} - -/* ---------- BraState ---------- */ - -#define BRA_BUF_SIZE (1 << 14) - -typedef struct -{ - size_t bufPos; - size_t bufConv; - size_t bufTotal; - - int encodeMode; - - UInt32 methodId; - UInt32 delta; - UInt32 ip; - UInt32 x86State; - Byte deltaState[DELTA_STATE_SIZE]; - - Byte buf[BRA_BUF_SIZE]; -} CBraState; - -static void BraState_Free(void *pp, ISzAllocPtr alloc) -{ - ISzAlloc_Free(alloc, pp); -} - -static SRes BraState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) -{ - CBraState *p = ((CBraState *)pp); - UNUSED_VAR(alloc); - p->ip = 0; - if (p->methodId == XZ_ID_Delta) - { - if (propSize != 1) - return SZ_ERROR_UNSUPPORTED; - p->delta = (unsigned)props[0] + 1; - } - else - { - if (propSize == 4) - { - UInt32 v = GetUi32(props); - switch (p->methodId) - { - case XZ_ID_PPC: - case XZ_ID_ARM: - case XZ_ID_SPARC: - if ((v & 3) != 0) - return SZ_ERROR_UNSUPPORTED; - break; - case XZ_ID_ARMT: - if ((v & 1) != 0) - return SZ_ERROR_UNSUPPORTED; - break; - case XZ_ID_IA64: - if ((v & 0xF) != 0) - return SZ_ERROR_UNSUPPORTED; - break; - } - p->ip = v; - } - else if (propSize != 0) - return SZ_ERROR_UNSUPPORTED; - } - return SZ_OK; -} - -static void BraState_Init(void *pp) -{ - CBraState *p = ((CBraState *)pp); - p->bufPos = p->bufConv = p->bufTotal = 0; - x86_Convert_Init(p->x86State); - if (p->methodId == XZ_ID_Delta) - Delta_Init(p->deltaState); -} - - -#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: size = isa ## _Convert(data, size, p->ip, p->encodeMode); break; - -static SizeT BraState_Filter(void *pp, Byte *data, SizeT size) -{ - CBraState *p = ((CBraState *)pp); - switch (p->methodId) - { - case XZ_ID_Delta: - if (p->encodeMode) - Delta_Encode(p->deltaState, p->delta, data, size); - else - Delta_Decode(p->deltaState, p->delta, data, size); - break; - case XZ_ID_X86: - size = x86_Convert(data, size, p->ip, &p->x86State, p->encodeMode); - break; - CASE_BRA_CONV(PPC) - CASE_BRA_CONV(IA64) - CASE_BRA_CONV(ARM) - CASE_BRA_CONV(ARMT) - CASE_BRA_CONV(SPARC) - } - p->ip += (UInt32)size; - return size; -} - - -static SRes BraState_Code2(void *pp, - Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, - // int *wasFinished - ECoderStatus *status) -{ - CBraState *p = ((CBraState *)pp); - SizeT destRem = *destLen; - SizeT srcRem = *srcLen; - UNUSED_VAR(finishMode); - - *destLen = 0; - *srcLen = 0; - // *wasFinished = False; - *status = CODER_STATUS_NOT_FINISHED; - - while (destRem > 0) - { - if (p->bufPos != p->bufConv) - { - size_t size = p->bufConv - p->bufPos; - if (size > destRem) - size = destRem; - memcpy(dest, p->buf + p->bufPos, size); - p->bufPos += size; - *destLen += size; - dest += size; - destRem -= size; - continue; - } - - p->bufTotal -= p->bufPos; - memmove(p->buf, p->buf + p->bufPos, p->bufTotal); - p->bufPos = 0; - p->bufConv = 0; - { - size_t size = BRA_BUF_SIZE - p->bufTotal; - if (size > srcRem) - size = srcRem; - memcpy(p->buf + p->bufTotal, src, size); - *srcLen += size; - src += size; - srcRem -= size; - p->bufTotal += size; - } - if (p->bufTotal == 0) - break; - - p->bufConv = BraState_Filter(pp, p->buf, p->bufTotal); - - if (p->bufConv == 0) - { - if (!srcWasFinished) - break; - p->bufConv = p->bufTotal; - } - } - - if (p->bufTotal == p->bufPos && srcRem == 0 && srcWasFinished) - { - *status = CODER_STATUS_FINISHED_WITH_MARK; - // *wasFinished = 1; - } - - return SZ_OK; -} - - -SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc) -{ - CBraState *decoder; - if (id < XZ_ID_Delta || id > XZ_ID_SPARC) - return SZ_ERROR_UNSUPPORTED; - decoder = (CBraState *)p->p; - if (!decoder) - { - decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); - if (!decoder) - return SZ_ERROR_MEM; - p->p = decoder; - p->Free = BraState_Free; - p->SetProps = BraState_SetProps; - p->Init = BraState_Init; - p->Code2 = BraState_Code2; - p->Filter = BraState_Filter; - } - decoder->methodId = (UInt32)id; - decoder->encodeMode = encodeMode; - return SZ_OK; -} - - - -/* ---------- SbState ---------- */ - -#ifdef USE_SUBBLOCK - -static void SbState_Free(void *pp, ISzAllocPtr alloc) -{ - CSbDec *p = (CSbDec *)pp; - SbDec_Free(p); - ISzAlloc_Free(alloc, pp); -} - -static SRes SbState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) -{ - UNUSED_VAR(pp); - UNUSED_VAR(props); - UNUSED_VAR(alloc); - return (propSize == 0) ? SZ_OK : SZ_ERROR_UNSUPPORTED; -} - -static void SbState_Init(void *pp) -{ - SbDec_Init((CSbDec *)pp); -} - -static SRes SbState_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, - // int *wasFinished - ECoderStatus *status) -{ - CSbDec *p = (CSbDec *)pp; - SRes res; - UNUSED_VAR(srcWasFinished); - p->dest = dest; - p->destLen = *destLen; - p->src = src; - p->srcLen = *srcLen; - p->finish = finishMode; /* change it */ - res = SbDec_Decode((CSbDec *)pp); - *destLen -= p->destLen; - *srcLen -= p->srcLen; - // *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ - *status = (*destLen == 0 && *srcLen == 0) ? - CODER_STATUS_FINISHED_WITH_MARK : - CODER_STATUS_NOT_FINISHED; - return res; -} - -static SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) -{ - CSbDec *decoder = (CSbDec *)p->p; - if (!decoder) - { - decoder = (CSbDec *)ISzAlloc_Alloc(alloc, sizeof(CSbDec)); - if (!decoder) - return SZ_ERROR_MEM; - p->p = decoder; - p->Free = SbState_Free; - p->SetProps = SbState_SetProps; - p->Init = SbState_Init; - p->Code2 = SbState_Code2; - p->Filter = NULL; - } - SbDec_Construct(decoder); - SbDec_SetAlloc(decoder, alloc); - return SZ_OK; -} - -#endif - - - -/* ---------- Lzma2 ---------- */ - -typedef struct -{ - CLzma2Dec decoder; - BoolInt outBufMode; -} CLzma2Dec_Spec; - - -static void Lzma2State_Free(void *pp, ISzAllocPtr alloc) -{ - CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; - if (p->outBufMode) - Lzma2Dec_FreeProbs(&p->decoder, alloc); - else - Lzma2Dec_Free(&p->decoder, alloc); - ISzAlloc_Free(alloc, pp); -} - -static SRes Lzma2State_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) -{ - if (propSize != 1) - return SZ_ERROR_UNSUPPORTED; - { - CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; - if (p->outBufMode) - return Lzma2Dec_AllocateProbs(&p->decoder, props[0], alloc); - else - return Lzma2Dec_Allocate(&p->decoder, props[0], alloc); - } -} - -static void Lzma2State_Init(void *pp) -{ - Lzma2Dec_Init(&((CLzma2Dec_Spec *)pp)->decoder); -} - - -/* - if (outBufMode), then (dest) is not used. Use NULL. - Data is unpacked to (spec->decoder.decoder.dic) output buffer. -*/ - -static SRes Lzma2State_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, - // int *wasFinished, - ECoderStatus *status) -{ - CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)pp; - ELzmaStatus status2; - /* ELzmaFinishMode fm = (finishMode == LZMA_FINISH_ANY) ? LZMA_FINISH_ANY : LZMA_FINISH_END; */ - SRes res; - UNUSED_VAR(srcWasFinished); - if (spec->outBufMode) - { - SizeT dicPos = spec->decoder.decoder.dicPos; - SizeT dicLimit = dicPos + *destLen; - res = Lzma2Dec_DecodeToDic(&spec->decoder, dicLimit, src, srcLen, (ELzmaFinishMode)finishMode, &status2); - *destLen = spec->decoder.decoder.dicPos - dicPos; - } - else - res = Lzma2Dec_DecodeToBuf(&spec->decoder, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status2); - // *wasFinished = (status2 == LZMA_STATUS_FINISHED_WITH_MARK); - // ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder - *status = (ECoderStatus)status2; - return res; -} - - -static SRes Lzma2State_SetFromMethod(IStateCoder *p, Byte *outBuf, size_t outBufSize, ISzAllocPtr alloc) -{ - CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; - if (!spec) - { - spec = (CLzma2Dec_Spec *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Dec_Spec)); - if (!spec) - return SZ_ERROR_MEM; - p->p = spec; - p->Free = Lzma2State_Free; - p->SetProps = Lzma2State_SetProps; - p->Init = Lzma2State_Init; - p->Code2 = Lzma2State_Code2; - p->Filter = NULL; - Lzma2Dec_Construct(&spec->decoder); - } - spec->outBufMode = False; - if (outBuf) - { - spec->outBufMode = True; - spec->decoder.decoder.dic = outBuf; - spec->decoder.decoder.dicBufSize = outBufSize; - } - return SZ_OK; -} - - -static SRes Lzma2State_ResetOutBuf(IStateCoder *p, Byte *outBuf, size_t outBufSize) -{ - CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; - if ((spec->outBufMode && !outBuf) || (!spec->outBufMode && outBuf)) - return SZ_ERROR_FAIL; - if (outBuf) - { - spec->decoder.decoder.dic = outBuf; - spec->decoder.decoder.dicBufSize = outBufSize; - } - return SZ_OK; -} - - - -static void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc) -{ - unsigned i; - p->alloc = alloc; - p->buf = NULL; - p->numCoders = 0; - - p->outBufSize = 0; - p->outBuf = NULL; - // p->SingleBufMode = False; - - for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) - p->coders[i].p = NULL; -} - - -static void MixCoder_Free(CMixCoder *p) -{ - unsigned i; - p->numCoders = 0; - for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) - { - IStateCoder *sc = &p->coders[i]; - if (sc->p) - { - sc->Free(sc->p, p->alloc); - sc->p = NULL; - } - } - if (p->buf) - { - ISzAlloc_Free(p->alloc, p->buf); - p->buf = NULL; /* 9.31: the BUG was fixed */ - } -} - -static void MixCoder_Init(CMixCoder *p) -{ - unsigned i; - for (i = 0; i < MIXCODER_NUM_FILTERS_MAX - 1; i++) - { - p->size[i] = 0; - p->pos[i] = 0; - p->finished[i] = 0; - } - for (i = 0; i < p->numCoders; i++) - { - IStateCoder *coder = &p->coders[i]; - coder->Init(coder->p); - p->results[i] = SZ_OK; - } - p->outWritten = 0; - p->wasFinished = False; - p->res = SZ_OK; - p->status = CODER_STATUS_NOT_SPECIFIED; -} - - -static SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) -{ - IStateCoder *sc = &p->coders[coderIndex]; - p->ids[coderIndex] = methodId; - switch (methodId) - { - case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, outBuf, outBufSize, p->alloc); - #ifdef USE_SUBBLOCK - case XZ_ID_Subblock: return SbState_SetFromMethod(sc, p->alloc); - #endif - } - if (coderIndex == 0) - return SZ_ERROR_UNSUPPORTED; - return BraState_SetFromMethod(sc, methodId, 0, p->alloc); -} - - -static SRes MixCoder_ResetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) -{ - IStateCoder *sc = &p->coders[coderIndex]; - switch (methodId) - { - case XZ_ID_LZMA2: return Lzma2State_ResetOutBuf(sc, outBuf, outBufSize); - } - return SZ_ERROR_UNSUPPORTED; -} - - - -/* - if (destFinish) - then unpack data block is finished at (*destLen) position, - and we can return data that were not processed by filter - -output (status) can be : - CODER_STATUS_NOT_FINISHED - CODER_STATUS_FINISHED_WITH_MARK - CODER_STATUS_NEEDS_MORE_INPUT - not implemented still -*/ - -static SRes MixCoder_Code(CMixCoder *p, - Byte *dest, SizeT *destLen, int destFinish, - const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode) -{ - SizeT destLenOrig = *destLen; - SizeT srcLenOrig = *srcLen; - - *destLen = 0; - *srcLen = 0; - - if (p->wasFinished) - return p->res; - - p->status = CODER_STATUS_NOT_FINISHED; - - // if (p->SingleBufMode) - if (p->outBuf) - { - SRes res; - SizeT destLen2, srcLen2; - int wasFinished; - - PRF_STR("------- MixCoder Single ----------"); - - srcLen2 = srcLenOrig; - destLen2 = destLenOrig; - - { - IStateCoder *coder = &p->coders[0]; - res = coder->Code2(coder->p, NULL, &destLen2, src, &srcLen2, srcWasFinished, finishMode, - // &wasFinished, - &p->status); - wasFinished = (p->status == CODER_STATUS_FINISHED_WITH_MARK); - } - - p->res = res; - - /* - if (wasFinished) - p->status = CODER_STATUS_FINISHED_WITH_MARK; - else - { - if (res == SZ_OK) - if (destLen2 != destLenOrig) - p->status = CODER_STATUS_NEEDS_MORE_INPUT; - } - */ - - - *srcLen = srcLen2; - src += srcLen2; - p->outWritten += destLen2; - - if (res != SZ_OK || srcWasFinished || wasFinished) - p->wasFinished = True; - - if (p->numCoders == 1) - *destLen = destLen2; - else if (p->wasFinished) - { - unsigned i; - size_t processed = p->outWritten; - - for (i = 1; i < p->numCoders; i++) - { - IStateCoder *coder = &p->coders[i]; - processed = coder->Filter(coder->p, p->outBuf, processed); - if (wasFinished || (destFinish && p->outWritten == destLenOrig)) - processed = p->outWritten; - PRF_STR_INT("filter", i); - } - *destLen = processed; - } - return res; - } - - PRF_STR("standard mix"); - - if (p->numCoders != 1) - { - if (!p->buf) - { - p->buf = (Byte *)ISzAlloc_Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); - if (!p->buf) - return SZ_ERROR_MEM; - } - - finishMode = CODER_FINISH_ANY; - } - - for (;;) - { - BoolInt processed = False; - BoolInt allFinished = True; - SRes resMain = SZ_OK; - unsigned i; - - p->status = CODER_STATUS_NOT_FINISHED; - /* - if (p->numCoders == 1 && *destLen == destLenOrig && finishMode == LZMA_FINISH_ANY) - break; - */ - - for (i = 0; i < p->numCoders; i++) - { - SRes res; - IStateCoder *coder = &p->coders[i]; - Byte *dest2; - SizeT destLen2, srcLen2; // destLen2_Orig; - const Byte *src2; - int srcFinished2; - int encodingWasFinished; - ECoderStatus status2; - - if (i == 0) - { - src2 = src; - srcLen2 = srcLenOrig - *srcLen; - srcFinished2 = srcWasFinished; - } - else - { - size_t k = i - 1; - src2 = p->buf + (CODER_BUF_SIZE * k) + p->pos[k]; - srcLen2 = p->size[k] - p->pos[k]; - srcFinished2 = p->finished[k]; - } - - if (i == p->numCoders - 1) - { - dest2 = dest; - destLen2 = destLenOrig - *destLen; - } - else - { - if (p->pos[i] != p->size[i]) - continue; - dest2 = p->buf + (CODER_BUF_SIZE * i); - destLen2 = CODER_BUF_SIZE; - } - - // destLen2_Orig = destLen2; - - if (p->results[i] != SZ_OK) - { - if (resMain == SZ_OK) - resMain = p->results[i]; - continue; - } - - res = coder->Code2(coder->p, - dest2, &destLen2, - src2, &srcLen2, srcFinished2, - finishMode, - // &encodingWasFinished, - &status2); - - if (res != SZ_OK) - { - p->results[i] = res; - if (resMain == SZ_OK) - resMain = res; - } - - encodingWasFinished = (status2 == CODER_STATUS_FINISHED_WITH_MARK); - - if (!encodingWasFinished) - { - allFinished = False; - if (p->numCoders == 1 && res == SZ_OK) - p->status = status2; - } - - if (i == 0) - { - *srcLen += srcLen2; - src += srcLen2; - } - else - p->pos[(size_t)i - 1] += srcLen2; - - if (i == p->numCoders - 1) - { - *destLen += destLen2; - dest += destLen2; - } - else - { - p->size[i] = destLen2; - p->pos[i] = 0; - p->finished[i] = encodingWasFinished; - } - - if (destLen2 != 0 || srcLen2 != 0) - processed = True; - } - - if (!processed) - { - if (allFinished) - p->status = CODER_STATUS_FINISHED_WITH_MARK; - return resMain; - } - } -} - - -SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf) -{ - *p = (CXzStreamFlags)GetBe16(buf + XZ_SIG_SIZE); - if (CrcCalc(buf + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE) != - GetUi32(buf + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE)) - return SZ_ERROR_NO_ARCHIVE; - return XzFlags_IsSupported(*p) ? SZ_OK : SZ_ERROR_UNSUPPORTED; -} - -static BoolInt Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) -{ - return indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) - && GetUi32(buf) == CrcCalc(buf + 4, 6) - && flags == GetBe16(buf + 8) - && buf[10] == XZ_FOOTER_SIG_0 - && buf[11] == XZ_FOOTER_SIG_1; -} - -#define READ_VARINT_AND_CHECK(buf, pos, size, res) \ - { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ - if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } - - -static BoolInt XzBlock_AreSupportedFilters(const CXzBlock *p) -{ - unsigned numFilters = XzBlock_GetNumFilters(p) - 1; - unsigned i; - { - const CXzFilter *f = &p->filters[numFilters]; - if (f->id != XZ_ID_LZMA2 || f->propsSize != 1 || f->props[0] > 40) - return False; - } - - for (i = 0; i < numFilters; i++) - { - const CXzFilter *f = &p->filters[i]; - if (f->id == XZ_ID_Delta) - { - if (f->propsSize != 1) - return False; - } - else if (f->id < XZ_ID_Delta - || f->id > XZ_ID_SPARC - || (f->propsSize != 0 && f->propsSize != 4)) - return False; - } - return True; -} - - -SRes XzBlock_Parse(CXzBlock *p, const Byte *header) -{ - unsigned pos; - unsigned numFilters, i; - unsigned headerSize = (unsigned)header[0] << 2; - - /* (headerSize != 0) : another code checks */ - - if (CrcCalc(header, headerSize) != GetUi32(header + headerSize)) - return SZ_ERROR_ARCHIVE; - - pos = 1; - p->flags = header[pos++]; - - p->packSize = (UInt64)(Int64)-1; - if (XzBlock_HasPackSize(p)) - { - READ_VARINT_AND_CHECK(header, pos, headerSize, &p->packSize); - if (p->packSize == 0 || p->packSize + headerSize >= (UInt64)1 << 63) - return SZ_ERROR_ARCHIVE; - } - - p->unpackSize = (UInt64)(Int64)-1; - if (XzBlock_HasUnpackSize(p)) - READ_VARINT_AND_CHECK(header, pos, headerSize, &p->unpackSize); - - numFilters = XzBlock_GetNumFilters(p); - for (i = 0; i < numFilters; i++) - { - CXzFilter *filter = p->filters + i; - UInt64 size; - READ_VARINT_AND_CHECK(header, pos, headerSize, &filter->id); - READ_VARINT_AND_CHECK(header, pos, headerSize, &size); - if (size > headerSize - pos || size > XZ_FILTER_PROPS_SIZE_MAX) - return SZ_ERROR_ARCHIVE; - filter->propsSize = (UInt32)size; - memcpy(filter->props, header + pos, (size_t)size); - pos += (unsigned)size; - - #ifdef XZ_DUMP - printf("\nf[%u] = %2X: ", i, (unsigned)filter->id); - { - unsigned i; - for (i = 0; i < size; i++) - printf(" %2X", filter->props[i]); - } - #endif - } - - if (XzBlock_HasUnsupportedFlags(p)) - return SZ_ERROR_UNSUPPORTED; - - while (pos < headerSize) - if (header[pos++] != 0) - return SZ_ERROR_ARCHIVE; - return SZ_OK; -} - - - - -static SRes XzDecMix_Init(CMixCoder *p, const CXzBlock *block, Byte *outBuf, size_t outBufSize) -{ - unsigned i; - BoolInt needReInit = True; - unsigned numFilters = XzBlock_GetNumFilters(block); - - if (numFilters == p->numCoders && ((p->outBuf && outBuf) || (!p->outBuf && !outBuf))) - { - needReInit = False; - for (i = 0; i < numFilters; i++) - if (p->ids[i] != block->filters[numFilters - 1 - i].id) - { - needReInit = True; - break; - } - } - - // p->SingleBufMode = (outBuf != NULL); - p->outBuf = outBuf; - p->outBufSize = outBufSize; - - // p->SingleBufMode = False; - // outBuf = NULL; - - if (needReInit) - { - MixCoder_Free(p); - for (i = 0; i < numFilters; i++) - { - RINOK(MixCoder_SetFromMethod(p, i, block->filters[numFilters - 1 - i].id, outBuf, outBufSize)); - } - p->numCoders = numFilters; - } - else - { - RINOK(MixCoder_ResetFromMethod(p, 0, block->filters[numFilters - 1].id, outBuf, outBufSize)); - } - - for (i = 0; i < numFilters; i++) - { - const CXzFilter *f = &block->filters[numFilters - 1 - i]; - IStateCoder *sc = &p->coders[i]; - RINOK(sc->SetProps(sc->p, f->props, f->propsSize, p->alloc)); - } - - MixCoder_Init(p); - return SZ_OK; -} - - - -void XzUnpacker_Init(CXzUnpacker *p) -{ - p->state = XZ_STATE_STREAM_HEADER; - p->pos = 0; - p->numStartedStreams = 0; - p->numFinishedStreams = 0; - p->numTotalBlocks = 0; - p->padSize = 0; - p->decodeOnlyOneBlock = 0; - - p->parseMode = False; - p->decodeToStreamSignature = False; - - // p->outBuf = NULL; - // p->outBufSize = 0; - p->outDataWritten = 0; -} - - -void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize) -{ - p->outBuf = outBuf; - p->outBufSize = outBufSize; -} - - -void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc) -{ - MixCoder_Construct(&p->decoder, alloc); - p->outBuf = NULL; - p->outBufSize = 0; - XzUnpacker_Init(p); -} - - -void XzUnpacker_Free(CXzUnpacker *p) -{ - MixCoder_Free(&p->decoder); -} - - -void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p) -{ - p->indexSize = 0; - p->numBlocks = 0; - Sha256_Init(&p->sha); - p->state = XZ_STATE_BLOCK_HEADER; - p->pos = 0; - p->decodeOnlyOneBlock = 1; -} - - -static void XzUnpacker_UpdateIndex(CXzUnpacker *p, UInt64 packSize, UInt64 unpackSize) -{ - Byte temp[32]; - unsigned num = Xz_WriteVarInt(temp, packSize); - num += Xz_WriteVarInt(temp + num, unpackSize); - Sha256_Update(&p->sha, temp, num); - p->indexSize += num; - p->numBlocks++; -} - - - -SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, int srcFinished, - ECoderFinishMode finishMode, ECoderStatus *status) -{ - SizeT destLenOrig = *destLen; - SizeT srcLenOrig = *srcLen; - *destLen = 0; - *srcLen = 0; - *status = CODER_STATUS_NOT_SPECIFIED; - - for (;;) - { - SizeT srcRem; - - if (p->state == XZ_STATE_BLOCK) - { - SizeT destLen2 = destLenOrig - *destLen; - SizeT srcLen2 = srcLenOrig - *srcLen; - SRes res; - - ECoderFinishMode finishMode2 = finishMode; - BoolInt srcFinished2 = srcFinished; - BoolInt destFinish = False; - - if (p->block.packSize != (UInt64)(Int64)-1) - { - UInt64 rem = p->block.packSize - p->packSize; - if (srcLen2 >= rem) - { - srcFinished2 = True; - srcLen2 = (SizeT)rem; - } - if (rem == 0 && p->block.unpackSize == p->unpackSize) - return SZ_ERROR_DATA; - } - - if (p->block.unpackSize != (UInt64)(Int64)-1) - { - UInt64 rem = p->block.unpackSize - p->unpackSize; - if (destLen2 >= rem) - { - destFinish = True; - finishMode2 = CODER_FINISH_END; - destLen2 = (SizeT)rem; - } - } - - /* - if (srcLen2 == 0 && destLen2 == 0) - { - *status = CODER_STATUS_NOT_FINISHED; - return SZ_OK; - } - */ - - { - res = MixCoder_Code(&p->decoder, - (p->outBuf ? NULL : dest), &destLen2, destFinish, - src, &srcLen2, srcFinished2, - finishMode2); - - *status = p->decoder.status; - XzCheck_Update(&p->check, (p->outBuf ? p->outBuf + p->outDataWritten : dest), destLen2); - if (!p->outBuf) - dest += destLen2; - p->outDataWritten += destLen2; - } - - (*srcLen) += srcLen2; - src += srcLen2; - p->packSize += srcLen2; - (*destLen) += destLen2; - p->unpackSize += destLen2; - - RINOK(res); - - if (*status != CODER_STATUS_FINISHED_WITH_MARK) - { - if (p->block.packSize == p->packSize - && *status == CODER_STATUS_NEEDS_MORE_INPUT) - { - PRF_STR("CODER_STATUS_NEEDS_MORE_INPUT"); - *status = CODER_STATUS_NOT_SPECIFIED; - return SZ_ERROR_DATA; - } - - return SZ_OK; - } - { - XzUnpacker_UpdateIndex(p, XzUnpacker_GetPackSizeForIndex(p), p->unpackSize); - p->state = XZ_STATE_BLOCK_FOOTER; - p->pos = 0; - p->alignPos = 0; - *status = CODER_STATUS_NOT_SPECIFIED; - - if ((p->block.packSize != (UInt64)(Int64)-1 && p->block.packSize != p->packSize) - || (p->block.unpackSize != (UInt64)(Int64)-1 && p->block.unpackSize != p->unpackSize)) - { - PRF_STR("ERROR: block.size mismatch"); - return SZ_ERROR_DATA; - } - } - // continue; - } - - srcRem = srcLenOrig - *srcLen; - - // XZ_STATE_BLOCK_FOOTER can transit to XZ_STATE_BLOCK_HEADER without input bytes - if (srcRem == 0 && p->state != XZ_STATE_BLOCK_FOOTER) - { - *status = CODER_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - - switch (p->state) - { - case XZ_STATE_STREAM_HEADER: - { - if (p->pos < XZ_STREAM_HEADER_SIZE) - { - if (p->pos < XZ_SIG_SIZE && *src != XZ_SIG[p->pos]) - return SZ_ERROR_NO_ARCHIVE; - if (p->decodeToStreamSignature) - return SZ_OK; - p->buf[p->pos++] = *src++; - (*srcLen)++; - } - else - { - RINOK(Xz_ParseHeader(&p->streamFlags, p->buf)); - p->numStartedStreams++; - p->indexSize = 0; - p->numBlocks = 0; - Sha256_Init(&p->sha); - p->state = XZ_STATE_BLOCK_HEADER; - p->pos = 0; - } - break; - } - - case XZ_STATE_BLOCK_HEADER: - { - if (p->pos == 0) - { - p->buf[p->pos++] = *src++; - (*srcLen)++; - if (p->buf[0] == 0) - { - if (p->decodeOnlyOneBlock) - return SZ_ERROR_DATA; - p->indexPreSize = 1 + Xz_WriteVarInt(p->buf + 1, p->numBlocks); - p->indexPos = p->indexPreSize; - p->indexSize += p->indexPreSize; - Sha256_Final(&p->sha, p->shaDigest); - Sha256_Init(&p->sha); - p->crc = CrcUpdate(CRC_INIT_VAL, p->buf, p->indexPreSize); - p->state = XZ_STATE_STREAM_INDEX; - break; - } - p->blockHeaderSize = ((UInt32)p->buf[0] << 2) + 4; - break; - } - - if (p->pos != p->blockHeaderSize) - { - UInt32 cur = p->blockHeaderSize - p->pos; - if (cur > srcRem) - cur = (UInt32)srcRem; - memcpy(p->buf + p->pos, src, cur); - p->pos += cur; - (*srcLen) += cur; - src += cur; - } - else - { - RINOK(XzBlock_Parse(&p->block, p->buf)); - if (!XzBlock_AreSupportedFilters(&p->block)) - return SZ_ERROR_UNSUPPORTED; - p->numTotalBlocks++; - p->state = XZ_STATE_BLOCK; - p->packSize = 0; - p->unpackSize = 0; - XzCheck_Init(&p->check, XzFlags_GetCheckType(p->streamFlags)); - if (p->parseMode) - { - p->headerParsedOk = True; - return SZ_OK; - } - RINOK(XzDecMix_Init(&p->decoder, &p->block, p->outBuf, p->outBufSize)); - } - break; - } - - case XZ_STATE_BLOCK_FOOTER: - { - if ((((unsigned)p->packSize + p->alignPos) & 3) != 0) - { - if (srcRem == 0) - { - *status = CODER_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - (*srcLen)++; - p->alignPos++; - if (*src++ != 0) - return SZ_ERROR_CRC; - } - else - { - UInt32 checkSize = XzFlags_GetCheckSize(p->streamFlags); - UInt32 cur = checkSize - p->pos; - if (cur != 0) - { - if (srcRem == 0) - { - *status = CODER_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (cur > srcRem) - cur = (UInt32)srcRem; - memcpy(p->buf + p->pos, src, cur); - p->pos += cur; - (*srcLen) += cur; - src += cur; - if (checkSize != p->pos) - break; - } - { - Byte digest[XZ_CHECK_SIZE_MAX]; - p->state = XZ_STATE_BLOCK_HEADER; - p->pos = 0; - if (XzCheck_Final(&p->check, digest) && memcmp(digest, p->buf, checkSize) != 0) - return SZ_ERROR_CRC; - if (p->decodeOnlyOneBlock) - { - *status = CODER_STATUS_FINISHED_WITH_MARK; - return SZ_OK; - } - } - } - break; - } - - case XZ_STATE_STREAM_INDEX: - { - if (p->pos < p->indexPreSize) - { - (*srcLen)++; - if (*src++ != p->buf[p->pos++]) - return SZ_ERROR_CRC; - } - else - { - if (p->indexPos < p->indexSize) - { - UInt64 cur = p->indexSize - p->indexPos; - if (srcRem > cur) - srcRem = (SizeT)cur; - p->crc = CrcUpdate(p->crc, src, srcRem); - Sha256_Update(&p->sha, src, srcRem); - (*srcLen) += srcRem; - src += srcRem; - p->indexPos += srcRem; - } - else if ((p->indexPos & 3) != 0) - { - Byte b = *src++; - p->crc = CRC_UPDATE_BYTE(p->crc, b); - (*srcLen)++; - p->indexPos++; - p->indexSize++; - if (b != 0) - return SZ_ERROR_CRC; - } - else - { - Byte digest[SHA256_DIGEST_SIZE]; - p->state = XZ_STATE_STREAM_INDEX_CRC; - p->indexSize += 4; - p->pos = 0; - Sha256_Final(&p->sha, digest); - if (memcmp(digest, p->shaDigest, SHA256_DIGEST_SIZE) != 0) - return SZ_ERROR_CRC; - } - } - break; - } - - case XZ_STATE_STREAM_INDEX_CRC: - { - if (p->pos < 4) - { - (*srcLen)++; - p->buf[p->pos++] = *src++; - } - else - { - p->state = XZ_STATE_STREAM_FOOTER; - p->pos = 0; - if (CRC_GET_DIGEST(p->crc) != GetUi32(p->buf)) - return SZ_ERROR_CRC; - } - break; - } - - case XZ_STATE_STREAM_FOOTER: - { - UInt32 cur = XZ_STREAM_FOOTER_SIZE - p->pos; - if (cur > srcRem) - cur = (UInt32)srcRem; - memcpy(p->buf + p->pos, src, cur); - p->pos += cur; - (*srcLen) += cur; - src += cur; - if (p->pos == XZ_STREAM_FOOTER_SIZE) - { - p->state = XZ_STATE_STREAM_PADDING; - p->numFinishedStreams++; - p->padSize = 0; - if (!Xz_CheckFooter(p->streamFlags, p->indexSize, p->buf)) - return SZ_ERROR_CRC; - } - break; - } - - case XZ_STATE_STREAM_PADDING: - { - if (*src != 0) - { - if (((UInt32)p->padSize & 3) != 0) - return SZ_ERROR_NO_ARCHIVE; - p->pos = 0; - p->state = XZ_STATE_STREAM_HEADER; - } - else - { - (*srcLen)++; - src++; - p->padSize++; - } - break; - } - - case XZ_STATE_BLOCK: break; /* to disable GCC warning */ - } - } - /* - if (p->state == XZ_STATE_FINISHED) - *status = CODER_STATUS_FINISHED_WITH_MARK; - return SZ_OK; - */ -} - - -SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, - ECoderFinishMode finishMode, ECoderStatus *status) -{ - XzUnpacker_Init(p); - XzUnpacker_SetOutBuf(p, dest, *destLen); - - return XzUnpacker_Code(p, - NULL, destLen, - src, srcLen, True, - finishMode, status); -} - - -BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p) -{ - return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0); -} - -BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p) -{ - return (p->state == XZ_STATE_STREAM_PADDING) && (((UInt32)p->padSize & 3) == 0); -} - -UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p) -{ - UInt64 num = 0; - if (p->state == XZ_STATE_STREAM_PADDING) - num = p->padSize; - else if (p->state == XZ_STATE_STREAM_HEADER) - num = p->padSize + p->pos; - return num; -} - - - - - - - - - - - - - - - - - - - - - -#ifndef _7ZIP_ST -#include "MtDec.h" -#endif - - -void XzDecMtProps_Init(CXzDecMtProps *p) -{ - p->inBufSize_ST = 1 << 18; - p->outStep_ST = 1 << 20; - p->ignoreErrors = False; - - #ifndef _7ZIP_ST - p->numThreads = 1; - p->inBufSize_MT = 1 << 18; - p->memUseMax = sizeof(size_t) << 28; - #endif -} - - - -#ifndef _7ZIP_ST - -/* ---------- CXzDecMtThread ---------- */ - -typedef struct -{ - Byte *outBuf; - size_t outBufSize; - size_t outPreSize; - size_t inPreSize; - size_t inPreHeaderSize; - size_t blockPackSize_for_Index; // including block header and checksum. - size_t blockPackTotal; // including stream header, block header and checksum. - size_t inCodeSize; - size_t outCodeSize; - ECoderStatus status; - SRes codeRes; - BoolInt skipMode; - // BoolInt finishedWithMark; - EMtDecParseState parseState; - BoolInt parsing_Truncated; - BoolInt atBlockHeader; - CXzStreamFlags streamFlags; - // UInt64 numFinishedStreams - UInt64 numStreams; - UInt64 numTotalBlocks; - UInt64 numBlocks; - - BoolInt dec_created; - CXzUnpacker dec; - - Byte mtPad[1 << 7]; -} CXzDecMtThread; - -#endif - - -/* ---------- CXzDecMt ---------- */ - -typedef struct -{ - CAlignOffsetAlloc alignOffsetAlloc; - ISzAllocPtr allocMid; - - CXzDecMtProps props; - size_t unpackBlockMaxSize; - - ISeqInStream *inStream; - ISeqOutStream *outStream; - ICompressProgress *progress; - // CXzStatInfo *stat; - - BoolInt finishMode; - BoolInt outSize_Defined; - UInt64 outSize; - - UInt64 outProcessed; - UInt64 inProcessed; - UInt64 readProcessed; - BoolInt readWasFinished; - SRes readRes; - SRes writeRes; - - Byte *outBuf; - size_t outBufSize; - Byte *inBuf; - size_t inBufSize; - - CXzUnpacker dec; - - ECoderStatus status; - SRes codeRes; - - #ifndef _7ZIP_ST - BoolInt mainDecoderWasCalled; - // int statErrorDefined; - int finishedDecoderIndex; - - // global values that are used in Parse stage - CXzStreamFlags streamFlags; - // UInt64 numFinishedStreams - UInt64 numStreams; - UInt64 numTotalBlocks; - UInt64 numBlocks; - - // UInt64 numBadBlocks; - SRes mainErrorCode; - - BoolInt isBlockHeaderState_Parse; - BoolInt isBlockHeaderState_Write; - UInt64 outProcessed_Parse; - BoolInt parsing_Truncated; - - BoolInt mtc_WasConstructed; - CMtDec mtc; - CXzDecMtThread coders[MTDEC__THREADS_MAX]; - #endif - -} CXzDecMt; - - - -CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) -{ - CXzDecMt *p = (CXzDecMt *)ISzAlloc_Alloc(alloc, sizeof(CXzDecMt)); - if (!p) - return NULL; - - AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); - p->alignOffsetAlloc.baseAlloc = alloc; - p->alignOffsetAlloc.numAlignBits = 7; - p->alignOffsetAlloc.offset = 0; - - p->allocMid = allocMid; - - p->outBuf = NULL; - p->outBufSize = 0; - p->inBuf = NULL; - p->inBufSize = 0; - - XzUnpacker_Construct(&p->dec, &p->alignOffsetAlloc.vt); - - p->unpackBlockMaxSize = 0; - - XzDecMtProps_Init(&p->props); - - #ifndef _7ZIP_ST - p->mtc_WasConstructed = False; - { - unsigned i; - for (i = 0; i < MTDEC__THREADS_MAX; i++) - { - CXzDecMtThread *coder = &p->coders[i]; - coder->dec_created = False; - coder->outBuf = NULL; - coder->outBufSize = 0; - } - } - #endif - - return p; -} - - -#ifndef _7ZIP_ST - -static void XzDecMt_FreeOutBufs(CXzDecMt *p) -{ - unsigned i; - for (i = 0; i < MTDEC__THREADS_MAX; i++) - { - CXzDecMtThread *coder = &p->coders[i]; - if (coder->outBuf) - { - ISzAlloc_Free(p->allocMid, coder->outBuf); - coder->outBuf = NULL; - coder->outBufSize = 0; - } - } - p->unpackBlockMaxSize = 0; -} - -#endif - - - -static void XzDecMt_FreeSt(CXzDecMt *p) -{ - XzUnpacker_Free(&p->dec); - - if (p->outBuf) - { - ISzAlloc_Free(p->allocMid, p->outBuf); - p->outBuf = NULL; - } - p->outBufSize = 0; - - if (p->inBuf) - { - ISzAlloc_Free(p->allocMid, p->inBuf); - p->inBuf = NULL; - } - p->inBufSize = 0; -} - - -void XzDecMt_Destroy(CXzDecMtHandle pp) -{ - CXzDecMt *p = (CXzDecMt *)pp; - - XzDecMt_FreeSt(p); - - #ifndef _7ZIP_ST - - if (p->mtc_WasConstructed) - { - MtDec_Destruct(&p->mtc); - p->mtc_WasConstructed = False; - } - { - unsigned i; - for (i = 0; i < MTDEC__THREADS_MAX; i++) - { - CXzDecMtThread *t = &p->coders[i]; - if (t->dec_created) - { - // we don't need to free dict here - XzUnpacker_Free(&t->dec); - t->dec_created = False; - } - } - } - XzDecMt_FreeOutBufs(p); - - #endif - - ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); -} - - - -#ifndef _7ZIP_ST - -static void XzDecMt_Callback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) -{ - CXzDecMt *me = (CXzDecMt *)obj; - CXzDecMtThread *coder = &me->coders[coderIndex]; - size_t srcSize = cc->srcSize; - - cc->srcSize = 0; - cc->outPos = 0; - cc->state = MTDEC_PARSE_CONTINUE; - - cc->canCreateNewThread = True; - - if (cc->startCall) - { - coder->outPreSize = 0; - coder->inPreSize = 0; - coder->inPreHeaderSize = 0; - coder->parseState = MTDEC_PARSE_CONTINUE; - coder->parsing_Truncated = False; - coder->skipMode = False; - coder->codeRes = SZ_OK; - coder->status = CODER_STATUS_NOT_SPECIFIED; - coder->inCodeSize = 0; - coder->outCodeSize = 0; - - coder->numStreams = me->numStreams; - coder->numTotalBlocks = me->numTotalBlocks; - coder->numBlocks = me->numBlocks; - - if (!coder->dec_created) - { - XzUnpacker_Construct(&coder->dec, &me->alignOffsetAlloc.vt); - coder->dec_created = True; - } - - XzUnpacker_Init(&coder->dec); - - if (me->isBlockHeaderState_Parse) - { - coder->dec.streamFlags = me->streamFlags; - coder->atBlockHeader = True; - XzUnpacker_PrepareToRandomBlockDecoding(&coder->dec); - } - else - { - coder->atBlockHeader = False; - me->isBlockHeaderState_Parse = True; - } - - coder->dec.numStartedStreams = me->numStreams; - coder->dec.numTotalBlocks = me->numTotalBlocks; - coder->dec.numBlocks = me->numBlocks; - } - - while (!coder->skipMode) - { - ECoderStatus status; - SRes res; - size_t srcSize2 = srcSize; - size_t destSize = (size_t)0 - 1; - - coder->dec.parseMode = True; - coder->dec.headerParsedOk = False; - - PRF_STR_INT("Parse", srcSize2); - - res = XzUnpacker_Code(&coder->dec, - NULL, &destSize, - cc->src, &srcSize2, cc->srcFinished, - CODER_FINISH_END, &status); - - // PRF(printf(" res = %d, srcSize2 = %d", res, (unsigned)srcSize2)); - - coder->codeRes = res; - coder->status = status; - cc->srcSize += srcSize2; - srcSize -= srcSize2; - coder->inPreHeaderSize += srcSize2; - coder->inPreSize = coder->inPreHeaderSize; - - if (res != SZ_OK) - { - cc->state = - coder->parseState = MTDEC_PARSE_END; - /* - if (res == SZ_ERROR_MEM) - return res; - return SZ_OK; - */ - return; // res; - } - - if (coder->dec.headerParsedOk) - { - const CXzBlock *block = &coder->dec.block; - if (XzBlock_HasUnpackSize(block) - // && block->unpackSize <= me->props.outBlockMax - && XzBlock_HasPackSize(block)) - { - { - if (block->unpackSize * 2 * me->mtc.numStartedThreads > me->props.memUseMax) - { - cc->state = MTDEC_PARSE_OVERFLOW; - return; // SZ_OK; - } - } - { - UInt64 packSize = block->packSize; - UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); - UInt32 checkSize = XzFlags_GetCheckSize(coder->dec.streamFlags); - UInt64 blockPackSum = coder->inPreSize + packSizeAligned + checkSize; - // if (blockPackSum <= me->props.inBlockMax) - // unpackBlockMaxSize - { - coder->blockPackSize_for_Index = (size_t)(coder->dec.blockHeaderSize + packSize + checkSize); - coder->blockPackTotal = (size_t)blockPackSum; - coder->outPreSize = (size_t)block->unpackSize; - coder->streamFlags = coder->dec.streamFlags; - me->streamFlags = coder->dec.streamFlags; - coder->skipMode = True; - break; - } - } - } - } - else - // if (coder->inPreSize <= me->props.inBlockMax) - { - if (!cc->srcFinished) - return; // SZ_OK; - cc->state = - coder->parseState = MTDEC_PARSE_END; - return; // SZ_OK; - } - cc->state = MTDEC_PARSE_OVERFLOW; - return; // SZ_OK; - } - - // ---------- skipMode ---------- - { - UInt64 rem = coder->blockPackTotal - coder->inPreSize; - size_t cur = srcSize; - if (cur > rem) - cur = (size_t)rem; - cc->srcSize += cur; - coder->inPreSize += cur; - srcSize -= cur; - - if (coder->inPreSize == coder->blockPackTotal) - { - if (srcSize == 0) - { - if (!cc->srcFinished) - return; // SZ_OK; - cc->state = MTDEC_PARSE_END; - } - else if ((cc->src)[cc->srcSize] == 0) // we check control byte of next block - cc->state = MTDEC_PARSE_END; - else - { - cc->state = MTDEC_PARSE_NEW; - - { - size_t blockMax = me->unpackBlockMaxSize; - if (blockMax < coder->outPreSize) - blockMax = coder->outPreSize; - { - UInt64 required = (UInt64)blockMax * (me->mtc.numStartedThreads + 1) * 2; - if (me->props.memUseMax < required) - cc->canCreateNewThread = False; - } - } - - if (me->outSize_Defined) - { - // next block can be zero size - const UInt64 rem2 = me->outSize - me->outProcessed_Parse; - if (rem2 < coder->outPreSize) - { - coder->parsing_Truncated = True; - cc->state = MTDEC_PARSE_END; - } - me->outProcessed_Parse += coder->outPreSize; - } - } - } - else if (cc->srcFinished) - cc->state = MTDEC_PARSE_END; - else - return; // SZ_OK; - - coder->parseState = cc->state; - cc->outPos = coder->outPreSize; - - me->numStreams = coder->dec.numStartedStreams; - me->numTotalBlocks = coder->dec.numTotalBlocks; - me->numBlocks = coder->dec.numBlocks + 1; - return; // SZ_OK; - } -} - - -static SRes XzDecMt_Callback_PreCode(void *pp, unsigned coderIndex) -{ - CXzDecMt *me = (CXzDecMt *)pp; - CXzDecMtThread *coder = &me->coders[coderIndex]; - Byte *dest; - - if (!coder->dec.headerParsedOk) - return SZ_OK; - - dest = coder->outBuf; - - if (!dest || coder->outBufSize < coder->outPreSize) - { - if (dest) - { - ISzAlloc_Free(me->allocMid, dest); - coder->outBuf = NULL; - coder->outBufSize = 0; - } - { - size_t outPreSize = coder->outPreSize; - if (outPreSize == 0) - outPreSize = 1; - dest = (Byte *)ISzAlloc_Alloc(me->allocMid, outPreSize); - } - if (!dest) - return SZ_ERROR_MEM; - coder->outBuf = dest; - coder->outBufSize = coder->outPreSize; - - if (coder->outBufSize > me->unpackBlockMaxSize) - me->unpackBlockMaxSize = coder->outBufSize; - } - - // return SZ_ERROR_MEM; - - XzUnpacker_SetOutBuf(&coder->dec, coder->outBuf, coder->outBufSize); - - { - SRes res = XzDecMix_Init(&coder->dec.decoder, &coder->dec.block, coder->outBuf, coder->outBufSize); - // res = SZ_ERROR_UNSUPPORTED; // to test - coder->codeRes = res; - if (res != SZ_OK) - { - // if (res == SZ_ERROR_MEM) return res; - if (me->props.ignoreErrors && res != SZ_ERROR_MEM) - return S_OK; - return res; - } - } - - return SZ_OK; -} - - -static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex, - const Byte *src, size_t srcSize, int srcFinished, - // int finished, int blockFinished, - UInt64 *inCodePos, UInt64 *outCodePos, int *stop) -{ - CXzDecMt *me = (CXzDecMt *)pp; - CXzDecMtThread *coder = &me->coders[coderIndex]; - - *inCodePos = coder->inCodeSize; - *outCodePos = coder->outCodeSize; - *stop = True; - - if (coder->inCodeSize < coder->inPreHeaderSize) - { - UInt64 rem = coder->inPreHeaderSize - coder->inCodeSize; - size_t step = srcSize; - if (step > rem) - step = (size_t)rem; - src += step; - srcSize -= step; - coder->inCodeSize += step; - if (coder->inCodeSize < coder->inPreHeaderSize) - { - *stop = False; - return SZ_OK; - } - } - - if (!coder->dec.headerParsedOk) - return SZ_OK; - if (!coder->outBuf) - return SZ_OK; - - if (coder->codeRes == SZ_OK) - { - ECoderStatus status; - SRes res; - size_t srcProcessed = srcSize; - size_t outSizeCur = coder->outPreSize - coder->dec.outDataWritten; - - // PRF(printf("\nCallback_Code: Code %d %d\n", (unsigned)srcSize, (unsigned)outSizeCur)); - - res = XzUnpacker_Code(&coder->dec, - NULL, &outSizeCur, - src, &srcProcessed, srcFinished, - // coder->finishedWithMark ? CODER_FINISH_END : CODER_FINISH_ANY, - CODER_FINISH_END, - &status); - - // PRF(printf(" res = %d, srcSize2 = %d, outSizeCur = %d", res, (unsigned)srcProcessed, (unsigned)outSizeCur)); - - coder->codeRes = res; - coder->status = status; - coder->inCodeSize += srcProcessed; - coder->outCodeSize = coder->dec.outDataWritten; - *inCodePos = coder->inCodeSize; - *outCodePos = coder->outCodeSize; - - if (res == SZ_OK) - { - if (srcProcessed == srcSize) - *stop = False; - return SZ_OK; - } - } - - if (me->props.ignoreErrors && coder->codeRes != SZ_ERROR_MEM) - { - *inCodePos = coder->inPreSize; - *outCodePos = coder->outPreSize; - return S_OK; - } - return coder->codeRes; -} - - -#define XZDECMT_STREAM_WRITE_STEP (1 << 24) - -static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, - BoolInt needWriteToStream, - const Byte *src, size_t srcSize, - // int srcFinished, - BoolInt *needContinue, - BoolInt *canRecode) -{ - CXzDecMt *me = (CXzDecMt *)pp; - const CXzDecMtThread *coder = &me->coders[coderIndex]; - - // PRF(printf("\nWrite processed = %d srcSize = %d\n", (unsigned)me->mtc.inProcessed, (unsigned)srcSize)); - - *needContinue = False; - *canRecode = True; - - if (!needWriteToStream) - return SZ_OK; - - if (!coder->dec.headerParsedOk || !coder->outBuf) - { - if (me->finishedDecoderIndex < 0) - me->finishedDecoderIndex = coderIndex; - return SZ_OK; - } - - if (me->finishedDecoderIndex >= 0) - return SZ_OK; - - me->mtc.inProcessed += coder->inCodeSize; - - *canRecode = False; - - { - SRes res; - size_t size = coder->outCodeSize; - Byte *data = coder->outBuf; - - // we use in me->dec: sha, numBlocks, indexSize - - if (!me->isBlockHeaderState_Write) - { - XzUnpacker_PrepareToRandomBlockDecoding(&me->dec); - me->dec.decodeOnlyOneBlock = False; - me->dec.numStartedStreams = coder->dec.numStartedStreams; - me->dec.streamFlags = coder->streamFlags; - - me->isBlockHeaderState_Write = True; - } - - me->dec.numTotalBlocks = coder->dec.numTotalBlocks; - XzUnpacker_UpdateIndex(&me->dec, coder->blockPackSize_for_Index, coder->outPreSize); - - if (coder->outPreSize != size) - { - if (me->props.ignoreErrors) - { - memset(data + size, 0, coder->outPreSize - size); - size = coder->outPreSize; - } - // me->numBadBlocks++; - if (me->mainErrorCode == SZ_OK) - { - if ((int)coder->status == LZMA_STATUS_NEEDS_MORE_INPUT) - me->mainErrorCode = SZ_ERROR_INPUT_EOF; - else - me->mainErrorCode = SZ_ERROR_DATA; - } - } - - if (me->writeRes != SZ_OK) - return me->writeRes; - - res = SZ_OK; - { - if (me->outSize_Defined) - { - const UInt64 rem = me->outSize - me->outProcessed; - if (size > rem) - size = (SizeT)rem; - } - - for (;;) - { - size_t cur = size; - size_t written; - if (cur > XZDECMT_STREAM_WRITE_STEP) - cur = XZDECMT_STREAM_WRITE_STEP; - - written = ISeqOutStream_Write(me->outStream, data, cur); - - // PRF(printf("\nWritten ask = %d written = %d\n", (unsigned)cur, (unsigned)written)); - - me->outProcessed += written; - if (written != cur) - { - me->writeRes = SZ_ERROR_WRITE; - res = me->writeRes; - break; - } - data += cur; - size -= cur; - // PRF_STR_INT("Written size =", size); - if (size == 0) - break; - res = MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0); - if (res != SZ_OK) - break; - } - } - - if (coder->codeRes != SZ_OK) - if (!me->props.ignoreErrors) - { - me->finishedDecoderIndex = coderIndex; - return res; - } - - RINOK(res); - - if (coder->inPreSize != coder->inCodeSize - || coder->blockPackTotal != coder->inCodeSize) - { - me->finishedDecoderIndex = coderIndex; - return SZ_OK; - } - - if (coder->parseState != MTDEC_PARSE_END) - { - *needContinue = True; - return SZ_OK; - } - } - - // (coder->state == MTDEC_PARSE_END) means that there are no other working threads - // so we can use mtc variables without lock - - PRF_STR_INT("Write MTDEC_PARSE_END", me->mtc.inProcessed); - - me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; - { - CXzUnpacker *dec = &me->dec; - - PRF_STR_INT("PostSingle", srcSize); - - { - size_t srcProcessed = srcSize; - ECoderStatus status; - size_t outSizeCur = 0; - SRes res; - - // dec->decodeOnlyOneBlock = False; - dec->decodeToStreamSignature = True; - - me->mainDecoderWasCalled = True; - - if (coder->parsing_Truncated) - { - me->parsing_Truncated = True; - return SZ_OK; - } - - res = XzUnpacker_Code(dec, - NULL, &outSizeCur, - src, &srcProcessed, - me->mtc.readWasFinished, // srcFinished - CODER_FINISH_END, // CODER_FINISH_ANY, - &status); - - me->status = status; - me->codeRes = res; - - me->mtc.inProcessed += srcProcessed; - me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; - - if (res != SZ_OK) - { - return S_OK; - // return res; - } - - if (dec->state == XZ_STATE_STREAM_HEADER) - { - *needContinue = True; - me->isBlockHeaderState_Parse = False; - me->isBlockHeaderState_Write = False; - { - Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); - if (!crossBuf) - return SZ_ERROR_MEM; - memcpy(crossBuf, src + srcProcessed, srcSize - srcProcessed); - } - me->mtc.crossStart = 0; - me->mtc.crossEnd = srcSize - srcProcessed; - return SZ_OK; - } - - if (status != CODER_STATUS_NEEDS_MORE_INPUT) - { - return E_FAIL; - } - - if (me->mtc.readWasFinished) - { - return SZ_OK; - } - } - - { - size_t inPos; - size_t inLim; - const Byte *inData; - UInt64 inProgressPrev = me->mtc.inProcessed; - - // XzDecMt_Prepare_InBuf_ST(p); - Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); - if (!crossBuf) - return SZ_ERROR_MEM; - - inPos = 0; - inLim = 0; - // outProcessed = 0; - - inData = crossBuf; - - for (;;) - { - SizeT inProcessed; - SizeT outProcessed; - ECoderStatus status; - SRes res; - - if (inPos == inLim) - { - if (!me->mtc.readWasFinished) - { - inPos = 0; - inLim = me->mtc.inBufSize; - me->mtc.readRes = ISeqInStream_Read(me->inStream, (void *)inData, &inLim); - me->mtc.readProcessed += inLim; - if (inLim == 0 || me->mtc.readRes != SZ_OK) - me->mtc.readWasFinished = True; - } - } - - inProcessed = inLim - inPos; - outProcessed = 0; - - res = XzUnpacker_Code(dec, - NULL, &outProcessed, - inData + inPos, &inProcessed, - (inProcessed == 0), // srcFinished - CODER_FINISH_END, &status); - - me->codeRes = res; - me->status = status; - inPos += inProcessed; - me->mtc.inProcessed += inProcessed; - me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; - - if (res != SZ_OK) - { - return S_OK; - // return res; - } - - if (dec->state == XZ_STATE_STREAM_HEADER) - { - *needContinue = True; - me->mtc.crossStart = inPos; - me->mtc.crossEnd = inLim; - me->isBlockHeaderState_Parse = False; - me->isBlockHeaderState_Write = False; - return SZ_OK; - } - - if (status != CODER_STATUS_NEEDS_MORE_INPUT) - return E_FAIL; - - if (me->mtc.progress) - { - UInt64 inDelta = me->mtc.inProcessed - inProgressPrev; - if (inDelta >= (1 << 22)) - { - RINOK(MtProgress_Progress_ST(&me->mtc.mtProgress)); - inProgressPrev = me->mtc.inProcessed; - } - } - if (me->mtc.readWasFinished) - return SZ_OK; - } - } - } -} - - -#endif - - - -void XzStatInfo_Clear(CXzStatInfo *p) -{ - p->InSize = 0; - p->OutSize = 0; - - p->NumStreams = 0; - p->NumBlocks = 0; - - p->UnpackSize_Defined = False; - - p->NumStreams_Defined = False; - p->NumBlocks_Defined = False; - - // p->IsArc = False; - // p->UnexpectedEnd = False; - // p->Unsupported = False; - // p->HeadersError = False; - // p->DataError = False; - // p->CrcError = False; - - p->DataAfterEnd = False; - p->DecodingTruncated = False; - - p->DecodeRes = SZ_OK; - p->ReadRes = SZ_OK; - p->ProgressRes = SZ_OK; - - p->CombinedRes = SZ_OK; - p->CombinedRes_Type = SZ_OK; -} - - - - -static SRes XzDecMt_Decode_ST(CXzDecMt *p - #ifndef _7ZIP_ST - , BoolInt tMode - #endif - , CXzStatInfo *stat) -{ - size_t outPos; - size_t inPos, inLim; - const Byte *inData; - UInt64 inPrev, outPrev; - - CXzUnpacker *dec; - - #ifndef _7ZIP_ST - if (tMode) - { - XzDecMt_FreeOutBufs(p); - tMode = MtDec_PrepareRead(&p->mtc); - } - #endif - - if (!p->outBuf || p->outBufSize != p->props.outStep_ST) - { - ISzAlloc_Free(p->allocMid, p->outBuf); - p->outBufSize = 0; - p->outBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.outStep_ST); - if (!p->outBuf) - return SZ_ERROR_MEM; - p->outBufSize = p->props.outStep_ST; - } - - if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) - { - ISzAlloc_Free(p->allocMid, p->inBuf); - p->inBufSize = 0; - p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); - if (!p->inBuf) - return SZ_ERROR_MEM; - p->inBufSize = p->props.inBufSize_ST; - } - - dec = &p->dec; - dec->decodeToStreamSignature = False; - // dec->decodeOnlyOneBlock = False; - - XzUnpacker_SetOutBuf(dec, NULL, 0); - - inPrev = p->inProcessed; - outPrev = p->outProcessed; - - inPos = 0; - inLim = 0; - inData = NULL; - outPos = 0; - - for (;;) - { - SizeT outSize; - BoolInt finished; - ECoderFinishMode finishMode; - SizeT inProcessed; - ECoderStatus status; - SRes res; - - SizeT outProcessed; - - - - if (inPos == inLim) - { - #ifndef _7ZIP_ST - if (tMode) - { - inData = MtDec_Read(&p->mtc, &inLim); - inPos = 0; - if (inData) - continue; - tMode = False; - inLim = 0; - } - #endif - - if (!p->readWasFinished) - { - inPos = 0; - inLim = p->inBufSize; - inData = p->inBuf; - p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); - p->readProcessed += inLim; - if (inLim == 0 || p->readRes != SZ_OK) - p->readWasFinished = True; - } - } - - outSize = p->props.outStep_ST - outPos; - - finishMode = CODER_FINISH_ANY; - if (p->outSize_Defined) - { - const UInt64 rem = p->outSize - p->outProcessed; - if (outSize >= rem) - { - outSize = (SizeT)rem; - if (p->finishMode) - finishMode = CODER_FINISH_END; - } - } - - inProcessed = inLim - inPos; - outProcessed = outSize; - - res = XzUnpacker_Code(dec, p->outBuf + outPos, &outProcessed, - inData + inPos, &inProcessed, - (inPos == inLim), // srcFinished - finishMode, &status); - - p->codeRes = res; - p->status = status; - - inPos += inProcessed; - outPos += outProcessed; - p->inProcessed += inProcessed; - p->outProcessed += outProcessed; - - finished = ((inProcessed == 0 && outProcessed == 0) || res != SZ_OK); - - if (finished || outProcessed >= outSize) - if (outPos != 0) - { - size_t written = ISeqOutStream_Write(p->outStream, p->outBuf, outPos); - p->outProcessed += written; - if (written != outPos) - { - stat->CombinedRes_Type = SZ_ERROR_WRITE; - return SZ_ERROR_WRITE; - } - outPos = 0; - } - - if (p->progress && res == SZ_OK) - { - UInt64 inDelta = p->inProcessed - inPrev; - UInt64 outDelta = p->outProcessed - outPrev; - if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) - { - res = ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed); - if (res != SZ_OK) - { - stat->CombinedRes_Type = SZ_ERROR_PROGRESS; - stat->ProgressRes = res; - return res; - } - inPrev = p->inProcessed; - outPrev = p->outProcessed; - } - } - - if (finished) - return res; - } -} - -static SRes XzStatInfo_SetStat(const CXzUnpacker *dec, - int finishMode, - UInt64 readProcessed, UInt64 inProcessed, - SRes res, ECoderStatus status, - BoolInt decodingTruncated, - CXzStatInfo *stat) -{ - UInt64 extraSize; - - stat->DecodingTruncated = (Byte)(decodingTruncated ? 1 : 0); - stat->InSize = inProcessed; - stat->NumStreams = dec->numStartedStreams; - stat->NumBlocks = dec->numTotalBlocks; - - stat->UnpackSize_Defined = True; - stat->NumStreams_Defined = True; - stat->NumBlocks_Defined = True; - - extraSize = XzUnpacker_GetExtraSize(dec); - - if (res == SZ_OK) - { - if (status == CODER_STATUS_NEEDS_MORE_INPUT) - { - // CODER_STATUS_NEEDS_MORE_INPUT is expected status for correct xz streams - extraSize = 0; - if (!XzUnpacker_IsStreamWasFinished(dec)) - res = SZ_ERROR_INPUT_EOF; - } - else if (!decodingTruncated || finishMode) // (status == CODER_STATUS_NOT_FINISHED) - res = SZ_ERROR_DATA; - } - else if (res == SZ_ERROR_NO_ARCHIVE) - { - /* - SZ_ERROR_NO_ARCHIVE is possible for 2 states: - XZ_STATE_STREAM_HEADER - if bad signature or bad CRC - XZ_STATE_STREAM_PADDING - if non-zero padding data - extraSize / inProcessed don't include "bad" byte - */ - if (inProcessed != extraSize) // if good streams before error - if (extraSize != 0 || readProcessed != inProcessed) - { - stat->DataAfterEnd = True; - // there is some good xz stream before. So we set SZ_OK - res = SZ_OK; - } - } - - stat->DecodeRes = res; - - stat->InSize -= extraSize; - return res; -} - - -SRes XzDecMt_Decode(CXzDecMtHandle pp, - const CXzDecMtProps *props, - const UInt64 *outDataSize, int finishMode, - ISeqOutStream *outStream, - // Byte *outBuf, size_t *outBufSize, - ISeqInStream *inStream, - // const Byte *inData, size_t inDataSize, - CXzStatInfo *stat, - int *isMT, - ICompressProgress *progress) -{ - CXzDecMt *p = (CXzDecMt *)pp; - #ifndef _7ZIP_ST - BoolInt tMode; - #endif - - XzStatInfo_Clear(stat); - - p->props = *props; - - p->inStream = inStream; - p->outStream = outStream; - p->progress = progress; - // p->stat = stat; - - p->outSize = 0; - p->outSize_Defined = False; - if (outDataSize) - { - p->outSize_Defined = True; - p->outSize = *outDataSize; - } - - p->finishMode = finishMode; - - // p->outSize = 457; p->outSize_Defined = True; p->finishMode = False; // for test - - p->writeRes = SZ_OK; - p->outProcessed = 0; - p->inProcessed = 0; - p->readProcessed = 0; - p->readWasFinished = False; - - p->codeRes = 0; - p->status = CODER_STATUS_NOT_SPECIFIED; - - XzUnpacker_Init(&p->dec); - - *isMT = False; - - /* - p->outBuf = NULL; - p->outBufSize = 0; - if (!outStream) - { - p->outBuf = outBuf; - p->outBufSize = *outBufSize; - *outBufSize = 0; - } - */ - - - #ifndef _7ZIP_ST - - p->isBlockHeaderState_Parse = False; - p->isBlockHeaderState_Write = False; - // p->numBadBlocks = 0; - p->mainErrorCode = SZ_OK; - p->mainDecoderWasCalled = False; - - tMode = False; - - if (p->props.numThreads > 1) - { - IMtDecCallback vt; - - // we just free ST buffers here - // but we still keep state variables, that was set in XzUnpacker_Init() - XzDecMt_FreeSt(p); - - p->outProcessed_Parse = 0; - p->parsing_Truncated = False; - - p->numStreams = 0; - p->numTotalBlocks = 0; - p->numBlocks = 0; - p->finishedDecoderIndex = -1; - - if (!p->mtc_WasConstructed) - { - p->mtc_WasConstructed = True; - MtDec_Construct(&p->mtc); - } - - p->mtc.mtCallback = &vt; - p->mtc.mtCallbackObject = p; - - p->mtc.progress = progress; - p->mtc.inStream = inStream; - p->mtc.alloc = &p->alignOffsetAlloc.vt; - // p->mtc.inData = inData; - // p->mtc.inDataSize = inDataSize; - p->mtc.inBufSize = p->props.inBufSize_MT; - // p->mtc.inBlockMax = p->props.inBlockMax; - p->mtc.numThreadsMax = p->props.numThreads; - - *isMT = True; - - vt.Parse = XzDecMt_Callback_Parse; - vt.PreCode = XzDecMt_Callback_PreCode; - vt.Code = XzDecMt_Callback_Code; - vt.Write = XzDecMt_Callback_Write; - - { - BoolInt needContinue; - - SRes res = MtDec_Code(&p->mtc); - - stat->InSize = p->mtc.inProcessed; - - p->inProcessed = p->mtc.inProcessed; - p->readRes = p->mtc.readRes; - p->readWasFinished = p->mtc.readWasFinished; - p->readProcessed = p->mtc.readProcessed; - - tMode = True; - needContinue = False; - - if (res == SZ_OK) - { - if (p->mtc.mtProgress.res != SZ_OK) - { - res = p->mtc.mtProgress.res; - stat->ProgressRes = res; - stat->CombinedRes_Type = SZ_ERROR_PROGRESS; - } - else - needContinue = p->mtc.needContinue; - } - - if (!needContinue) - { - SRes codeRes; - BoolInt truncated = False; - ECoderStatus status; - CXzUnpacker *dec; - - stat->OutSize = p->outProcessed; - - if (p->finishedDecoderIndex >= 0) - { - CXzDecMtThread *coder = &p->coders[(unsigned)p->finishedDecoderIndex]; - codeRes = coder->codeRes; - dec = &coder->dec; - status = coder->status; - } - else if (p->mainDecoderWasCalled) - { - codeRes = p->codeRes; - dec = &p->dec; - status = p->status; - truncated = p->parsing_Truncated; - } - else - return E_FAIL; - - XzStatInfo_SetStat(dec, p->finishMode, - p->mtc.readProcessed, p->mtc.inProcessed, - codeRes, status, - truncated, - stat); - - if (res == SZ_OK) - { - if (p->writeRes != SZ_OK) - { - res = p->writeRes; - stat->CombinedRes_Type = SZ_ERROR_WRITE; - } - else if (p->mtc.readRes != SZ_OK && p->mtc.inProcessed == p->mtc.readProcessed) - { - res = p->mtc.readRes; - stat->ReadRes = res; - stat->CombinedRes_Type = SZ_ERROR_READ; - } - else if (p->mainErrorCode != SZ_OK) - { - res = p->mainErrorCode; - } - } - - stat->CombinedRes = res; - if (stat->CombinedRes_Type == SZ_OK) - stat->CombinedRes_Type = res; - return res; - } - - PRF_STR("----- decoding ST -----"); - } - } - - #endif - - - *isMT = False; - - { - SRes res = XzDecMt_Decode_ST(p - #ifndef _7ZIP_ST - , tMode - #endif - , stat - ); - - XzStatInfo_SetStat(&p->dec, - p->finishMode, - p->readProcessed, p->inProcessed, - p->codeRes, p->status, - False, // truncated - stat); - - if (res == SZ_OK) - { - /* - if (p->writeRes != SZ_OK) - { - res = p->writeRes; - stat->CombinedRes_Type = SZ_ERROR_WRITE; - } - else - */ - if (p->readRes != SZ_OK && p->inProcessed == p->readProcessed) - { - res = p->readRes; - stat->ReadRes = res; - stat->CombinedRes_Type = SZ_ERROR_READ; - } - #ifndef _7ZIP_ST - else if (p->mainErrorCode != SZ_OK) - res = p->mainErrorCode; - #endif - } - - stat->CombinedRes = res; - if (stat->CombinedRes_Type == SZ_OK) - stat->CombinedRes_Type = res; - return res; - } -} diff --git a/code/ryzom/client/src/seven_zip/XzEnc.cpp b/code/ryzom/client/src/seven_zip/XzEnc.cpp deleted file mode 100644 index d0a8b4489..000000000 --- a/code/ryzom/client/src/seven_zip/XzEnc.cpp +++ /dev/null @@ -1,1329 +0,0 @@ -/* XzEnc.c -- Xz Encode -2019-02-02 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include -#include - -#include "7zCrc.h" -#include "Bra.h" -#include "CpuArch.h" - -#ifdef USE_SUBBLOCK -#include "Bcj3Enc.c" -#include "SbFind.c" -#include "SbEnc.c" -#endif - -#include "XzEnc.h" - -// #define _7ZIP_ST - -#ifndef _7ZIP_ST -#include "MtCoder.h" -#else -#define MTCODER__THREADS_MAX 1 -#define MTCODER__BLOCKS_MAX 1 -#endif - -#define XZ_GET_PAD_SIZE(dataSize) ((4 - ((unsigned)(dataSize) & 3)) & 3) - -/* max pack size for LZMA2 block + check-64bytrs: */ -#define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + 64) - -#define XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(unpackSize) (XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize)) - - -#define XzBlock_ClearFlags(p) (p)->flags = 0; -#define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1); -#define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE; -#define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE; - - -static SRes WriteBytes(ISeqOutStream *s, const void *buf, size_t size) -{ - return (ISeqOutStream_Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; -} - -static SRes WriteBytesUpdateCrc(ISeqOutStream *s, const void *buf, size_t size, UInt32 *crc) -{ - *crc = CrcUpdate(*crc, buf, size); - return WriteBytes(s, buf, size); -} - - -static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s) -{ - UInt32 crc; - Byte header[XZ_STREAM_HEADER_SIZE]; - memcpy(header, XZ_SIG, XZ_SIG_SIZE); - header[XZ_SIG_SIZE] = (Byte)(f >> 8); - header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF); - crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE); - SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc); - return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE); -} - - -static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s) -{ - Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; - - unsigned pos = 1; - unsigned numFilters, i; - header[pos++] = p->flags; - - if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize); - if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize); - numFilters = XzBlock_GetNumFilters(p); - - for (i = 0; i < numFilters; i++) - { - const CXzFilter *f = &p->filters[i]; - pos += Xz_WriteVarInt(header + pos, f->id); - pos += Xz_WriteVarInt(header + pos, f->propsSize); - memcpy(header + pos, f->props, f->propsSize); - pos += f->propsSize; - } - - while ((pos & 3) != 0) - header[pos++] = 0; - - header[0] = (Byte)(pos >> 2); - SetUi32(header + pos, CrcCalc(header, pos)); - return WriteBytes(s, header, pos + 4); -} - - - - -typedef struct -{ - size_t numBlocks; - size_t size; - size_t allocated; - Byte *blocks; -} CXzEncIndex; - - -static void XzEncIndex_Construct(CXzEncIndex *p) -{ - p->numBlocks = 0; - p->size = 0; - p->allocated = 0; - p->blocks = NULL; -} - -static void XzEncIndex_Init(CXzEncIndex *p) -{ - p->numBlocks = 0; - p->size = 0; -} - -static void XzEncIndex_Free(CXzEncIndex *p, ISzAllocPtr alloc) -{ - if (p->blocks) - { - ISzAlloc_Free(alloc, p->blocks); - p->blocks = NULL; - } - p->numBlocks = 0; - p->size = 0; - p->allocated = 0; -} - - -static SRes XzEncIndex_ReAlloc(CXzEncIndex *p, size_t newSize, ISzAllocPtr alloc) -{ - Byte *blocks = (Byte *)ISzAlloc_Alloc(alloc, newSize); - if (!blocks) - return SZ_ERROR_MEM; - if (p->size != 0) - memcpy(blocks, p->blocks, p->size); - if (p->blocks) - ISzAlloc_Free(alloc, p->blocks); - p->blocks = blocks; - p->allocated = newSize; - return SZ_OK; -} - - -static SRes XzEncIndex_PreAlloc(CXzEncIndex *p, UInt64 numBlocks, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) -{ - UInt64 pos; - { - Byte buf[32]; - unsigned pos2 = Xz_WriteVarInt(buf, totalSize); - pos2 += Xz_WriteVarInt(buf + pos2, unpackSize); - pos = numBlocks * pos2; - } - - if (pos <= p->allocated - p->size) - return SZ_OK; - { - UInt64 newSize64 = p->size + pos; - size_t newSize = (size_t)newSize64; - if (newSize != newSize64) - return SZ_ERROR_MEM; - return XzEncIndex_ReAlloc(p, newSize, alloc); - } -} - - -static SRes XzEncIndex_AddIndexRecord(CXzEncIndex *p, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) -{ - Byte buf[32]; - unsigned pos = Xz_WriteVarInt(buf, totalSize); - pos += Xz_WriteVarInt(buf + pos, unpackSize); - - if (pos > p->allocated - p->size) - { - size_t newSize = p->allocated * 2 + 16 * 2; - if (newSize < p->size + pos) - return SZ_ERROR_MEM; - RINOK(XzEncIndex_ReAlloc(p, newSize, alloc)); - } - memcpy(p->blocks + p->size, buf, pos); - p->size += pos; - p->numBlocks++; - return SZ_OK; -} - - -static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, ISeqOutStream *s) -{ - Byte buf[32]; - UInt64 globalPos; - UInt32 crc = CRC_INIT_VAL; - unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); - - globalPos = pos; - buf[0] = 0; - RINOK(WriteBytesUpdateCrc(s, buf, pos, &crc)); - RINOK(WriteBytesUpdateCrc(s, p->blocks, p->size, &crc)); - globalPos += p->size; - - pos = XZ_GET_PAD_SIZE(globalPos); - buf[1] = 0; - buf[2] = 0; - buf[3] = 0; - globalPos += pos; - - crc = CrcUpdate(crc, buf + 4 - pos, pos); - SetUi32(buf + 4, CRC_GET_DIGEST(crc)); - - SetUi32(buf + 8 + 4, (UInt32)(globalPos >> 2)); - buf[8 + 8] = (Byte)(flags >> 8); - buf[8 + 9] = (Byte)(flags & 0xFF); - SetUi32(buf + 8, CrcCalc(buf + 8 + 4, 6)); - buf[8 + 10] = XZ_FOOTER_SIG_0; - buf[8 + 11] = XZ_FOOTER_SIG_1; - - return WriteBytes(s, buf + 4 - pos, pos + 4 + 12); -} - - - -/* ---------- CSeqCheckInStream ---------- */ - -typedef struct -{ - ISeqInStream vt; - ISeqInStream *realStream; - const Byte *data; - UInt64 limit; - UInt64 processed; - int realStreamFinished; - CXzCheck check; -} CSeqCheckInStream; - -static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned checkMode) -{ - p->limit = (UInt64)(Int64)-1; - p->processed = 0; - p->realStreamFinished = 0; - XzCheck_Init(&p->check, checkMode); -} - -static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest) -{ - XzCheck_Final(&p->check, digest); -} - -static SRes SeqCheckInStream_Read(const ISeqInStream *pp, void *data, size_t *size) -{ - CSeqCheckInStream *p = CONTAINER_FROM_VTBL(pp, CSeqCheckInStream, vt); - size_t size2 = *size; - SRes res = SZ_OK; - - if (p->limit != (UInt64)(Int64)-1) - { - UInt64 rem = p->limit - p->processed; - if (size2 > rem) - size2 = (size_t)rem; - } - if (size2 != 0) - { - if (p->realStream) - { - res = ISeqInStream_Read(p->realStream, data, &size2); - p->realStreamFinished = (size2 == 0) ? 1 : 0; - } - else - memcpy(data, p->data + (size_t)p->processed, size2); - XzCheck_Update(&p->check, data, size2); - p->processed += size2; - } - *size = size2; - return res; -} - - -/* ---------- CSeqSizeOutStream ---------- */ - -typedef struct -{ - ISeqOutStream vt; - ISeqOutStream *realStream; - Byte *outBuf; - size_t outBufLimit; - UInt64 processed; -} CSeqSizeOutStream; - -static size_t SeqSizeOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) -{ - CSeqSizeOutStream *p = CONTAINER_FROM_VTBL(pp, CSeqSizeOutStream, vt); - if (p->realStream) - size = ISeqOutStream_Write(p->realStream, data, size); - else - { - if (size > p->outBufLimit - (size_t)p->processed) - return 0; - memcpy(p->outBuf + (size_t)p->processed, data, size); - } - p->processed += size; - return size; -} - - -/* ---------- CSeqInFilter ---------- */ - -#define FILTER_BUF_SIZE (1 << 20) - -typedef struct -{ - ISeqInStream p; - ISeqInStream *realStream; - IStateCoder StateCoder; - Byte *buf; - size_t curPos; - size_t endPos; - int srcWasFinished; -} CSeqInFilter; - - -SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc); - -static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPtr alloc) -{ - if (!p->buf) - { - p->buf = (Byte *)ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); - if (!p->buf) - return SZ_ERROR_MEM; - } - p->curPos = p->endPos = 0; - p->srcWasFinished = 0; - RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, alloc)); - RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, alloc)); - p->StateCoder.Init(p->StateCoder.p); - return SZ_OK; -} - - -static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) -{ - CSeqInFilter *p = CONTAINER_FROM_VTBL(pp, CSeqInFilter, p); - size_t sizeOriginal = *size; - if (sizeOriginal == 0) - return SZ_OK; - *size = 0; - - for (;;) - { - if (!p->srcWasFinished && p->curPos == p->endPos) - { - p->curPos = 0; - p->endPos = FILTER_BUF_SIZE; - RINOK(ISeqInStream_Read(p->realStream, p->buf, &p->endPos)); - if (p->endPos == 0) - p->srcWasFinished = 1; - } - { - SizeT srcLen = p->endPos - p->curPos; - ECoderStatus status; - SRes res; - *size = sizeOriginal; - res = p->StateCoder.Code2(p->StateCoder.p, - (Byte *)data, size, - p->buf + p->curPos, &srcLen, - p->srcWasFinished, CODER_FINISH_ANY, - &status); - p->curPos += srcLen; - if (*size != 0 || srcLen == 0 || res != SZ_OK) - return res; - } - } -} - -static void SeqInFilter_Construct(CSeqInFilter *p) -{ - p->buf = NULL; - p->StateCoder.p = NULL; - p->p.Read = SeqInFilter_Read; -} - -static void SeqInFilter_Free(CSeqInFilter *p, ISzAllocPtr alloc) -{ - if (p->StateCoder.p) - { - p->StateCoder.Free(p->StateCoder.p, alloc); - p->StateCoder.p = NULL; - } - if (p->buf) - { - ISzAlloc_Free(alloc, p->buf); - p->buf = NULL; - } -} - - -/* ---------- CSbEncInStream ---------- */ - -#ifdef USE_SUBBLOCK - -typedef struct -{ - ISeqInStream vt; - ISeqInStream *inStream; - CSbEnc enc; -} CSbEncInStream; - -static SRes SbEncInStream_Read(const ISeqInStream *pp, void *data, size_t *size) -{ - CSbEncInStream *p = CONTAINER_FROM_VTBL(pp, CSbEncInStream, vt); - size_t sizeOriginal = *size; - if (sizeOriginal == 0) - return SZ_OK; - - for (;;) - { - if (p->enc.needRead && !p->enc.readWasFinished) - { - size_t processed = p->enc.needReadSizeMax; - RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed)); - p->enc.readPos += processed; - if (processed == 0) - { - p->enc.readWasFinished = True; - p->enc.isFinalFinished = True; - } - p->enc.needRead = False; - } - - *size = sizeOriginal; - RINOK(SbEnc_Read(&p->enc, data, size)); - if (*size != 0 || !p->enc.needRead) - return SZ_OK; - } -} - -void SbEncInStream_Construct(CSbEncInStream *p, ISzAllocPtr alloc) -{ - SbEnc_Construct(&p->enc, alloc); - p->vt.Read = SbEncInStream_Read; -} - -SRes SbEncInStream_Init(CSbEncInStream *p) -{ - return SbEnc_Init(&p->enc); -} - -void SbEncInStream_Free(CSbEncInStream *p) -{ - SbEnc_Free(&p->enc); -} - -#endif - - - -/* ---------- CXzProps ---------- */ - - -void XzFilterProps_Init(CXzFilterProps *p) -{ - p->id = 0; - p->delta = 0; - p->ip = 0; - p->ipDefined = False; -} - -void XzProps_Init(CXzProps *p) -{ - p->checkId = XZ_CHECK_CRC32; - p->blockSize = XZ_PROPS__BLOCK_SIZE__AUTO; - p->numBlockThreads_Reduced = -1; - p->numBlockThreads_Max = -1; - p->numTotalThreads = -1; - p->reduceSize = (UInt64)(Int64)-1; - p->forceWriteSizesInHeader = 0; - // p->forceWriteSizesInHeader = 1; - - XzFilterProps_Init(&p->filterProps); - Lzma2EncProps_Init(&p->lzma2Props); -} - - -static void XzEncProps_Normalize_Fixed(CXzProps *p) -{ - UInt64 fileSize; - int t1, t1n, t2, t2r, t3; - { - CLzma2EncProps tp = p->lzma2Props; - if (tp.numTotalThreads <= 0) - tp.numTotalThreads = p->numTotalThreads; - Lzma2EncProps_Normalize(&tp); - t1n = tp.numTotalThreads; - } - - t1 = p->lzma2Props.numTotalThreads; - t2 = p->numBlockThreads_Max; - t3 = p->numTotalThreads; - - if (t2 > MTCODER__THREADS_MAX) - t2 = MTCODER__THREADS_MAX; - - if (t3 <= 0) - { - if (t2 <= 0) - t2 = 1; - t3 = t1n * t2; - } - else if (t2 <= 0) - { - t2 = t3 / t1n; - if (t2 == 0) - { - t1 = 1; - t2 = t3; - } - if (t2 > MTCODER__THREADS_MAX) - t2 = MTCODER__THREADS_MAX; - } - else if (t1 <= 0) - { - t1 = t3 / t2; - if (t1 == 0) - t1 = 1; - } - else - t3 = t1n * t2; - - p->lzma2Props.numTotalThreads = t1; - - t2r = t2; - - fileSize = p->reduceSize; - - if ((p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) - p->lzma2Props.lzmaProps.reduceSize = p->blockSize; - - Lzma2EncProps_Normalize(&p->lzma2Props); - - t1 = p->lzma2Props.numTotalThreads; - - { - if (t2 > 1 && fileSize != (UInt64)(Int64)-1) - { - UInt64 numBlocks = fileSize / p->blockSize; - if (numBlocks * p->blockSize != fileSize) - numBlocks++; - if (numBlocks < (unsigned)t2) - { - t2r = (unsigned)numBlocks; - if (t2r == 0) - t2r = 1; - t3 = t1 * t2r; - } - } - } - - p->numBlockThreads_Max = t2; - p->numBlockThreads_Reduced = t2r; - p->numTotalThreads = t3; -} - - -static void XzProps_Normalize(CXzProps *p) -{ - /* we normalize xzProps properties, but we normalize only some of CXzProps::lzma2Props properties. - Lzma2Enc_SetProps() will normalize lzma2Props later. */ - - if (p->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID) - { - p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; - p->numBlockThreads_Reduced = 1; - p->numBlockThreads_Max = 1; - if (p->lzma2Props.numTotalThreads <= 0) - p->lzma2Props.numTotalThreads = p->numTotalThreads; - return; - } - else - { - CLzma2EncProps *lzma2 = &p->lzma2Props; - if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) - { - // xz-auto - p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; - - if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) - { - // if (xz-auto && lzma2-solid) - we use solid for both - p->blockSize = XZ_PROPS__BLOCK_SIZE__SOLID; - p->numBlockThreads_Reduced = 1; - p->numBlockThreads_Max = 1; - if (p->lzma2Props.numTotalThreads <= 0) - p->lzma2Props.numTotalThreads = p->numTotalThreads; - } - else - { - // if (xz-auto && (lzma2-auto || lzma2-fixed_) - // we calculate block size for lzma2 and use that block size for xz, lzma2 uses single-chunk per block - CLzma2EncProps tp = p->lzma2Props; - if (tp.numTotalThreads <= 0) - tp.numTotalThreads = p->numTotalThreads; - - Lzma2EncProps_Normalize(&tp); - - p->blockSize = tp.blockSize; // fixed or solid - p->numBlockThreads_Reduced = tp.numBlockThreads_Reduced; - p->numBlockThreads_Max = tp.numBlockThreads_Max; - if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) - lzma2->blockSize = tp.blockSize; // fixed or solid, LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID - if (lzma2->lzmaProps.reduceSize > tp.blockSize && tp.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) - lzma2->lzmaProps.reduceSize = tp.blockSize; - lzma2->numBlockThreads_Reduced = 1; - lzma2->numBlockThreads_Max = 1; - return; - } - } - else - { - // xz-fixed - // we can use xz::reduceSize or xz::blockSize as base for lzmaProps::reduceSize - - p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; - { - UInt64 r = p->reduceSize; - if (r > p->blockSize || r == (UInt64)(Int64)-1) - r = p->blockSize; - lzma2->lzmaProps.reduceSize = r; - } - if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) - lzma2->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; - else if (lzma2->blockSize > p->blockSize && lzma2->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) - lzma2->blockSize = p->blockSize; - - XzEncProps_Normalize_Fixed(p); - } - } -} - - -/* ---------- CLzma2WithFilters ---------- */ - -typedef struct -{ - CLzma2EncHandle lzma2; - CSeqInFilter filter; - - #ifdef USE_SUBBLOCK - CSbEncInStream sb; - #endif -} CLzma2WithFilters; - - -static void Lzma2WithFilters_Construct(CLzma2WithFilters *p) -{ - p->lzma2 = NULL; - SeqInFilter_Construct(&p->filter); - - #ifdef USE_SUBBLOCK - SbEncInStream_Construct(&p->sb, alloc); - #endif -} - - -static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p, ISzAllocPtr alloc, ISzAllocPtr bigAlloc) -{ - if (!p->lzma2) - { - p->lzma2 = Lzma2Enc_Create(alloc, bigAlloc); - if (!p->lzma2) - return SZ_ERROR_MEM; - } - return SZ_OK; -} - - -static void Lzma2WithFilters_Free(CLzma2WithFilters *p, ISzAllocPtr alloc) -{ - #ifdef USE_SUBBLOCK - SbEncInStream_Free(&p->sb); - #endif - - SeqInFilter_Free(&p->filter, alloc); - if (p->lzma2) - { - Lzma2Enc_Destroy(p->lzma2); - p->lzma2 = NULL; - } -} - - -typedef struct -{ - UInt64 unpackSize; - UInt64 totalSize; - size_t headerSize; -} CXzEncBlockInfo; - - -static SRes Xz_CompressBlock( - CLzma2WithFilters *lzmaf, - - ISeqOutStream *outStream, - Byte *outBufHeader, - Byte *outBufData, size_t outBufDataLimit, - - ISeqInStream *inStream, - // UInt64 expectedSize, - const Byte *inBuf, // used if (!inStream) - size_t inBufSize, // used if (!inStream), it's block size, props->blockSize is ignored - - const CXzProps *props, - ICompressProgress *progress, - int *inStreamFinished, /* only for inStream version */ - CXzEncBlockInfo *blockSizes, - ISzAllocPtr alloc, - ISzAllocPtr allocBig) -{ - CSeqCheckInStream checkInStream; - CSeqSizeOutStream seqSizeOutStream; - CXzBlock block; - unsigned filterIndex = 0; - CXzFilter *filter = NULL; - const CXzFilterProps *fp = &props->filterProps; - if (fp->id == 0) - fp = NULL; - - *inStreamFinished = False; - - RINOK(Lzma2WithFilters_Create(lzmaf, alloc, allocBig)); - - RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, &props->lzma2Props)); - - XzBlock_ClearFlags(&block); - XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0)); - - if (fp) - { - filter = &block.filters[filterIndex++]; - filter->id = fp->id; - filter->propsSize = 0; - - if (fp->id == XZ_ID_Delta) - { - filter->props[0] = (Byte)(fp->delta - 1); - filter->propsSize = 1; - } - else if (fp->ipDefined) - { - SetUi32(filter->props, fp->ip); - filter->propsSize = 4; - } - } - - { - CXzFilter *f = &block.filters[filterIndex++]; - f->id = XZ_ID_LZMA2; - f->propsSize = 1; - f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); - } - - seqSizeOutStream.vt.Write = SeqSizeOutStream_Write; - seqSizeOutStream.realStream = outStream; - seqSizeOutStream.outBuf = outBufData; - seqSizeOutStream.outBufLimit = outBufDataLimit; - seqSizeOutStream.processed = 0; - - /* - if (expectedSize != (UInt64)(Int64)-1) - { - block.unpackSize = expectedSize; - if (props->blockSize != (UInt64)(Int64)-1) - if (expectedSize > props->blockSize) - block.unpackSize = props->blockSize; - XzBlock_SetHasUnpackSize(&block); - } - */ - - if (outStream) - { - RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); - } - - checkInStream.vt.Read = SeqCheckInStream_Read; - SeqCheckInStream_Init(&checkInStream, props->checkId); - - checkInStream.realStream = inStream; - checkInStream.data = inBuf; - checkInStream.limit = props->blockSize; - if (!inStream) - checkInStream.limit = inBufSize; - - if (fp) - { - #ifdef USE_SUBBLOCK - if (fp->id == XZ_ID_Subblock) - { - lzmaf->sb.inStream = &checkInStream.vt; - RINOK(SbEncInStream_Init(&lzmaf->sb)); - } - else - #endif - { - lzmaf->filter.realStream = &checkInStream.vt; - RINOK(SeqInFilter_Init(&lzmaf->filter, filter, alloc)); - } - } - - { - SRes res; - Byte *outBuf = NULL; - size_t outSize = 0; - BoolInt useStream = (fp || inStream); - // useStream = True; - - if (!useStream) - { - XzCheck_Update(&checkInStream.check, inBuf, inBufSize); - checkInStream.processed = inBufSize; - } - - if (!outStream) - { - outBuf = seqSizeOutStream.outBuf; // + (size_t)seqSizeOutStream.processed; - outSize = seqSizeOutStream.outBufLimit; // - (size_t)seqSizeOutStream.processed; - } - - res = Lzma2Enc_Encode2(lzmaf->lzma2, - outBuf ? NULL : &seqSizeOutStream.vt, - outBuf, - outBuf ? &outSize : NULL, - - useStream ? - (fp ? - ( - #ifdef USE_SUBBLOCK - (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.vt: - #endif - &lzmaf->filter.p) : - &checkInStream.vt) : NULL, - - useStream ? NULL : inBuf, - useStream ? 0 : inBufSize, - - progress); - - if (outBuf) - seqSizeOutStream.processed += outSize; - - RINOK(res); - blockSizes->unpackSize = checkInStream.processed; - } - { - Byte buf[4 + 64]; - unsigned padSize = XZ_GET_PAD_SIZE(seqSizeOutStream.processed); - UInt64 packSize = seqSizeOutStream.processed; - - buf[0] = 0; - buf[1] = 0; - buf[2] = 0; - buf[3] = 0; - - SeqCheckInStream_GetDigest(&checkInStream, buf + 4); - RINOK(WriteBytes(&seqSizeOutStream.vt, buf + (4 - padSize), padSize + XzFlags_GetCheckSize((CXzStreamFlags)props->checkId))); - - blockSizes->totalSize = seqSizeOutStream.processed - padSize; - - if (!outStream) - { - seqSizeOutStream.outBuf = outBufHeader; - seqSizeOutStream.outBufLimit = XZ_BLOCK_HEADER_SIZE_MAX; - seqSizeOutStream.processed = 0; - - block.unpackSize = blockSizes->unpackSize; - XzBlock_SetHasUnpackSize(&block); - - block.packSize = packSize; - XzBlock_SetHasPackSize(&block); - - RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); - - blockSizes->headerSize = (size_t)seqSizeOutStream.processed; - blockSizes->totalSize += seqSizeOutStream.processed; - } - } - - if (inStream) - *inStreamFinished = checkInStream.realStreamFinished; - else - { - *inStreamFinished = False; - if (checkInStream.processed != inBufSize) - return SZ_ERROR_FAIL; - } - - return SZ_OK; -} - - - -typedef struct -{ - ICompressProgress vt; - ICompressProgress *progress; - UInt64 inOffset; - UInt64 outOffset; -} CCompressProgress_XzEncOffset; - - -static SRes CompressProgress_XzEncOffset_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) -{ - const CCompressProgress_XzEncOffset *p = CONTAINER_FROM_VTBL(pp, CCompressProgress_XzEncOffset, vt); - inSize += p->inOffset; - outSize += p->outOffset; - return ICompressProgress_Progress(p->progress, inSize, outSize); -} - - - - -typedef struct -{ - ISzAllocPtr alloc; - ISzAllocPtr allocBig; - - CXzProps xzProps; - UInt64 expectedDataSize; - - CXzEncIndex xzIndex; - - CLzma2WithFilters lzmaf_Items[MTCODER__THREADS_MAX]; - - size_t outBufSize; /* size of allocated outBufs[i] */ - Byte *outBufs[MTCODER__BLOCKS_MAX]; - - #ifndef _7ZIP_ST - unsigned checkType; - ISeqOutStream *outStream; - BoolInt mtCoder_WasConstructed; - CMtCoder mtCoder; - CXzEncBlockInfo EncBlocks[MTCODER__BLOCKS_MAX]; - #endif - -} CXzEnc; - - -static void XzEnc_Construct(CXzEnc *p) -{ - unsigned i; - - XzEncIndex_Construct(&p->xzIndex); - - for (i = 0; i < MTCODER__THREADS_MAX; i++) - Lzma2WithFilters_Construct(&p->lzmaf_Items[i]); - - #ifndef _7ZIP_ST - p->mtCoder_WasConstructed = False; - { - for (i = 0; i < MTCODER__BLOCKS_MAX; i++) - p->outBufs[i] = NULL; - p->outBufSize = 0; - } - #endif -} - - -static void XzEnc_FreeOutBufs(CXzEnc *p) -{ - unsigned i; - for (i = 0; i < MTCODER__BLOCKS_MAX; i++) - if (p->outBufs[i]) - { - ISzAlloc_Free(p->alloc, p->outBufs[i]); - p->outBufs[i] = NULL; - } - p->outBufSize = 0; -} - - -static void XzEnc_Free(CXzEnc *p, ISzAllocPtr alloc) -{ - unsigned i; - - XzEncIndex_Free(&p->xzIndex, alloc); - - for (i = 0; i < MTCODER__THREADS_MAX; i++) - Lzma2WithFilters_Free(&p->lzmaf_Items[i], alloc); - - #ifndef _7ZIP_ST - if (p->mtCoder_WasConstructed) - { - MtCoder_Destruct(&p->mtCoder); - p->mtCoder_WasConstructed = False; - } - XzEnc_FreeOutBufs(p); - #endif -} - - -CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) -{ - CXzEnc *p = (CXzEnc *)ISzAlloc_Alloc(alloc, sizeof(CXzEnc)); - if (!p) - return NULL; - XzEnc_Construct(p); - XzProps_Init(&p->xzProps); - XzProps_Normalize(&p->xzProps); - p->expectedDataSize = (UInt64)(Int64)-1; - p->alloc = alloc; - p->allocBig = allocBig; - return p; -} - - -void XzEnc_Destroy(CXzEncHandle pp) -{ - CXzEnc *p = (CXzEnc *)pp; - XzEnc_Free(p, p->alloc); - ISzAlloc_Free(p->alloc, p); -} - - -SRes XzEnc_SetProps(CXzEncHandle pp, const CXzProps *props) -{ - CXzEnc *p = (CXzEnc *)pp; - p->xzProps = *props; - XzProps_Normalize(&p->xzProps); - return SZ_OK; -} - - -void XzEnc_SetDataSize(CXzEncHandle pp, UInt64 expectedDataSiize) -{ - CXzEnc *p = (CXzEnc *)pp; - p->expectedDataSize = expectedDataSiize; -} - - - - -#ifndef _7ZIP_ST - -static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, - const Byte *src, size_t srcSize, int finished) -{ - CXzEnc *me = (CXzEnc *)pp; - SRes res; - CMtProgressThunk progressThunk; - - Byte *dest = me->outBufs[outBufIndex]; - - UNUSED_VAR(finished) - - { - CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; - bInfo->totalSize = 0; - bInfo->unpackSize = 0; - bInfo->headerSize = 0; - } - - if (!dest) - { - dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); - if (!dest) - return SZ_ERROR_MEM; - me->outBufs[outBufIndex] = dest; - } - - MtProgressThunk_CreateVTable(&progressThunk); - progressThunk.mtProgress = &me->mtCoder.mtProgress; - MtProgressThunk_Init(&progressThunk); - - { - CXzEncBlockInfo blockSizes; - int inStreamFinished; - - res = Xz_CompressBlock( - &me->lzmaf_Items[coderIndex], - - NULL, - dest, - dest + XZ_BLOCK_HEADER_SIZE_MAX, me->outBufSize - XZ_BLOCK_HEADER_SIZE_MAX, - - NULL, - // srcSize, // expectedSize - src, srcSize, - - &me->xzProps, - &progressThunk.vt, - &inStreamFinished, - &blockSizes, - me->alloc, - me->allocBig); - - if (res == SZ_OK) - me->EncBlocks[outBufIndex] = blockSizes; - - return res; - } -} - - -static SRes XzEnc_MtCallback_Write(void *pp, unsigned outBufIndex) -{ - CXzEnc *me = (CXzEnc *)pp; - - const CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; - const Byte *data = me->outBufs[outBufIndex]; - - RINOK(WriteBytes(me->outStream, data, bInfo->headerSize)); - - { - UInt64 totalPackFull = bInfo->totalSize + XZ_GET_PAD_SIZE(bInfo->totalSize); - RINOK(WriteBytes(me->outStream, data + XZ_BLOCK_HEADER_SIZE_MAX, (size_t)totalPackFull - bInfo->headerSize)); - } - - return XzEncIndex_AddIndexRecord(&me->xzIndex, bInfo->unpackSize, bInfo->totalSize, me->alloc); -} - -#endif - - - -SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) -{ - CXzEnc *p = (CXzEnc *)pp; - - const CXzProps *props = &p->xzProps; - - XzEncIndex_Init(&p->xzIndex); - { - UInt64 numBlocks = 1; - UInt64 blockSize = props->blockSize; - - if (blockSize != XZ_PROPS__BLOCK_SIZE__SOLID - && props->reduceSize != (UInt64)(Int64)-1) - { - numBlocks = props->reduceSize / blockSize; - if (numBlocks * blockSize != props->reduceSize) - numBlocks++; - } - else - blockSize = (UInt64)1 << 62; - - RINOK(XzEncIndex_PreAlloc(&p->xzIndex, numBlocks, blockSize, XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(blockSize), p->alloc)); - } - - RINOK(Xz_WriteHeader((CXzStreamFlags)props->checkId, outStream)); - - - #ifndef _7ZIP_ST - if (props->numBlockThreads_Reduced > 1) - { - IMtCoderCallback2 vt; - - if (!p->mtCoder_WasConstructed) - { - p->mtCoder_WasConstructed = True; - MtCoder_Construct(&p->mtCoder); - } - - vt.Code = XzEnc_MtCallback_Code; - vt.Write = XzEnc_MtCallback_Write; - - p->checkType = props->checkId; - p->xzProps = *props; - - p->outStream = outStream; - - p->mtCoder.allocBig = p->allocBig; - p->mtCoder.progress = progress; - p->mtCoder.inStream = inStream; - p->mtCoder.inData = NULL; - p->mtCoder.inDataSize = 0; - p->mtCoder.mtCallback = &vt; - p->mtCoder.mtCallbackObject = p; - - if ( props->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID - || props->blockSize == XZ_PROPS__BLOCK_SIZE__AUTO) - return SZ_ERROR_FAIL; - - p->mtCoder.blockSize = (size_t)props->blockSize; - if (p->mtCoder.blockSize != props->blockSize) - return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ - - { - size_t destBlockSize = XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(p->mtCoder.blockSize); - if (destBlockSize < p->mtCoder.blockSize) - return SZ_ERROR_PARAM; - if (p->outBufSize != destBlockSize) - XzEnc_FreeOutBufs(p); - p->outBufSize = destBlockSize; - } - - p->mtCoder.numThreadsMax = props->numBlockThreads_Max; - p->mtCoder.expectedDataSize = p->expectedDataSize; - - RINOK(MtCoder_Code(&p->mtCoder)); - } - else - #endif - { - int writeStartSizes; - CCompressProgress_XzEncOffset progress2; - Byte *bufData = NULL; - size_t bufSize = 0; - - progress2.vt.Progress = CompressProgress_XzEncOffset_Progress; - progress2.inOffset = 0; - progress2.outOffset = 0; - progress2.progress = progress; - - writeStartSizes = 0; - - if (props->blockSize != XZ_PROPS__BLOCK_SIZE__SOLID) - { - writeStartSizes = (props->forceWriteSizesInHeader > 0); - - if (writeStartSizes) - { - size_t t2; - size_t t = (size_t)props->blockSize; - if (t != props->blockSize) - return SZ_ERROR_PARAM; - t = XZ_GET_MAX_BLOCK_PACK_SIZE(t); - if (t < props->blockSize) - return SZ_ERROR_PARAM; - t2 = XZ_BLOCK_HEADER_SIZE_MAX + t; - if (!p->outBufs[0] || t2 != p->outBufSize) - { - XzEnc_FreeOutBufs(p); - p->outBufs[0] = (Byte *)ISzAlloc_Alloc(p->alloc, t2); - if (!p->outBufs[0]) - return SZ_ERROR_MEM; - p->outBufSize = t2; - } - bufData = p->outBufs[0] + XZ_BLOCK_HEADER_SIZE_MAX; - bufSize = t; - } - } - - for (;;) - { - CXzEncBlockInfo blockSizes; - int inStreamFinished; - - /* - UInt64 rem = (UInt64)(Int64)-1; - if (props->reduceSize != (UInt64)(Int64)-1 - && props->reduceSize >= progress2.inOffset) - rem = props->reduceSize - progress2.inOffset; - */ - - blockSizes.headerSize = 0; // for GCC - - RINOK(Xz_CompressBlock( - &p->lzmaf_Items[0], - - writeStartSizes ? NULL : outStream, - writeStartSizes ? p->outBufs[0] : NULL, - bufData, bufSize, - - inStream, - // rem, - NULL, 0, - - props, - progress ? &progress2.vt : NULL, - &inStreamFinished, - &blockSizes, - p->alloc, - p->allocBig)); - - { - UInt64 totalPackFull = blockSizes.totalSize + XZ_GET_PAD_SIZE(blockSizes.totalSize); - - if (writeStartSizes) - { - RINOK(WriteBytes(outStream, p->outBufs[0], blockSizes.headerSize)); - RINOK(WriteBytes(outStream, bufData, (size_t)totalPackFull - blockSizes.headerSize)); - } - - RINOK(XzEncIndex_AddIndexRecord(&p->xzIndex, blockSizes.unpackSize, blockSizes.totalSize, p->alloc)); - - progress2.inOffset += blockSizes.unpackSize; - progress2.outOffset += totalPackFull; - } - - if (inStreamFinished) - break; - } - } - - return XzEncIndex_WriteFooter(&p->xzIndex, (CXzStreamFlags)props->checkId, outStream); -} - - -#include "Alloc.h" - -SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, - const CXzProps *props, ICompressProgress *progress) -{ - SRes res; - CXzEncHandle xz = XzEnc_Create(&g_Alloc, &g_BigAlloc); - if (!xz) - return SZ_ERROR_MEM; - res = XzEnc_SetProps(xz, props); - if (res == SZ_OK) - res = XzEnc_Encode(xz, outStream, inStream, progress); - XzEnc_Destroy(xz); - return res; -} - - -SRes Xz_EncodeEmpty(ISeqOutStream *outStream) -{ - SRes res; - CXzEncIndex xzIndex; - XzEncIndex_Construct(&xzIndex); - res = Xz_WriteHeader((CXzStreamFlags)0, outStream); - if (res == SZ_OK) - res = XzEncIndex_WriteFooter(&xzIndex, (CXzStreamFlags)0, outStream); - XzEncIndex_Free(&xzIndex, NULL); // g_Alloc - return res; -} diff --git a/code/ryzom/client/src/seven_zip/XzEnc.h b/code/ryzom/client/src/seven_zip/XzEnc.h deleted file mode 100644 index 0c29e7e1e..000000000 --- a/code/ryzom/client/src/seven_zip/XzEnc.h +++ /dev/null @@ -1,60 +0,0 @@ -/* XzEnc.h -- Xz Encode -2017-06-27 : Igor Pavlov : Public domain */ - -#ifndef __XZ_ENC_H -#define __XZ_ENC_H - -#include "Lzma2Enc.h" - -#include "Xz.h" - -EXTERN_C_BEGIN - - -#define XZ_PROPS__BLOCK_SIZE__AUTO LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO -#define XZ_PROPS__BLOCK_SIZE__SOLID LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID - - -typedef struct -{ - UInt32 id; - UInt32 delta; - UInt32 ip; - int ipDefined; -} CXzFilterProps; - -void XzFilterProps_Init(CXzFilterProps *p); - - -typedef struct -{ - CLzma2EncProps lzma2Props; - CXzFilterProps filterProps; - unsigned checkId; - UInt64 blockSize; - int numBlockThreads_Reduced; - int numBlockThreads_Max; - int numTotalThreads; - int forceWriteSizesInHeader; - UInt64 reduceSize; -} CXzProps; - -void XzProps_Init(CXzProps *p); - - -typedef void * CXzEncHandle; - -CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); -void XzEnc_Destroy(CXzEncHandle p); -SRes XzEnc_SetProps(CXzEncHandle p, const CXzProps *props); -void XzEnc_SetDataSize(CXzEncHandle p, UInt64 expectedDataSiize); -SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress); - -SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, - const CXzProps *props, ICompressProgress *progress); - -SRes Xz_EncodeEmpty(ISeqOutStream *outStream); - -EXTERN_C_END - -#endif diff --git a/code/ryzom/client/src/seven_zip/XzIn.cpp b/code/ryzom/client/src/seven_zip/XzIn.cpp deleted file mode 100644 index ff48e2dd4..000000000 --- a/code/ryzom/client/src/seven_zip/XzIn.cpp +++ /dev/null @@ -1,319 +0,0 @@ -/* XzIn.c - Xz input -2018-07-04 : Igor Pavlov : Public domain */ - -#include "Precomp.h" - -#include - -#include "7zCrc.h" -#include "CpuArch.h" -#include "Xz.h" - -/* -#define XZ_FOOTER_SIG_CHECK(p) (memcmp((p), XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0) -*/ -#define XZ_FOOTER_SIG_CHECK(p) ((p)[0] == XZ_FOOTER_SIG_0 && (p)[1] == XZ_FOOTER_SIG_1) - - -SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream) -{ - Byte sig[XZ_STREAM_HEADER_SIZE]; - RINOK(SeqInStream_Read2(inStream, sig, XZ_STREAM_HEADER_SIZE, SZ_ERROR_NO_ARCHIVE)); - if (memcmp(sig, XZ_SIG, XZ_SIG_SIZE) != 0) - return SZ_ERROR_NO_ARCHIVE; - return Xz_ParseHeader(p, sig); -} - -#define READ_VARINT_AND_CHECK(buf, pos, size, res) \ - { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ - if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } - -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes) -{ - Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; - unsigned headerSize; - *headerSizeRes = 0; - RINOK(SeqInStream_ReadByte(inStream, &header[0])); - headerSize = (unsigned)header[0]; - if (headerSize == 0) - { - *headerSizeRes = 1; - *isIndex = True; - return SZ_OK; - } - - *isIndex = False; - headerSize = (headerSize << 2) + 4; - *headerSizeRes = headerSize; - RINOK(SeqInStream_Read(inStream, header + 1, headerSize - 1)); - return XzBlock_Parse(p, header); -} - -#define ADD_SIZE_CHECK(size, val) \ - { UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; } - -UInt64 Xz_GetUnpackSize(const CXzStream *p) -{ - UInt64 size = 0; - size_t i; - for (i = 0; i < p->numBlocks; i++) - ADD_SIZE_CHECK(size, p->blocks[i].unpackSize); - return size; -} - -UInt64 Xz_GetPackSize(const CXzStream *p) -{ - UInt64 size = 0; - size_t i; - for (i = 0; i < p->numBlocks; i++) - ADD_SIZE_CHECK(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3); - return size; -} - -/* -SRes XzBlock_ReadFooter(CXzBlock *p, CXzStreamFlags f, ISeqInStream *inStream) -{ - return SeqInStream_Read(inStream, p->check, XzFlags_GetCheckSize(f)); -} -*/ - -static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPtr alloc) -{ - size_t numBlocks, pos = 1; - UInt32 crc; - - if (size < 5 || buf[0] != 0) - return SZ_ERROR_ARCHIVE; - - size -= 4; - crc = CrcCalc(buf, size); - if (crc != GetUi32(buf + size)) - return SZ_ERROR_ARCHIVE; - - { - UInt64 numBlocks64; - READ_VARINT_AND_CHECK(buf, pos, size, &numBlocks64); - numBlocks = (size_t)numBlocks64; - if (numBlocks != numBlocks64 || numBlocks * 2 > size) - return SZ_ERROR_ARCHIVE; - } - - Xz_Free(p, alloc); - if (numBlocks != 0) - { - size_t i; - p->numBlocks = numBlocks; - p->blocks = (CXzBlockSizes *)ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); - if (!p->blocks) - return SZ_ERROR_MEM; - for (i = 0; i < numBlocks; i++) - { - CXzBlockSizes *block = &p->blocks[i]; - READ_VARINT_AND_CHECK(buf, pos, size, &block->totalSize); - READ_VARINT_AND_CHECK(buf, pos, size, &block->unpackSize); - if (block->totalSize == 0) - return SZ_ERROR_ARCHIVE; - } - } - while ((pos & 3) != 0) - if (buf[pos++] != 0) - return SZ_ERROR_ARCHIVE; - return (pos == size) ? SZ_OK : SZ_ERROR_ARCHIVE; -} - -static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, ISzAllocPtr alloc) -{ - SRes res; - size_t size; - Byte *buf; - if (indexSize > ((UInt32)1 << 31)) - return SZ_ERROR_UNSUPPORTED; - size = (size_t)indexSize; - if (size != indexSize) - return SZ_ERROR_UNSUPPORTED; - buf = (Byte *)ISzAlloc_Alloc(alloc, size); - if (!buf) - return SZ_ERROR_MEM; - res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED); - if (res == SZ_OK) - res = Xz_ReadIndex2(p, buf, size, alloc); - ISzAlloc_Free(alloc, buf); - return res; -} - -static SRes LookInStream_SeekRead_ForArc(ILookInStream *stream, UInt64 offset, void *buf, size_t size) -{ - RINOK(LookInStream_SeekTo(stream, offset)); - return LookInStream_Read(stream, buf, size); - /* return LookInStream_Read2(stream, buf, size, SZ_ERROR_NO_ARCHIVE); */ -} - -static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOffset, ISzAllocPtr alloc) -{ - UInt64 indexSize; - Byte buf[XZ_STREAM_FOOTER_SIZE]; - UInt64 pos = *startOffset; - - if ((pos & 3) != 0 || pos < XZ_STREAM_FOOTER_SIZE) - return SZ_ERROR_NO_ARCHIVE; - - pos -= XZ_STREAM_FOOTER_SIZE; - RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); - - if (!XZ_FOOTER_SIG_CHECK(buf + 10)) - { - UInt32 total = 0; - pos += XZ_STREAM_FOOTER_SIZE; - - for (;;) - { - size_t i; - #define TEMP_BUF_SIZE (1 << 10) - Byte temp[TEMP_BUF_SIZE]; - - i = (pos > TEMP_BUF_SIZE) ? TEMP_BUF_SIZE : (size_t)pos; - pos -= i; - RINOK(LookInStream_SeekRead_ForArc(stream, pos, temp, i)); - total += (UInt32)i; - for (; i != 0; i--) - if (temp[i - 1] != 0) - break; - if (i != 0) - { - if ((i & 3) != 0) - return SZ_ERROR_NO_ARCHIVE; - pos += i; - break; - } - if (pos < XZ_STREAM_FOOTER_SIZE || total > (1 << 16)) - return SZ_ERROR_NO_ARCHIVE; - } - - if (pos < XZ_STREAM_FOOTER_SIZE) - return SZ_ERROR_NO_ARCHIVE; - pos -= XZ_STREAM_FOOTER_SIZE; - RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); - if (!XZ_FOOTER_SIG_CHECK(buf + 10)) - return SZ_ERROR_NO_ARCHIVE; - } - - p->flags = (CXzStreamFlags)GetBe16(buf + 8); - - if (!XzFlags_IsSupported(p->flags)) - return SZ_ERROR_UNSUPPORTED; - - if (GetUi32(buf) != CrcCalc(buf + 4, 6)) - return SZ_ERROR_ARCHIVE; - - indexSize = ((UInt64)GetUi32(buf + 4) + 1) << 2; - - if (pos < indexSize) - return SZ_ERROR_ARCHIVE; - - pos -= indexSize; - RINOK(LookInStream_SeekTo(stream, pos)); - RINOK(Xz_ReadIndex(p, stream, indexSize, alloc)); - - { - UInt64 totalSize = Xz_GetPackSize(p); - if (totalSize == XZ_SIZE_OVERFLOW - || totalSize >= ((UInt64)1 << 63) - || pos < totalSize + XZ_STREAM_HEADER_SIZE) - return SZ_ERROR_ARCHIVE; - pos -= (totalSize + XZ_STREAM_HEADER_SIZE); - RINOK(LookInStream_SeekTo(stream, pos)); - *startOffset = pos; - } - { - CXzStreamFlags headerFlags; - CSecToRead secToRead; - SecToRead_CreateVTable(&secToRead); - secToRead.realStream = stream; - - RINOK(Xz_ReadHeader(&headerFlags, &secToRead.vt)); - return (p->flags == headerFlags) ? SZ_OK : SZ_ERROR_ARCHIVE; - } -} - - -/* ---------- Xz Streams ---------- */ - -void Xzs_Construct(CXzs *p) -{ - p->num = p->numAllocated = 0; - p->streams = 0; -} - -void Xzs_Free(CXzs *p, ISzAllocPtr alloc) -{ - size_t i; - for (i = 0; i < p->num; i++) - Xz_Free(&p->streams[i], alloc); - ISzAlloc_Free(alloc, p->streams); - p->num = p->numAllocated = 0; - p->streams = 0; -} - -UInt64 Xzs_GetNumBlocks(const CXzs *p) -{ - UInt64 num = 0; - size_t i; - for (i = 0; i < p->num; i++) - num += p->streams[i].numBlocks; - return num; -} - -UInt64 Xzs_GetUnpackSize(const CXzs *p) -{ - UInt64 size = 0; - size_t i; - for (i = 0; i < p->num; i++) - ADD_SIZE_CHECK(size, Xz_GetUnpackSize(&p->streams[i])); - return size; -} - -/* -UInt64 Xzs_GetPackSize(const CXzs *p) -{ - UInt64 size = 0; - size_t i; - for (i = 0; i < p->num; i++) - ADD_SIZE_CHECK(size, Xz_GetTotalSize(&p->streams[i])); - return size; -} -*/ - -SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc) -{ - Int64 endOffset = 0; - RINOK(ILookInStream_Seek(stream, &endOffset, SZ_SEEK_END)); - *startOffset = endOffset; - for (;;) - { - CXzStream st; - SRes res; - Xz_Construct(&st); - res = Xz_ReadBackward(&st, stream, startOffset, alloc); - st.startOffset = *startOffset; - RINOK(res); - if (p->num == p->numAllocated) - { - size_t newNum = p->num + p->num / 4 + 1; - Byte *data = (Byte *)ISzAlloc_Alloc(alloc, newNum * sizeof(CXzStream)); - if (!data) - return SZ_ERROR_MEM; - p->numAllocated = newNum; - if (p->num != 0) - memcpy(data, p->streams, p->num * sizeof(CXzStream)); - ISzAlloc_Free(alloc, p->streams); - p->streams = (CXzStream *)data; - } - p->streams[p->num++] = st; - if (*startOffset == 0) - break; - RINOK(LookInStream_SeekTo(stream, *startOffset)); - if (progress && ICompressProgress_Progress(progress, endOffset - *startOffset, (UInt64)(Int64)-1) != SZ_OK) - return SZ_ERROR_PROGRESS; - } - return SZ_OK; -} diff --git a/code/ryzom/tools/client/client_patcher/CMakeLists.txt b/code/ryzom/tools/client/client_patcher/CMakeLists.txt index ad2eeb629..810f4ccfd 100644 --- a/code/ryzom/tools/client/client_patcher/CMakeLists.txt +++ b/code/ryzom/tools/client/client_patcher/CMakeLists.txt @@ -23,7 +23,7 @@ TARGET_LINK_LIBRARIES(ryzom_client_patcher nelmisc nelnet ryzom_gameshare - ryzom_sevenzip + nel_sevenzip ${CURL_LIBRARIES}) IF(APPLE) diff --git a/code/ryzom/tools/client/ryzom_installer/CMakeLists.txt b/code/ryzom/tools/client/ryzom_installer/CMakeLists.txt index f3949055d..bdd076aec 100644 --- a/code/ryzom/tools/client/ryzom_installer/CMakeLists.txt +++ b/code/ryzom/tools/client/ryzom_installer/CMakeLists.txt @@ -1,5 +1,5 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR} ${NEL_INCLUDE_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/ryzom/client/src/seven_zip ${ZLIB_INCLUDE_DIR}) +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/nel/3rdparty/seven_zip ${ZLIB_INCLUDE_DIR}) FILE(GLOB SRC src/*.cpp src/*.h res/*.rc) FILE(GLOB CLIENT_INSTALL_HDR src/*.h) @@ -74,7 +74,7 @@ ENDIF() NL_DEFAULT_PROPS(ryzom_installer_qt "Ryzom, Tools: Ryzom Installer" ) NL_ADD_RUNTIME_FLAGS(ryzom_installer_qt) NL_ADD_LIB_SUFFIX(ryzom_installer_qt) -TARGET_LINK_LIBRARIES(ryzom_installer_qt nelmisc ryzom_sevenzip ${QT_LIBRARIES}) +TARGET_LINK_LIBRARIES(ryzom_installer_qt nelmisc nel_sevenzip ${QT_LIBRARIES}) IF(WIN32) TARGET_LINK_LIBRARIES(ryzom_installer_qt nelmisc ${WINSDK_LIBRARY_DIR}/Version.lib) diff --git a/code/ryzom/tools/patch_gen/CMakeLists.txt b/code/ryzom/tools/patch_gen/CMakeLists.txt index 330733cc2..477e13c26 100644 --- a/code/ryzom/tools/patch_gen/CMakeLists.txt +++ b/code/ryzom/tools/patch_gen/CMakeLists.txt @@ -1,15 +1,15 @@ -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/ryzom/client/src/seven_zip) +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/nel/3rdparty/seven_zip) SET(MAIN_SRC patch_gen_common.cpp patch_gen_main.cpp patch_gen_main.h patch_gen.rc) SET(SERVICE_SRC patch_gen_common.cpp patch_gen_service.cpp patch_gen_service.h patch_gen_service.rc) ADD_EXECUTABLE(patch_gen ${MAIN_SRC} ${RECS}) -TARGET_LINK_LIBRARIES(patch_gen ryzom_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges) +TARGET_LINK_LIBRARIES(patch_gen nel_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges) NL_DEFAULT_PROPS(patch_gen "Ryzom, Tools: Patch Generator") NL_ADD_RUNTIME_FLAGS(patch_gen) ADD_EXECUTABLE(patch_gen_service WIN32 ${SERVICE_SRC} ${RECS}) -TARGET_LINK_LIBRARIES(patch_gen_service ryzom_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges) +TARGET_LINK_LIBRARIES(patch_gen_service nel_sevenzip ryzom_gameshare nelmisc nelnet nelligo nelgeorges) NL_DEFAULT_PROPS(patch_gen_service "Ryzom, Tools: Patch Generator Service") NL_ADD_RUNTIME_FLAGS(patch_gen_service) From cf2d5e939f66d4cd8507ee874e253d2c994cc434 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 13 Nov 2019 18:30:44 +0800 Subject: [PATCH 23/27] Fix build --- code/ryzom/client/src/login_patch.cpp | 2 +- code/ryzom/common/src/game_share/bnp_patch.cpp | 6 +++--- code/ryzom/common/src/game_share/bnp_patch.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/code/ryzom/client/src/login_patch.cpp b/code/ryzom/client/src/login_patch.cpp index 908f12977..a21f7cab1 100644 --- a/code/ryzom/client/src/login_patch.cpp +++ b/code/ryzom/client/src/login_patch.cpp @@ -48,6 +48,7 @@ #include "nel/misc/big_file.h" #include "nel/misc/i18n.h" #include "nel/misc/cmd_args.h" +#include "nel/misc/seven_zip.h" #include "game_share/bg_downloader_msg.h" @@ -55,7 +56,6 @@ #include "login.h" #include "user_agent.h" -#include "seven_zip/seven_zip.h" #ifndef RY_BG_DOWNLOADER #include "client_cfg.h" diff --git a/code/ryzom/common/src/game_share/bnp_patch.cpp b/code/ryzom/common/src/game_share/bnp_patch.cpp index baac733e8..72c484c84 100644 --- a/code/ryzom/common/src/game_share/bnp_patch.cpp +++ b/code/ryzom/common/src/game_share/bnp_patch.cpp @@ -74,7 +74,7 @@ bool CBNPFileVersion::setup(const std::string &fileName, uint32 versionNumber) BOMB_IF(!NLMISC::CFile::fileExists(fileName),("File not found: "+fileName).c_str(),return false); // generate a hash key for the file and store it in a vector of uint32 - CHashKey hashKey= getSHA1(fileName); + NLMISC::CHashKey hashKey= NLMISC::getSHA1(fileName); nlassert(hashKey.HashKeyString.size()==20); _HashKey.clear(); for (uint32 i=0;i<5;++i) @@ -137,10 +137,10 @@ uint32 CBNPFileVersion::getPatchSize() const return _PatchSize; } -CHashKey CBNPFileVersion::getHashKey() const +NLMISC::CHashKey CBNPFileVersion::getHashKey() const { nlassert(_HashKey.size()==5); - CHashKey hashKey; + NLMISC::CHashKey hashKey; for (uint32 i=0;i<5;++i) { *(uint32*)&hashKey.HashKeyString[4*i]=_HashKey[i]; diff --git a/code/ryzom/common/src/game_share/bnp_patch.h b/code/ryzom/common/src/game_share/bnp_patch.h index 395727bf0..b2d602194 100644 --- a/code/ryzom/common/src/game_share/bnp_patch.h +++ b/code/ryzom/common/src/game_share/bnp_patch.h @@ -72,7 +72,7 @@ public: uint32 get7ZFileSize() const; uint32 getFileSize() const; uint32 getPatchSize() const; - CHashKey getHashKey() const; + NLMISC::CHashKey getHashKey() const; // == operator bool operator==(const CBNPFileVersion& other) const; From dedb0cbc087ac6b1cdc00eb97ce1292a6fc539ce Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 13 Nov 2019 18:45:36 +0800 Subject: [PATCH 24/27] Revert unused changes --- code/nel/include/nel/misc/sha1.h | 30 ------- code/nel/src/misc/sha1.cpp | 142 ++++++++++++------------------- 2 files changed, 53 insertions(+), 119 deletions(-) diff --git a/code/nel/include/nel/misc/sha1.h b/code/nel/include/nel/misc/sha1.h index 9b7fe937b..33db14f75 100644 --- a/code/nel/include/nel/misc/sha1.h +++ b/code/nel/include/nel/misc/sha1.h @@ -101,27 +101,6 @@ inline bool operator <(const struct CHashKey &a,const struct CHashKey &b) return a.HashKeyString= 16 bits - * - */ - -typedef unsigned int uint32_t; -typedef unsigned char uint8_t; -typedef short int_least16_t; +static const int bufferSize = 100000; +static uint8 buffer[bufferSize]; -namespace NLMISC { +#define SHA1HashSize 20 // -// Constantes +// Types // -static const int bufferSize = 100000; -static uint8_t buffer[bufferSize]; +/* +* This structure will hold context information for the SHA-1 +* hashing operation +*/ +typedef struct SHA1Context +{ + uint32 Intermediate_Hash[SHA1HashSize/4]; /* Message Digest */ + + uint32 Length_Low; /* Message length in bits */ + uint32 Length_High; /* Message length in bits */ + + /* Index into message block array */ + sint16 Message_Block_Index; + uint8 Message_Block[64]; /* 512-bit message blocks */ + + int Computed; /* Is the digest computed? */ + int Corrupted; /* Is the message digest corrupted? */ +} SHA1Context; enum { @@ -75,8 +82,8 @@ enum // int SHA1Reset (SHA1Context *); -int SHA1Input (SHA1Context *, const uint8_t *, unsigned int); -int SHA1Result (SHA1Context *, uint8_t Message_Digest[SHA1HashSize]); +int SHA1Input (SHA1Context *, const uint8 *, unsigned int); +int SHA1Result (SHA1Context *, uint8 Message_Digest[SHA1HashSize]); // // Functions @@ -86,7 +93,7 @@ CHashKey getSHA1(const uint8 *buffer, uint32 size) { SHA1Context sha; int err; - uint8_t Message_Digest[20]; + uint8 Message_Digest[20]; err = SHA1Reset(&sha); if (err) @@ -95,7 +102,7 @@ CHashKey getSHA1(const uint8 *buffer, uint32 size) return CHashKey(); } - err = SHA1Input(&sha, (const uint8_t*)buffer, size); + err = SHA1Input(&sha, (const uint8*)buffer, size); if (err) { nlwarning ("SHA: SHA1Input Error %d.\n", err); @@ -117,7 +124,7 @@ CHashKey getSHA1(const string &filename, bool forcePath) { SHA1Context sha; int err; - uint8_t Message_Digest[20]; + uint8 Message_Digest[20]; //printf("reading '%s'\n", findData.cFileName); CIFile ifile; @@ -168,50 +175,7 @@ CHashKey getSHA1(const string &filename, bool forcePath) return CHashKey(); } - CHashKey hk(Message_Digest); - return hk; -} - -// Begin calculating a SHA1 hash -bool resetSHA1(SHA1Context *context) -{ - int err; - err = SHA1Reset(context); - if (err) - { - nlwarning("SHA: SHA1Reset Error %d.\n", err ); - return false; - } - return true; -} - -// Add data to a SHA1 hash calculation -bool inputSHA1(SHA1Context *context, const uint8 *buffer, uint32 size) -{ - int err; - err = SHA1Input(context, buffer, size); - if (err) - { - nlwarning ("SHA: SHA1Input Error %d.\n", err); - return false; - } - return true; -} - -// Get the SHA1 result -CHashKey getSHA1(SHA1Context *context) -{ - int err; - uint8_t messageDigest[20]; - - err = SHA1Result(context, messageDigest); - if (err) - { - nlwarning("SHA: SHA1Result Error %d, could not compute message digest.\n", err ); - return CHashKey(); - } - - CHashKey hk(messageDigest); + CHashKey hk (Message_Digest); return hk; } @@ -225,9 +189,9 @@ CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint3 { SHA1Context sha; - uint8_t SHA1_Key[64]; - uint8_t SHA1_Key1[20]; - uint8_t SHA1_Key2[20]; + uint8 SHA1_Key[64]; + uint8 SHA1_Key1[20]; + uint8 SHA1_Key2[20]; string buffer1; string buffer2; @@ -238,9 +202,9 @@ CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint3 // If lenght of key > 64 use sha1 hash if (key_len > 64) { - uint8_t SHA1_Key0[20]; + uint8 SHA1_Key0[20]; SHA1Reset(&sha); - SHA1Input(&sha, (const uint8_t*)key, key_len); + SHA1Input(&sha, (const uint8*)key, key_len); SHA1Result(&sha, SHA1_Key0); CHashKey hk0 (SHA1_Key0); for (uint i = 0; i < 20; i++) @@ -260,7 +224,7 @@ CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint3 // Get hash SHA1Reset(&sha); - SHA1Input(&sha, (const uint8_t*)buffer1.c_str(), (uint)buffer1.size()); + SHA1Input(&sha, (const uint8*)buffer1.c_str(), (uint)buffer1.size()); SHA1Result(&sha, SHA1_Key1); CHashKey hk1 (SHA1_Key1); @@ -274,7 +238,7 @@ CHashKey getHMacSHA1(const uint8 *text, uint32 text_len, const uint8 *key, uint3 // Get new hash SHA1Reset(&sha); - SHA1Input(&sha, (const uint8_t*)buffer2.c_str(), (uint)buffer2.size()); + SHA1Input(&sha, (const uint8*)buffer2.c_str(), (uint)buffer2.size()); SHA1Result(&sha, SHA1_Key2); CHashKey hk (SHA1_Key2); @@ -354,7 +318,7 @@ int SHA1Reset(SHA1Context *context) * */ int SHA1Result( SHA1Context *context, - uint8_t Message_Digest[SHA1HashSize]) + uint8 Message_Digest[SHA1HashSize]) { int i; @@ -384,7 +348,7 @@ int SHA1Result( SHA1Context *context, for(i = 0; i < SHA1HashSize; ++i) { - Message_Digest[i] = uint8_t((context->Intermediate_Hash[i>>2] + Message_Digest[i] = uint8((context->Intermediate_Hash[i>>2] >> 8 * ( 3 - ( i & 0x03 ) ) ) & 0xff ); } @@ -412,7 +376,7 @@ int SHA1Result( SHA1Context *context, * */ int SHA1Input( SHA1Context *context, - const uint8_t *message_array, + const uint8 *message_array, unsigned length) { if (!length) @@ -486,16 +450,16 @@ int SHA1Input( SHA1Context *context, */ void SHA1ProcessMessageBlock(SHA1Context *context) { - const uint32_t K[] = { /* Constants defined in SHA-1 */ + const uint32 K[] = { /* Constants defined in SHA-1 */ 0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC, 0xCA62C1D6 }; int t; /* Loop counter */ - uint32_t temp; /* Temporary word value */ - uint32_t W[80]; /* Word sequence */ - uint32_t A, B, C, D, E; /* Word buffers */ + uint32 temp; /* Temporary word value */ + uint32 W[80]; /* Word sequence */ + uint32 A, B, C, D, E; /* Word buffers */ /* * Initialize the first 16 words in the array W @@ -631,14 +595,14 @@ void SHA1PadMessage(SHA1Context *context) /* * Store the message length as the last 8 octets */ - context->Message_Block[56] = uint8_t((context->Length_High >> 24)&0xff); - context->Message_Block[57] = uint8_t((context->Length_High >> 16)&0xff); - context->Message_Block[58] = uint8_t((context->Length_High >> 8)&0xff); - context->Message_Block[59] = uint8_t((context->Length_High)&0xff); - context->Message_Block[60] = uint8_t((context->Length_Low >> 24)&0xff); - context->Message_Block[61] = uint8_t((context->Length_Low >> 16)&0xff); - context->Message_Block[62] = uint8_t((context->Length_Low >> 8)&0xff); - context->Message_Block[63] = uint8_t((context->Length_Low)&0xff); + context->Message_Block[56] = uint8((context->Length_High >> 24)&0xff); + context->Message_Block[57] = uint8((context->Length_High >> 16)&0xff); + context->Message_Block[58] = uint8((context->Length_High >> 8)&0xff); + context->Message_Block[59] = uint8((context->Length_High)&0xff); + context->Message_Block[60] = uint8((context->Length_Low >> 24)&0xff); + context->Message_Block[61] = uint8((context->Length_Low >> 16)&0xff); + context->Message_Block[62] = uint8((context->Length_Low >> 8)&0xff); + context->Message_Block[63] = uint8((context->Length_Low)&0xff); SHA1ProcessMessageBlock(context); } From e9c175d3f320e68d1d8f378bab18aec54a9ccf96 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 13 Nov 2019 20:04:01 +0800 Subject: [PATCH 25/27] Add ignored files --- .gitignore | 1 - code/nel/3rdparty/seven_zip/7z.h | 202 ++ code/nel/3rdparty/seven_zip/7zAlloc.c | 80 + code/nel/3rdparty/seven_zip/7zArcIn.c | 1771 ++++++++++ code/nel/3rdparty/seven_zip/7zBuf.c | 36 + code/nel/3rdparty/seven_zip/7zBuf.h | 35 + code/nel/3rdparty/seven_zip/7zBuf2.c | 52 + code/nel/3rdparty/seven_zip/7zCrc.c | 128 + code/nel/3rdparty/seven_zip/7zCrcOpt.c | 115 + code/nel/3rdparty/seven_zip/7zDec.c | 591 ++++ code/nel/3rdparty/seven_zip/7zFile.c | 286 ++ code/nel/3rdparty/seven_zip/7zFile.h | 83 + code/nel/3rdparty/seven_zip/7zStream.c | 176 + code/nel/3rdparty/seven_zip/7zVersion.h | 27 + code/nel/3rdparty/seven_zip/7zVersion.rc | 55 + code/nel/3rdparty/seven_zip/Aes.c | 306 ++ code/nel/3rdparty/seven_zip/Aes.h | 38 + code/nel/3rdparty/seven_zip/AesOpt.c | 184 + code/nel/3rdparty/seven_zip/Alloc.c | 455 +++ code/nel/3rdparty/seven_zip/Alloc.h | 51 + code/nel/3rdparty/seven_zip/Bcj2.c | 257 ++ code/nel/3rdparty/seven_zip/Bcj2.h | 146 + code/nel/3rdparty/seven_zip/Bcj2Enc.c | 311 ++ code/nel/3rdparty/seven_zip/Bra.c | 230 ++ code/nel/3rdparty/seven_zip/Bra.h | 64 + code/nel/3rdparty/seven_zip/Bra86.c | 82 + code/nel/3rdparty/seven_zip/BraIA64.c | 53 + code/nel/3rdparty/seven_zip/CMakeLists.txt | 2 +- code/nel/3rdparty/seven_zip/Compiler.h | 33 + code/nel/3rdparty/seven_zip/CpuArch.c | 218 ++ code/nel/3rdparty/seven_zip/CpuArch.h | 336 ++ code/nel/3rdparty/seven_zip/Delta.c | 64 + code/nel/3rdparty/seven_zip/Delta.h | 19 + code/nel/3rdparty/seven_zip/DllSecur.c | 108 + code/nel/3rdparty/seven_zip/DllSecur.h | 20 + code/nel/3rdparty/seven_zip/LzFind.c | 1127 +++++++ code/nel/3rdparty/seven_zip/LzFind.h | 121 + code/nel/3rdparty/seven_zip/LzHash.h | 57 + code/nel/3rdparty/seven_zip/Lzma2Dec.c | 488 +++ code/nel/3rdparty/seven_zip/Lzma2Dec.h | 120 + code/nel/3rdparty/seven_zip/Lzma2DecMt.c | 1082 ++++++ code/nel/3rdparty/seven_zip/Lzma2DecMt.h | 79 + code/nel/3rdparty/seven_zip/Lzma2Enc.c | 803 +++++ code/nel/3rdparty/seven_zip/Lzma2Enc.h | 55 + code/nel/3rdparty/seven_zip/Lzma86.h | 111 + code/nel/3rdparty/seven_zip/Lzma86Dec.c | 54 + code/nel/3rdparty/seven_zip/Lzma86Enc.c | 106 + code/nel/3rdparty/seven_zip/LzmaDec.c | 1185 +++++++ code/nel/3rdparty/seven_zip/LzmaDec.h | 234 ++ code/nel/3rdparty/seven_zip/LzmaEnc.c | 2976 +++++++++++++++++ code/nel/3rdparty/seven_zip/LzmaEnc.h | 76 + code/nel/3rdparty/seven_zip/LzmaLib.c | 40 + code/nel/3rdparty/seven_zip/LzmaLib.h | 131 + code/nel/3rdparty/seven_zip/LzmaUtil.c | 258 ++ code/nel/3rdparty/seven_zip/Ppmd.h | 85 + code/nel/3rdparty/seven_zip/Ppmd7.c | 712 ++++ code/nel/3rdparty/seven_zip/Ppmd7.h | 142 + code/nel/3rdparty/seven_zip/Ppmd7Dec.c | 191 ++ code/nel/3rdparty/seven_zip/Ppmd7Enc.c | 187 ++ code/nel/3rdparty/seven_zip/Precomp.h | 10 + code/nel/3rdparty/seven_zip/RotateDefs.h | 30 + code/nel/3rdparty/seven_zip/Sha256.c | 248 ++ code/nel/3rdparty/seven_zip/Sha256.h | 26 + code/nel/3rdparty/seven_zip/Sort.c | 141 + code/nel/3rdparty/seven_zip/Sort.h | 18 + code/nel/3rdparty/seven_zip/Xz.c | 90 + code/nel/3rdparty/seven_zip/Xz.h | 460 +++ code/nel/3rdparty/seven_zip/XzCrc64.c | 86 + code/nel/3rdparty/seven_zip/XzCrc64.h | 26 + code/nel/3rdparty/seven_zip/XzCrc64Opt.c | 69 + code/nel/3rdparty/seven_zip/XzDec.c | 2766 +++++++++++++++ code/nel/3rdparty/seven_zip/XzEnc.c | 1329 ++++++++ code/nel/3rdparty/seven_zip/XzEnc.h | 60 + code/nel/3rdparty/seven_zip/XzIn.c | 319 ++ .../tools/patch_gen/patch_gen_common.cpp | 2 +- 75 files changed, 22712 insertions(+), 3 deletions(-) create mode 100644 code/nel/3rdparty/seven_zip/7z.h create mode 100644 code/nel/3rdparty/seven_zip/7zAlloc.c create mode 100644 code/nel/3rdparty/seven_zip/7zArcIn.c create mode 100644 code/nel/3rdparty/seven_zip/7zBuf.c create mode 100644 code/nel/3rdparty/seven_zip/7zBuf.h create mode 100644 code/nel/3rdparty/seven_zip/7zBuf2.c create mode 100644 code/nel/3rdparty/seven_zip/7zCrc.c create mode 100644 code/nel/3rdparty/seven_zip/7zCrcOpt.c create mode 100644 code/nel/3rdparty/seven_zip/7zDec.c create mode 100644 code/nel/3rdparty/seven_zip/7zFile.c create mode 100644 code/nel/3rdparty/seven_zip/7zFile.h create mode 100644 code/nel/3rdparty/seven_zip/7zStream.c create mode 100644 code/nel/3rdparty/seven_zip/7zVersion.h create mode 100644 code/nel/3rdparty/seven_zip/7zVersion.rc create mode 100644 code/nel/3rdparty/seven_zip/Aes.c create mode 100644 code/nel/3rdparty/seven_zip/Aes.h create mode 100644 code/nel/3rdparty/seven_zip/AesOpt.c create mode 100644 code/nel/3rdparty/seven_zip/Alloc.c create mode 100644 code/nel/3rdparty/seven_zip/Alloc.h create mode 100644 code/nel/3rdparty/seven_zip/Bcj2.c create mode 100644 code/nel/3rdparty/seven_zip/Bcj2.h create mode 100644 code/nel/3rdparty/seven_zip/Bcj2Enc.c create mode 100644 code/nel/3rdparty/seven_zip/Bra.c create mode 100644 code/nel/3rdparty/seven_zip/Bra.h create mode 100644 code/nel/3rdparty/seven_zip/Bra86.c create mode 100644 code/nel/3rdparty/seven_zip/BraIA64.c create mode 100644 code/nel/3rdparty/seven_zip/Compiler.h create mode 100644 code/nel/3rdparty/seven_zip/CpuArch.c create mode 100644 code/nel/3rdparty/seven_zip/CpuArch.h create mode 100644 code/nel/3rdparty/seven_zip/Delta.c create mode 100644 code/nel/3rdparty/seven_zip/Delta.h create mode 100644 code/nel/3rdparty/seven_zip/DllSecur.c create mode 100644 code/nel/3rdparty/seven_zip/DllSecur.h create mode 100644 code/nel/3rdparty/seven_zip/LzFind.c create mode 100644 code/nel/3rdparty/seven_zip/LzFind.h create mode 100644 code/nel/3rdparty/seven_zip/LzHash.h create mode 100644 code/nel/3rdparty/seven_zip/Lzma2Dec.c create mode 100644 code/nel/3rdparty/seven_zip/Lzma2Dec.h create mode 100644 code/nel/3rdparty/seven_zip/Lzma2DecMt.c create mode 100644 code/nel/3rdparty/seven_zip/Lzma2DecMt.h create mode 100644 code/nel/3rdparty/seven_zip/Lzma2Enc.c create mode 100644 code/nel/3rdparty/seven_zip/Lzma2Enc.h create mode 100644 code/nel/3rdparty/seven_zip/Lzma86.h create mode 100644 code/nel/3rdparty/seven_zip/Lzma86Dec.c create mode 100644 code/nel/3rdparty/seven_zip/Lzma86Enc.c create mode 100644 code/nel/3rdparty/seven_zip/LzmaDec.c create mode 100644 code/nel/3rdparty/seven_zip/LzmaDec.h create mode 100644 code/nel/3rdparty/seven_zip/LzmaEnc.c create mode 100644 code/nel/3rdparty/seven_zip/LzmaEnc.h create mode 100644 code/nel/3rdparty/seven_zip/LzmaLib.c create mode 100644 code/nel/3rdparty/seven_zip/LzmaLib.h create mode 100644 code/nel/3rdparty/seven_zip/LzmaUtil.c create mode 100644 code/nel/3rdparty/seven_zip/Ppmd.h create mode 100644 code/nel/3rdparty/seven_zip/Ppmd7.c create mode 100644 code/nel/3rdparty/seven_zip/Ppmd7.h create mode 100644 code/nel/3rdparty/seven_zip/Ppmd7Dec.c create mode 100644 code/nel/3rdparty/seven_zip/Ppmd7Enc.c create mode 100644 code/nel/3rdparty/seven_zip/Precomp.h create mode 100644 code/nel/3rdparty/seven_zip/RotateDefs.h create mode 100644 code/nel/3rdparty/seven_zip/Sha256.c create mode 100644 code/nel/3rdparty/seven_zip/Sha256.h create mode 100644 code/nel/3rdparty/seven_zip/Sort.c create mode 100644 code/nel/3rdparty/seven_zip/Sort.h create mode 100644 code/nel/3rdparty/seven_zip/Xz.c create mode 100644 code/nel/3rdparty/seven_zip/Xz.h create mode 100644 code/nel/3rdparty/seven_zip/XzCrc64.c create mode 100644 code/nel/3rdparty/seven_zip/XzCrc64.h create mode 100644 code/nel/3rdparty/seven_zip/XzCrc64Opt.c create mode 100644 code/nel/3rdparty/seven_zip/XzDec.c create mode 100644 code/nel/3rdparty/seven_zip/XzEnc.c create mode 100644 code/nel/3rdparty/seven_zip/XzEnc.h create mode 100644 code/nel/3rdparty/seven_zip/XzIn.c diff --git a/.gitignore b/.gitignore index 1f3f7abba..22089c19f 100644 --- a/.gitignore +++ b/.gitignore @@ -142,7 +142,6 @@ moc_*.cpp *.cache *.patch *.7z -3rdParty .svn thumbs.db Thumbs.db diff --git a/code/nel/3rdparty/seven_zip/7z.h b/code/nel/3rdparty/seven_zip/7z.h new file mode 100644 index 000000000..6c7886e38 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7z.h @@ -0,0 +1,202 @@ +/* 7z.h -- 7z interface +2017-04-03 : Igor Pavlov : Public domain */ + +#ifndef __7Z_H +#define __7Z_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define k7zStartHeaderSize 0x20 +#define k7zSignatureSize 6 + +extern const Byte k7zSignature[k7zSignatureSize]; + +typedef struct +{ + const Byte *Data; + size_t Size; +} CSzData; + +/* CSzCoderInfo & CSzFolder support only default methods */ + +typedef struct +{ + size_t PropsOffset; + UInt32 MethodID; + Byte NumStreams; + Byte PropsSize; +} CSzCoderInfo; + +typedef struct +{ + UInt32 InIndex; + UInt32 OutIndex; +} CSzBond; + +#define SZ_NUM_CODERS_IN_FOLDER_MAX 4 +#define SZ_NUM_BONDS_IN_FOLDER_MAX 3 +#define SZ_NUM_PACK_STREAMS_IN_FOLDER_MAX 4 + +typedef struct +{ + UInt32 NumCoders; + UInt32 NumBonds; + UInt32 NumPackStreams; + UInt32 UnpackStream; + UInt32 PackStreams[SZ_NUM_PACK_STREAMS_IN_FOLDER_MAX]; + CSzBond Bonds[SZ_NUM_BONDS_IN_FOLDER_MAX]; + CSzCoderInfo Coders[SZ_NUM_CODERS_IN_FOLDER_MAX]; +} CSzFolder; + + +SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd); + +typedef struct +{ + UInt32 Low; + UInt32 High; +} CNtfsFileTime; + +typedef struct +{ + Byte *Defs; /* MSB 0 bit numbering */ + UInt32 *Vals; +} CSzBitUi32s; + +typedef struct +{ + Byte *Defs; /* MSB 0 bit numbering */ + // UInt64 *Vals; + CNtfsFileTime *Vals; +} CSzBitUi64s; + +#define SzBitArray_Check(p, i) (((p)[(i) >> 3] & (0x80 >> ((i) & 7))) != 0) + +#define SzBitWithVals_Check(p, i) ((p)->Defs && ((p)->Defs[(i) >> 3] & (0x80 >> ((i) & 7))) != 0) + +typedef struct +{ + UInt32 NumPackStreams; + UInt32 NumFolders; + + UInt64 *PackPositions; // NumPackStreams + 1 + CSzBitUi32s FolderCRCs; // NumFolders + + size_t *FoCodersOffsets; // NumFolders + 1 + UInt32 *FoStartPackStreamIndex; // NumFolders + 1 + UInt32 *FoToCoderUnpackSizes; // NumFolders + 1 + Byte *FoToMainUnpackSizeIndex; // NumFolders + UInt64 *CoderUnpackSizes; // for all coders in all folders + + Byte *CodersData; +} CSzAr; + +UInt64 SzAr_GetFolderUnpackSize(const CSzAr *p, UInt32 folderIndex); + +SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, + ILookInStream *stream, UInt64 startPos, + Byte *outBuffer, size_t outSize, + ISzAllocPtr allocMain); + +typedef struct +{ + CSzAr db; + + UInt64 startPosAfterHeader; + UInt64 dataPos; + + UInt32 NumFiles; + + UInt64 *UnpackPositions; // NumFiles + 1 + // Byte *IsEmptyFiles; + Byte *IsDirs; + CSzBitUi32s CRCs; + + CSzBitUi32s Attribs; + // CSzBitUi32s Parents; + CSzBitUi64s MTime; + CSzBitUi64s CTime; + + UInt32 *FolderToFile; // NumFolders + 1 + UInt32 *FileToFolder; // NumFiles + + size_t *FileNameOffsets; /* in 2-byte steps */ + Byte *FileNames; /* UTF-16-LE */ +} CSzArEx; + +#define SzArEx_IsDir(p, i) (SzBitArray_Check((p)->IsDirs, i)) + +#define SzArEx_GetFileSize(p, i) ((p)->UnpackPositions[(i) + 1] - (p)->UnpackPositions[i]) + +void SzArEx_Init(CSzArEx *p); +void SzArEx_Free(CSzArEx *p, ISzAllocPtr alloc); +UInt64 SzArEx_GetFolderStreamPos(const CSzArEx *p, UInt32 folderIndex, UInt32 indexInFolder); +int SzArEx_GetFolderFullPackSize(const CSzArEx *p, UInt32 folderIndex, UInt64 *resSize); + +/* +if dest == NULL, the return value specifies the required size of the buffer, + in 16-bit characters, including the null-terminating character. +if dest != NULL, the return value specifies the number of 16-bit characters that + are written to the dest, including the null-terminating character. */ + +size_t SzArEx_GetFileNameUtf16(const CSzArEx *p, size_t fileIndex, UInt16 *dest); + +/* +size_t SzArEx_GetFullNameLen(const CSzArEx *p, size_t fileIndex); +UInt16 *SzArEx_GetFullNameUtf16_Back(const CSzArEx *p, size_t fileIndex, UInt16 *dest); +*/ + + + +/* + SzArEx_Extract extracts file from archive + + *outBuffer must be 0 before first call for each new archive. + + Extracting cache: + If you need to decompress more than one file, you can send + these values from previous call: + *blockIndex, + *outBuffer, + *outBufferSize + You can consider "*outBuffer" as cache of solid block. If your archive is solid, + it will increase decompression speed. + + If you use external function, you can declare these 3 cache variables + (blockIndex, outBuffer, outBufferSize) as static in that external function. + + Free *outBuffer and set *outBuffer to 0, if you want to flush cache. +*/ + +SRes SzArEx_Extract( + const CSzArEx *db, + ILookInStream *inStream, + UInt32 fileIndex, /* index of file */ + UInt32 *blockIndex, /* index of solid block */ + Byte **outBuffer, /* pointer to pointer to output buffer (allocated with allocMain) */ + size_t *outBufferSize, /* buffer size for output buffer */ + size_t *offset, /* offset of stream for required file in *outBuffer */ + size_t *outSizeProcessed, /* size of file in *outBuffer */ + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp); + + +/* +SzArEx_Open Errors: +SZ_ERROR_NO_ARCHIVE +SZ_ERROR_ARCHIVE +SZ_ERROR_UNSUPPORTED +SZ_ERROR_MEM +SZ_ERROR_CRC +SZ_ERROR_INPUT_EOF +SZ_ERROR_FAIL +*/ + +SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, + ISzAllocPtr allocMain, ISzAllocPtr allocTemp); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/7zAlloc.c b/code/nel/3rdparty/seven_zip/7zAlloc.c new file mode 100644 index 000000000..c924a529f --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zAlloc.c @@ -0,0 +1,80 @@ +/* 7zAlloc.c -- Allocation functions +2017-04-03 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +#include "7zAlloc.h" + +/* #define _SZ_ALLOC_DEBUG */ +/* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ + +#ifdef _SZ_ALLOC_DEBUG + +#ifdef _WIN32 +#include +#endif + +#include +int g_allocCount = 0; +int g_allocCountTemp = 0; + +#endif + +void *SzAlloc(ISzAllocPtr p, size_t size) +{ + UNUSED_VAR(p); + if (size == 0) + return 0; + #ifdef _SZ_ALLOC_DEBUG + fprintf(stderr, "\nAlloc %10u bytes; count = %10d", (unsigned)size, g_allocCount); + g_allocCount++; + #endif + return malloc(size); +} + +void SzFree(ISzAllocPtr p, void *address) +{ + UNUSED_VAR(p); + #ifdef _SZ_ALLOC_DEBUG + if (address != 0) + { + g_allocCount--; + fprintf(stderr, "\nFree; count = %10d", g_allocCount); + } + #endif + free(address); +} + +void *SzAllocTemp(ISzAllocPtr p, size_t size) +{ + UNUSED_VAR(p); + if (size == 0) + return 0; + #ifdef _SZ_ALLOC_DEBUG + fprintf(stderr, "\nAlloc_temp %10u bytes; count = %10d", (unsigned)size, g_allocCountTemp); + g_allocCountTemp++; + #ifdef _WIN32 + return HeapAlloc(GetProcessHeap(), 0, size); + #endif + #endif + return malloc(size); +} + +void SzFreeTemp(ISzAllocPtr p, void *address) +{ + UNUSED_VAR(p); + #ifdef _SZ_ALLOC_DEBUG + if (address != 0) + { + g_allocCountTemp--; + fprintf(stderr, "\nFree_temp; count = %10d", g_allocCountTemp); + } + #ifdef _WIN32 + HeapFree(GetProcessHeap(), 0, address); + return; + #endif + #endif + free(address); +} diff --git a/code/nel/3rdparty/seven_zip/7zArcIn.c b/code/nel/3rdparty/seven_zip/7zArcIn.c new file mode 100644 index 000000000..f74d0fad5 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zArcIn.c @@ -0,0 +1,1771 @@ +/* 7zArcIn.c -- 7z Input functions +2018-12-31 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +#include "7z.h" +#include "7zBuf.h" +#include "7zCrc.h" +#include "CpuArch.h" + +#define MY_ALLOC(T, p, size, alloc) { \ + if ((p = (T *)ISzAlloc_Alloc(alloc, (size) * sizeof(T))) == NULL) return SZ_ERROR_MEM; } + +#define MY_ALLOC_ZE(T, p, size, alloc) { if ((size) == 0) p = NULL; else MY_ALLOC(T, p, size, alloc) } + +#define MY_ALLOC_AND_CPY(to, size, from, alloc) \ + { MY_ALLOC(Byte, to, size, alloc); memcpy(to, from, size); } + +#define MY_ALLOC_ZE_AND_CPY(to, size, from, alloc) \ + { if ((size) == 0) to = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } } + +#define k7zMajorVersion 0 + +enum EIdEnum +{ + k7zIdEnd, + k7zIdHeader, + k7zIdArchiveProperties, + k7zIdAdditionalStreamsInfo, + k7zIdMainStreamsInfo, + k7zIdFilesInfo, + k7zIdPackInfo, + k7zIdUnpackInfo, + k7zIdSubStreamsInfo, + k7zIdSize, + k7zIdCRC, + k7zIdFolder, + k7zIdCodersUnpackSize, + k7zIdNumUnpackStream, + k7zIdEmptyStream, + k7zIdEmptyFile, + k7zIdAnti, + k7zIdName, + k7zIdCTime, + k7zIdATime, + k7zIdMTime, + k7zIdWinAttrib, + k7zIdComment, + k7zIdEncodedHeader, + k7zIdStartPos, + k7zIdDummy + // k7zNtSecure, + // k7zParent, + // k7zIsReal +}; + +const Byte k7zSignature[k7zSignatureSize] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C}; + +#define SzBitUi32s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } + +static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAllocPtr alloc) +{ + if (num == 0) + { + p->Defs = NULL; + p->Vals = NULL; + } + else + { + MY_ALLOC(Byte, p->Defs, (num + 7) >> 3, alloc); + MY_ALLOC(UInt32, p->Vals, num, alloc); + } + return SZ_OK; +} + +void SzBitUi32s_Free(CSzBitUi32s *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->Defs); p->Defs = NULL; + ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; +} + +#define SzBitUi64s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } + +void SzBitUi64s_Free(CSzBitUi64s *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->Defs); p->Defs = NULL; + ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; +} + + +static void SzAr_Init(CSzAr *p) +{ + p->NumPackStreams = 0; + p->NumFolders = 0; + + p->PackPositions = NULL; + SzBitUi32s_Init(&p->FolderCRCs); + + p->FoCodersOffsets = NULL; + p->FoStartPackStreamIndex = NULL; + p->FoToCoderUnpackSizes = NULL; + p->FoToMainUnpackSizeIndex = NULL; + p->CoderUnpackSizes = NULL; + + p->CodersData = NULL; +} + +static void SzAr_Free(CSzAr *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->PackPositions); + SzBitUi32s_Free(&p->FolderCRCs, alloc); + + ISzAlloc_Free(alloc, p->FoCodersOffsets); + ISzAlloc_Free(alloc, p->FoStartPackStreamIndex); + ISzAlloc_Free(alloc, p->FoToCoderUnpackSizes); + ISzAlloc_Free(alloc, p->FoToMainUnpackSizeIndex); + ISzAlloc_Free(alloc, p->CoderUnpackSizes); + + ISzAlloc_Free(alloc, p->CodersData); + + SzAr_Init(p); +} + + +void SzArEx_Init(CSzArEx *p) +{ + SzAr_Init(&p->db); + + p->NumFiles = 0; + p->dataPos = 0; + + p->UnpackPositions = NULL; + p->IsDirs = NULL; + + p->FolderToFile = NULL; + p->FileToFolder = NULL; + + p->FileNameOffsets = NULL; + p->FileNames = NULL; + + SzBitUi32s_Init(&p->CRCs); + SzBitUi32s_Init(&p->Attribs); + // SzBitUi32s_Init(&p->Parents); + SzBitUi64s_Init(&p->MTime); + SzBitUi64s_Init(&p->CTime); +} + +void SzArEx_Free(CSzArEx *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->UnpackPositions); + ISzAlloc_Free(alloc, p->IsDirs); + + ISzAlloc_Free(alloc, p->FolderToFile); + ISzAlloc_Free(alloc, p->FileToFolder); + + ISzAlloc_Free(alloc, p->FileNameOffsets); + ISzAlloc_Free(alloc, p->FileNames); + + SzBitUi32s_Free(&p->CRCs, alloc); + SzBitUi32s_Free(&p->Attribs, alloc); + // SzBitUi32s_Free(&p->Parents, alloc); + SzBitUi64s_Free(&p->MTime, alloc); + SzBitUi64s_Free(&p->CTime, alloc); + + SzAr_Free(&p->db, alloc); + SzArEx_Init(p); +} + + +static int TestSignatureCandidate(const Byte *testBytes) +{ + unsigned i; + for (i = 0; i < k7zSignatureSize; i++) + if (testBytes[i] != k7zSignature[i]) + return 0; + return 1; +} + +#define SzData_Clear(p) { (p)->Data = NULL; (p)->Size = 0; } + +#define SZ_READ_BYTE_SD(_sd_, dest) if ((_sd_)->Size == 0) return SZ_ERROR_ARCHIVE; (_sd_)->Size--; dest = *(_sd_)->Data++; +#define SZ_READ_BYTE(dest) SZ_READ_BYTE_SD(sd, dest) +#define SZ_READ_BYTE_2(dest) if (sd.Size == 0) return SZ_ERROR_ARCHIVE; sd.Size--; dest = *sd.Data++; + +#define SKIP_DATA(sd, size) { sd->Size -= (size_t)(size); sd->Data += (size_t)(size); } +#define SKIP_DATA2(sd, size) { sd.Size -= (size_t)(size); sd.Data += (size_t)(size); } + +#define SZ_READ_32(dest) if (sd.Size < 4) return SZ_ERROR_ARCHIVE; \ + dest = GetUi32(sd.Data); SKIP_DATA2(sd, 4); + +static MY_NO_INLINE SRes ReadNumber(CSzData *sd, UInt64 *value) +{ + Byte firstByte, mask; + unsigned i; + UInt32 v; + + SZ_READ_BYTE(firstByte); + if ((firstByte & 0x80) == 0) + { + *value = firstByte; + return SZ_OK; + } + SZ_READ_BYTE(v); + if ((firstByte & 0x40) == 0) + { + *value = (((UInt32)firstByte & 0x3F) << 8) | v; + return SZ_OK; + } + SZ_READ_BYTE(mask); + *value = v | ((UInt32)mask << 8); + mask = 0x20; + for (i = 2; i < 8; i++) + { + Byte b; + if ((firstByte & mask) == 0) + { + UInt64 highPart = (unsigned)firstByte & (unsigned)(mask - 1); + *value |= (highPart << (8 * i)); + return SZ_OK; + } + SZ_READ_BYTE(b); + *value |= ((UInt64)b << (8 * i)); + mask >>= 1; + } + return SZ_OK; +} + + +static MY_NO_INLINE SRes SzReadNumber32(CSzData *sd, UInt32 *value) +{ + Byte firstByte; + UInt64 value64; + if (sd->Size == 0) + return SZ_ERROR_ARCHIVE; + firstByte = *sd->Data; + if ((firstByte & 0x80) == 0) + { + *value = firstByte; + sd->Data++; + sd->Size--; + return SZ_OK; + } + RINOK(ReadNumber(sd, &value64)); + if (value64 >= (UInt32)0x80000000 - 1) + return SZ_ERROR_UNSUPPORTED; + if (value64 >= ((UInt64)(1) << ((sizeof(size_t) - 1) * 8 + 4))) + return SZ_ERROR_UNSUPPORTED; + *value = (UInt32)value64; + return SZ_OK; +} + +#define ReadID(sd, value) ReadNumber(sd, value) + +static SRes SkipData(CSzData *sd) +{ + UInt64 size; + RINOK(ReadNumber(sd, &size)); + if (size > sd->Size) + return SZ_ERROR_ARCHIVE; + SKIP_DATA(sd, size); + return SZ_OK; +} + +static SRes WaitId(CSzData *sd, UInt32 id) +{ + for (;;) + { + UInt64 type; + RINOK(ReadID(sd, &type)); + if (type == id) + return SZ_OK; + if (type == k7zIdEnd) + return SZ_ERROR_ARCHIVE; + RINOK(SkipData(sd)); + } +} + +static SRes RememberBitVector(CSzData *sd, UInt32 numItems, const Byte **v) +{ + UInt32 numBytes = (numItems + 7) >> 3; + if (numBytes > sd->Size) + return SZ_ERROR_ARCHIVE; + *v = sd->Data; + SKIP_DATA(sd, numBytes); + return SZ_OK; +} + +static UInt32 CountDefinedBits(const Byte *bits, UInt32 numItems) +{ + Byte b = 0; + unsigned m = 0; + UInt32 sum = 0; + for (; numItems != 0; numItems--) + { + if (m == 0) + { + b = *bits++; + m = 8; + } + m--; + sum += ((b >> m) & 1); + } + return sum; +} + +static MY_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, ISzAllocPtr alloc) +{ + Byte allAreDefined; + Byte *v2; + UInt32 numBytes = (numItems + 7) >> 3; + *v = NULL; + SZ_READ_BYTE(allAreDefined); + if (numBytes == 0) + return SZ_OK; + if (allAreDefined == 0) + { + if (numBytes > sd->Size) + return SZ_ERROR_ARCHIVE; + MY_ALLOC_AND_CPY(*v, numBytes, sd->Data, alloc); + SKIP_DATA(sd, numBytes); + return SZ_OK; + } + MY_ALLOC(Byte, *v, numBytes, alloc); + v2 = *v; + memset(v2, 0xFF, (size_t)numBytes); + { + unsigned numBits = (unsigned)numItems & 7; + if (numBits != 0) + v2[(size_t)numBytes - 1] = (Byte)((((UInt32)1 << numBits) - 1) << (8 - numBits)); + } + return SZ_OK; +} + +static MY_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) +{ + UInt32 i; + CSzData sd; + UInt32 *vals; + const Byte *defs; + MY_ALLOC_ZE(UInt32, crcs->Vals, numItems, alloc); + sd = *sd2; + defs = crcs->Defs; + vals = crcs->Vals; + for (i = 0; i < numItems; i++) + if (SzBitArray_Check(defs, i)) + { + SZ_READ_32(vals[i]); + } + else + vals[i] = 0; + *sd2 = sd; + return SZ_OK; +} + +static SRes ReadBitUi32s(CSzData *sd, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) +{ + SzBitUi32s_Free(crcs, alloc); + RINOK(ReadBitVector(sd, numItems, &crcs->Defs, alloc)); + return ReadUi32s(sd, numItems, crcs, alloc); +} + +static SRes SkipBitUi32s(CSzData *sd, UInt32 numItems) +{ + Byte allAreDefined; + UInt32 numDefined = numItems; + SZ_READ_BYTE(allAreDefined); + if (!allAreDefined) + { + size_t numBytes = (numItems + 7) >> 3; + if (numBytes > sd->Size) + return SZ_ERROR_ARCHIVE; + numDefined = CountDefinedBits(sd->Data, numItems); + SKIP_DATA(sd, numBytes); + } + if (numDefined > (sd->Size >> 2)) + return SZ_ERROR_ARCHIVE; + SKIP_DATA(sd, (size_t)numDefined * 4); + return SZ_OK; +} + +static SRes ReadPackInfo(CSzAr *p, CSzData *sd, ISzAllocPtr alloc) +{ + RINOK(SzReadNumber32(sd, &p->NumPackStreams)); + + RINOK(WaitId(sd, k7zIdSize)); + MY_ALLOC(UInt64, p->PackPositions, (size_t)p->NumPackStreams + 1, alloc); + { + UInt64 sum = 0; + UInt32 i; + UInt32 numPackStreams = p->NumPackStreams; + for (i = 0; i < numPackStreams; i++) + { + UInt64 packSize; + p->PackPositions[i] = sum; + RINOK(ReadNumber(sd, &packSize)); + sum += packSize; + if (sum < packSize) + return SZ_ERROR_ARCHIVE; + } + p->PackPositions[i] = sum; + } + + for (;;) + { + UInt64 type; + RINOK(ReadID(sd, &type)); + if (type == k7zIdEnd) + return SZ_OK; + if (type == k7zIdCRC) + { + /* CRC of packed streams is unused now */ + RINOK(SkipBitUi32s(sd, p->NumPackStreams)); + continue; + } + RINOK(SkipData(sd)); + } +} + +/* +static SRes SzReadSwitch(CSzData *sd) +{ + Byte external; + RINOK(SzReadByte(sd, &external)); + return (external == 0) ? SZ_OK: SZ_ERROR_UNSUPPORTED; +} +*/ + +#define k_NumCodersStreams_in_Folder_MAX (SZ_NUM_BONDS_IN_FOLDER_MAX + SZ_NUM_PACK_STREAMS_IN_FOLDER_MAX) + +SRes SzGetNextFolderItem(CSzFolder *f, CSzData *sd) +{ + UInt32 numCoders, i; + UInt32 numInStreams = 0; + const Byte *dataStart = sd->Data; + + f->NumCoders = 0; + f->NumBonds = 0; + f->NumPackStreams = 0; + f->UnpackStream = 0; + + RINOK(SzReadNumber32(sd, &numCoders)); + if (numCoders == 0 || numCoders > SZ_NUM_CODERS_IN_FOLDER_MAX) + return SZ_ERROR_UNSUPPORTED; + + for (i = 0; i < numCoders; i++) + { + Byte mainByte; + CSzCoderInfo *coder = f->Coders + i; + unsigned idSize, j; + UInt64 id; + + SZ_READ_BYTE(mainByte); + if ((mainByte & 0xC0) != 0) + return SZ_ERROR_UNSUPPORTED; + + idSize = (unsigned)(mainByte & 0xF); + if (idSize > sizeof(id)) + return SZ_ERROR_UNSUPPORTED; + if (idSize > sd->Size) + return SZ_ERROR_ARCHIVE; + id = 0; + for (j = 0; j < idSize; j++) + { + id = ((id << 8) | *sd->Data); + sd->Data++; + sd->Size--; + } + if (id > (UInt32)0xFFFFFFFF) + return SZ_ERROR_UNSUPPORTED; + coder->MethodID = (UInt32)id; + + coder->NumStreams = 1; + coder->PropsOffset = 0; + coder->PropsSize = 0; + + if ((mainByte & 0x10) != 0) + { + UInt32 numStreams; + + RINOK(SzReadNumber32(sd, &numStreams)); + if (numStreams > k_NumCodersStreams_in_Folder_MAX) + return SZ_ERROR_UNSUPPORTED; + coder->NumStreams = (Byte)numStreams; + + RINOK(SzReadNumber32(sd, &numStreams)); + if (numStreams != 1) + return SZ_ERROR_UNSUPPORTED; + } + + numInStreams += coder->NumStreams; + + if (numInStreams > k_NumCodersStreams_in_Folder_MAX) + return SZ_ERROR_UNSUPPORTED; + + if ((mainByte & 0x20) != 0) + { + UInt32 propsSize = 0; + RINOK(SzReadNumber32(sd, &propsSize)); + if (propsSize > sd->Size) + return SZ_ERROR_ARCHIVE; + if (propsSize >= 0x80) + return SZ_ERROR_UNSUPPORTED; + coder->PropsOffset = sd->Data - dataStart; + coder->PropsSize = (Byte)propsSize; + sd->Data += (size_t)propsSize; + sd->Size -= (size_t)propsSize; + } + } + + /* + if (numInStreams == 1 && numCoders == 1) + { + f->NumPackStreams = 1; + f->PackStreams[0] = 0; + } + else + */ + { + Byte streamUsed[k_NumCodersStreams_in_Folder_MAX]; + UInt32 numBonds, numPackStreams; + + numBonds = numCoders - 1; + if (numInStreams < numBonds) + return SZ_ERROR_ARCHIVE; + if (numBonds > SZ_NUM_BONDS_IN_FOLDER_MAX) + return SZ_ERROR_UNSUPPORTED; + f->NumBonds = numBonds; + + numPackStreams = numInStreams - numBonds; + if (numPackStreams > SZ_NUM_PACK_STREAMS_IN_FOLDER_MAX) + return SZ_ERROR_UNSUPPORTED; + f->NumPackStreams = numPackStreams; + + for (i = 0; i < numInStreams; i++) + streamUsed[i] = False; + + if (numBonds != 0) + { + Byte coderUsed[SZ_NUM_CODERS_IN_FOLDER_MAX]; + + for (i = 0; i < numCoders; i++) + coderUsed[i] = False; + + for (i = 0; i < numBonds; i++) + { + CSzBond *bp = f->Bonds + i; + + RINOK(SzReadNumber32(sd, &bp->InIndex)); + if (bp->InIndex >= numInStreams || streamUsed[bp->InIndex]) + return SZ_ERROR_ARCHIVE; + streamUsed[bp->InIndex] = True; + + RINOK(SzReadNumber32(sd, &bp->OutIndex)); + if (bp->OutIndex >= numCoders || coderUsed[bp->OutIndex]) + return SZ_ERROR_ARCHIVE; + coderUsed[bp->OutIndex] = True; + } + + for (i = 0; i < numCoders; i++) + if (!coderUsed[i]) + { + f->UnpackStream = i; + break; + } + + if (i == numCoders) + return SZ_ERROR_ARCHIVE; + } + + if (numPackStreams == 1) + { + for (i = 0; i < numInStreams; i++) + if (!streamUsed[i]) + break; + if (i == numInStreams) + return SZ_ERROR_ARCHIVE; + f->PackStreams[0] = i; + } + else + for (i = 0; i < numPackStreams; i++) + { + UInt32 index; + RINOK(SzReadNumber32(sd, &index)); + if (index >= numInStreams || streamUsed[index]) + return SZ_ERROR_ARCHIVE; + streamUsed[index] = True; + f->PackStreams[i] = index; + } + } + + f->NumCoders = numCoders; + + return SZ_OK; +} + + +static MY_NO_INLINE SRes SkipNumbers(CSzData *sd2, UInt32 num) +{ + CSzData sd; + sd = *sd2; + for (; num != 0; num--) + { + Byte firstByte, mask; + unsigned i; + SZ_READ_BYTE_2(firstByte); + if ((firstByte & 0x80) == 0) + continue; + if ((firstByte & 0x40) == 0) + { + if (sd.Size == 0) + return SZ_ERROR_ARCHIVE; + sd.Size--; + sd.Data++; + continue; + } + mask = 0x20; + for (i = 2; i < 8 && (firstByte & mask) != 0; i++) + mask >>= 1; + if (i > sd.Size) + return SZ_ERROR_ARCHIVE; + SKIP_DATA2(sd, i); + } + *sd2 = sd; + return SZ_OK; +} + + +#define k_Scan_NumCoders_MAX 64 +#define k_Scan_NumCodersStreams_in_Folder_MAX 64 + + +static SRes ReadUnpackInfo(CSzAr *p, + CSzData *sd2, + UInt32 numFoldersMax, + const CBuf *tempBufs, UInt32 numTempBufs, + ISzAllocPtr alloc) +{ + CSzData sd; + + UInt32 fo, numFolders, numCodersOutStreams, packStreamIndex; + const Byte *startBufPtr; + Byte external; + + RINOK(WaitId(sd2, k7zIdFolder)); + + RINOK(SzReadNumber32(sd2, &numFolders)); + if (numFolders > numFoldersMax) + return SZ_ERROR_UNSUPPORTED; + p->NumFolders = numFolders; + + SZ_READ_BYTE_SD(sd2, external); + if (external == 0) + sd = *sd2; + else + { + UInt32 index; + RINOK(SzReadNumber32(sd2, &index)); + if (index >= numTempBufs) + return SZ_ERROR_ARCHIVE; + sd.Data = tempBufs[index].data; + sd.Size = tempBufs[index].size; + } + + MY_ALLOC(size_t, p->FoCodersOffsets, (size_t)numFolders + 1, alloc); + MY_ALLOC(UInt32, p->FoStartPackStreamIndex, (size_t)numFolders + 1, alloc); + MY_ALLOC(UInt32, p->FoToCoderUnpackSizes, (size_t)numFolders + 1, alloc); + MY_ALLOC_ZE(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); + + startBufPtr = sd.Data; + + packStreamIndex = 0; + numCodersOutStreams = 0; + + for (fo = 0; fo < numFolders; fo++) + { + UInt32 numCoders, ci, numInStreams = 0; + + p->FoCodersOffsets[fo] = sd.Data - startBufPtr; + + RINOK(SzReadNumber32(&sd, &numCoders)); + if (numCoders == 0 || numCoders > k_Scan_NumCoders_MAX) + return SZ_ERROR_UNSUPPORTED; + + for (ci = 0; ci < numCoders; ci++) + { + Byte mainByte; + unsigned idSize; + UInt32 coderInStreams; + + SZ_READ_BYTE_2(mainByte); + if ((mainByte & 0xC0) != 0) + return SZ_ERROR_UNSUPPORTED; + idSize = (mainByte & 0xF); + if (idSize > 8) + return SZ_ERROR_UNSUPPORTED; + if (idSize > sd.Size) + return SZ_ERROR_ARCHIVE; + SKIP_DATA2(sd, idSize); + + coderInStreams = 1; + + if ((mainByte & 0x10) != 0) + { + UInt32 coderOutStreams; + RINOK(SzReadNumber32(&sd, &coderInStreams)); + RINOK(SzReadNumber32(&sd, &coderOutStreams)); + if (coderInStreams > k_Scan_NumCodersStreams_in_Folder_MAX || coderOutStreams != 1) + return SZ_ERROR_UNSUPPORTED; + } + + numInStreams += coderInStreams; + + if ((mainByte & 0x20) != 0) + { + UInt32 propsSize; + RINOK(SzReadNumber32(&sd, &propsSize)); + if (propsSize > sd.Size) + return SZ_ERROR_ARCHIVE; + SKIP_DATA2(sd, propsSize); + } + } + + { + UInt32 indexOfMainStream = 0; + UInt32 numPackStreams = 1; + + if (numCoders != 1 || numInStreams != 1) + { + Byte streamUsed[k_Scan_NumCodersStreams_in_Folder_MAX]; + Byte coderUsed[k_Scan_NumCoders_MAX]; + + UInt32 i; + UInt32 numBonds = numCoders - 1; + if (numInStreams < numBonds) + return SZ_ERROR_ARCHIVE; + + if (numInStreams > k_Scan_NumCodersStreams_in_Folder_MAX) + return SZ_ERROR_UNSUPPORTED; + + for (i = 0; i < numInStreams; i++) + streamUsed[i] = False; + for (i = 0; i < numCoders; i++) + coderUsed[i] = False; + + for (i = 0; i < numBonds; i++) + { + UInt32 index; + + RINOK(SzReadNumber32(&sd, &index)); + if (index >= numInStreams || streamUsed[index]) + return SZ_ERROR_ARCHIVE; + streamUsed[index] = True; + + RINOK(SzReadNumber32(&sd, &index)); + if (index >= numCoders || coderUsed[index]) + return SZ_ERROR_ARCHIVE; + coderUsed[index] = True; + } + + numPackStreams = numInStreams - numBonds; + + if (numPackStreams != 1) + for (i = 0; i < numPackStreams; i++) + { + UInt32 index; + RINOK(SzReadNumber32(&sd, &index)); + if (index >= numInStreams || streamUsed[index]) + return SZ_ERROR_ARCHIVE; + streamUsed[index] = True; + } + + for (i = 0; i < numCoders; i++) + if (!coderUsed[i]) + { + indexOfMainStream = i; + break; + } + + if (i == numCoders) + return SZ_ERROR_ARCHIVE; + } + + p->FoStartPackStreamIndex[fo] = packStreamIndex; + p->FoToCoderUnpackSizes[fo] = numCodersOutStreams; + p->FoToMainUnpackSizeIndex[fo] = (Byte)indexOfMainStream; + numCodersOutStreams += numCoders; + if (numCodersOutStreams < numCoders) + return SZ_ERROR_UNSUPPORTED; + if (numPackStreams > p->NumPackStreams - packStreamIndex) + return SZ_ERROR_ARCHIVE; + packStreamIndex += numPackStreams; + } + } + + p->FoToCoderUnpackSizes[fo] = numCodersOutStreams; + + { + size_t dataSize = sd.Data - startBufPtr; + p->FoStartPackStreamIndex[fo] = packStreamIndex; + p->FoCodersOffsets[fo] = dataSize; + MY_ALLOC_ZE_AND_CPY(p->CodersData, dataSize, startBufPtr, alloc); + } + + if (external != 0) + { + if (sd.Size != 0) + return SZ_ERROR_ARCHIVE; + sd = *sd2; + } + + RINOK(WaitId(&sd, k7zIdCodersUnpackSize)); + + MY_ALLOC_ZE(UInt64, p->CoderUnpackSizes, (size_t)numCodersOutStreams, alloc); + { + UInt32 i; + for (i = 0; i < numCodersOutStreams; i++) + { + RINOK(ReadNumber(&sd, p->CoderUnpackSizes + i)); + } + } + + for (;;) + { + UInt64 type; + RINOK(ReadID(&sd, &type)); + if (type == k7zIdEnd) + { + *sd2 = sd; + return SZ_OK; + } + if (type == k7zIdCRC) + { + RINOK(ReadBitUi32s(&sd, numFolders, &p->FolderCRCs, alloc)); + continue; + } + RINOK(SkipData(&sd)); + } +} + + +UInt64 SzAr_GetFolderUnpackSize(const CSzAr *p, UInt32 folderIndex) +{ + return p->CoderUnpackSizes[p->FoToCoderUnpackSizes[folderIndex] + p->FoToMainUnpackSizeIndex[folderIndex]]; +} + + +typedef struct +{ + UInt32 NumTotalSubStreams; + UInt32 NumSubDigests; + CSzData sdNumSubStreams; + CSzData sdSizes; + CSzData sdCRCs; +} CSubStreamInfo; + + +static SRes ReadSubStreamsInfo(CSzAr *p, CSzData *sd, CSubStreamInfo *ssi) +{ + UInt64 type = 0; + UInt32 numSubDigests = 0; + UInt32 numFolders = p->NumFolders; + UInt32 numUnpackStreams = numFolders; + UInt32 numUnpackSizesInData = 0; + + for (;;) + { + RINOK(ReadID(sd, &type)); + if (type == k7zIdNumUnpackStream) + { + UInt32 i; + ssi->sdNumSubStreams.Data = sd->Data; + numUnpackStreams = 0; + numSubDigests = 0; + for (i = 0; i < numFolders; i++) + { + UInt32 numStreams; + RINOK(SzReadNumber32(sd, &numStreams)); + if (numUnpackStreams > numUnpackStreams + numStreams) + return SZ_ERROR_UNSUPPORTED; + numUnpackStreams += numStreams; + if (numStreams != 0) + numUnpackSizesInData += (numStreams - 1); + if (numStreams != 1 || !SzBitWithVals_Check(&p->FolderCRCs, i)) + numSubDigests += numStreams; + } + ssi->sdNumSubStreams.Size = sd->Data - ssi->sdNumSubStreams.Data; + continue; + } + if (type == k7zIdCRC || type == k7zIdSize || type == k7zIdEnd) + break; + RINOK(SkipData(sd)); + } + + if (!ssi->sdNumSubStreams.Data) + { + numSubDigests = numFolders; + if (p->FolderCRCs.Defs) + numSubDigests = numFolders - CountDefinedBits(p->FolderCRCs.Defs, numFolders); + } + + ssi->NumTotalSubStreams = numUnpackStreams; + ssi->NumSubDigests = numSubDigests; + + if (type == k7zIdSize) + { + ssi->sdSizes.Data = sd->Data; + RINOK(SkipNumbers(sd, numUnpackSizesInData)); + ssi->sdSizes.Size = sd->Data - ssi->sdSizes.Data; + RINOK(ReadID(sd, &type)); + } + + for (;;) + { + if (type == k7zIdEnd) + return SZ_OK; + if (type == k7zIdCRC) + { + ssi->sdCRCs.Data = sd->Data; + RINOK(SkipBitUi32s(sd, numSubDigests)); + ssi->sdCRCs.Size = sd->Data - ssi->sdCRCs.Data; + } + else + { + RINOK(SkipData(sd)); + } + RINOK(ReadID(sd, &type)); + } +} + +static SRes SzReadStreamsInfo(CSzAr *p, + CSzData *sd, + UInt32 numFoldersMax, const CBuf *tempBufs, UInt32 numTempBufs, + UInt64 *dataOffset, + CSubStreamInfo *ssi, + ISzAllocPtr alloc) +{ + UInt64 type; + + SzData_Clear(&ssi->sdSizes); + SzData_Clear(&ssi->sdCRCs); + SzData_Clear(&ssi->sdNumSubStreams); + + *dataOffset = 0; + RINOK(ReadID(sd, &type)); + if (type == k7zIdPackInfo) + { + RINOK(ReadNumber(sd, dataOffset)); + RINOK(ReadPackInfo(p, sd, alloc)); + RINOK(ReadID(sd, &type)); + } + if (type == k7zIdUnpackInfo) + { + RINOK(ReadUnpackInfo(p, sd, numFoldersMax, tempBufs, numTempBufs, alloc)); + RINOK(ReadID(sd, &type)); + } + if (type == k7zIdSubStreamsInfo) + { + RINOK(ReadSubStreamsInfo(p, sd, ssi)); + RINOK(ReadID(sd, &type)); + } + else + { + ssi->NumTotalSubStreams = p->NumFolders; + // ssi->NumSubDigests = 0; + } + + return (type == k7zIdEnd ? SZ_OK : SZ_ERROR_UNSUPPORTED); +} + +static SRes SzReadAndDecodePackedStreams( + ILookInStream *inStream, + CSzData *sd, + CBuf *tempBufs, + UInt32 numFoldersMax, + UInt64 baseOffset, + CSzAr *p, + ISzAllocPtr allocTemp) +{ + UInt64 dataStartPos; + UInt32 fo; + CSubStreamInfo ssi; + + RINOK(SzReadStreamsInfo(p, sd, numFoldersMax, NULL, 0, &dataStartPos, &ssi, allocTemp)); + + dataStartPos += baseOffset; + if (p->NumFolders == 0) + return SZ_ERROR_ARCHIVE; + + for (fo = 0; fo < p->NumFolders; fo++) + Buf_Init(tempBufs + fo); + + for (fo = 0; fo < p->NumFolders; fo++) + { + CBuf *tempBuf = tempBufs + fo; + UInt64 unpackSize = SzAr_GetFolderUnpackSize(p, fo); + if ((size_t)unpackSize != unpackSize) + return SZ_ERROR_MEM; + if (!Buf_Create(tempBuf, (size_t)unpackSize, allocTemp)) + return SZ_ERROR_MEM; + } + + for (fo = 0; fo < p->NumFolders; fo++) + { + const CBuf *tempBuf = tempBufs + fo; + RINOK(LookInStream_SeekTo(inStream, dataStartPos)); + RINOK(SzAr_DecodeFolder(p, fo, inStream, dataStartPos, tempBuf->data, tempBuf->size, allocTemp)); + } + + return SZ_OK; +} + +static SRes SzReadFileNames(const Byte *data, size_t size, UInt32 numFiles, size_t *offsets) +{ + size_t pos = 0; + *offsets++ = 0; + if (numFiles == 0) + return (size == 0) ? SZ_OK : SZ_ERROR_ARCHIVE; + if (size < 2) + return SZ_ERROR_ARCHIVE; + if (data[size - 2] != 0 || data[size - 1] != 0) + return SZ_ERROR_ARCHIVE; + do + { + const Byte *p; + if (pos == size) + return SZ_ERROR_ARCHIVE; + for (p = data + pos; + #ifdef _WIN32 + *(const UInt16 *)p != 0 + #else + p[0] != 0 || p[1] != 0 + #endif + ; p += 2); + pos = p - data + 2; + *offsets++ = (pos >> 1); + } + while (--numFiles); + return (pos == size) ? SZ_OK : SZ_ERROR_ARCHIVE; +} + +static MY_NO_INLINE SRes ReadTime(CSzBitUi64s *p, UInt32 num, + CSzData *sd2, + const CBuf *tempBufs, UInt32 numTempBufs, + ISzAllocPtr alloc) +{ + CSzData sd; + UInt32 i; + CNtfsFileTime *vals; + Byte *defs; + Byte external; + + RINOK(ReadBitVector(sd2, num, &p->Defs, alloc)); + + SZ_READ_BYTE_SD(sd2, external); + if (external == 0) + sd = *sd2; + else + { + UInt32 index; + RINOK(SzReadNumber32(sd2, &index)); + if (index >= numTempBufs) + return SZ_ERROR_ARCHIVE; + sd.Data = tempBufs[index].data; + sd.Size = tempBufs[index].size; + } + + MY_ALLOC_ZE(CNtfsFileTime, p->Vals, num, alloc); + vals = p->Vals; + defs = p->Defs; + for (i = 0; i < num; i++) + if (SzBitArray_Check(defs, i)) + { + if (sd.Size < 8) + return SZ_ERROR_ARCHIVE; + vals[i].Low = GetUi32(sd.Data); + vals[i].High = GetUi32(sd.Data + 4); + SKIP_DATA2(sd, 8); + } + else + vals[i].High = vals[i].Low = 0; + + if (external == 0) + *sd2 = sd; + + return SZ_OK; +} + + +#define NUM_ADDITIONAL_STREAMS_MAX 8 + + +static SRes SzReadHeader2( + CSzArEx *p, /* allocMain */ + CSzData *sd, + ILookInStream *inStream, + CBuf *tempBufs, UInt32 *numTempBufs, + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp + ) +{ + CSubStreamInfo ssi; + +{ + UInt64 type; + + SzData_Clear(&ssi.sdSizes); + SzData_Clear(&ssi.sdCRCs); + SzData_Clear(&ssi.sdNumSubStreams); + + ssi.NumSubDigests = 0; + ssi.NumTotalSubStreams = 0; + + RINOK(ReadID(sd, &type)); + + if (type == k7zIdArchiveProperties) + { + for (;;) + { + UInt64 type2; + RINOK(ReadID(sd, &type2)); + if (type2 == k7zIdEnd) + break; + RINOK(SkipData(sd)); + } + RINOK(ReadID(sd, &type)); + } + + if (type == k7zIdAdditionalStreamsInfo) + { + CSzAr tempAr; + SRes res; + + SzAr_Init(&tempAr); + res = SzReadAndDecodePackedStreams(inStream, sd, tempBufs, NUM_ADDITIONAL_STREAMS_MAX, + p->startPosAfterHeader, &tempAr, allocTemp); + *numTempBufs = tempAr.NumFolders; + SzAr_Free(&tempAr, allocTemp); + + if (res != SZ_OK) + return res; + RINOK(ReadID(sd, &type)); + } + + if (type == k7zIdMainStreamsInfo) + { + RINOK(SzReadStreamsInfo(&p->db, sd, (UInt32)1 << 30, tempBufs, *numTempBufs, + &p->dataPos, &ssi, allocMain)); + p->dataPos += p->startPosAfterHeader; + RINOK(ReadID(sd, &type)); + } + + if (type == k7zIdEnd) + { + return SZ_OK; + } + + if (type != k7zIdFilesInfo) + return SZ_ERROR_ARCHIVE; +} + +{ + UInt32 numFiles = 0; + UInt32 numEmptyStreams = 0; + const Byte *emptyStreams = NULL; + const Byte *emptyFiles = NULL; + + RINOK(SzReadNumber32(sd, &numFiles)); + p->NumFiles = numFiles; + + for (;;) + { + UInt64 type; + UInt64 size; + RINOK(ReadID(sd, &type)); + if (type == k7zIdEnd) + break; + RINOK(ReadNumber(sd, &size)); + if (size > sd->Size) + return SZ_ERROR_ARCHIVE; + + if (type >= ((UInt32)1 << 8)) + { + SKIP_DATA(sd, size); + } + else switch ((unsigned)type) + { + case k7zIdName: + { + size_t namesSize; + const Byte *namesData; + Byte external; + + SZ_READ_BYTE(external); + if (external == 0) + { + namesSize = (size_t)size - 1; + namesData = sd->Data; + } + else + { + UInt32 index; + RINOK(SzReadNumber32(sd, &index)); + if (index >= *numTempBufs) + return SZ_ERROR_ARCHIVE; + namesData = (tempBufs)[index].data; + namesSize = (tempBufs)[index].size; + } + + if ((namesSize & 1) != 0) + return SZ_ERROR_ARCHIVE; + MY_ALLOC(size_t, p->FileNameOffsets, numFiles + 1, allocMain); + MY_ALLOC_ZE_AND_CPY(p->FileNames, namesSize, namesData, allocMain); + RINOK(SzReadFileNames(p->FileNames, namesSize, numFiles, p->FileNameOffsets)) + if (external == 0) + { + SKIP_DATA(sd, namesSize); + } + break; + } + case k7zIdEmptyStream: + { + RINOK(RememberBitVector(sd, numFiles, &emptyStreams)); + numEmptyStreams = CountDefinedBits(emptyStreams, numFiles); + emptyFiles = NULL; + break; + } + case k7zIdEmptyFile: + { + RINOK(RememberBitVector(sd, numEmptyStreams, &emptyFiles)); + break; + } + case k7zIdWinAttrib: + { + Byte external; + CSzData sdSwitch; + CSzData *sdPtr; + SzBitUi32s_Free(&p->Attribs, allocMain); + RINOK(ReadBitVector(sd, numFiles, &p->Attribs.Defs, allocMain)); + + SZ_READ_BYTE(external); + if (external == 0) + sdPtr = sd; + else + { + UInt32 index; + RINOK(SzReadNumber32(sd, &index)); + if (index >= *numTempBufs) + return SZ_ERROR_ARCHIVE; + sdSwitch.Data = (tempBufs)[index].data; + sdSwitch.Size = (tempBufs)[index].size; + sdPtr = &sdSwitch; + } + RINOK(ReadUi32s(sdPtr, numFiles, &p->Attribs, allocMain)); + break; + } + /* + case k7zParent: + { + SzBitUi32s_Free(&p->Parents, allocMain); + RINOK(ReadBitVector(sd, numFiles, &p->Parents.Defs, allocMain)); + RINOK(SzReadSwitch(sd)); + RINOK(ReadUi32s(sd, numFiles, &p->Parents, allocMain)); + break; + } + */ + case k7zIdMTime: RINOK(ReadTime(&p->MTime, numFiles, sd, tempBufs, *numTempBufs, allocMain)); break; + case k7zIdCTime: RINOK(ReadTime(&p->CTime, numFiles, sd, tempBufs, *numTempBufs, allocMain)); break; + default: + { + SKIP_DATA(sd, size); + } + } + } + + if (numFiles - numEmptyStreams != ssi.NumTotalSubStreams) + return SZ_ERROR_ARCHIVE; + + for (;;) + { + UInt64 type; + RINOK(ReadID(sd, &type)); + if (type == k7zIdEnd) + break; + RINOK(SkipData(sd)); + } + + { + UInt32 i; + UInt32 emptyFileIndex = 0; + UInt32 folderIndex = 0; + UInt32 remSubStreams = 0; + UInt32 numSubStreams = 0; + UInt64 unpackPos = 0; + const Byte *digestsDefs = NULL; + const Byte *digestsVals = NULL; + UInt32 digestsValsIndex = 0; + UInt32 digestIndex; + Byte allDigestsDefined = 0; + Byte isDirMask = 0; + Byte crcMask = 0; + Byte mask = 0x80; + + MY_ALLOC(UInt32, p->FolderToFile, p->db.NumFolders + 1, allocMain); + MY_ALLOC_ZE(UInt32, p->FileToFolder, p->NumFiles, allocMain); + MY_ALLOC(UInt64, p->UnpackPositions, p->NumFiles + 1, allocMain); + MY_ALLOC_ZE(Byte, p->IsDirs, (p->NumFiles + 7) >> 3, allocMain); + + RINOK(SzBitUi32s_Alloc(&p->CRCs, p->NumFiles, allocMain)); + + if (ssi.sdCRCs.Size != 0) + { + SZ_READ_BYTE_SD(&ssi.sdCRCs, allDigestsDefined); + if (allDigestsDefined) + digestsVals = ssi.sdCRCs.Data; + else + { + size_t numBytes = (ssi.NumSubDigests + 7) >> 3; + digestsDefs = ssi.sdCRCs.Data; + digestsVals = digestsDefs + numBytes; + } + } + + digestIndex = 0; + + for (i = 0; i < numFiles; i++, mask >>= 1) + { + if (mask == 0) + { + UInt32 byteIndex = (i - 1) >> 3; + p->IsDirs[byteIndex] = isDirMask; + p->CRCs.Defs[byteIndex] = crcMask; + isDirMask = 0; + crcMask = 0; + mask = 0x80; + } + + p->UnpackPositions[i] = unpackPos; + p->CRCs.Vals[i] = 0; + + if (emptyStreams && SzBitArray_Check(emptyStreams, i)) + { + if (emptyFiles) + { + if (!SzBitArray_Check(emptyFiles, emptyFileIndex)) + isDirMask |= mask; + emptyFileIndex++; + } + else + isDirMask |= mask; + if (remSubStreams == 0) + { + p->FileToFolder[i] = (UInt32)-1; + continue; + } + } + + if (remSubStreams == 0) + { + for (;;) + { + if (folderIndex >= p->db.NumFolders) + return SZ_ERROR_ARCHIVE; + p->FolderToFile[folderIndex] = i; + numSubStreams = 1; + if (ssi.sdNumSubStreams.Data) + { + RINOK(SzReadNumber32(&ssi.sdNumSubStreams, &numSubStreams)); + } + remSubStreams = numSubStreams; + if (numSubStreams != 0) + break; + { + UInt64 folderUnpackSize = SzAr_GetFolderUnpackSize(&p->db, folderIndex); + unpackPos += folderUnpackSize; + if (unpackPos < folderUnpackSize) + return SZ_ERROR_ARCHIVE; + } + + folderIndex++; + } + } + + p->FileToFolder[i] = folderIndex; + + if (emptyStreams && SzBitArray_Check(emptyStreams, i)) + continue; + + if (--remSubStreams == 0) + { + UInt64 folderUnpackSize = SzAr_GetFolderUnpackSize(&p->db, folderIndex); + UInt64 startFolderUnpackPos = p->UnpackPositions[p->FolderToFile[folderIndex]]; + if (folderUnpackSize < unpackPos - startFolderUnpackPos) + return SZ_ERROR_ARCHIVE; + unpackPos = startFolderUnpackPos + folderUnpackSize; + if (unpackPos < folderUnpackSize) + return SZ_ERROR_ARCHIVE; + + if (numSubStreams == 1 && SzBitWithVals_Check(&p->db.FolderCRCs, i)) + { + p->CRCs.Vals[i] = p->db.FolderCRCs.Vals[folderIndex]; + crcMask |= mask; + } + else if (allDigestsDefined || (digestsDefs && SzBitArray_Check(digestsDefs, digestIndex))) + { + p->CRCs.Vals[i] = GetUi32(digestsVals + (size_t)digestsValsIndex * 4); + digestsValsIndex++; + crcMask |= mask; + } + + folderIndex++; + } + else + { + UInt64 v; + RINOK(ReadNumber(&ssi.sdSizes, &v)); + unpackPos += v; + if (unpackPos < v) + return SZ_ERROR_ARCHIVE; + if (allDigestsDefined || (digestsDefs && SzBitArray_Check(digestsDefs, digestIndex))) + { + p->CRCs.Vals[i] = GetUi32(digestsVals + (size_t)digestsValsIndex * 4); + digestsValsIndex++; + crcMask |= mask; + } + } + } + + if (mask != 0x80) + { + UInt32 byteIndex = (i - 1) >> 3; + p->IsDirs[byteIndex] = isDirMask; + p->CRCs.Defs[byteIndex] = crcMask; + } + + p->UnpackPositions[i] = unpackPos; + + if (remSubStreams != 0) + return SZ_ERROR_ARCHIVE; + + for (;;) + { + p->FolderToFile[folderIndex] = i; + if (folderIndex >= p->db.NumFolders) + break; + if (!ssi.sdNumSubStreams.Data) + return SZ_ERROR_ARCHIVE; + RINOK(SzReadNumber32(&ssi.sdNumSubStreams, &numSubStreams)); + if (numSubStreams != 0) + return SZ_ERROR_ARCHIVE; + /* + { + UInt64 folderUnpackSize = SzAr_GetFolderUnpackSize(&p->db, folderIndex); + unpackPos += folderUnpackSize; + if (unpackPos < folderUnpackSize) + return SZ_ERROR_ARCHIVE; + } + */ + folderIndex++; + } + + if (ssi.sdNumSubStreams.Data && ssi.sdNumSubStreams.Size != 0) + return SZ_ERROR_ARCHIVE; + } +} + return SZ_OK; +} + + +static SRes SzReadHeader( + CSzArEx *p, + CSzData *sd, + ILookInStream *inStream, + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp) +{ + UInt32 i; + UInt32 numTempBufs = 0; + SRes res; + CBuf tempBufs[NUM_ADDITIONAL_STREAMS_MAX]; + + for (i = 0; i < NUM_ADDITIONAL_STREAMS_MAX; i++) + Buf_Init(tempBufs + i); + + res = SzReadHeader2(p, sd, inStream, + tempBufs, &numTempBufs, + allocMain, allocTemp); + + for (i = 0; i < NUM_ADDITIONAL_STREAMS_MAX; i++) + Buf_Free(tempBufs + i, allocTemp); + + RINOK(res); + + if (sd->Size != 0) + return SZ_ERROR_FAIL; + + return res; +} + +static SRes SzArEx_Open2( + CSzArEx *p, + ILookInStream *inStream, + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp) +{ + Byte header[k7zStartHeaderSize]; + Int64 startArcPos; + UInt64 nextHeaderOffset, nextHeaderSize; + size_t nextHeaderSizeT; + UInt32 nextHeaderCRC; + CBuf buf; + SRes res; + + startArcPos = 0; + RINOK(ILookInStream_Seek(inStream, &startArcPos, SZ_SEEK_CUR)); + + RINOK(LookInStream_Read2(inStream, header, k7zStartHeaderSize, SZ_ERROR_NO_ARCHIVE)); + + if (!TestSignatureCandidate(header)) + return SZ_ERROR_NO_ARCHIVE; + if (header[6] != k7zMajorVersion) + return SZ_ERROR_UNSUPPORTED; + + nextHeaderOffset = GetUi64(header + 12); + nextHeaderSize = GetUi64(header + 20); + nextHeaderCRC = GetUi32(header + 28); + + p->startPosAfterHeader = startArcPos + k7zStartHeaderSize; + + if (CrcCalc(header + 12, 20) != GetUi32(header + 8)) + return SZ_ERROR_CRC; + + nextHeaderSizeT = (size_t)nextHeaderSize; + if (nextHeaderSizeT != nextHeaderSize) + return SZ_ERROR_MEM; + if (nextHeaderSizeT == 0) + return SZ_OK; + if (nextHeaderOffset > nextHeaderOffset + nextHeaderSize || + nextHeaderOffset > nextHeaderOffset + nextHeaderSize + k7zStartHeaderSize) + return SZ_ERROR_NO_ARCHIVE; + + { + Int64 pos = 0; + RINOK(ILookInStream_Seek(inStream, &pos, SZ_SEEK_END)); + if ((UInt64)pos < startArcPos + nextHeaderOffset || + (UInt64)pos < startArcPos + k7zStartHeaderSize + nextHeaderOffset || + (UInt64)pos < startArcPos + k7zStartHeaderSize + nextHeaderOffset + nextHeaderSize) + return SZ_ERROR_INPUT_EOF; + } + + RINOK(LookInStream_SeekTo(inStream, startArcPos + k7zStartHeaderSize + nextHeaderOffset)); + + if (!Buf_Create(&buf, nextHeaderSizeT, allocTemp)) + return SZ_ERROR_MEM; + + res = LookInStream_Read(inStream, buf.data, nextHeaderSizeT); + + if (res == SZ_OK) + { + res = SZ_ERROR_ARCHIVE; + if (CrcCalc(buf.data, nextHeaderSizeT) == nextHeaderCRC) + { + CSzData sd; + UInt64 type; + sd.Data = buf.data; + sd.Size = buf.size; + + res = ReadID(&sd, &type); + + if (res == SZ_OK && type == k7zIdEncodedHeader) + { + CSzAr tempAr; + CBuf tempBuf; + Buf_Init(&tempBuf); + + SzAr_Init(&tempAr); + res = SzReadAndDecodePackedStreams(inStream, &sd, &tempBuf, 1, p->startPosAfterHeader, &tempAr, allocTemp); + SzAr_Free(&tempAr, allocTemp); + + if (res != SZ_OK) + { + Buf_Free(&tempBuf, allocTemp); + } + else + { + Buf_Free(&buf, allocTemp); + buf.data = tempBuf.data; + buf.size = tempBuf.size; + sd.Data = buf.data; + sd.Size = buf.size; + res = ReadID(&sd, &type); + } + } + + if (res == SZ_OK) + { + if (type == k7zIdHeader) + { + /* + CSzData sd2; + unsigned ttt; + for (ttt = 0; ttt < 40000; ttt++) + { + SzArEx_Free(p, allocMain); + sd2 = sd; + res = SzReadHeader(p, &sd2, inStream, allocMain, allocTemp); + if (res != SZ_OK) + break; + } + */ + res = SzReadHeader(p, &sd, inStream, allocMain, allocTemp); + } + else + res = SZ_ERROR_UNSUPPORTED; + } + } + } + + Buf_Free(&buf, allocTemp); + return res; +} + + +SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, + ISzAllocPtr allocMain, ISzAllocPtr allocTemp) +{ + SRes res = SzArEx_Open2(p, inStream, allocMain, allocTemp); + if (res != SZ_OK) + SzArEx_Free(p, allocMain); + return res; +} + + +SRes SzArEx_Extract( + const CSzArEx *p, + ILookInStream *inStream, + UInt32 fileIndex, + UInt32 *blockIndex, + Byte **tempBuf, + size_t *outBufferSize, + size_t *offset, + size_t *outSizeProcessed, + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp) +{ + UInt32 folderIndex = p->FileToFolder[fileIndex]; + SRes res = SZ_OK; + + *offset = 0; + *outSizeProcessed = 0; + + if (folderIndex == (UInt32)-1) + { + ISzAlloc_Free(allocMain, *tempBuf); + *blockIndex = folderIndex; + *tempBuf = NULL; + *outBufferSize = 0; + return SZ_OK; + } + + if (*tempBuf == NULL || *blockIndex != folderIndex) + { + UInt64 unpackSizeSpec = SzAr_GetFolderUnpackSize(&p->db, folderIndex); + /* + UInt64 unpackSizeSpec = + p->UnpackPositions[p->FolderToFile[(size_t)folderIndex + 1]] - + p->UnpackPositions[p->FolderToFile[folderIndex]]; + */ + size_t unpackSize = (size_t)unpackSizeSpec; + + if (unpackSize != unpackSizeSpec) + return SZ_ERROR_MEM; + *blockIndex = folderIndex; + ISzAlloc_Free(allocMain, *tempBuf); + *tempBuf = NULL; + + if (res == SZ_OK) + { + *outBufferSize = unpackSize; + if (unpackSize != 0) + { + *tempBuf = (Byte *)ISzAlloc_Alloc(allocMain, unpackSize); + if (*tempBuf == NULL) + res = SZ_ERROR_MEM; + } + + if (res == SZ_OK) + { + res = SzAr_DecodeFolder(&p->db, folderIndex, + inStream, p->dataPos, *tempBuf, unpackSize, allocTemp); + } + } + } + + if (res == SZ_OK) + { + UInt64 unpackPos = p->UnpackPositions[fileIndex]; + *offset = (size_t)(unpackPos - p->UnpackPositions[p->FolderToFile[folderIndex]]); + *outSizeProcessed = (size_t)(p->UnpackPositions[(size_t)fileIndex + 1] - unpackPos); + if (*offset + *outSizeProcessed > *outBufferSize) + return SZ_ERROR_FAIL; + if (SzBitWithVals_Check(&p->CRCs, fileIndex)) + if (CrcCalc(*tempBuf + *offset, *outSizeProcessed) != p->CRCs.Vals[fileIndex]) + res = SZ_ERROR_CRC; + } + + return res; +} + + +size_t SzArEx_GetFileNameUtf16(const CSzArEx *p, size_t fileIndex, UInt16 *dest) +{ + size_t offs = p->FileNameOffsets[fileIndex]; + size_t len = p->FileNameOffsets[fileIndex + 1] - offs; + if (dest != 0) + { + size_t i; + const Byte *src = p->FileNames + offs * 2; + for (i = 0; i < len; i++) + dest[i] = GetUi16(src + i * 2); + } + return len; +} + +/* +size_t SzArEx_GetFullNameLen(const CSzArEx *p, size_t fileIndex) +{ + size_t len; + if (!p->FileNameOffsets) + return 1; + len = 0; + for (;;) + { + UInt32 parent = (UInt32)(Int32)-1; + len += p->FileNameOffsets[fileIndex + 1] - p->FileNameOffsets[fileIndex]; + if SzBitWithVals_Check(&p->Parents, fileIndex) + parent = p->Parents.Vals[fileIndex]; + if (parent == (UInt32)(Int32)-1) + return len; + fileIndex = parent; + } +} + +UInt16 *SzArEx_GetFullNameUtf16_Back(const CSzArEx *p, size_t fileIndex, UInt16 *dest) +{ + BoolInt needSlash; + if (!p->FileNameOffsets) + { + *(--dest) = 0; + return dest; + } + needSlash = False; + for (;;) + { + UInt32 parent = (UInt32)(Int32)-1; + size_t curLen = p->FileNameOffsets[fileIndex + 1] - p->FileNameOffsets[fileIndex]; + SzArEx_GetFileNameUtf16(p, fileIndex, dest - curLen); + if (needSlash) + *(dest - 1) = '/'; + needSlash = True; + dest -= curLen; + + if SzBitWithVals_Check(&p->Parents, fileIndex) + parent = p->Parents.Vals[fileIndex]; + if (parent == (UInt32)(Int32)-1) + return dest; + fileIndex = parent; + } +} +*/ diff --git a/code/nel/3rdparty/seven_zip/7zBuf.c b/code/nel/3rdparty/seven_zip/7zBuf.c new file mode 100644 index 000000000..8865c32a8 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zBuf.c @@ -0,0 +1,36 @@ +/* 7zBuf.c -- Byte Buffer +2017-04-03 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "7zBuf.h" + +void Buf_Init(CBuf *p) +{ + p->data = 0; + p->size = 0; +} + +int Buf_Create(CBuf *p, size_t size, ISzAllocPtr alloc) +{ + p->size = 0; + if (size == 0) + { + p->data = 0; + return 1; + } + p->data = (Byte *)ISzAlloc_Alloc(alloc, size); + if (p->data) + { + p->size = size; + return 1; + } + return 0; +} + +void Buf_Free(CBuf *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->data); + p->data = 0; + p->size = 0; +} diff --git a/code/nel/3rdparty/seven_zip/7zBuf.h b/code/nel/3rdparty/seven_zip/7zBuf.h new file mode 100644 index 000000000..81d1b5b64 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zBuf.h @@ -0,0 +1,35 @@ +/* 7zBuf.h -- Byte Buffer +2017-04-03 : Igor Pavlov : Public domain */ + +#ifndef __7Z_BUF_H +#define __7Z_BUF_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +typedef struct +{ + Byte *data; + size_t size; +} CBuf; + +void Buf_Init(CBuf *p); +int Buf_Create(CBuf *p, size_t size, ISzAllocPtr alloc); +void Buf_Free(CBuf *p, ISzAllocPtr alloc); + +typedef struct +{ + Byte *data; + size_t size; + size_t pos; +} CDynBuf; + +void DynBuf_Construct(CDynBuf *p); +void DynBuf_SeekToBeg(CDynBuf *p); +int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAllocPtr alloc); +void DynBuf_Free(CDynBuf *p, ISzAllocPtr alloc); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/7zBuf2.c b/code/nel/3rdparty/seven_zip/7zBuf2.c new file mode 100644 index 000000000..208347416 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zBuf2.c @@ -0,0 +1,52 @@ +/* 7zBuf2.c -- Byte Buffer +2017-04-03 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +#include "7zBuf.h" + +void DynBuf_Construct(CDynBuf *p) +{ + p->data = 0; + p->size = 0; + p->pos = 0; +} + +void DynBuf_SeekToBeg(CDynBuf *p) +{ + p->pos = 0; +} + +int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAllocPtr alloc) +{ + if (size > p->size - p->pos) + { + size_t newSize = p->pos + size; + Byte *data; + newSize += newSize / 4; + data = (Byte *)ISzAlloc_Alloc(alloc, newSize); + if (!data) + return 0; + p->size = newSize; + if (p->pos != 0) + memcpy(data, p->data, p->pos); + ISzAlloc_Free(alloc, p->data); + p->data = data; + } + if (size != 0) + { + memcpy(p->data + p->pos, buf, size); + p->pos += size; + } + return 1; +} + +void DynBuf_Free(CDynBuf *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->data); + p->data = 0; + p->size = 0; + p->pos = 0; +} diff --git a/code/nel/3rdparty/seven_zip/7zCrc.c b/code/nel/3rdparty/seven_zip/7zCrc.c new file mode 100644 index 000000000..b4d84f023 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zCrc.c @@ -0,0 +1,128 @@ +/* 7zCrc.c -- CRC32 init +2017-06-06 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "7zCrc.h" +#include "CpuArch.h" + +#define kCrcPoly 0xEDB88320 + +#ifdef MY_CPU_LE + #define CRC_NUM_TABLES 8 +#else + #define CRC_NUM_TABLES 9 + + #define CRC_UINT32_SWAP(v) ((v >> 24) | ((v >> 8) & 0xFF00) | ((v << 8) & 0xFF0000) | (v << 24)) + + UInt32 MY_FAST_CALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, const UInt32 *table); + UInt32 MY_FAST_CALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, const UInt32 *table); +#endif + +#ifndef MY_CPU_BE + UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table); + UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table); +#endif + +typedef UInt32 (MY_FAST_CALL *CRC_FUNC)(UInt32 v, const void *data, size_t size, const UInt32 *table); + +CRC_FUNC g_CrcUpdateT4; +CRC_FUNC g_CrcUpdateT8; +CRC_FUNC g_CrcUpdate; + +UInt32 g_CrcTable[256 * CRC_NUM_TABLES]; + +UInt32 MY_FAST_CALL CrcUpdate(UInt32 v, const void *data, size_t size) +{ + return g_CrcUpdate(v, data, size, g_CrcTable); +} + +UInt32 MY_FAST_CALL CrcCalc(const void *data, size_t size) +{ + return g_CrcUpdate(CRC_INIT_VAL, data, size, g_CrcTable) ^ CRC_INIT_VAL; +} + +#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) + +UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table) +{ + const Byte *p = (const Byte *)data; + const Byte *pEnd = p + size; + for (; p != pEnd; p++) + v = CRC_UPDATE_BYTE_2(v, *p); + return v; +} + +void MY_FAST_CALL CrcGenerateTable() +{ + UInt32 i; + for (i = 0; i < 256; i++) + { + UInt32 r = i; + unsigned j; + for (j = 0; j < 8; j++) + r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); + g_CrcTable[i] = r; + } + for (i = 256; i < 256 * CRC_NUM_TABLES; i++) + { + UInt32 r = g_CrcTable[(size_t)i - 256]; + g_CrcTable[i] = g_CrcTable[r & 0xFF] ^ (r >> 8); + } + + #if CRC_NUM_TABLES < 4 + + g_CrcUpdate = CrcUpdateT1; + + #else + + #ifdef MY_CPU_LE + + g_CrcUpdateT4 = CrcUpdateT4; + g_CrcUpdate = CrcUpdateT4; + + #if CRC_NUM_TABLES >= 8 + g_CrcUpdateT8 = CrcUpdateT8; + + #ifdef MY_CPU_X86_OR_AMD64 + if (!CPU_Is_InOrder()) + #endif + g_CrcUpdate = CrcUpdateT8; + #endif + + #else + { + #ifndef MY_CPU_BE + UInt32 k = 0x01020304; + const Byte *p = (const Byte *)&k; + if (p[0] == 4 && p[1] == 3) + { + g_CrcUpdateT4 = CrcUpdateT4; + g_CrcUpdate = CrcUpdateT4; + #if CRC_NUM_TABLES >= 8 + g_CrcUpdateT8 = CrcUpdateT8; + g_CrcUpdate = CrcUpdateT8; + #endif + } + else if (p[0] != 1 || p[1] != 2) + g_CrcUpdate = CrcUpdateT1; + else + #endif + { + for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--) + { + UInt32 x = g_CrcTable[(size_t)i - 256]; + g_CrcTable[i] = CRC_UINT32_SWAP(x); + } + g_CrcUpdateT4 = CrcUpdateT1_BeT4; + g_CrcUpdate = CrcUpdateT1_BeT4; + #if CRC_NUM_TABLES >= 8 + g_CrcUpdateT8 = CrcUpdateT1_BeT8; + g_CrcUpdate = CrcUpdateT1_BeT8; + #endif + } + } + #endif + + #endif +} diff --git a/code/nel/3rdparty/seven_zip/7zCrcOpt.c b/code/nel/3rdparty/seven_zip/7zCrcOpt.c new file mode 100644 index 000000000..73beba298 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zCrcOpt.c @@ -0,0 +1,115 @@ +/* 7zCrcOpt.c -- CRC32 calculation +2017-04-03 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "CpuArch.h" + +#ifndef MY_CPU_BE + +#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) + +UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table) +{ + const Byte *p = (const Byte *)data; + for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) + v = CRC_UPDATE_BYTE_2(v, *p); + for (; size >= 4; size -= 4, p += 4) + { + v ^= *(const UInt32 *)p; + v = + (table + 0x300)[((v ) & 0xFF)] + ^ (table + 0x200)[((v >> 8) & 0xFF)] + ^ (table + 0x100)[((v >> 16) & 0xFF)] + ^ (table + 0x000)[((v >> 24))]; + } + for (; size > 0; size--, p++) + v = CRC_UPDATE_BYTE_2(v, *p); + return v; +} + +UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table) +{ + const Byte *p = (const Byte *)data; + for (; size > 0 && ((unsigned)(ptrdiff_t)p & 7) != 0; size--, p++) + v = CRC_UPDATE_BYTE_2(v, *p); + for (; size >= 8; size -= 8, p += 8) + { + UInt32 d; + v ^= *(const UInt32 *)p; + v = + (table + 0x700)[((v ) & 0xFF)] + ^ (table + 0x600)[((v >> 8) & 0xFF)] + ^ (table + 0x500)[((v >> 16) & 0xFF)] + ^ (table + 0x400)[((v >> 24))]; + d = *((const UInt32 *)p + 1); + v ^= + (table + 0x300)[((d ) & 0xFF)] + ^ (table + 0x200)[((d >> 8) & 0xFF)] + ^ (table + 0x100)[((d >> 16) & 0xFF)] + ^ (table + 0x000)[((d >> 24))]; + } + for (; size > 0; size--, p++) + v = CRC_UPDATE_BYTE_2(v, *p); + return v; +} + +#endif + + +#ifndef MY_CPU_LE + +#define CRC_UINT32_SWAP(v) ((v >> 24) | ((v >> 8) & 0xFF00) | ((v << 8) & 0xFF0000) | (v << 24)) + +#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[(((crc) >> 24) ^ (b))] ^ ((crc) << 8)) + +UInt32 MY_FAST_CALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, const UInt32 *table) +{ + const Byte *p = (const Byte *)data; + table += 0x100; + v = CRC_UINT32_SWAP(v); + for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) + v = CRC_UPDATE_BYTE_2_BE(v, *p); + for (; size >= 4; size -= 4, p += 4) + { + v ^= *(const UInt32 *)p; + v = + (table + 0x000)[((v ) & 0xFF)] + ^ (table + 0x100)[((v >> 8) & 0xFF)] + ^ (table + 0x200)[((v >> 16) & 0xFF)] + ^ (table + 0x300)[((v >> 24))]; + } + for (; size > 0; size--, p++) + v = CRC_UPDATE_BYTE_2_BE(v, *p); + return CRC_UINT32_SWAP(v); +} + +UInt32 MY_FAST_CALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, const UInt32 *table) +{ + const Byte *p = (const Byte *)data; + table += 0x100; + v = CRC_UINT32_SWAP(v); + for (; size > 0 && ((unsigned)(ptrdiff_t)p & 7) != 0; size--, p++) + v = CRC_UPDATE_BYTE_2_BE(v, *p); + for (; size >= 8; size -= 8, p += 8) + { + UInt32 d; + v ^= *(const UInt32 *)p; + v = + (table + 0x400)[((v ) & 0xFF)] + ^ (table + 0x500)[((v >> 8) & 0xFF)] + ^ (table + 0x600)[((v >> 16) & 0xFF)] + ^ (table + 0x700)[((v >> 24))]; + d = *((const UInt32 *)p + 1); + v ^= + (table + 0x000)[((d ) & 0xFF)] + ^ (table + 0x100)[((d >> 8) & 0xFF)] + ^ (table + 0x200)[((d >> 16) & 0xFF)] + ^ (table + 0x300)[((d >> 24))]; + } + for (; size > 0; size--, p++) + v = CRC_UPDATE_BYTE_2_BE(v, *p); + return CRC_UINT32_SWAP(v); +} + +#endif diff --git a/code/nel/3rdparty/seven_zip/7zDec.c b/code/nel/3rdparty/seven_zip/7zDec.c new file mode 100644 index 000000000..7c4635211 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zDec.c @@ -0,0 +1,591 @@ +/* 7zDec.c -- Decoding from 7z folder +2019-02-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +/* #define _7ZIP_PPMD_SUPPPORT */ + +#include "7z.h" +#include "7zCrc.h" + +#include "Bcj2.h" +#include "Bra.h" +#include "CpuArch.h" +#include "Delta.h" +#include "LzmaDec.h" +#include "Lzma2Dec.h" +#ifdef _7ZIP_PPMD_SUPPPORT +#include "Ppmd7.h" +#endif + +#define k_Copy 0 +#define k_Delta 3 +#define k_LZMA2 0x21 +#define k_LZMA 0x30101 +#define k_BCJ 0x3030103 +#define k_BCJ2 0x303011B +#define k_PPC 0x3030205 +#define k_IA64 0x3030401 +#define k_ARM 0x3030501 +#define k_ARMT 0x3030701 +#define k_SPARC 0x3030805 + + +#ifdef _7ZIP_PPMD_SUPPPORT + +#define k_PPMD 0x30401 + +typedef struct +{ + IByteIn vt; + const Byte *cur; + const Byte *end; + const Byte *begin; + UInt64 processed; + BoolInt extra; + SRes res; + const ILookInStream *inStream; +} CByteInToLook; + +static Byte ReadByte(const IByteIn *pp) +{ + CByteInToLook *p = CONTAINER_FROM_VTBL(pp, CByteInToLook, vt); + if (p->cur != p->end) + return *p->cur++; + if (p->res == SZ_OK) + { + size_t size = p->cur - p->begin; + p->processed += size; + p->res = ILookInStream_Skip(p->inStream, size); + size = (1 << 25); + p->res = ILookInStream_Look(p->inStream, (const void **)&p->begin, &size); + p->cur = p->begin; + p->end = p->begin + size; + if (size != 0) + return *p->cur++;; + } + p->extra = True; + return 0; +} + +static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, const ILookInStream *inStream, + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) +{ + CPpmd7 ppmd; + CByteInToLook s; + SRes res = SZ_OK; + + s.vt.Read = ReadByte; + s.inStream = inStream; + s.begin = s.end = s.cur = NULL; + s.extra = False; + s.res = SZ_OK; + s.processed = 0; + + if (propsSize != 5) + return SZ_ERROR_UNSUPPORTED; + + { + unsigned order = props[0]; + UInt32 memSize = GetUi32(props + 1); + if (order < PPMD7_MIN_ORDER || + order > PPMD7_MAX_ORDER || + memSize < PPMD7_MIN_MEM_SIZE || + memSize > PPMD7_MAX_MEM_SIZE) + return SZ_ERROR_UNSUPPORTED; + Ppmd7_Construct(&ppmd); + if (!Ppmd7_Alloc(&ppmd, memSize, allocMain)) + return SZ_ERROR_MEM; + Ppmd7_Init(&ppmd, order); + } + { + CPpmd7z_RangeDec rc; + Ppmd7z_RangeDec_CreateVTable(&rc); + rc.Stream = &s.vt; + if (!Ppmd7z_RangeDec_Init(&rc)) + res = SZ_ERROR_DATA; + else if (s.extra) + res = (s.res != SZ_OK ? s.res : SZ_ERROR_DATA); + else + { + SizeT i; + for (i = 0; i < outSize; i++) + { + int sym = Ppmd7_DecodeSymbol(&ppmd, &rc.vt); + if (s.extra || sym < 0) + break; + outBuffer[i] = (Byte)sym; + } + if (i != outSize) + res = (s.res != SZ_OK ? s.res : SZ_ERROR_DATA); + else if (s.processed + (s.cur - s.begin) != inSize || !Ppmd7z_RangeDec_IsFinishedOK(&rc)) + res = SZ_ERROR_DATA; + } + } + Ppmd7_Free(&ppmd, allocMain); + return res; +} + +#endif + + +static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) +{ + CLzmaDec state; + SRes res = SZ_OK; + + LzmaDec_Construct(&state); + RINOK(LzmaDec_AllocateProbs(&state, props, propsSize, allocMain)); + state.dic = outBuffer; + state.dicBufSize = outSize; + LzmaDec_Init(&state); + + for (;;) + { + const void *inBuf = NULL; + size_t lookahead = (1 << 18); + if (lookahead > inSize) + lookahead = (size_t)inSize; + res = ILookInStream_Look(inStream, &inBuf, &lookahead); + if (res != SZ_OK) + break; + + { + SizeT inProcessed = (SizeT)lookahead, dicPos = state.dicPos; + ELzmaStatus status; + res = LzmaDec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); + lookahead -= inProcessed; + inSize -= inProcessed; + if (res != SZ_OK) + break; + + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + { + if (outSize != state.dicPos || inSize != 0) + res = SZ_ERROR_DATA; + break; + } + + if (outSize == state.dicPos && inSize == 0 && status == LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) + break; + + if (inProcessed == 0 && dicPos == state.dicPos) + { + res = SZ_ERROR_DATA; + break; + } + + res = ILookInStream_Skip(inStream, inProcessed); + if (res != SZ_OK) + break; + } + } + + LzmaDec_FreeProbs(&state, allocMain); + return res; +} + + +#ifndef _7Z_NO_METHOD_LZMA2 + +static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) +{ + CLzma2Dec state; + SRes res = SZ_OK; + + Lzma2Dec_Construct(&state); + if (propsSize != 1) + return SZ_ERROR_DATA; + RINOK(Lzma2Dec_AllocateProbs(&state, props[0], allocMain)); + state.decoder.dic = outBuffer; + state.decoder.dicBufSize = outSize; + Lzma2Dec_Init(&state); + + for (;;) + { + const void *inBuf = NULL; + size_t lookahead = (1 << 18); + if (lookahead > inSize) + lookahead = (size_t)inSize; + res = ILookInStream_Look(inStream, &inBuf, &lookahead); + if (res != SZ_OK) + break; + + { + SizeT inProcessed = (SizeT)lookahead, dicPos = state.decoder.dicPos; + ELzmaStatus status; + res = Lzma2Dec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); + lookahead -= inProcessed; + inSize -= inProcessed; + if (res != SZ_OK) + break; + + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + { + if (outSize != state.decoder.dicPos || inSize != 0) + res = SZ_ERROR_DATA; + break; + } + + if (inProcessed == 0 && dicPos == state.decoder.dicPos) + { + res = SZ_ERROR_DATA; + break; + } + + res = ILookInStream_Skip(inStream, inProcessed); + if (res != SZ_OK) + break; + } + } + + Lzma2Dec_FreeProbs(&state, allocMain); + return res; +} + +#endif + + +static SRes SzDecodeCopy(UInt64 inSize, ILookInStream *inStream, Byte *outBuffer) +{ + while (inSize > 0) + { + const void *inBuf; + size_t curSize = (1 << 18); + if (curSize > inSize) + curSize = (size_t)inSize; + RINOK(ILookInStream_Look(inStream, &inBuf, &curSize)); + if (curSize == 0) + return SZ_ERROR_INPUT_EOF; + memcpy(outBuffer, inBuf, curSize); + outBuffer += curSize; + inSize -= curSize; + RINOK(ILookInStream_Skip(inStream, curSize)); + } + return SZ_OK; +} + +static BoolInt IS_MAIN_METHOD(UInt32 m) +{ + switch (m) + { + case k_Copy: + case k_LZMA: + #ifndef _7Z_NO_METHOD_LZMA2 + case k_LZMA2: + #endif + #ifdef _7ZIP_PPMD_SUPPPORT + case k_PPMD: + #endif + return True; + } + return False; +} + +static BoolInt IS_SUPPORTED_CODER(const CSzCoderInfo *c) +{ + return + c->NumStreams == 1 + /* && c->MethodID <= (UInt32)0xFFFFFFFF */ + && IS_MAIN_METHOD((UInt32)c->MethodID); +} + +#define IS_BCJ2(c) ((c)->MethodID == k_BCJ2 && (c)->NumStreams == 4) + +static SRes CheckSupportedFolder(const CSzFolder *f) +{ + if (f->NumCoders < 1 || f->NumCoders > 4) + return SZ_ERROR_UNSUPPORTED; + if (!IS_SUPPORTED_CODER(&f->Coders[0])) + return SZ_ERROR_UNSUPPORTED; + if (f->NumCoders == 1) + { + if (f->NumPackStreams != 1 || f->PackStreams[0] != 0 || f->NumBonds != 0) + return SZ_ERROR_UNSUPPORTED; + return SZ_OK; + } + + + #ifndef _7Z_NO_METHODS_FILTERS + + if (f->NumCoders == 2) + { + const CSzCoderInfo *c = &f->Coders[1]; + if ( + /* c->MethodID > (UInt32)0xFFFFFFFF || */ + c->NumStreams != 1 + || f->NumPackStreams != 1 + || f->PackStreams[0] != 0 + || f->NumBonds != 1 + || f->Bonds[0].InIndex != 1 + || f->Bonds[0].OutIndex != 0) + return SZ_ERROR_UNSUPPORTED; + switch ((UInt32)c->MethodID) + { + case k_Delta: + case k_BCJ: + case k_PPC: + case k_IA64: + case k_SPARC: + case k_ARM: + case k_ARMT: + break; + default: + return SZ_ERROR_UNSUPPORTED; + } + return SZ_OK; + } + + #endif + + + if (f->NumCoders == 4) + { + if (!IS_SUPPORTED_CODER(&f->Coders[1]) + || !IS_SUPPORTED_CODER(&f->Coders[2]) + || !IS_BCJ2(&f->Coders[3])) + return SZ_ERROR_UNSUPPORTED; + if (f->NumPackStreams != 4 + || f->PackStreams[0] != 2 + || f->PackStreams[1] != 6 + || f->PackStreams[2] != 1 + || f->PackStreams[3] != 0 + || f->NumBonds != 3 + || f->Bonds[0].InIndex != 5 || f->Bonds[0].OutIndex != 0 + || f->Bonds[1].InIndex != 4 || f->Bonds[1].OutIndex != 1 + || f->Bonds[2].InIndex != 3 || f->Bonds[2].OutIndex != 2) + return SZ_ERROR_UNSUPPORTED; + return SZ_OK; + } + + return SZ_ERROR_UNSUPPORTED; +} + +#define CASE_BRA_CONV(isa) case k_ ## isa: isa ## _Convert(outBuffer, outSize, 0, 0); break; + +static SRes SzFolder_Decode2(const CSzFolder *folder, + const Byte *propsData, + const UInt64 *unpackSizes, + const UInt64 *packPositions, + ILookInStream *inStream, UInt64 startPos, + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain, + Byte *tempBuf[]) +{ + UInt32 ci; + SizeT tempSizes[3] = { 0, 0, 0}; + SizeT tempSize3 = 0; + Byte *tempBuf3 = 0; + + RINOK(CheckSupportedFolder(folder)); + + for (ci = 0; ci < folder->NumCoders; ci++) + { + const CSzCoderInfo *coder = &folder->Coders[ci]; + + if (IS_MAIN_METHOD((UInt32)coder->MethodID)) + { + UInt32 si = 0; + UInt64 offset; + UInt64 inSize; + Byte *outBufCur = outBuffer; + SizeT outSizeCur = outSize; + if (folder->NumCoders == 4) + { + UInt32 indices[] = { 3, 2, 0 }; + UInt64 unpackSize = unpackSizes[ci]; + si = indices[ci]; + if (ci < 2) + { + Byte *temp; + outSizeCur = (SizeT)unpackSize; + if (outSizeCur != unpackSize) + return SZ_ERROR_MEM; + temp = (Byte *)ISzAlloc_Alloc(allocMain, outSizeCur); + if (!temp && outSizeCur != 0) + return SZ_ERROR_MEM; + outBufCur = tempBuf[1 - ci] = temp; + tempSizes[1 - ci] = outSizeCur; + } + else if (ci == 2) + { + if (unpackSize > outSize) /* check it */ + return SZ_ERROR_PARAM; + tempBuf3 = outBufCur = outBuffer + (outSize - (size_t)unpackSize); + tempSize3 = outSizeCur = (SizeT)unpackSize; + } + else + return SZ_ERROR_UNSUPPORTED; + } + offset = packPositions[si]; + inSize = packPositions[(size_t)si + 1] - offset; + RINOK(LookInStream_SeekTo(inStream, startPos + offset)); + + if (coder->MethodID == k_Copy) + { + if (inSize != outSizeCur) /* check it */ + return SZ_ERROR_DATA; + RINOK(SzDecodeCopy(inSize, inStream, outBufCur)); + } + else if (coder->MethodID == k_LZMA) + { + RINOK(SzDecodeLzma(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain)); + } + #ifndef _7Z_NO_METHOD_LZMA2 + else if (coder->MethodID == k_LZMA2) + { + RINOK(SzDecodeLzma2(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain)); + } + #endif + #ifdef _7ZIP_PPMD_SUPPPORT + else if (coder->MethodID == k_PPMD) + { + RINOK(SzDecodePpmd(propsData + coder->PropsOffset, coder->PropsSize, inSize, inStream, outBufCur, outSizeCur, allocMain)); + } + #endif + else + return SZ_ERROR_UNSUPPORTED; + } + else if (coder->MethodID == k_BCJ2) + { + UInt64 offset = packPositions[1]; + UInt64 s3Size = packPositions[2] - offset; + + if (ci != 3) + return SZ_ERROR_UNSUPPORTED; + + tempSizes[2] = (SizeT)s3Size; + if (tempSizes[2] != s3Size) + return SZ_ERROR_MEM; + tempBuf[2] = (Byte *)ISzAlloc_Alloc(allocMain, tempSizes[2]); + if (!tempBuf[2] && tempSizes[2] != 0) + return SZ_ERROR_MEM; + + RINOK(LookInStream_SeekTo(inStream, startPos + offset)); + RINOK(SzDecodeCopy(s3Size, inStream, tempBuf[2])); + + if ((tempSizes[0] & 3) != 0 || + (tempSizes[1] & 3) != 0 || + tempSize3 + tempSizes[0] + tempSizes[1] != outSize) + return SZ_ERROR_DATA; + + { + CBcj2Dec p; + + p.bufs[0] = tempBuf3; p.lims[0] = tempBuf3 + tempSize3; + p.bufs[1] = tempBuf[0]; p.lims[1] = tempBuf[0] + tempSizes[0]; + p.bufs[2] = tempBuf[1]; p.lims[2] = tempBuf[1] + tempSizes[1]; + p.bufs[3] = tempBuf[2]; p.lims[3] = tempBuf[2] + tempSizes[2]; + + p.dest = outBuffer; + p.destLim = outBuffer + outSize; + + Bcj2Dec_Init(&p); + RINOK(Bcj2Dec_Decode(&p)); + + { + unsigned i; + for (i = 0; i < 4; i++) + if (p.bufs[i] != p.lims[i]) + return SZ_ERROR_DATA; + + if (!Bcj2Dec_IsFinished(&p)) + return SZ_ERROR_DATA; + + if (p.dest != p.destLim + || p.state != BCJ2_STREAM_MAIN) + return SZ_ERROR_DATA; + } + } + } + #ifndef _7Z_NO_METHODS_FILTERS + else if (ci == 1) + { + if (coder->MethodID == k_Delta) + { + if (coder->PropsSize != 1) + return SZ_ERROR_UNSUPPORTED; + { + Byte state[DELTA_STATE_SIZE]; + Delta_Init(state); + Delta_Decode(state, (unsigned)(propsData[coder->PropsOffset]) + 1, outBuffer, outSize); + } + } + else + { + if (coder->PropsSize != 0) + return SZ_ERROR_UNSUPPORTED; + switch (coder->MethodID) + { + case k_BCJ: + { + UInt32 state; + x86_Convert_Init(state); + x86_Convert(outBuffer, outSize, 0, &state, 0); + break; + } + CASE_BRA_CONV(PPC) + CASE_BRA_CONV(IA64) + CASE_BRA_CONV(SPARC) + CASE_BRA_CONV(ARM) + CASE_BRA_CONV(ARMT) + default: + return SZ_ERROR_UNSUPPORTED; + } + } + } + #endif + else + return SZ_ERROR_UNSUPPORTED; + } + + return SZ_OK; +} + + +SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, + ILookInStream *inStream, UInt64 startPos, + Byte *outBuffer, size_t outSize, + ISzAllocPtr allocMain) +{ + SRes res; + CSzFolder folder; + CSzData sd; + + const Byte *data = p->CodersData + p->FoCodersOffsets[folderIndex]; + sd.Data = data; + sd.Size = p->FoCodersOffsets[(size_t)folderIndex + 1] - p->FoCodersOffsets[folderIndex]; + + res = SzGetNextFolderItem(&folder, &sd); + + if (res != SZ_OK) + return res; + + if (sd.Size != 0 + || folder.UnpackStream != p->FoToMainUnpackSizeIndex[folderIndex] + || outSize != SzAr_GetFolderUnpackSize(p, folderIndex)) + return SZ_ERROR_FAIL; + { + unsigned i; + Byte *tempBuf[3] = { 0, 0, 0}; + + res = SzFolder_Decode2(&folder, data, + &p->CoderUnpackSizes[p->FoToCoderUnpackSizes[folderIndex]], + p->PackPositions + p->FoStartPackStreamIndex[folderIndex], + inStream, startPos, + outBuffer, (SizeT)outSize, allocMain, tempBuf); + + for (i = 0; i < 3; i++) + ISzAlloc_Free(allocMain, tempBuf[i]); + + if (res == SZ_OK) + if (SzBitWithVals_Check(&p->FolderCRCs, folderIndex)) + if (CrcCalc(outBuffer, outSize) != p->FolderCRCs.Vals[folderIndex]) + res = SZ_ERROR_CRC; + + return res; + } +} diff --git a/code/nel/3rdparty/seven_zip/7zFile.c b/code/nel/3rdparty/seven_zip/7zFile.c new file mode 100644 index 000000000..8992fb1c5 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zFile.c @@ -0,0 +1,286 @@ +/* 7zFile.c -- File IO +2017-04-03 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "7zFile.h" + +#ifndef USE_WINDOWS_FILE + +#ifndef UNDER_CE +#include +#endif + +#else + +/* + ReadFile and WriteFile functions in Windows have BUG: + If you Read or Write 64MB or more (probably min_failure_size = 64MB - 32KB + 1) + from/to Network file, it returns ERROR_NO_SYSTEM_RESOURCES + (Insufficient system resources exist to complete the requested service). + Probably in some version of Windows there are problems with other sizes: + for 32 MB (maybe also for 16 MB). + And message can be "Network connection was lost" +*/ + +#define kChunkSizeMax (1 << 22) + +#endif + +void File_Construct(CSzFile *p) +{ + #ifdef USE_WINDOWS_FILE + p->handle = INVALID_HANDLE_VALUE; + #else + p->file = NULL; + #endif +} + +#if !defined(UNDER_CE) || !defined(USE_WINDOWS_FILE) +static WRes File_Open(CSzFile *p, const char *name, int writeMode) +{ + #ifdef USE_WINDOWS_FILE + p->handle = CreateFileA(name, + writeMode ? GENERIC_WRITE : GENERIC_READ, + FILE_SHARE_READ, NULL, + writeMode ? CREATE_ALWAYS : OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL); + return (p->handle != INVALID_HANDLE_VALUE) ? 0 : GetLastError(); + #else + p->file = fopen(name, writeMode ? "wb+" : "rb"); + return (p->file != 0) ? 0 : + #ifdef UNDER_CE + 2; /* ENOENT */ + #else + errno; + #endif + #endif +} + +WRes InFile_Open(CSzFile *p, const char *name) { return File_Open(p, name, 0); } +WRes OutFile_Open(CSzFile *p, const char *name) { return File_Open(p, name, 1); } +#endif + +#ifdef USE_WINDOWS_FILE +static WRes File_OpenW(CSzFile *p, const WCHAR *name, int writeMode) +{ + p->handle = CreateFileW(name, + writeMode ? GENERIC_WRITE : GENERIC_READ, + FILE_SHARE_READ, NULL, + writeMode ? CREATE_ALWAYS : OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL); + return (p->handle != INVALID_HANDLE_VALUE) ? 0 : GetLastError(); +} +WRes InFile_OpenW(CSzFile *p, const WCHAR *name) { return File_OpenW(p, name, 0); } +WRes OutFile_OpenW(CSzFile *p, const WCHAR *name) { return File_OpenW(p, name, 1); } +#endif + +WRes File_Close(CSzFile *p) +{ + #ifdef USE_WINDOWS_FILE + if (p->handle != INVALID_HANDLE_VALUE) + { + if (!CloseHandle(p->handle)) + return GetLastError(); + p->handle = INVALID_HANDLE_VALUE; + } + #else + if (p->file != NULL) + { + int res = fclose(p->file); + if (res != 0) + return res; + p->file = NULL; + } + #endif + return 0; +} + +WRes File_Read(CSzFile *p, void *data, size_t *size) +{ + size_t originalSize = *size; + if (originalSize == 0) + return 0; + + #ifdef USE_WINDOWS_FILE + + *size = 0; + do + { + DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; + DWORD processed = 0; + BOOL res = ReadFile(p->handle, data, curSize, &processed, NULL); + data = (void *)((Byte *)data + processed); + originalSize -= processed; + *size += processed; + if (!res) + return GetLastError(); + if (processed == 0) + break; + } + while (originalSize > 0); + return 0; + + #else + + *size = fread(data, 1, originalSize, p->file); + if (*size == originalSize) + return 0; + return ferror(p->file); + + #endif +} + +WRes File_Write(CSzFile *p, const void *data, size_t *size) +{ + size_t originalSize = *size; + if (originalSize == 0) + return 0; + + #ifdef USE_WINDOWS_FILE + + *size = 0; + do + { + DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; + DWORD processed = 0; + BOOL res = WriteFile(p->handle, data, curSize, &processed, NULL); + data = (void *)((Byte *)data + processed); + originalSize -= processed; + *size += processed; + if (!res) + return GetLastError(); + if (processed == 0) + break; + } + while (originalSize > 0); + return 0; + + #else + + *size = fwrite(data, 1, originalSize, p->file); + if (*size == originalSize) + return 0; + return ferror(p->file); + + #endif +} + +WRes File_Seek(CSzFile *p, Int64 *pos, ESzSeek origin) +{ + #ifdef USE_WINDOWS_FILE + + LARGE_INTEGER value; + DWORD moveMethod; + value.LowPart = (DWORD)*pos; + value.HighPart = (LONG)((UInt64)*pos >> 16 >> 16); /* for case when UInt64 is 32-bit only */ + switch (origin) + { + case SZ_SEEK_SET: moveMethod = FILE_BEGIN; break; + case SZ_SEEK_CUR: moveMethod = FILE_CURRENT; break; + case SZ_SEEK_END: moveMethod = FILE_END; break; + default: return ERROR_INVALID_PARAMETER; + } + value.LowPart = SetFilePointer(p->handle, value.LowPart, &value.HighPart, moveMethod); + if (value.LowPart == 0xFFFFFFFF) + { + WRes res = GetLastError(); + if (res != NO_ERROR) + return res; + } + *pos = ((Int64)value.HighPart << 32) | value.LowPart; + return 0; + + #else + + int moveMethod; + int res; + switch (origin) + { + case SZ_SEEK_SET: moveMethod = SEEK_SET; break; + case SZ_SEEK_CUR: moveMethod = SEEK_CUR; break; + case SZ_SEEK_END: moveMethod = SEEK_END; break; + default: return 1; + } + res = fseek(p->file, (long)*pos, moveMethod); + *pos = ftell(p->file); + return res; + + #endif +} + +WRes File_GetLength(CSzFile *p, UInt64 *length) +{ + #ifdef USE_WINDOWS_FILE + + DWORD sizeHigh; + DWORD sizeLow = GetFileSize(p->handle, &sizeHigh); + if (sizeLow == 0xFFFFFFFF) + { + DWORD res = GetLastError(); + if (res != NO_ERROR) + return res; + } + *length = (((UInt64)sizeHigh) << 32) + sizeLow; + return 0; + + #else + + long pos = ftell(p->file); + int res = fseek(p->file, 0, SEEK_END); + *length = ftell(p->file); + fseek(p->file, pos, SEEK_SET); + return res; + + #endif +} + + +/* ---------- FileSeqInStream ---------- */ + +static SRes FileSeqInStream_Read(const ISeqInStream *pp, void *buf, size_t *size) +{ + CFileSeqInStream *p = CONTAINER_FROM_VTBL(pp, CFileSeqInStream, vt); + return File_Read(&p->file, buf, size) == 0 ? SZ_OK : SZ_ERROR_READ; +} + +void FileSeqInStream_CreateVTable(CFileSeqInStream *p) +{ + p->vt.Read = FileSeqInStream_Read; +} + + +/* ---------- FileInStream ---------- */ + +static SRes FileInStream_Read(const ISeekInStream *pp, void *buf, size_t *size) +{ + CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); + return (File_Read(&p->file, buf, size) == 0) ? SZ_OK : SZ_ERROR_READ; +} + +static SRes FileInStream_Seek(const ISeekInStream *pp, Int64 *pos, ESzSeek origin) +{ + CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); + return File_Seek(&p->file, pos, origin); +} + +void FileInStream_CreateVTable(CFileInStream *p) +{ + p->vt.Read = FileInStream_Read; + p->vt.Seek = FileInStream_Seek; +} + + +/* ---------- FileOutStream ---------- */ + +static size_t FileOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) +{ + CFileOutStream *p = CONTAINER_FROM_VTBL(pp, CFileOutStream, vt); + File_Write(&p->file, data, &size); + return size; +} + +void FileOutStream_CreateVTable(CFileOutStream *p) +{ + p->vt.Write = FileOutStream_Write; +} diff --git a/code/nel/3rdparty/seven_zip/7zFile.h b/code/nel/3rdparty/seven_zip/7zFile.h new file mode 100644 index 000000000..0e7925382 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zFile.h @@ -0,0 +1,83 @@ +/* 7zFile.h -- File IO +2017-04-03 : Igor Pavlov : Public domain */ + +#ifndef __7Z_FILE_H +#define __7Z_FILE_H + +#ifdef _WIN32 +#define USE_WINDOWS_FILE +#endif + +#ifdef USE_WINDOWS_FILE +#include +#else +#include +#endif + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +/* ---------- File ---------- */ + +typedef struct +{ + #ifdef USE_WINDOWS_FILE + HANDLE handle; + #else + FILE *file; + #endif +} CSzFile; + +void File_Construct(CSzFile *p); +#if !defined(UNDER_CE) || !defined(USE_WINDOWS_FILE) +WRes InFile_Open(CSzFile *p, const char *name); +WRes OutFile_Open(CSzFile *p, const char *name); +#endif +#ifdef USE_WINDOWS_FILE +WRes InFile_OpenW(CSzFile *p, const WCHAR *name); +WRes OutFile_OpenW(CSzFile *p, const WCHAR *name); +#endif +WRes File_Close(CSzFile *p); + +/* reads max(*size, remain file's size) bytes */ +WRes File_Read(CSzFile *p, void *data, size_t *size); + +/* writes *size bytes */ +WRes File_Write(CSzFile *p, const void *data, size_t *size); + +WRes File_Seek(CSzFile *p, Int64 *pos, ESzSeek origin); +WRes File_GetLength(CSzFile *p, UInt64 *length); + + +/* ---------- FileInStream ---------- */ + +typedef struct +{ + ISeqInStream vt; + CSzFile file; +} CFileSeqInStream; + +void FileSeqInStream_CreateVTable(CFileSeqInStream *p); + + +typedef struct +{ + ISeekInStream vt; + CSzFile file; +} CFileInStream; + +void FileInStream_CreateVTable(CFileInStream *p); + + +typedef struct +{ + ISeqOutStream vt; + CSzFile file; +} CFileOutStream; + +void FileOutStream_CreateVTable(CFileOutStream *p); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/7zStream.c b/code/nel/3rdparty/seven_zip/7zStream.c new file mode 100644 index 000000000..6b5aa1621 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zStream.c @@ -0,0 +1,176 @@ +/* 7zStream.c -- 7z Stream functions +2017-04-03 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +#include "7zTypes.h" + +SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes errorType) +{ + while (size != 0) + { + size_t processed = size; + RINOK(ISeqInStream_Read(stream, buf, &processed)); + if (processed == 0) + return errorType; + buf = (void *)((Byte *)buf + processed); + size -= processed; + } + return SZ_OK; +} + +SRes SeqInStream_Read(const ISeqInStream *stream, void *buf, size_t size) +{ + return SeqInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); +} + +SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf) +{ + size_t processed = 1; + RINOK(ISeqInStream_Read(stream, buf, &processed)); + return (processed == 1) ? SZ_OK : SZ_ERROR_INPUT_EOF; +} + + + +SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset) +{ + Int64 t = offset; + return ILookInStream_Seek(stream, &t, SZ_SEEK_SET); +} + +SRes LookInStream_LookRead(const ILookInStream *stream, void *buf, size_t *size) +{ + const void *lookBuf; + if (*size == 0) + return SZ_OK; + RINOK(ILookInStream_Look(stream, &lookBuf, size)); + memcpy(buf, lookBuf, *size); + return ILookInStream_Skip(stream, *size); +} + +SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRes errorType) +{ + while (size != 0) + { + size_t processed = size; + RINOK(ILookInStream_Read(stream, buf, &processed)); + if (processed == 0) + return errorType; + buf = (void *)((Byte *)buf + processed); + size -= processed; + } + return SZ_OK; +} + +SRes LookInStream_Read(const ILookInStream *stream, void *buf, size_t size) +{ + return LookInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); +} + + + +#define GET_LookToRead2 CLookToRead2 *p = CONTAINER_FROM_VTBL(pp, CLookToRead2, vt); + +static SRes LookToRead2_Look_Lookahead(const ILookInStream *pp, const void **buf, size_t *size) +{ + SRes res = SZ_OK; + GET_LookToRead2 + size_t size2 = p->size - p->pos; + if (size2 == 0 && *size != 0) + { + p->pos = 0; + p->size = 0; + size2 = p->bufSize; + res = ISeekInStream_Read(p->realStream, p->buf, &size2); + p->size = size2; + } + if (*size > size2) + *size = size2; + *buf = p->buf + p->pos; + return res; +} + +static SRes LookToRead2_Look_Exact(const ILookInStream *pp, const void **buf, size_t *size) +{ + SRes res = SZ_OK; + GET_LookToRead2 + size_t size2 = p->size - p->pos; + if (size2 == 0 && *size != 0) + { + p->pos = 0; + p->size = 0; + if (*size > p->bufSize) + *size = p->bufSize; + res = ISeekInStream_Read(p->realStream, p->buf, size); + size2 = p->size = *size; + } + if (*size > size2) + *size = size2; + *buf = p->buf + p->pos; + return res; +} + +static SRes LookToRead2_Skip(const ILookInStream *pp, size_t offset) +{ + GET_LookToRead2 + p->pos += offset; + return SZ_OK; +} + +static SRes LookToRead2_Read(const ILookInStream *pp, void *buf, size_t *size) +{ + GET_LookToRead2 + size_t rem = p->size - p->pos; + if (rem == 0) + return ISeekInStream_Read(p->realStream, buf, size); + if (rem > *size) + rem = *size; + memcpy(buf, p->buf + p->pos, rem); + p->pos += rem; + *size = rem; + return SZ_OK; +} + +static SRes LookToRead2_Seek(const ILookInStream *pp, Int64 *pos, ESzSeek origin) +{ + GET_LookToRead2 + p->pos = p->size = 0; + return ISeekInStream_Seek(p->realStream, pos, origin); +} + +void LookToRead2_CreateVTable(CLookToRead2 *p, int lookahead) +{ + p->vt.Look = lookahead ? + LookToRead2_Look_Lookahead : + LookToRead2_Look_Exact; + p->vt.Skip = LookToRead2_Skip; + p->vt.Read = LookToRead2_Read; + p->vt.Seek = LookToRead2_Seek; +} + + + +static SRes SecToLook_Read(const ISeqInStream *pp, void *buf, size_t *size) +{ + CSecToLook *p = CONTAINER_FROM_VTBL(pp, CSecToLook, vt); + return LookInStream_LookRead(p->realStream, buf, size); +} + +void SecToLook_CreateVTable(CSecToLook *p) +{ + p->vt.Read = SecToLook_Read; +} + +static SRes SecToRead_Read(const ISeqInStream *pp, void *buf, size_t *size) +{ + CSecToRead *p = CONTAINER_FROM_VTBL(pp, CSecToRead, vt); + return ILookInStream_Read(p->realStream, buf, size); +} + +void SecToRead_CreateVTable(CSecToRead *p) +{ + p->vt.Read = SecToRead_Read; +} diff --git a/code/nel/3rdparty/seven_zip/7zVersion.h b/code/nel/3rdparty/seven_zip/7zVersion.h new file mode 100644 index 000000000..c176823a4 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zVersion.h @@ -0,0 +1,27 @@ +#define MY_VER_MAJOR 19 +#define MY_VER_MINOR 00 +#define MY_VER_BUILD 0 +#define MY_VERSION_NUMBERS "19.00" +#define MY_VERSION MY_VERSION_NUMBERS + +#ifdef MY_CPU_NAME + #define MY_VERSION_CPU MY_VERSION " (" MY_CPU_NAME ")" +#else + #define MY_VERSION_CPU MY_VERSION +#endif + +#define MY_DATE "2019-02-21" +#undef MY_COPYRIGHT +#undef MY_VERSION_COPYRIGHT_DATE +#define MY_AUTHOR_NAME "Igor Pavlov" +#define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2018 Igor Pavlov" + +#ifdef USE_COPYRIGHT_CR + #define MY_COPYRIGHT MY_COPYRIGHT_CR +#else + #define MY_COPYRIGHT MY_COPYRIGHT_PD +#endif + +#define MY_COPYRIGHT_DATE MY_COPYRIGHT " : " MY_DATE +#define MY_VERSION_COPYRIGHT_DATE MY_VERSION_CPU " : " MY_COPYRIGHT " : " MY_DATE diff --git a/code/nel/3rdparty/seven_zip/7zVersion.rc b/code/nel/3rdparty/seven_zip/7zVersion.rc new file mode 100644 index 000000000..e520995dd --- /dev/null +++ b/code/nel/3rdparty/seven_zip/7zVersion.rc @@ -0,0 +1,55 @@ +#define MY_VS_FFI_FILEFLAGSMASK 0x0000003FL +#define MY_VOS_NT_WINDOWS32 0x00040004L +#define MY_VOS_CE_WINDOWS32 0x00050004L + +#define MY_VFT_APP 0x00000001L +#define MY_VFT_DLL 0x00000002L + +// #include + +#ifndef MY_VERSION +#include "7zVersion.h" +#endif + +#define MY_VER MY_VER_MAJOR,MY_VER_MINOR,MY_VER_BUILD,0 + +#ifdef DEBUG +#define DBG_FL VS_FF_DEBUG +#else +#define DBG_FL 0 +#endif + +#define MY_VERSION_INFO(fileType, descr, intName, origName) \ +LANGUAGE 9, 1 \ +1 VERSIONINFO \ + FILEVERSION MY_VER \ + PRODUCTVERSION MY_VER \ + FILEFLAGSMASK MY_VS_FFI_FILEFLAGSMASK \ + FILEFLAGS DBG_FL \ + FILEOS MY_VOS_NT_WINDOWS32 \ + FILETYPE fileType \ + FILESUBTYPE 0x0L \ +BEGIN \ + BLOCK "StringFileInfo" \ + BEGIN \ + BLOCK "040904b0" \ + BEGIN \ + VALUE "CompanyName", "Igor Pavlov" \ + VALUE "FileDescription", descr \ + VALUE "FileVersion", MY_VERSION \ + VALUE "InternalName", intName \ + VALUE "LegalCopyright", MY_COPYRIGHT \ + VALUE "OriginalFilename", origName \ + VALUE "ProductName", "7-Zip" \ + VALUE "ProductVersion", MY_VERSION \ + END \ + END \ + BLOCK "VarFileInfo" \ + BEGIN \ + VALUE "Translation", 0x409, 1200 \ + END \ +END + +#define MY_VERSION_INFO_APP(descr, intName) MY_VERSION_INFO(MY_VFT_APP, descr, intName, intName ".exe") + +#define MY_VERSION_INFO_DLL(descr, intName) MY_VERSION_INFO(MY_VFT_DLL, descr, intName, intName ".dll") diff --git a/code/nel/3rdparty/seven_zip/Aes.c b/code/nel/3rdparty/seven_zip/Aes.c new file mode 100644 index 000000000..1cdd0e787 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Aes.c @@ -0,0 +1,306 @@ +/* Aes.c -- AES encryption / decryption +2017-01-24 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "Aes.h" +#include "CpuArch.h" + +static UInt32 T[256 * 4]; +static const Byte Sbox[256] = { + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16}; + +void MY_FAST_CALL AesCbc_Encode(UInt32 *ivAes, Byte *data, size_t numBlocks); +void MY_FAST_CALL AesCbc_Decode(UInt32 *ivAes, Byte *data, size_t numBlocks); +void MY_FAST_CALL AesCtr_Code(UInt32 *ivAes, Byte *data, size_t numBlocks); + +void MY_FAST_CALL AesCbc_Encode_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); +void MY_FAST_CALL AesCbc_Decode_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); +void MY_FAST_CALL AesCtr_Code_Intel(UInt32 *ivAes, Byte *data, size_t numBlocks); + +AES_CODE_FUNC g_AesCbc_Encode; +AES_CODE_FUNC g_AesCbc_Decode; +AES_CODE_FUNC g_AesCtr_Code; + +static UInt32 D[256 * 4]; +static Byte InvS[256]; + +static const Byte Rcon[11] = { 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36 }; + +#define xtime(x) ((((x) << 1) ^ (((x) & 0x80) != 0 ? 0x1B : 0)) & 0xFF) + +#define Ui32(a0, a1, a2, a3) ((UInt32)(a0) | ((UInt32)(a1) << 8) | ((UInt32)(a2) << 16) | ((UInt32)(a3) << 24)) + +#define gb0(x) ( (x) & 0xFF) +#define gb1(x) (((x) >> ( 8)) & 0xFF) +#define gb2(x) (((x) >> (16)) & 0xFF) +#define gb3(x) (((x) >> (24))) + +#define gb(n, x) gb ## n(x) + +#define TT(x) (T + (x << 8)) +#define DD(x) (D + (x << 8)) + + +void AesGenTables(void) +{ + unsigned i; + for (i = 0; i < 256; i++) + InvS[Sbox[i]] = (Byte)i; + + for (i = 0; i < 256; i++) + { + { + UInt32 a1 = Sbox[i]; + UInt32 a2 = xtime(a1); + UInt32 a3 = a2 ^ a1; + TT(0)[i] = Ui32(a2, a1, a1, a3); + TT(1)[i] = Ui32(a3, a2, a1, a1); + TT(2)[i] = Ui32(a1, a3, a2, a1); + TT(3)[i] = Ui32(a1, a1, a3, a2); + } + { + UInt32 a1 = InvS[i]; + UInt32 a2 = xtime(a1); + UInt32 a4 = xtime(a2); + UInt32 a8 = xtime(a4); + UInt32 a9 = a8 ^ a1; + UInt32 aB = a8 ^ a2 ^ a1; + UInt32 aD = a8 ^ a4 ^ a1; + UInt32 aE = a8 ^ a4 ^ a2; + DD(0)[i] = Ui32(aE, a9, aD, aB); + DD(1)[i] = Ui32(aB, aE, a9, aD); + DD(2)[i] = Ui32(aD, aB, aE, a9); + DD(3)[i] = Ui32(a9, aD, aB, aE); + } + } + + g_AesCbc_Encode = AesCbc_Encode; + g_AesCbc_Decode = AesCbc_Decode; + g_AesCtr_Code = AesCtr_Code; + + #ifdef MY_CPU_X86_OR_AMD64 + if (CPU_Is_Aes_Supported()) + { + g_AesCbc_Encode = AesCbc_Encode_Intel; + g_AesCbc_Decode = AesCbc_Decode_Intel; + g_AesCtr_Code = AesCtr_Code_Intel; + } + #endif +} + + +#define HT(i, x, s) TT(x)[gb(x, s[(i + x) & 3])] + +#define HT4(m, i, s, p) m[i] = \ + HT(i, 0, s) ^ \ + HT(i, 1, s) ^ \ + HT(i, 2, s) ^ \ + HT(i, 3, s) ^ w[p + i] + +#define HT16(m, s, p) \ + HT4(m, 0, s, p); \ + HT4(m, 1, s, p); \ + HT4(m, 2, s, p); \ + HT4(m, 3, s, p); \ + +#define FT(i, x) Sbox[gb(x, m[(i + x) & 3])] +#define FT4(i) dest[i] = Ui32(FT(i, 0), FT(i, 1), FT(i, 2), FT(i, 3)) ^ w[i]; + + +#define HD(i, x, s) DD(x)[gb(x, s[(i - x) & 3])] + +#define HD4(m, i, s, p) m[i] = \ + HD(i, 0, s) ^ \ + HD(i, 1, s) ^ \ + HD(i, 2, s) ^ \ + HD(i, 3, s) ^ w[p + i]; + +#define HD16(m, s, p) \ + HD4(m, 0, s, p); \ + HD4(m, 1, s, p); \ + HD4(m, 2, s, p); \ + HD4(m, 3, s, p); \ + +#define FD(i, x) InvS[gb(x, m[(i - x) & 3])] +#define FD4(i) dest[i] = Ui32(FD(i, 0), FD(i, 1), FD(i, 2), FD(i, 3)) ^ w[i]; + +void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize) +{ + unsigned i, wSize; + wSize = keySize + 28; + keySize /= 4; + w[0] = ((UInt32)keySize / 2) + 3; + w += 4; + + for (i = 0; i < keySize; i++, key += 4) + w[i] = GetUi32(key); + + for (; i < wSize; i++) + { + UInt32 t = w[(size_t)i - 1]; + unsigned rem = i % keySize; + if (rem == 0) + t = Ui32(Sbox[gb1(t)] ^ Rcon[i / keySize], Sbox[gb2(t)], Sbox[gb3(t)], Sbox[gb0(t)]); + else if (keySize > 6 && rem == 4) + t = Ui32(Sbox[gb0(t)], Sbox[gb1(t)], Sbox[gb2(t)], Sbox[gb3(t)]); + w[i] = w[i - keySize] ^ t; + } +} + +void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *w, const Byte *key, unsigned keySize) +{ + unsigned i, num; + Aes_SetKey_Enc(w, key, keySize); + num = keySize + 20; + w += 8; + for (i = 0; i < num; i++) + { + UInt32 r = w[i]; + w[i] = + DD(0)[Sbox[gb0(r)]] ^ + DD(1)[Sbox[gb1(r)]] ^ + DD(2)[Sbox[gb2(r)]] ^ + DD(3)[Sbox[gb3(r)]]; + } +} + +/* Aes_Encode and Aes_Decode functions work with little-endian words. + src and dest are pointers to 4 UInt32 words. + src and dest can point to same block */ + +static void Aes_Encode(const UInt32 *w, UInt32 *dest, const UInt32 *src) +{ + UInt32 s[4]; + UInt32 m[4]; + UInt32 numRounds2 = w[0]; + w += 4; + s[0] = src[0] ^ w[0]; + s[1] = src[1] ^ w[1]; + s[2] = src[2] ^ w[2]; + s[3] = src[3] ^ w[3]; + w += 4; + for (;;) + { + HT16(m, s, 0); + if (--numRounds2 == 0) + break; + HT16(s, m, 4); + w += 8; + } + w += 4; + FT4(0); FT4(1); FT4(2); FT4(3); +} + +static void Aes_Decode(const UInt32 *w, UInt32 *dest, const UInt32 *src) +{ + UInt32 s[4]; + UInt32 m[4]; + UInt32 numRounds2 = w[0]; + w += 4 + numRounds2 * 8; + s[0] = src[0] ^ w[0]; + s[1] = src[1] ^ w[1]; + s[2] = src[2] ^ w[2]; + s[3] = src[3] ^ w[3]; + for (;;) + { + w -= 8; + HD16(m, s, 4); + if (--numRounds2 == 0) + break; + HD16(s, m, 0); + } + FD4(0); FD4(1); FD4(2); FD4(3); +} + +void AesCbc_Init(UInt32 *p, const Byte *iv) +{ + unsigned i; + for (i = 0; i < 4; i++) + p[i] = GetUi32(iv + i * 4); +} + +void MY_FAST_CALL AesCbc_Encode(UInt32 *p, Byte *data, size_t numBlocks) +{ + for (; numBlocks != 0; numBlocks--, data += AES_BLOCK_SIZE) + { + p[0] ^= GetUi32(data); + p[1] ^= GetUi32(data + 4); + p[2] ^= GetUi32(data + 8); + p[3] ^= GetUi32(data + 12); + + Aes_Encode(p + 4, p, p); + + SetUi32(data, p[0]); + SetUi32(data + 4, p[1]); + SetUi32(data + 8, p[2]); + SetUi32(data + 12, p[3]); + } +} + +void MY_FAST_CALL AesCbc_Decode(UInt32 *p, Byte *data, size_t numBlocks) +{ + UInt32 in[4], out[4]; + for (; numBlocks != 0; numBlocks--, data += AES_BLOCK_SIZE) + { + in[0] = GetUi32(data); + in[1] = GetUi32(data + 4); + in[2] = GetUi32(data + 8); + in[3] = GetUi32(data + 12); + + Aes_Decode(p + 4, out, in); + + SetUi32(data, p[0] ^ out[0]); + SetUi32(data + 4, p[1] ^ out[1]); + SetUi32(data + 8, p[2] ^ out[2]); + SetUi32(data + 12, p[3] ^ out[3]); + + p[0] = in[0]; + p[1] = in[1]; + p[2] = in[2]; + p[3] = in[3]; + } +} + +void MY_FAST_CALL AesCtr_Code(UInt32 *p, Byte *data, size_t numBlocks) +{ + for (; numBlocks != 0; numBlocks--) + { + UInt32 temp[4]; + unsigned i; + + if (++p[0] == 0) + p[1]++; + + Aes_Encode(p + 4, temp, p); + + for (i = 0; i < 4; i++, data += 4) + { + UInt32 t = temp[i]; + + #ifdef MY_CPU_LE_UNALIGN + *((UInt32 *)data) ^= t; + #else + data[0] ^= (t & 0xFF); + data[1] ^= ((t >> 8) & 0xFF); + data[2] ^= ((t >> 16) & 0xFF); + data[3] ^= ((t >> 24)); + #endif + } + } +} diff --git a/code/nel/3rdparty/seven_zip/Aes.h b/code/nel/3rdparty/seven_zip/Aes.h new file mode 100644 index 000000000..64979b5bc --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Aes.h @@ -0,0 +1,38 @@ +/* Aes.h -- AES encryption / decryption +2013-01-18 : Igor Pavlov : Public domain */ + +#ifndef __AES_H +#define __AES_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define AES_BLOCK_SIZE 16 + +/* Call AesGenTables one time before other AES functions */ +void AesGenTables(void); + +/* UInt32 pointers must be 16-byte aligned */ + +/* 16-byte (4 * 32-bit words) blocks: 1 (IV) + 1 (keyMode) + 15 (AES-256 roundKeys) */ +#define AES_NUM_IVMRK_WORDS ((1 + 1 + 15) * 4) + +/* aes - 16-byte aligned pointer to keyMode+roundKeys sequence */ +/* keySize = 16 or 24 or 32 (bytes) */ +typedef void (MY_FAST_CALL *AES_SET_KEY_FUNC)(UInt32 *aes, const Byte *key, unsigned keySize); +void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *aes, const Byte *key, unsigned keySize); +void MY_FAST_CALL Aes_SetKey_Dec(UInt32 *aes, const Byte *key, unsigned keySize); + +/* ivAes - 16-byte aligned pointer to iv+keyMode+roundKeys sequence: UInt32[AES_NUM_IVMRK_WORDS] */ +void AesCbc_Init(UInt32 *ivAes, const Byte *iv); /* iv size is AES_BLOCK_SIZE */ +/* data - 16-byte aligned pointer to data */ +/* numBlocks - the number of 16-byte blocks in data array */ +typedef void (MY_FAST_CALL *AES_CODE_FUNC)(UInt32 *ivAes, Byte *data, size_t numBlocks); +extern AES_CODE_FUNC g_AesCbc_Encode; +extern AES_CODE_FUNC g_AesCbc_Decode; +extern AES_CODE_FUNC g_AesCtr_Code; + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/AesOpt.c b/code/nel/3rdparty/seven_zip/AesOpt.c new file mode 100644 index 000000000..9571c467f --- /dev/null +++ b/code/nel/3rdparty/seven_zip/AesOpt.c @@ -0,0 +1,184 @@ +/* AesOpt.c -- Intel's AES +2017-06-08 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "CpuArch.h" + +#ifdef MY_CPU_X86_OR_AMD64 +#if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729) +#define USE_INTEL_AES +#endif +#endif + +#ifdef USE_INTEL_AES + +#include + +void MY_FAST_CALL AesCbc_Encode_Intel(__m128i *p, __m128i *data, size_t numBlocks) +{ + __m128i m = *p; + for (; numBlocks != 0; numBlocks--, data++) + { + UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; + const __m128i *w = p + 3; + m = _mm_xor_si128(m, *data); + m = _mm_xor_si128(m, p[2]); + do + { + m = _mm_aesenc_si128(m, w[0]); + m = _mm_aesenc_si128(m, w[1]); + w += 2; + } + while (--numRounds2 != 0); + m = _mm_aesenc_si128(m, w[0]); + m = _mm_aesenclast_si128(m, w[1]); + *data = m; + } + *p = m; +} + +#define NUM_WAYS 3 + +#define AES_OP_W(op, n) { \ + const __m128i t = w[n]; \ + m0 = op(m0, t); \ + m1 = op(m1, t); \ + m2 = op(m2, t); \ + } + +#define AES_DEC(n) AES_OP_W(_mm_aesdec_si128, n) +#define AES_DEC_LAST(n) AES_OP_W(_mm_aesdeclast_si128, n) +#define AES_ENC(n) AES_OP_W(_mm_aesenc_si128, n) +#define AES_ENC_LAST(n) AES_OP_W(_mm_aesenclast_si128, n) + +void MY_FAST_CALL AesCbc_Decode_Intel(__m128i *p, __m128i *data, size_t numBlocks) +{ + __m128i iv = *p; + for (; numBlocks >= NUM_WAYS; numBlocks -= NUM_WAYS, data += NUM_WAYS) + { + UInt32 numRounds2 = *(const UInt32 *)(p + 1); + const __m128i *w = p + numRounds2 * 2; + __m128i m0, m1, m2; + { + const __m128i t = w[2]; + m0 = _mm_xor_si128(t, data[0]); + m1 = _mm_xor_si128(t, data[1]); + m2 = _mm_xor_si128(t, data[2]); + } + numRounds2--; + do + { + AES_DEC(1) + AES_DEC(0) + w -= 2; + } + while (--numRounds2 != 0); + AES_DEC(1) + AES_DEC_LAST(0) + + { + __m128i t; + t = _mm_xor_si128(m0, iv); iv = data[0]; data[0] = t; + t = _mm_xor_si128(m1, iv); iv = data[1]; data[1] = t; + t = _mm_xor_si128(m2, iv); iv = data[2]; data[2] = t; + } + } + for (; numBlocks != 0; numBlocks--, data++) + { + UInt32 numRounds2 = *(const UInt32 *)(p + 1); + const __m128i *w = p + numRounds2 * 2; + __m128i m = _mm_xor_si128(w[2], *data); + numRounds2--; + do + { + m = _mm_aesdec_si128(m, w[1]); + m = _mm_aesdec_si128(m, w[0]); + w -= 2; + } + while (--numRounds2 != 0); + m = _mm_aesdec_si128(m, w[1]); + m = _mm_aesdeclast_si128(m, w[0]); + + m = _mm_xor_si128(m, iv); + iv = *data; + *data = m; + } + *p = iv; +} + +void MY_FAST_CALL AesCtr_Code_Intel(__m128i *p, __m128i *data, size_t numBlocks) +{ + __m128i ctr = *p; + __m128i one; + one.m128i_u64[0] = 1; + one.m128i_u64[1] = 0; + for (; numBlocks >= NUM_WAYS; numBlocks -= NUM_WAYS, data += NUM_WAYS) + { + UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; + const __m128i *w = p; + __m128i m0, m1, m2; + { + const __m128i t = w[2]; + ctr = _mm_add_epi64(ctr, one); m0 = _mm_xor_si128(ctr, t); + ctr = _mm_add_epi64(ctr, one); m1 = _mm_xor_si128(ctr, t); + ctr = _mm_add_epi64(ctr, one); m2 = _mm_xor_si128(ctr, t); + } + w += 3; + do + { + AES_ENC(0) + AES_ENC(1) + w += 2; + } + while (--numRounds2 != 0); + AES_ENC(0) + AES_ENC_LAST(1) + data[0] = _mm_xor_si128(data[0], m0); + data[1] = _mm_xor_si128(data[1], m1); + data[2] = _mm_xor_si128(data[2], m2); + } + for (; numBlocks != 0; numBlocks--, data++) + { + UInt32 numRounds2 = *(const UInt32 *)(p + 1) - 1; + const __m128i *w = p; + __m128i m; + ctr = _mm_add_epi64(ctr, one); + m = _mm_xor_si128(ctr, p[2]); + w += 3; + do + { + m = _mm_aesenc_si128(m, w[0]); + m = _mm_aesenc_si128(m, w[1]); + w += 2; + } + while (--numRounds2 != 0); + m = _mm_aesenc_si128(m, w[0]); + m = _mm_aesenclast_si128(m, w[1]); + *data = _mm_xor_si128(*data, m); + } + *p = ctr; +} + +#else + +void MY_FAST_CALL AesCbc_Encode(UInt32 *ivAes, Byte *data, size_t numBlocks); +void MY_FAST_CALL AesCbc_Decode(UInt32 *ivAes, Byte *data, size_t numBlocks); +void MY_FAST_CALL AesCtr_Code(UInt32 *ivAes, Byte *data, size_t numBlocks); + +void MY_FAST_CALL AesCbc_Encode_Intel(UInt32 *p, Byte *data, size_t numBlocks) +{ + AesCbc_Encode(p, data, numBlocks); +} + +void MY_FAST_CALL AesCbc_Decode_Intel(UInt32 *p, Byte *data, size_t numBlocks) +{ + AesCbc_Decode(p, data, numBlocks); +} + +void MY_FAST_CALL AesCtr_Code_Intel(UInt32 *p, Byte *data, size_t numBlocks) +{ + AesCtr_Code(p, data, numBlocks); +} + +#endif diff --git a/code/nel/3rdparty/seven_zip/Alloc.c b/code/nel/3rdparty/seven_zip/Alloc.c new file mode 100644 index 000000000..bcede4b85 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Alloc.c @@ -0,0 +1,455 @@ +/* Alloc.c -- Memory allocation functions +2018-04-27 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +#ifdef _WIN32 +#include +#endif +#include + +#include "Alloc.h" + +/* #define _SZ_ALLOC_DEBUG */ + +/* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ +#ifdef _SZ_ALLOC_DEBUG + +#include +int g_allocCount = 0; +int g_allocCountMid = 0; +int g_allocCountBig = 0; + + +#define CONVERT_INT_TO_STR(charType, tempSize) \ + unsigned char temp[tempSize]; unsigned i = 0; \ + while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \ + *s++ = (charType)('0' + (unsigned)val); \ + while (i != 0) { i--; *s++ = temp[i]; } \ + *s = 0; + +static void ConvertUInt64ToString(UInt64 val, char *s) +{ + CONVERT_INT_TO_STR(char, 24); +} + +#define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) + +static void ConvertUInt64ToHex(UInt64 val, char *s) +{ + UInt64 v = val; + unsigned i; + for (i = 1;; i++) + { + v >>= 4; + if (v == 0) + break; + } + s[i] = 0; + do + { + unsigned t = (unsigned)(val & 0xF); + val >>= 4; + s[--i] = GET_HEX_CHAR(t); + } + while (i); +} + +#define DEBUG_OUT_STREAM stderr + +static void Print(const char *s) +{ + fputs(s, DEBUG_OUT_STREAM); +} + +static void PrintAligned(const char *s, size_t align) +{ + size_t len = strlen(s); + for(;;) + { + fputc(' ', DEBUG_OUT_STREAM); + if (len >= align) + break; + ++len; + } + Print(s); +} + +static void PrintLn() +{ + Print("\n"); +} + +static void PrintHex(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToHex(v, s); + PrintAligned(s, align); +} + +static void PrintDec(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToString(v, s); + PrintAligned(s, align); +} + +static void PrintAddr(void *p) +{ + PrintHex((UInt64)(size_t)(ptrdiff_t)p, 12); +} + + +#define PRINT_ALLOC(name, cnt, size, ptr) \ + Print(name " "); \ + PrintDec(cnt++, 10); \ + PrintHex(size, 10); \ + PrintAddr(ptr); \ + PrintLn(); + +#define PRINT_FREE(name, cnt, ptr) if (ptr) { \ + Print(name " "); \ + PrintDec(--cnt, 10); \ + PrintAddr(ptr); \ + PrintLn(); } + +#else + +#define PRINT_ALLOC(name, cnt, size, ptr) +#define PRINT_FREE(name, cnt, ptr) +#define Print(s) +#define PrintLn() +#define PrintHex(v, align) +#define PrintDec(v, align) +#define PrintAddr(p) + +#endif + + + +void *MyAlloc(size_t size) +{ + if (size == 0) + return NULL; + #ifdef _SZ_ALLOC_DEBUG + { + void *p = malloc(size); + PRINT_ALLOC("Alloc ", g_allocCount, size, p); + return p; + } + #else + return malloc(size); + #endif +} + +void MyFree(void *address) +{ + PRINT_FREE("Free ", g_allocCount, address); + + free(address); +} + +#ifdef _WIN32 + +void *MidAlloc(size_t size) +{ + if (size == 0) + return NULL; + + PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, NULL); + + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); +} + +void MidFree(void *address) +{ + PRINT_FREE("Free-Mid", g_allocCountMid, address); + + if (!address) + return; + VirtualFree(address, 0, MEM_RELEASE); +} + +#ifndef MEM_LARGE_PAGES +#undef _7ZIP_LARGE_PAGES +#endif + +#ifdef _7ZIP_LARGE_PAGES +SIZE_T g_LargePageSize = 0; +typedef SIZE_T (WINAPI *GetLargePageMinimumP)(); +#endif + +void SetLargePageSize() +{ + #ifdef _7ZIP_LARGE_PAGES + SIZE_T size; + GetLargePageMinimumP largePageMinimum = (GetLargePageMinimumP) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum"); + if (!largePageMinimum) + return; + size = largePageMinimum(); + if (size == 0 || (size & (size - 1)) != 0) + return; + g_LargePageSize = size; + #endif +} + + +void *BigAlloc(size_t size) +{ + if (size == 0) + return NULL; + + PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL); + + #ifdef _7ZIP_LARGE_PAGES + { + SIZE_T ps = g_LargePageSize; + if (ps != 0 && ps <= (1 << 30) && size > (ps / 2)) + { + size_t size2; + ps--; + size2 = (size + ps) & ~ps; + if (size2 >= size) + { + void *res = VirtualAlloc(NULL, size2, MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); + if (res) + return res; + } + } + } + #endif + + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); +} + +void BigFree(void *address) +{ + PRINT_FREE("Free-Big", g_allocCountBig, address); + + if (!address) + return; + VirtualFree(address, 0, MEM_RELEASE); +} + +#endif + + +static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } +static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MyFree(address); } +const ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MidAlloc(size); } +static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MidFree(address); } +const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; + +static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } +static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); BigFree(address); } +const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; + + +/* + uintptr_t : C99 (optional) + : unsupported in VS6 +*/ + +#ifdef _WIN32 + typedef UINT_PTR UIntPtr; +#else + /* + typedef uintptr_t UIntPtr; + */ + typedef ptrdiff_t UIntPtr; +#endif + + +#define ADJUST_ALLOC_SIZE 0 +/* +#define ADJUST_ALLOC_SIZE (sizeof(void *) - 1) +*/ +/* + Use (ADJUST_ALLOC_SIZE = (sizeof(void *) - 1)), if + MyAlloc() can return address that is NOT multiple of sizeof(void *). +*/ + + +/* +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((char *)(p) - ((size_t)(UIntPtr)(p) & ((align) - 1)))) +*/ +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1)))) + +#define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align) + + +#if (_POSIX_C_SOURCE >= 200112L) && !defined(_WIN32) + #define USE_posix_memalign +#endif + +/* + This posix_memalign() is for test purposes only. + We also need special Free() function instead of free(), + if this posix_memalign() is used. +*/ + +/* +static int posix_memalign(void **ptr, size_t align, size_t size) +{ + size_t newSize = size + align; + void *p; + void *pAligned; + *ptr = NULL; + if (newSize < size) + return 12; // ENOMEM + p = MyAlloc(newSize); + if (!p) + return 12; // ENOMEM + pAligned = MY_ALIGN_PTR_UP_PLUS(p, align); + ((void **)pAligned)[-1] = p; + *ptr = pAligned; + return 0; +} +*/ + +/* + ALLOC_ALIGN_SIZE >= sizeof(void *) + ALLOC_ALIGN_SIZE >= cache_line_size +*/ + +#define ALLOC_ALIGN_SIZE ((size_t)1 << 7) + +static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) +{ + #ifndef USE_posix_memalign + + void *p; + void *pAligned; + size_t newSize; + UNUSED_VAR(pp); + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + + newSize = size + ALLOC_ALIGN_SIZE * 1 + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + p = MyAlloc(newSize); + + if (!p) + return NULL; + pAligned = MY_ALIGN_PTR_UP_PLUS(p, ALLOC_ALIGN_SIZE); + + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(p); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + ((void **)pAligned)[-1] = p; + + return pAligned; + + #else + + void *p; + UNUSED_VAR(pp); + if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size)) + return NULL; + + Print(" posix_memalign="); PrintAddr(p); + PrintLn(); + + return p; + + #endif +} + + +static void SzAlignedFree(ISzAllocPtr pp, void *address) +{ + UNUSED_VAR(pp); + #ifndef USE_posix_memalign + if (address) + MyFree(((void **)address)[-1]); + #else + free(address); + #endif +} + + +const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree }; + + + +#define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) + +/* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */ +#define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] +/* +#define REAL_BLOCK_PTR_VAR(p) ((void **)(p))[-1] +*/ + +static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) +{ + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + void *adr; + void *pAligned; + size_t newSize; + size_t extra; + size_t alignSize = (size_t)1 << p->numAlignBits; + + if (alignSize < sizeof(void *)) + alignSize = sizeof(void *); + + if (p->offset >= alignSize) + return NULL; + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + extra = p->offset & (sizeof(void *) - 1); + newSize = size + alignSize + extra + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + adr = ISzAlloc_Alloc(p->baseAlloc, newSize); + + if (!adr) + return NULL; + + pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr + + alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset; + + PrintLn(); + Print("- Aligned: "); + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(adr); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + REAL_BLOCK_PTR_VAR(pAligned) = adr; + + return pAligned; +} + + +static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address) +{ + if (address) + { + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + PrintLn(); + Print("- Aligned Free: "); + PrintLn(); + ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address)); + } +} + + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p) +{ + p->vt.Alloc = AlignOffsetAlloc_Alloc; + p->vt.Free = AlignOffsetAlloc_Free; +} diff --git a/code/nel/3rdparty/seven_zip/Alloc.h b/code/nel/3rdparty/seven_zip/Alloc.h new file mode 100644 index 000000000..648237646 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Alloc.h @@ -0,0 +1,51 @@ +/* Alloc.h -- Memory allocation functions +2018-02-19 : Igor Pavlov : Public domain */ + +#ifndef __COMMON_ALLOC_H +#define __COMMON_ALLOC_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +void *MyAlloc(size_t size); +void MyFree(void *address); + +#ifdef _WIN32 + +void SetLargePageSize(); + +void *MidAlloc(size_t size); +void MidFree(void *address); +void *BigAlloc(size_t size); +void BigFree(void *address); + +#else + +#define MidAlloc(size) MyAlloc(size) +#define MidFree(address) MyFree(address) +#define BigAlloc(size) MyAlloc(size) +#define BigFree(address) MyFree(address) + +#endif + +extern const ISzAlloc g_Alloc; +extern const ISzAlloc g_BigAlloc; +extern const ISzAlloc g_MidAlloc; +extern const ISzAlloc g_AlignedAlloc; + + +typedef struct +{ + ISzAlloc vt; + ISzAllocPtr baseAlloc; + unsigned numAlignBits; /* ((1 << numAlignBits) >= sizeof(void *)) */ + size_t offset; /* (offset == (k * sizeof(void *)) && offset < (1 << numAlignBits) */ +} CAlignOffsetAlloc; + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p); + + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/Bcj2.c b/code/nel/3rdparty/seven_zip/Bcj2.c new file mode 100644 index 000000000..9a0046a65 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Bcj2.c @@ -0,0 +1,257 @@ +/* Bcj2.c -- BCJ2 Decoder (Converter for x86 code) +2018-04-28 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "Bcj2.h" +#include "CpuArch.h" + +#define CProb UInt16 + +#define kTopValue ((UInt32)1 << 24) +#define kNumModelBits 11 +#define kBitModelTotal (1 << kNumModelBits) +#define kNumMoveBits 5 + +#define _IF_BIT_0 ttt = *prob; bound = (p->range >> kNumModelBits) * ttt; if (p->code < bound) +#define _UPDATE_0 p->range = bound; *prob = (CProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); +#define _UPDATE_1 p->range -= bound; p->code -= bound; *prob = (CProb)(ttt - (ttt >> kNumMoveBits)); + +void Bcj2Dec_Init(CBcj2Dec *p) +{ + unsigned i; + + p->state = BCJ2_DEC_STATE_OK; + p->ip = 0; + p->temp[3] = 0; + p->range = 0; + p->code = 0; + for (i = 0; i < sizeof(p->probs) / sizeof(p->probs[0]); i++) + p->probs[i] = kBitModelTotal >> 1; +} + +SRes Bcj2Dec_Decode(CBcj2Dec *p) +{ + if (p->range <= 5) + { + p->state = BCJ2_DEC_STATE_OK; + for (; p->range != 5; p->range++) + { + if (p->range == 1 && p->code != 0) + return SZ_ERROR_DATA; + + if (p->bufs[BCJ2_STREAM_RC] == p->lims[BCJ2_STREAM_RC]) + { + p->state = BCJ2_STREAM_RC; + return SZ_OK; + } + + p->code = (p->code << 8) | *(p->bufs[BCJ2_STREAM_RC])++; + } + + if (p->code == 0xFFFFFFFF) + return SZ_ERROR_DATA; + + p->range = 0xFFFFFFFF; + } + else if (p->state >= BCJ2_DEC_STATE_ORIG_0) + { + while (p->state <= BCJ2_DEC_STATE_ORIG_3) + { + Byte *dest = p->dest; + if (dest == p->destLim) + return SZ_OK; + *dest = p->temp[(size_t)p->state - BCJ2_DEC_STATE_ORIG_0]; + p->state++; + p->dest = dest + 1; + } + } + + /* + if (BCJ2_IS_32BIT_STREAM(p->state)) + { + const Byte *cur = p->bufs[p->state]; + if (cur == p->lims[p->state]) + return SZ_OK; + p->bufs[p->state] = cur + 4; + + { + UInt32 val; + Byte *dest; + SizeT rem; + + p->ip += 4; + val = GetBe32(cur) - p->ip; + dest = p->dest; + rem = p->destLim - dest; + if (rem < 4) + { + SizeT i; + SetUi32(p->temp, val); + for (i = 0; i < rem; i++) + dest[i] = p->temp[i]; + p->dest = dest + rem; + p->state = BCJ2_DEC_STATE_ORIG_0 + (unsigned)rem; + return SZ_OK; + } + SetUi32(dest, val); + p->temp[3] = (Byte)(val >> 24); + p->dest = dest + 4; + p->state = BCJ2_DEC_STATE_OK; + } + } + */ + + for (;;) + { + if (BCJ2_IS_32BIT_STREAM(p->state)) + p->state = BCJ2_DEC_STATE_OK; + else + { + if (p->range < kTopValue) + { + if (p->bufs[BCJ2_STREAM_RC] == p->lims[BCJ2_STREAM_RC]) + { + p->state = BCJ2_STREAM_RC; + return SZ_OK; + } + p->range <<= 8; + p->code = (p->code << 8) | *(p->bufs[BCJ2_STREAM_RC])++; + } + + { + const Byte *src = p->bufs[BCJ2_STREAM_MAIN]; + const Byte *srcLim; + Byte *dest; + SizeT num = p->lims[BCJ2_STREAM_MAIN] - src; + + if (num == 0) + { + p->state = BCJ2_STREAM_MAIN; + return SZ_OK; + } + + dest = p->dest; + if (num > (SizeT)(p->destLim - dest)) + { + num = p->destLim - dest; + if (num == 0) + { + p->state = BCJ2_DEC_STATE_ORIG; + return SZ_OK; + } + } + + srcLim = src + num; + + if (p->temp[3] == 0x0F && (src[0] & 0xF0) == 0x80) + *dest = src[0]; + else for (;;) + { + Byte b = *src; + *dest = b; + if (b != 0x0F) + { + if ((b & 0xFE) == 0xE8) + break; + dest++; + if (++src != srcLim) + continue; + break; + } + dest++; + if (++src == srcLim) + break; + if ((*src & 0xF0) != 0x80) + continue; + *dest = *src; + break; + } + + num = src - p->bufs[BCJ2_STREAM_MAIN]; + + if (src == srcLim) + { + p->temp[3] = src[-1]; + p->bufs[BCJ2_STREAM_MAIN] = src; + p->ip += (UInt32)num; + p->dest += num; + p->state = + p->bufs[BCJ2_STREAM_MAIN] == + p->lims[BCJ2_STREAM_MAIN] ? + (unsigned)BCJ2_STREAM_MAIN : + (unsigned)BCJ2_DEC_STATE_ORIG; + return SZ_OK; + } + + { + UInt32 bound, ttt; + CProb *prob; + Byte b = src[0]; + Byte prev = (Byte)(num == 0 ? p->temp[3] : src[-1]); + + p->temp[3] = b; + p->bufs[BCJ2_STREAM_MAIN] = src + 1; + num++; + p->ip += (UInt32)num; + p->dest += num; + + prob = p->probs + (unsigned)(b == 0xE8 ? 2 + (unsigned)prev : (b == 0xE9 ? 1 : 0)); + + _IF_BIT_0 + { + _UPDATE_0 + continue; + } + _UPDATE_1 + + } + } + } + + { + UInt32 val; + unsigned cj = (p->temp[3] == 0xE8) ? BCJ2_STREAM_CALL : BCJ2_STREAM_JUMP; + const Byte *cur = p->bufs[cj]; + Byte *dest; + SizeT rem; + + if (cur == p->lims[cj]) + { + p->state = cj; + break; + } + + val = GetBe32(cur); + p->bufs[cj] = cur + 4; + + p->ip += 4; + val -= p->ip; + dest = p->dest; + rem = p->destLim - dest; + + if (rem < 4) + { + p->temp[0] = (Byte)val; if (rem > 0) dest[0] = (Byte)val; val >>= 8; + p->temp[1] = (Byte)val; if (rem > 1) dest[1] = (Byte)val; val >>= 8; + p->temp[2] = (Byte)val; if (rem > 2) dest[2] = (Byte)val; val >>= 8; + p->temp[3] = (Byte)val; + p->dest = dest + rem; + p->state = BCJ2_DEC_STATE_ORIG_0 + (unsigned)rem; + break; + } + + SetUi32(dest, val); + p->temp[3] = (Byte)(val >> 24); + p->dest = dest + 4; + } + } + + if (p->range < kTopValue && p->bufs[BCJ2_STREAM_RC] != p->lims[BCJ2_STREAM_RC]) + { + p->range <<= 8; + p->code = (p->code << 8) | *(p->bufs[BCJ2_STREAM_RC])++; + } + + return SZ_OK; +} diff --git a/code/nel/3rdparty/seven_zip/Bcj2.h b/code/nel/3rdparty/seven_zip/Bcj2.h new file mode 100644 index 000000000..8824080ac --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Bcj2.h @@ -0,0 +1,146 @@ +/* Bcj2.h -- BCJ2 Converter for x86 code +2014-11-10 : Igor Pavlov : Public domain */ + +#ifndef __BCJ2_H +#define __BCJ2_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define BCJ2_NUM_STREAMS 4 + +enum +{ + BCJ2_STREAM_MAIN, + BCJ2_STREAM_CALL, + BCJ2_STREAM_JUMP, + BCJ2_STREAM_RC +}; + +enum +{ + BCJ2_DEC_STATE_ORIG_0 = BCJ2_NUM_STREAMS, + BCJ2_DEC_STATE_ORIG_1, + BCJ2_DEC_STATE_ORIG_2, + BCJ2_DEC_STATE_ORIG_3, + + BCJ2_DEC_STATE_ORIG, + BCJ2_DEC_STATE_OK +}; + +enum +{ + BCJ2_ENC_STATE_ORIG = BCJ2_NUM_STREAMS, + BCJ2_ENC_STATE_OK +}; + + +#define BCJ2_IS_32BIT_STREAM(s) ((s) == BCJ2_STREAM_CALL || (s) == BCJ2_STREAM_JUMP) + +/* +CBcj2Dec / CBcj2Enc +bufs sizes: + BUF_SIZE(n) = lims[n] - bufs[n] +bufs sizes for BCJ2_STREAM_CALL and BCJ2_STREAM_JUMP must be mutliply of 4: + (BUF_SIZE(BCJ2_STREAM_CALL) & 3) == 0 + (BUF_SIZE(BCJ2_STREAM_JUMP) & 3) == 0 +*/ + +/* +CBcj2Dec: +dest is allowed to overlap with bufs[BCJ2_STREAM_MAIN], with the following conditions: + bufs[BCJ2_STREAM_MAIN] >= dest && + bufs[BCJ2_STREAM_MAIN] - dest >= tempReserv + + BUF_SIZE(BCJ2_STREAM_CALL) + + BUF_SIZE(BCJ2_STREAM_JUMP) + tempReserv = 0 : for first call of Bcj2Dec_Decode + tempReserv = 4 : for any other calls of Bcj2Dec_Decode + overlap with offset = 1 is not allowed +*/ + +typedef struct +{ + const Byte *bufs[BCJ2_NUM_STREAMS]; + const Byte *lims[BCJ2_NUM_STREAMS]; + Byte *dest; + const Byte *destLim; + + unsigned state; /* BCJ2_STREAM_MAIN has more priority than BCJ2_STATE_ORIG */ + + UInt32 ip; + Byte temp[4]; + UInt32 range; + UInt32 code; + UInt16 probs[2 + 256]; +} CBcj2Dec; + +void Bcj2Dec_Init(CBcj2Dec *p); + +/* Returns: SZ_OK or SZ_ERROR_DATA */ +SRes Bcj2Dec_Decode(CBcj2Dec *p); + +#define Bcj2Dec_IsFinished(_p_) ((_p_)->code == 0) + + + +typedef enum +{ + BCJ2_ENC_FINISH_MODE_CONTINUE, + BCJ2_ENC_FINISH_MODE_END_BLOCK, + BCJ2_ENC_FINISH_MODE_END_STREAM +} EBcj2Enc_FinishMode; + +typedef struct +{ + Byte *bufs[BCJ2_NUM_STREAMS]; + const Byte *lims[BCJ2_NUM_STREAMS]; + const Byte *src; + const Byte *srcLim; + + unsigned state; + EBcj2Enc_FinishMode finishMode; + + Byte prevByte; + + Byte cache; + UInt32 range; + UInt64 low; + UInt64 cacheSize; + + UInt32 ip; + + /* 32-bit ralative offset in JUMP/CALL commands is + - (mod 4 GB) in 32-bit mode + - signed Int32 in 64-bit mode + We use (mod 4 GB) check for fileSize. + Use fileSize up to 2 GB, if you want to support 32-bit and 64-bit code conversion. */ + UInt32 fileIp; + UInt32 fileSize; /* (fileSize <= ((UInt32)1 << 31)), 0 means no_limit */ + UInt32 relatLimit; /* (relatLimit <= ((UInt32)1 << 31)), 0 means desable_conversion */ + + UInt32 tempTarget; + unsigned tempPos; + Byte temp[4 * 2]; + + unsigned flushPos; + + UInt16 probs[2 + 256]; +} CBcj2Enc; + +void Bcj2Enc_Init(CBcj2Enc *p); +void Bcj2Enc_Encode(CBcj2Enc *p); + +#define Bcj2Enc_Get_InputData_Size(p) ((SizeT)((p)->srcLim - (p)->src) + (p)->tempPos) +#define Bcj2Enc_IsFinished(p) ((p)->flushPos == 5) + + +#define BCJ2_RELAT_LIMIT_NUM_BITS 26 +#define BCJ2_RELAT_LIMIT ((UInt32)1 << BCJ2_RELAT_LIMIT_NUM_BITS) + +/* limit for CBcj2Enc::fileSize variable */ +#define BCJ2_FileSize_MAX ((UInt32)1 << 31) + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/Bcj2Enc.c b/code/nel/3rdparty/seven_zip/Bcj2Enc.c new file mode 100644 index 000000000..bfbeb8e49 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Bcj2Enc.c @@ -0,0 +1,311 @@ +/* Bcj2Enc.c -- BCJ2 Encoder (Converter for x86 code) +2019-02-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +/* #define SHOW_STAT */ + +#ifdef SHOW_STAT +#include +#define PRF(x) x +#else +#define PRF(x) +#endif + +#include + +#include "Bcj2.h" +#include "CpuArch.h" + +#define CProb UInt16 + +#define kTopValue ((UInt32)1 << 24) +#define kNumModelBits 11 +#define kBitModelTotal (1 << kNumModelBits) +#define kNumMoveBits 5 + +void Bcj2Enc_Init(CBcj2Enc *p) +{ + unsigned i; + + p->state = BCJ2_ENC_STATE_OK; + p->finishMode = BCJ2_ENC_FINISH_MODE_CONTINUE; + + p->prevByte = 0; + + p->cache = 0; + p->range = 0xFFFFFFFF; + p->low = 0; + p->cacheSize = 1; + + p->ip = 0; + + p->fileIp = 0; + p->fileSize = 0; + p->relatLimit = BCJ2_RELAT_LIMIT; + + p->tempPos = 0; + + p->flushPos = 0; + + for (i = 0; i < sizeof(p->probs) / sizeof(p->probs[0]); i++) + p->probs[i] = kBitModelTotal >> 1; +} + +static BoolInt MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) +{ + if ((UInt32)p->low < (UInt32)0xFF000000 || (UInt32)(p->low >> 32) != 0) + { + Byte *buf = p->bufs[BCJ2_STREAM_RC]; + do + { + if (buf == p->lims[BCJ2_STREAM_RC]) + { + p->state = BCJ2_STREAM_RC; + p->bufs[BCJ2_STREAM_RC] = buf; + return True; + } + *buf++ = (Byte)(p->cache + (Byte)(p->low >> 32)); + p->cache = 0xFF; + } + while (--p->cacheSize); + p->bufs[BCJ2_STREAM_RC] = buf; + p->cache = (Byte)((UInt32)p->low >> 24); + } + p->cacheSize++; + p->low = (UInt32)p->low << 8; + return False; +} + +static void Bcj2Enc_Encode_2(CBcj2Enc *p) +{ + if (BCJ2_IS_32BIT_STREAM(p->state)) + { + Byte *cur = p->bufs[p->state]; + if (cur == p->lims[p->state]) + return; + SetBe32(cur, p->tempTarget); + p->bufs[p->state] = cur + 4; + } + + p->state = BCJ2_ENC_STATE_ORIG; + + for (;;) + { + if (p->range < kTopValue) + { + if (RangeEnc_ShiftLow(p)) + return; + p->range <<= 8; + } + + { + { + const Byte *src = p->src; + const Byte *srcLim; + Byte *dest; + SizeT num = p->srcLim - src; + + if (p->finishMode == BCJ2_ENC_FINISH_MODE_CONTINUE) + { + if (num <= 4) + return; + num -= 4; + } + else if (num == 0) + break; + + dest = p->bufs[BCJ2_STREAM_MAIN]; + if (num > (SizeT)(p->lims[BCJ2_STREAM_MAIN] - dest)) + { + num = p->lims[BCJ2_STREAM_MAIN] - dest; + if (num == 0) + { + p->state = BCJ2_STREAM_MAIN; + return; + } + } + + srcLim = src + num; + + if (p->prevByte == 0x0F && (src[0] & 0xF0) == 0x80) + *dest = src[0]; + else for (;;) + { + Byte b = *src; + *dest = b; + if (b != 0x0F) + { + if ((b & 0xFE) == 0xE8) + break; + dest++; + if (++src != srcLim) + continue; + break; + } + dest++; + if (++src == srcLim) + break; + if ((*src & 0xF0) != 0x80) + continue; + *dest = *src; + break; + } + + num = src - p->src; + + if (src == srcLim) + { + p->prevByte = src[-1]; + p->bufs[BCJ2_STREAM_MAIN] = dest; + p->src = src; + p->ip += (UInt32)num; + continue; + } + + { + Byte context = (Byte)(num == 0 ? p->prevByte : src[-1]); + BoolInt needConvert; + + p->bufs[BCJ2_STREAM_MAIN] = dest + 1; + p->ip += (UInt32)num + 1; + src++; + + needConvert = False; + + if ((SizeT)(p->srcLim - src) >= 4) + { + UInt32 relatVal = GetUi32(src); + if ((p->fileSize == 0 || (UInt32)(p->ip + 4 + relatVal - p->fileIp) < p->fileSize) + && ((relatVal + p->relatLimit) >> 1) < p->relatLimit) + needConvert = True; + } + + { + UInt32 bound; + unsigned ttt; + Byte b = src[-1]; + CProb *prob = p->probs + (unsigned)(b == 0xE8 ? 2 + (unsigned)context : (b == 0xE9 ? 1 : 0)); + + ttt = *prob; + bound = (p->range >> kNumModelBits) * ttt; + + if (!needConvert) + { + p->range = bound; + *prob = (CProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); + p->src = src; + p->prevByte = b; + continue; + } + + p->low += bound; + p->range -= bound; + *prob = (CProb)(ttt - (ttt >> kNumMoveBits)); + + { + UInt32 relatVal = GetUi32(src); + UInt32 absVal; + p->ip += 4; + absVal = p->ip + relatVal; + p->prevByte = src[3]; + src += 4; + p->src = src; + { + unsigned cj = (b == 0xE8) ? BCJ2_STREAM_CALL : BCJ2_STREAM_JUMP; + Byte *cur = p->bufs[cj]; + if (cur == p->lims[cj]) + { + p->state = cj; + p->tempTarget = absVal; + return; + } + SetBe32(cur, absVal); + p->bufs[cj] = cur + 4; + } + } + } + } + } + } + } + + if (p->finishMode != BCJ2_ENC_FINISH_MODE_END_STREAM) + return; + + for (; p->flushPos < 5; p->flushPos++) + if (RangeEnc_ShiftLow(p)) + return; + p->state = BCJ2_ENC_STATE_OK; +} + + +void Bcj2Enc_Encode(CBcj2Enc *p) +{ + PRF(printf("\n")); + PRF(printf("---- ip = %8d tempPos = %8d src = %8d\n", p->ip, p->tempPos, p->srcLim - p->src)); + + if (p->tempPos != 0) + { + unsigned extra = 0; + + for (;;) + { + const Byte *src = p->src; + const Byte *srcLim = p->srcLim; + EBcj2Enc_FinishMode finishMode = p->finishMode; + + p->src = p->temp; + p->srcLim = p->temp + p->tempPos; + if (src != srcLim) + p->finishMode = BCJ2_ENC_FINISH_MODE_CONTINUE; + + PRF(printf(" ip = %8d tempPos = %8d src = %8d\n", p->ip, p->tempPos, p->srcLim - p->src)); + + Bcj2Enc_Encode_2(p); + + { + unsigned num = (unsigned)(p->src - p->temp); + unsigned tempPos = p->tempPos - num; + unsigned i; + p->tempPos = tempPos; + for (i = 0; i < tempPos; i++) + p->temp[i] = p->temp[(size_t)i + num]; + + p->src = src; + p->srcLim = srcLim; + p->finishMode = finishMode; + + if (p->state != BCJ2_ENC_STATE_ORIG || src == srcLim) + return; + + if (extra >= tempPos) + { + p->src = src - tempPos; + p->tempPos = 0; + break; + } + + p->temp[tempPos] = src[0]; + p->tempPos = tempPos + 1; + p->src = src + 1; + extra++; + } + } + } + + PRF(printf("++++ ip = %8d tempPos = %8d src = %8d\n", p->ip, p->tempPos, p->srcLim - p->src)); + + Bcj2Enc_Encode_2(p); + + if (p->state == BCJ2_ENC_STATE_ORIG) + { + const Byte *src = p->src; + unsigned rem = (unsigned)(p->srcLim - src); + unsigned i; + for (i = 0; i < rem; i++) + p->temp[i] = src[i]; + p->tempPos = rem; + p->src = src + rem; + } +} diff --git a/code/nel/3rdparty/seven_zip/Bra.c b/code/nel/3rdparty/seven_zip/Bra.c new file mode 100644 index 000000000..aed17e330 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Bra.c @@ -0,0 +1,230 @@ +/* Bra.c -- Converters for RISC code +2017-04-04 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "CpuArch.h" +#include "Bra.h" + +SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +{ + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip += 4; + p = data; + lim = data + size; + + if (encoding) + + for (;;) + { + for (;;) + { + if (p >= lim) + return p - data; + p += 4; + if (p[-1] == 0xEB) + break; + } + { + UInt32 v = GetUi32(p - 4); + v <<= 2; + v += ip + (UInt32)(p - data); + v >>= 2; + v &= 0x00FFFFFF; + v |= 0xEB000000; + SetUi32(p - 4, v); + } + } + + for (;;) + { + for (;;) + { + if (p >= lim) + return p - data; + p += 4; + if (p[-1] == 0xEB) + break; + } + { + UInt32 v = GetUi32(p - 4); + v <<= 2; + v -= ip + (UInt32)(p - data); + v >>= 2; + v &= 0x00FFFFFF; + v |= 0xEB000000; + SetUi32(p - 4, v); + } + } +} + + +SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +{ + Byte *p; + const Byte *lim; + size &= ~(size_t)1; + p = data; + lim = data + size - 4; + + if (encoding) + + for (;;) + { + UInt32 b1; + for (;;) + { + UInt32 b3; + if (p > lim) + return p - data; + b1 = p[1]; + b3 = p[3]; + p += 2; + b1 ^= 8; + if ((b3 & b1) >= 0xF8) + break; + } + { + UInt32 v = + ((UInt32)b1 << 19) + + (((UInt32)p[1] & 0x7) << 8) + + (((UInt32)p[-2] << 11)) + + (p[0]); + + p += 2; + { + UInt32 cur = (ip + (UInt32)(p - data)) >> 1; + v += cur; + } + + p[-4] = (Byte)(v >> 11); + p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); + p[-2] = (Byte)v; + p[-1] = (Byte)(0xF8 | (v >> 8)); + } + } + + for (;;) + { + UInt32 b1; + for (;;) + { + UInt32 b3; + if (p > lim) + return p - data; + b1 = p[1]; + b3 = p[3]; + p += 2; + b1 ^= 8; + if ((b3 & b1) >= 0xF8) + break; + } + { + UInt32 v = + ((UInt32)b1 << 19) + + (((UInt32)p[1] & 0x7) << 8) + + (((UInt32)p[-2] << 11)) + + (p[0]); + + p += 2; + { + UInt32 cur = (ip + (UInt32)(p - data)) >> 1; + v -= cur; + } + + /* + SetUi16(p - 4, (UInt16)(((v >> 11) & 0x7FF) | 0xF000)); + SetUi16(p - 2, (UInt16)(v | 0xF800)); + */ + + p[-4] = (Byte)(v >> 11); + p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); + p[-2] = (Byte)v; + p[-1] = (Byte)(0xF8 | (v >> 8)); + } + } +} + + +SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +{ + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip -= 4; + p = data; + lim = data + size; + + for (;;) + { + for (;;) + { + if (p >= lim) + return p - data; + p += 4; + /* if ((v & 0xFC000003) == 0x48000001) */ + if ((p[-4] & 0xFC) == 0x48 && (p[-1] & 3) == 1) + break; + } + { + UInt32 v = GetBe32(p - 4); + if (encoding) + v += ip + (UInt32)(p - data); + else + v -= ip + (UInt32)(p - data); + v &= 0x03FFFFFF; + v |= 0x48000000; + SetBe32(p - 4, v); + } + } +} + + +SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +{ + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip -= 4; + p = data; + lim = data + size; + + for (;;) + { + for (;;) + { + if (p >= lim) + return p - data; + /* + v = GetBe32(p); + p += 4; + m = v + ((UInt32)5 << 29); + m ^= (UInt32)7 << 29; + m += (UInt32)1 << 22; + if ((m & ((UInt32)0x1FF << 23)) == 0) + break; + */ + p += 4; + if ((p[-4] == 0x40 && (p[-3] & 0xC0) == 0) || + (p[-4] == 0x7F && (p[-3] >= 0xC0))) + break; + } + { + UInt32 v = GetBe32(p - 4); + v <<= 2; + if (encoding) + v += ip + (UInt32)(p - data); + else + v -= ip + (UInt32)(p - data); + + v &= 0x01FFFFFF; + v -= (UInt32)1 << 24; + v ^= 0xFF000000; + v >>= 2; + v |= 0x40000000; + SetBe32(p - 4, v); + } + } +} diff --git a/code/nel/3rdparty/seven_zip/Bra.h b/code/nel/3rdparty/seven_zip/Bra.h new file mode 100644 index 000000000..855e37a6b --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Bra.h @@ -0,0 +1,64 @@ +/* Bra.h -- Branch converters for executables +2013-01-18 : Igor Pavlov : Public domain */ + +#ifndef __BRA_H +#define __BRA_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +/* +These functions convert relative addresses to absolute addresses +in CALL instructions to increase the compression ratio. + + In: + data - data buffer + size - size of data + ip - current virtual Instruction Pinter (IP) value + state - state variable for x86 converter + encoding - 0 (for decoding), 1 (for encoding) + + Out: + state - state variable for x86 converter + + Returns: + The number of processed bytes. If you call these functions with multiple calls, + you must start next call with first byte after block of processed bytes. + + Type Endian Alignment LookAhead + + x86 little 1 4 + ARMT little 2 2 + ARM little 4 0 + PPC big 4 0 + SPARC big 4 0 + IA64 little 16 0 + + size must be >= Alignment + LookAhead, if it's not last block. + If (size < Alignment + LookAhead), converter returns 0. + + Example: + + UInt32 ip = 0; + for () + { + ; size must be >= Alignment + LookAhead, if it's not last block + SizeT processed = Convert(data, size, ip, 1); + data += processed; + size -= processed; + ip += processed; + } +*/ + +#define x86_Convert_Init(state) { state = 0; } +SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding); +SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); +SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); +SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); +SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); +SizeT IA64_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/Bra86.c b/code/nel/3rdparty/seven_zip/Bra86.c new file mode 100644 index 000000000..93ed4d762 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Bra86.c @@ -0,0 +1,82 @@ +/* Bra86.c -- Converter for x86 code (BCJ) +2017-04-03 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "Bra.h" + +#define Test86MSByte(b) ((((b) + 1) & 0xFE) == 0) + +SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding) +{ + SizeT pos = 0; + UInt32 mask = *state & 7; + if (size < 5) + return 0; + size -= 4; + ip += 5; + + for (;;) + { + Byte *p = data + pos; + const Byte *limit = data + size; + for (; p < limit; p++) + if ((*p & 0xFE) == 0xE8) + break; + + { + SizeT d = (SizeT)(p - data - pos); + pos = (SizeT)(p - data); + if (p >= limit) + { + *state = (d > 2 ? 0 : mask >> (unsigned)d); + return pos; + } + if (d > 2) + mask = 0; + else + { + mask >>= (unsigned)d; + if (mask != 0 && (mask > 4 || mask == 3 || Test86MSByte(p[(size_t)(mask >> 1) + 1]))) + { + mask = (mask >> 1) | 4; + pos++; + continue; + } + } + } + + if (Test86MSByte(p[4])) + { + UInt32 v = ((UInt32)p[4] << 24) | ((UInt32)p[3] << 16) | ((UInt32)p[2] << 8) | ((UInt32)p[1]); + UInt32 cur = ip + (UInt32)pos; + pos += 5; + if (encoding) + v += cur; + else + v -= cur; + if (mask != 0) + { + unsigned sh = (mask & 6) << 2; + if (Test86MSByte((Byte)(v >> sh))) + { + v ^= (((UInt32)0x100 << sh) - 1); + if (encoding) + v += cur; + else + v -= cur; + } + mask = 0; + } + p[1] = (Byte)v; + p[2] = (Byte)(v >> 8); + p[3] = (Byte)(v >> 16); + p[4] = (Byte)(0 - ((v >> 24) & 1)); + } + else + { + mask = (mask >> 1) | 4; + pos++; + } + } +} diff --git a/code/nel/3rdparty/seven_zip/BraIA64.c b/code/nel/3rdparty/seven_zip/BraIA64.c new file mode 100644 index 000000000..d1dbc62c5 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/BraIA64.c @@ -0,0 +1,53 @@ +/* BraIA64.c -- Converter for IA-64 code +2017-01-26 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "CpuArch.h" +#include "Bra.h" + +SizeT IA64_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) +{ + SizeT i; + if (size < 16) + return 0; + size -= 16; + i = 0; + do + { + unsigned m = ((UInt32)0x334B0000 >> (data[i] & 0x1E)) & 3; + if (m) + { + m++; + do + { + Byte *p = data + (i + (size_t)m * 5 - 8); + if (((p[3] >> m) & 15) == 5 + && (((p[-1] | ((UInt32)p[0] << 8)) >> m) & 0x70) == 0) + { + unsigned raw = GetUi32(p); + unsigned v = raw >> m; + v = (v & 0xFFFFF) | ((v & (1 << 23)) >> 3); + + v <<= 4; + if (encoding) + v += ip + (UInt32)i; + else + v -= ip + (UInt32)i; + v >>= 4; + + v &= 0x1FFFFF; + v += 0x700000; + v &= 0x8FFFFF; + raw &= ~((UInt32)0x8FFFFF << m); + raw |= (v << m); + SetUi32(p, raw); + } + } + while (++m <= 4); + } + i += 16; + } + while (i <= size); + return i; +} diff --git a/code/nel/3rdparty/seven_zip/CMakeLists.txt b/code/nel/3rdparty/seven_zip/CMakeLists.txt index 95e93438d..dfbf00aa2 100644 --- a/code/nel/3rdparty/seven_zip/CMakeLists.txt +++ b/code/nel/3rdparty/seven_zip/CMakeLists.txt @@ -10,7 +10,7 @@ NL_DEFAULT_PROPS(nel_sevenzip "NeL, 3rd Party: Seven Zip") NL_ADD_RUNTIME_FLAGS(nel_sevenzip) NL_ADD_LIB_SUFFIX(nel_sevenzip) -ADD_DEFINITIONS(-D_SZ_ONE_DIRECTORY) +ADD_DEFINITIONS(-D_7ZIP_ST) IF((WITH_INSTALL_LIBRARIES AND WITH_STATIC) OR NOT WITH_STATIC) INSTALL(TARGETS nel_sevenzip LIBRARY DESTINATION ${NL_LIB_PREFIX} ARCHIVE DESTINATION ${NL_LIB_PREFIX} COMPONENT libraries) diff --git a/code/nel/3rdparty/seven_zip/Compiler.h b/code/nel/3rdparty/seven_zip/Compiler.h new file mode 100644 index 000000000..0cc409d8a --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Compiler.h @@ -0,0 +1,33 @@ +/* Compiler.h +2017-04-03 : Igor Pavlov : Public domain */ + +#ifndef __7Z_COMPILER_H +#define __7Z_COMPILER_H + +#ifdef _MSC_VER + + #ifdef UNDER_CE + #define RPC_NO_WINDOWS_H + /* #pragma warning(disable : 4115) // '_RPC_ASYNC_STATE' : named type definition in parentheses */ + #pragma warning(disable : 4201) // nonstandard extension used : nameless struct/union + #pragma warning(disable : 4214) // nonstandard extension used : bit field types other than int + #endif + + #if _MSC_VER >= 1300 + #pragma warning(disable : 4996) // This function or variable may be unsafe + #else + #pragma warning(disable : 4511) // copy constructor could not be generated + #pragma warning(disable : 4512) // assignment operator could not be generated + #pragma warning(disable : 4514) // unreferenced inline function has been removed + #pragma warning(disable : 4702) // unreachable code + #pragma warning(disable : 4710) // not inlined + #pragma warning(disable : 4714) // function marked as __forceinline not inlined + #pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information + #endif + +#endif + +#define UNUSED_VAR(x) (void)x; +/* #define UNUSED_VAR(x) x=x; */ + +#endif diff --git a/code/nel/3rdparty/seven_zip/CpuArch.c b/code/nel/3rdparty/seven_zip/CpuArch.c new file mode 100644 index 000000000..02e482e08 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/CpuArch.c @@ -0,0 +1,218 @@ +/* CpuArch.c -- CPU specific code +2018-02-18: Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "CpuArch.h" + +#ifdef MY_CPU_X86_OR_AMD64 + +#if (defined(_MSC_VER) && !defined(MY_CPU_AMD64)) || defined(__GNUC__) +#define USE_ASM +#endif + +#if !defined(USE_ASM) && _MSC_VER >= 1500 +#include +#endif + +#if defined(USE_ASM) && !defined(MY_CPU_AMD64) +static UInt32 CheckFlag(UInt32 flag) +{ + #ifdef _MSC_VER + __asm pushfd; + __asm pop EAX; + __asm mov EDX, EAX; + __asm xor EAX, flag; + __asm push EAX; + __asm popfd; + __asm pushfd; + __asm pop EAX; + __asm xor EAX, EDX; + __asm push EDX; + __asm popfd; + __asm and flag, EAX; + #else + __asm__ __volatile__ ( + "pushf\n\t" + "pop %%EAX\n\t" + "movl %%EAX,%%EDX\n\t" + "xorl %0,%%EAX\n\t" + "push %%EAX\n\t" + "popf\n\t" + "pushf\n\t" + "pop %%EAX\n\t" + "xorl %%EDX,%%EAX\n\t" + "push %%EDX\n\t" + "popf\n\t" + "andl %%EAX, %0\n\t": + "=c" (flag) : "c" (flag) : + "%eax", "%edx"); + #endif + return flag; +} +#define CHECK_CPUID_IS_SUPPORTED if (CheckFlag(1 << 18) == 0 || CheckFlag(1 << 21) == 0) return False; +#else +#define CHECK_CPUID_IS_SUPPORTED +#endif + +void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d) +{ + #ifdef USE_ASM + + #ifdef _MSC_VER + + UInt32 a2, b2, c2, d2; + __asm xor EBX, EBX; + __asm xor ECX, ECX; + __asm xor EDX, EDX; + __asm mov EAX, function; + __asm cpuid; + __asm mov a2, EAX; + __asm mov b2, EBX; + __asm mov c2, ECX; + __asm mov d2, EDX; + + *a = a2; + *b = b2; + *c = c2; + *d = d2; + + #else + + __asm__ __volatile__ ( + #if defined(MY_CPU_AMD64) && defined(__PIC__) + "mov %%rbx, %%rdi;" + "cpuid;" + "xchg %%rbx, %%rdi;" + : "=a" (*a) , + "=D" (*b) , + #elif defined(MY_CPU_X86) && defined(__PIC__) + "mov %%ebx, %%edi;" + "cpuid;" + "xchgl %%ebx, %%edi;" + : "=a" (*a) , + "=D" (*b) , + #else + "cpuid" + : "=a" (*a) , + "=b" (*b) , + #endif + "=c" (*c) , + "=d" (*d) + : "0" (function)) ; + + #endif + + #else + + int CPUInfo[4]; + __cpuid(CPUInfo, function); + *a = CPUInfo[0]; + *b = CPUInfo[1]; + *c = CPUInfo[2]; + *d = CPUInfo[3]; + + #endif +} + +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p) +{ + CHECK_CPUID_IS_SUPPORTED + MyCPUID(0, &p->maxFunc, &p->vendor[0], &p->vendor[2], &p->vendor[1]); + MyCPUID(1, &p->ver, &p->b, &p->c, &p->d); + return True; +} + +static const UInt32 kVendors[][3] = +{ + { 0x756E6547, 0x49656E69, 0x6C65746E}, + { 0x68747541, 0x69746E65, 0x444D4163}, + { 0x746E6543, 0x48727561, 0x736C7561} +}; + +int x86cpuid_GetFirm(const Cx86cpuid *p) +{ + unsigned i; + for (i = 0; i < sizeof(kVendors) / sizeof(kVendors[i]); i++) + { + const UInt32 *v = kVendors[i]; + if (v[0] == p->vendor[0] && + v[1] == p->vendor[1] && + v[2] == p->vendor[2]) + return (int)i; + } + return -1; +} + +BoolInt CPU_Is_InOrder() +{ + Cx86cpuid p; + int firm; + UInt32 family, model; + if (!x86cpuid_CheckAndRead(&p)) + return True; + + family = x86cpuid_GetFamily(p.ver); + model = x86cpuid_GetModel(p.ver); + + firm = x86cpuid_GetFirm(&p); + + switch (firm) + { + case CPU_FIRM_INTEL: return (family < 6 || (family == 6 && ( + /* In-Order Atom CPU */ + model == 0x1C /* 45 nm, N4xx, D4xx, N5xx, D5xx, 230, 330 */ + || model == 0x26 /* 45 nm, Z6xx */ + || model == 0x27 /* 32 nm, Z2460 */ + || model == 0x35 /* 32 nm, Z2760 */ + || model == 0x36 /* 32 nm, N2xxx, D2xxx */ + ))); + case CPU_FIRM_AMD: return (family < 5 || (family == 5 && (model < 6 || model == 0xA))); + case CPU_FIRM_VIA: return (family < 6 || (family == 6 && model < 0xF)); + } + return True; +} + +#if !defined(MY_CPU_AMD64) && defined(_WIN32) +#include +static BoolInt CPU_Sys_Is_SSE_Supported() +{ + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + if (!GetVersionEx(&vi)) + return False; + return (vi.dwMajorVersion >= 5); +} +#define CHECK_SYS_SSE_SUPPORT if (!CPU_Sys_Is_SSE_Supported()) return False; +#else +#define CHECK_SYS_SSE_SUPPORT +#endif + +BoolInt CPU_Is_Aes_Supported() +{ + Cx86cpuid p; + CHECK_SYS_SSE_SUPPORT + if (!x86cpuid_CheckAndRead(&p)) + return False; + return (p.c >> 25) & 1; +} + +BoolInt CPU_IsSupported_PageGB() +{ + Cx86cpuid cpuid; + if (!x86cpuid_CheckAndRead(&cpuid)) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000000, &d[0], &d[1], &d[2], &d[3]); + if (d[0] < 0x80000001) + return False; + } + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000001, &d[0], &d[1], &d[2], &d[3]); + return (d[3] >> 26) & 1; + } +} + +#endif diff --git a/code/nel/3rdparty/seven_zip/CpuArch.h b/code/nel/3rdparty/seven_zip/CpuArch.h new file mode 100644 index 000000000..bd4293880 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/CpuArch.h @@ -0,0 +1,336 @@ +/* CpuArch.h -- CPU specific code +2018-02-18 : Igor Pavlov : Public domain */ + +#ifndef __CPU_ARCH_H +#define __CPU_ARCH_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +/* +MY_CPU_LE means that CPU is LITTLE ENDIAN. +MY_CPU_BE means that CPU is BIG ENDIAN. +If MY_CPU_LE and MY_CPU_BE are not defined, we don't know about ENDIANNESS of platform. + +MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned memory accesses. +*/ + +#if defined(_M_X64) \ + || defined(_M_AMD64) \ + || defined(__x86_64__) \ + || defined(__AMD64__) \ + || defined(__amd64__) + #define MY_CPU_AMD64 + #ifdef __ILP32__ + #define MY_CPU_NAME "x32" + #else + #define MY_CPU_NAME "x64" + #endif + #define MY_CPU_64BIT +#endif + + +#if defined(_M_IX86) \ + || defined(__i386__) + #define MY_CPU_X86 + #define MY_CPU_NAME "x86" + #define MY_CPU_32BIT +#endif + + +#if defined(_M_ARM64) \ + || defined(__AARCH64EL__) \ + || defined(__AARCH64EB__) \ + || defined(__aarch64__) + #define MY_CPU_ARM64 + #define MY_CPU_NAME "arm64" + #define MY_CPU_64BIT +#endif + + +#if defined(_M_ARM) \ + || defined(_M_ARM_NT) \ + || defined(_M_ARMT) \ + || defined(__arm__) \ + || defined(__thumb__) \ + || defined(__ARMEL__) \ + || defined(__ARMEB__) \ + || defined(__THUMBEL__) \ + || defined(__THUMBEB__) + #define MY_CPU_ARM + #define MY_CPU_NAME "arm" + #define MY_CPU_32BIT +#endif + + +#if defined(_M_IA64) \ + || defined(__ia64__) + #define MY_CPU_IA64 + #define MY_CPU_NAME "ia64" + #define MY_CPU_64BIT +#endif + + +#if defined(__mips64) \ + || defined(__mips64__) \ + || (defined(__mips) && (__mips == 64 || __mips == 4 || __mips == 3)) + #define MY_CPU_NAME "mips64" + #define MY_CPU_64BIT +#elif defined(__mips__) + #define MY_CPU_NAME "mips" + /* #define MY_CPU_32BIT */ +#endif + + +#if defined(__ppc64__) \ + || defined(__powerpc64__) + #ifdef __ILP32__ + #define MY_CPU_NAME "ppc64-32" + #else + #define MY_CPU_NAME "ppc64" + #endif + #define MY_CPU_64BIT +#elif defined(__ppc__) \ + || defined(__powerpc__) + #define MY_CPU_NAME "ppc" + #define MY_CPU_32BIT +#endif + + +#if defined(__sparc64__) + #define MY_CPU_NAME "sparc64" + #define MY_CPU_64BIT +#elif defined(__sparc__) + #define MY_CPU_NAME "sparc" + /* #define MY_CPU_32BIT */ +#endif + + +#if defined(MY_CPU_X86) || defined(MY_CPU_AMD64) +#define MY_CPU_X86_OR_AMD64 +#endif + + +#ifdef _WIN32 + + #ifdef MY_CPU_ARM + #define MY_CPU_ARM_LE + #endif + + #ifdef MY_CPU_ARM64 + #define MY_CPU_ARM64_LE + #endif + + #ifdef _M_IA64 + #define MY_CPU_IA64_LE + #endif + +#endif + + +#if defined(MY_CPU_X86_OR_AMD64) \ + || defined(MY_CPU_ARM_LE) \ + || defined(MY_CPU_ARM64_LE) \ + || defined(MY_CPU_IA64_LE) \ + || defined(__LITTLE_ENDIAN__) \ + || defined(__ARMEL__) \ + || defined(__THUMBEL__) \ + || defined(__AARCH64EL__) \ + || defined(__MIPSEL__) \ + || defined(__MIPSEL) \ + || defined(_MIPSEL) \ + || defined(__BFIN__) \ + || (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + #define MY_CPU_LE +#endif + +#if defined(__BIG_ENDIAN__) \ + || defined(__ARMEB__) \ + || defined(__THUMBEB__) \ + || defined(__AARCH64EB__) \ + || defined(__MIPSEB__) \ + || defined(__MIPSEB) \ + || defined(_MIPSEB) \ + || defined(__m68k__) \ + || defined(__s390__) \ + || defined(__s390x__) \ + || defined(__zarch__) \ + || (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) + #define MY_CPU_BE +#endif + + +#if defined(MY_CPU_LE) && defined(MY_CPU_BE) + #error Stop_Compiling_Bad_Endian +#endif + + +#if defined(MY_CPU_32BIT) && defined(MY_CPU_64BIT) + #error Stop_Compiling_Bad_32_64_BIT +#endif + + +#ifndef MY_CPU_NAME + #ifdef MY_CPU_LE + #define MY_CPU_NAME "LE" + #elif defined(MY_CPU_BE) + #define MY_CPU_NAME "BE" + #else + /* + #define MY_CPU_NAME "" + */ + #endif +#endif + + + + + +#ifdef MY_CPU_LE + #if defined(MY_CPU_X86_OR_AMD64) \ + || defined(MY_CPU_ARM64) \ + || defined(__ARM_FEATURE_UNALIGNED) + #define MY_CPU_LE_UNALIGN + #endif +#endif + + +#ifdef MY_CPU_LE_UNALIGN + +#define GetUi16(p) (*(const UInt16 *)(const void *)(p)) +#define GetUi32(p) (*(const UInt32 *)(const void *)(p)) +#define GetUi64(p) (*(const UInt64 *)(const void *)(p)) + +#define SetUi16(p, v) { *(UInt16 *)(p) = (v); } +#define SetUi32(p, v) { *(UInt32 *)(p) = (v); } +#define SetUi64(p, v) { *(UInt64 *)(p) = (v); } + +#else + +#define GetUi16(p) ( (UInt16) ( \ + ((const Byte *)(p))[0] | \ + ((UInt16)((const Byte *)(p))[1] << 8) )) + +#define GetUi32(p) ( \ + ((const Byte *)(p))[0] | \ + ((UInt32)((const Byte *)(p))[1] << 8) | \ + ((UInt32)((const Byte *)(p))[2] << 16) | \ + ((UInt32)((const Byte *)(p))[3] << 24)) + +#define GetUi64(p) (GetUi32(p) | ((UInt64)GetUi32(((const Byte *)(p)) + 4) << 32)) + +#define SetUi16(p, v) { Byte *_ppp_ = (Byte *)(p); UInt32 _vvv_ = (v); \ + _ppp_[0] = (Byte)_vvv_; \ + _ppp_[1] = (Byte)(_vvv_ >> 8); } + +#define SetUi32(p, v) { Byte *_ppp_ = (Byte *)(p); UInt32 _vvv_ = (v); \ + _ppp_[0] = (Byte)_vvv_; \ + _ppp_[1] = (Byte)(_vvv_ >> 8); \ + _ppp_[2] = (Byte)(_vvv_ >> 16); \ + _ppp_[3] = (Byte)(_vvv_ >> 24); } + +#define SetUi64(p, v) { Byte *_ppp2_ = (Byte *)(p); UInt64 _vvv2_ = (v); \ + SetUi32(_ppp2_ , (UInt32)_vvv2_); \ + SetUi32(_ppp2_ + 4, (UInt32)(_vvv2_ >> 32)); } + +#endif + +#ifdef __has_builtin + #define MY__has_builtin(x) __has_builtin(x) +#else + #define MY__has_builtin(x) 0 +#endif + +#if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ (_MSC_VER >= 1300) + +/* Note: we use bswap instruction, that is unsupported in 386 cpu */ + +#include + +#pragma intrinsic(_byteswap_ushort) +#pragma intrinsic(_byteswap_ulong) +#pragma intrinsic(_byteswap_uint64) + +/* #define GetBe16(p) _byteswap_ushort(*(const UInt16 *)(const Byte *)(p)) */ +#define GetBe32(p) _byteswap_ulong(*(const UInt32 *)(const Byte *)(p)) +#define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const Byte *)(p)) + +#define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = _byteswap_ulong(v) + +#elif defined(MY_CPU_LE_UNALIGN) && ( \ + (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \ + || (defined(__clang__) && MY__has_builtin(__builtin_bswap16)) ) + +/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const Byte *)(p)) */ +#define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const Byte *)(p)) +#define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const Byte *)(p)) + +#define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = __builtin_bswap32(v) + +#else + +#define GetBe32(p) ( \ + ((UInt32)((const Byte *)(p))[0] << 24) | \ + ((UInt32)((const Byte *)(p))[1] << 16) | \ + ((UInt32)((const Byte *)(p))[2] << 8) | \ + ((const Byte *)(p))[3] ) + +#define GetBe64(p) (((UInt64)GetBe32(p) << 32) | GetBe32(((const Byte *)(p)) + 4)) + +#define SetBe32(p, v) { Byte *_ppp_ = (Byte *)(p); UInt32 _vvv_ = (v); \ + _ppp_[0] = (Byte)(_vvv_ >> 24); \ + _ppp_[1] = (Byte)(_vvv_ >> 16); \ + _ppp_[2] = (Byte)(_vvv_ >> 8); \ + _ppp_[3] = (Byte)_vvv_; } + +#endif + + +#ifndef GetBe16 + +#define GetBe16(p) ( (UInt16) ( \ + ((UInt16)((const Byte *)(p))[0] << 8) | \ + ((const Byte *)(p))[1] )) + +#endif + + + +#ifdef MY_CPU_X86_OR_AMD64 + +typedef struct +{ + UInt32 maxFunc; + UInt32 vendor[3]; + UInt32 ver; + UInt32 b; + UInt32 c; + UInt32 d; +} Cx86cpuid; + +enum +{ + CPU_FIRM_INTEL, + CPU_FIRM_AMD, + CPU_FIRM_VIA +}; + +void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d); + +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p); +int x86cpuid_GetFirm(const Cx86cpuid *p); + +#define x86cpuid_GetFamily(ver) (((ver >> 16) & 0xFF0) | ((ver >> 8) & 0xF)) +#define x86cpuid_GetModel(ver) (((ver >> 12) & 0xF0) | ((ver >> 4) & 0xF)) +#define x86cpuid_GetStepping(ver) (ver & 0xF) + +BoolInt CPU_Is_InOrder(); +BoolInt CPU_Is_Aes_Supported(); +BoolInt CPU_IsSupported_PageGB(); + +#endif + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/Delta.c b/code/nel/3rdparty/seven_zip/Delta.c new file mode 100644 index 000000000..e3edd21ed --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Delta.c @@ -0,0 +1,64 @@ +/* Delta.c -- Delta converter +2009-05-26 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "Delta.h" + +void Delta_Init(Byte *state) +{ + unsigned i; + for (i = 0; i < DELTA_STATE_SIZE; i++) + state[i] = 0; +} + +static void MyMemCpy(Byte *dest, const Byte *src, unsigned size) +{ + unsigned i; + for (i = 0; i < size; i++) + dest[i] = src[i]; +} + +void Delta_Encode(Byte *state, unsigned delta, Byte *data, SizeT size) +{ + Byte buf[DELTA_STATE_SIZE]; + unsigned j = 0; + MyMemCpy(buf, state, delta); + { + SizeT i; + for (i = 0; i < size;) + { + for (j = 0; j < delta && i < size; i++, j++) + { + Byte b = data[i]; + data[i] = (Byte)(b - buf[j]); + buf[j] = b; + } + } + } + if (j == delta) + j = 0; + MyMemCpy(state, buf + j, delta - j); + MyMemCpy(state + delta - j, buf, j); +} + +void Delta_Decode(Byte *state, unsigned delta, Byte *data, SizeT size) +{ + Byte buf[DELTA_STATE_SIZE]; + unsigned j = 0; + MyMemCpy(buf, state, delta); + { + SizeT i; + for (i = 0; i < size;) + { + for (j = 0; j < delta && i < size; i++, j++) + { + buf[j] = data[i] = (Byte)(buf[j] + data[i]); + } + } + } + if (j == delta) + j = 0; + MyMemCpy(state, buf + j, delta - j); + MyMemCpy(state + delta - j, buf, j); +} diff --git a/code/nel/3rdparty/seven_zip/Delta.h b/code/nel/3rdparty/seven_zip/Delta.h new file mode 100644 index 000000000..2fa54ad67 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Delta.h @@ -0,0 +1,19 @@ +/* Delta.h -- Delta converter +2013-01-18 : Igor Pavlov : Public domain */ + +#ifndef __DELTA_H +#define __DELTA_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define DELTA_STATE_SIZE 256 + +void Delta_Init(Byte *state); +void Delta_Encode(Byte *state, unsigned delta, Byte *data, SizeT size); +void Delta_Decode(Byte *state, unsigned delta, Byte *data, SizeT size); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/DllSecur.c b/code/nel/3rdparty/seven_zip/DllSecur.c new file mode 100644 index 000000000..5ea108ab8 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/DllSecur.c @@ -0,0 +1,108 @@ +/* DllSecur.c -- DLL loading security +2018-02-21 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#ifdef _WIN32 + +#include + +#include "DllSecur.h" + +#ifndef UNDER_CE + +typedef BOOL (WINAPI *Func_SetDefaultDllDirectories)(DWORD DirectoryFlags); + +#define MY_LOAD_LIBRARY_SEARCH_USER_DIRS 0x400 +#define MY_LOAD_LIBRARY_SEARCH_SYSTEM32 0x800 + +static const char * const g_Dlls = + #ifndef _CONSOLE + "UXTHEME\0" + #endif + "USERENV\0" + "SETUPAPI\0" + "APPHELP\0" + "PROPSYS\0" + "DWMAPI\0" + "CRYPTBASE\0" + "OLEACC\0" + "CLBCATQ\0" + "VERSION\0" + ; + +#endif + +void My_SetDefaultDllDirectories() +{ + #ifndef UNDER_CE + + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + GetVersionEx(&vi); + if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) + { + Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + if (setDllDirs) + if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) + return; + } + + #endif +} + + +void LoadSecurityDlls() +{ + #ifndef UNDER_CE + + wchar_t buf[MAX_PATH + 100]; + + { + // at Vista (ver 6.0) : CoCreateInstance(CLSID_ShellLink, ...) doesn't work after SetDefaultDllDirectories() : Check it ??? + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) + { + Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + if (setDllDirs) + if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) + return; + } + } + + { + unsigned len = GetSystemDirectoryW(buf, MAX_PATH + 2); + if (len == 0 || len > MAX_PATH) + return; + } + { + const char *dll; + unsigned pos = (unsigned)lstrlenW(buf); + + if (buf[pos - 1] != '\\') + buf[pos++] = '\\'; + + for (dll = g_Dlls; dll[0] != 0;) + { + unsigned k = 0; + for (;;) + { + char c = *dll++; + buf[pos + k] = (Byte)c; + k++; + if (c == 0) + break; + } + + lstrcatW(buf, L".dll"); + LoadLibraryExW(buf, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); + } + } + + #endif +} + +#endif diff --git a/code/nel/3rdparty/seven_zip/DllSecur.h b/code/nel/3rdparty/seven_zip/DllSecur.h new file mode 100644 index 000000000..e2a049ad2 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/DllSecur.h @@ -0,0 +1,20 @@ +/* DllSecur.h -- DLL loading for security +2018-02-19 : Igor Pavlov : Public domain */ + +#ifndef __DLL_SECUR_H +#define __DLL_SECUR_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#ifdef _WIN32 + +void My_SetDefaultDllDirectories(); +void LoadSecurityDlls(); + +#endif + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/LzFind.c b/code/nel/3rdparty/seven_zip/LzFind.c new file mode 100644 index 000000000..df55e86c1 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzFind.c @@ -0,0 +1,1127 @@ +/* LzFind.c -- Match finder for LZ algorithms +2018-07-08 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +#include "LzFind.h" +#include "LzHash.h" + +#define kEmptyHashValue 0 +#define kMaxValForNormalize ((UInt32)0xFFFFFFFF) +#define kNormalizeStepMin (1 << 10) /* it must be power of 2 */ +#define kNormalizeMask (~(UInt32)(kNormalizeStepMin - 1)) +#define kMaxHistorySize ((UInt32)7 << 29) + +#define kStartMaxLen 3 + +static void LzInWindow_Free(CMatchFinder *p, ISzAllocPtr alloc) +{ + if (!p->directInput) + { + ISzAlloc_Free(alloc, p->bufferBase); + p->bufferBase = NULL; + } +} + +/* keepSizeBefore + keepSizeAfter + keepSizeReserv must be < 4G) */ + +static int LzInWindow_Create(CMatchFinder *p, UInt32 keepSizeReserv, ISzAllocPtr alloc) +{ + UInt32 blockSize = p->keepSizeBefore + p->keepSizeAfter + keepSizeReserv; + if (p->directInput) + { + p->blockSize = blockSize; + return 1; + } + if (!p->bufferBase || p->blockSize != blockSize) + { + LzInWindow_Free(p, alloc); + p->blockSize = blockSize; + p->bufferBase = (Byte *)ISzAlloc_Alloc(alloc, (size_t)blockSize); + } + return (p->bufferBase != NULL); +} + +Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } + +UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return p->streamPos - p->pos; } + +void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue) +{ + p->posLimit -= subValue; + p->pos -= subValue; + p->streamPos -= subValue; +} + +static void MatchFinder_ReadBlock(CMatchFinder *p) +{ + if (p->streamEndWasReached || p->result != SZ_OK) + return; + + /* We use (p->streamPos - p->pos) value. (p->streamPos < p->pos) is allowed. */ + + if (p->directInput) + { + UInt32 curSize = 0xFFFFFFFF - (p->streamPos - p->pos); + if (curSize > p->directInputRem) + curSize = (UInt32)p->directInputRem; + p->directInputRem -= curSize; + p->streamPos += curSize; + if (p->directInputRem == 0) + p->streamEndWasReached = 1; + return; + } + + for (;;) + { + Byte *dest = p->buffer + (p->streamPos - p->pos); + size_t size = (p->bufferBase + p->blockSize - dest); + if (size == 0) + return; + + p->result = ISeqInStream_Read(p->stream, dest, &size); + if (p->result != SZ_OK) + return; + if (size == 0) + { + p->streamEndWasReached = 1; + return; + } + p->streamPos += (UInt32)size; + if (p->streamPos - p->pos > p->keepSizeAfter) + return; + } +} + +void MatchFinder_MoveBlock(CMatchFinder *p) +{ + memmove(p->bufferBase, + p->buffer - p->keepSizeBefore, + (size_t)(p->streamPos - p->pos) + p->keepSizeBefore); + p->buffer = p->bufferBase + p->keepSizeBefore; +} + +int MatchFinder_NeedMove(CMatchFinder *p) +{ + if (p->directInput) + return 0; + /* if (p->streamEndWasReached) return 0; */ + return ((size_t)(p->bufferBase + p->blockSize - p->buffer) <= p->keepSizeAfter); +} + +void MatchFinder_ReadIfRequired(CMatchFinder *p) +{ + if (p->streamEndWasReached) + return; + if (p->keepSizeAfter >= p->streamPos - p->pos) + MatchFinder_ReadBlock(p); +} + +static void MatchFinder_CheckAndMoveAndRead(CMatchFinder *p) +{ + if (MatchFinder_NeedMove(p)) + MatchFinder_MoveBlock(p); + MatchFinder_ReadBlock(p); +} + +static void MatchFinder_SetDefaultSettings(CMatchFinder *p) +{ + p->cutValue = 32; + p->btMode = 1; + p->numHashBytes = 4; + p->bigHash = 0; +} + +#define kCrcPoly 0xEDB88320 + +void MatchFinder_Construct(CMatchFinder *p) +{ + unsigned i; + p->bufferBase = NULL; + p->directInput = 0; + p->hash = NULL; + p->expectedDataSize = (UInt64)(Int64)-1; + MatchFinder_SetDefaultSettings(p); + + for (i = 0; i < 256; i++) + { + UInt32 r = (UInt32)i; + unsigned j; + for (j = 0; j < 8; j++) + r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); + p->crc[i] = r; + } +} + +static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->hash); + p->hash = NULL; +} + +void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc) +{ + MatchFinder_FreeThisClassMemory(p, alloc); + LzInWindow_Free(p, alloc); +} + +static CLzRef* AllocRefs(size_t num, ISzAllocPtr alloc) +{ + size_t sizeInBytes = (size_t)num * sizeof(CLzRef); + if (sizeInBytes / sizeof(CLzRef) != num) + return NULL; + return (CLzRef *)ISzAlloc_Alloc(alloc, sizeInBytes); +} + +int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, + UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, + ISzAllocPtr alloc) +{ + UInt32 sizeReserv; + + if (historySize > kMaxHistorySize) + { + MatchFinder_Free(p, alloc); + return 0; + } + + sizeReserv = historySize >> 1; + if (historySize >= ((UInt32)3 << 30)) sizeReserv = historySize >> 3; + else if (historySize >= ((UInt32)2 << 30)) sizeReserv = historySize >> 2; + + sizeReserv += (keepAddBufferBefore + matchMaxLen + keepAddBufferAfter) / 2 + (1 << 19); + + p->keepSizeBefore = historySize + keepAddBufferBefore + 1; + p->keepSizeAfter = matchMaxLen + keepAddBufferAfter; + + /* we need one additional byte, since we use MoveBlock after pos++ and before dictionary using */ + + if (LzInWindow_Create(p, sizeReserv, alloc)) + { + UInt32 newCyclicBufferSize = historySize + 1; + UInt32 hs; + p->matchMaxLen = matchMaxLen; + { + p->fixedHashSize = 0; + if (p->numHashBytes == 2) + hs = (1 << 16) - 1; + else + { + hs = historySize; + if (hs > p->expectedDataSize) + hs = (UInt32)p->expectedDataSize; + if (hs != 0) + hs--; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + hs >>= 1; + hs |= 0xFFFF; /* don't change it! It's required for Deflate */ + if (hs > (1 << 24)) + { + if (p->numHashBytes == 3) + hs = (1 << 24) - 1; + else + hs >>= 1; + /* if (bigHash) mode, GetHeads4b() in LzFindMt.c needs (hs >= ((1 << 24) - 1))) */ + } + } + p->hashMask = hs; + hs++; + if (p->numHashBytes > 2) p->fixedHashSize += kHash2Size; + if (p->numHashBytes > 3) p->fixedHashSize += kHash3Size; + if (p->numHashBytes > 4) p->fixedHashSize += kHash4Size; + hs += p->fixedHashSize; + } + + { + size_t newSize; + size_t numSons; + p->historySize = historySize; + p->hashSizeSum = hs; + p->cyclicBufferSize = newCyclicBufferSize; + + numSons = newCyclicBufferSize; + if (p->btMode) + numSons <<= 1; + newSize = hs + numSons; + + if (p->hash && p->numRefs == newSize) + return 1; + + MatchFinder_FreeThisClassMemory(p, alloc); + p->numRefs = newSize; + p->hash = AllocRefs(newSize, alloc); + + if (p->hash) + { + p->son = p->hash + p->hashSizeSum; + return 1; + } + } + } + + MatchFinder_Free(p, alloc); + return 0; +} + +static void MatchFinder_SetLimits(CMatchFinder *p) +{ + UInt32 limit = kMaxValForNormalize - p->pos; + UInt32 limit2 = p->cyclicBufferSize - p->cyclicBufferPos; + + if (limit2 < limit) + limit = limit2; + limit2 = p->streamPos - p->pos; + + if (limit2 <= p->keepSizeAfter) + { + if (limit2 > 0) + limit2 = 1; + } + else + limit2 -= p->keepSizeAfter; + + if (limit2 < limit) + limit = limit2; + + { + UInt32 lenLimit = p->streamPos - p->pos; + if (lenLimit > p->matchMaxLen) + lenLimit = p->matchMaxLen; + p->lenLimit = lenLimit; + } + p->posLimit = p->pos + limit; +} + + +void MatchFinder_Init_LowHash(CMatchFinder *p) +{ + size_t i; + CLzRef *items = p->hash; + size_t numItems = p->fixedHashSize; + for (i = 0; i < numItems; i++) + items[i] = kEmptyHashValue; +} + + +void MatchFinder_Init_HighHash(CMatchFinder *p) +{ + size_t i; + CLzRef *items = p->hash + p->fixedHashSize; + size_t numItems = (size_t)p->hashMask + 1; + for (i = 0; i < numItems; i++) + items[i] = kEmptyHashValue; +} + + +void MatchFinder_Init_3(CMatchFinder *p, int readData) +{ + p->cyclicBufferPos = 0; + p->buffer = p->bufferBase; + p->pos = + p->streamPos = p->cyclicBufferSize; + p->result = SZ_OK; + p->streamEndWasReached = 0; + + if (readData) + MatchFinder_ReadBlock(p); + + MatchFinder_SetLimits(p); +} + + +void MatchFinder_Init(CMatchFinder *p) +{ + MatchFinder_Init_HighHash(p); + MatchFinder_Init_LowHash(p); + MatchFinder_Init_3(p, True); +} + + +static UInt32 MatchFinder_GetSubValue(CMatchFinder *p) +{ + return (p->pos - p->historySize - 1) & kNormalizeMask; +} + +void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems) +{ + size_t i; + for (i = 0; i < numItems; i++) + { + UInt32 value = items[i]; + if (value <= subValue) + value = kEmptyHashValue; + else + value -= subValue; + items[i] = value; + } +} + +static void MatchFinder_Normalize(CMatchFinder *p) +{ + UInt32 subValue = MatchFinder_GetSubValue(p); + MatchFinder_Normalize3(subValue, p->hash, p->numRefs); + MatchFinder_ReduceOffsets(p, subValue); +} + + +MY_NO_INLINE +static void MatchFinder_CheckLimits(CMatchFinder *p) +{ + if (p->pos == kMaxValForNormalize) + MatchFinder_Normalize(p); + if (!p->streamEndWasReached && p->keepSizeAfter == p->streamPos - p->pos) + MatchFinder_CheckAndMoveAndRead(p); + if (p->cyclicBufferPos == p->cyclicBufferSize) + p->cyclicBufferPos = 0; + MatchFinder_SetLimits(p); +} + + +/* + (lenLimit > maxLen) +*/ +MY_FORCE_INLINE +static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, + UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, + UInt32 *distances, unsigned maxLen) +{ + /* + son[_cyclicBufferPos] = curMatch; + for (;;) + { + UInt32 delta = pos - curMatch; + if (cutValue-- == 0 || delta >= _cyclicBufferSize) + return distances; + { + const Byte *pb = cur - delta; + curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; + if (pb[maxLen] == cur[maxLen] && *pb == *cur) + { + UInt32 len = 0; + while (++len != lenLimit) + if (pb[len] != cur[len]) + break; + if (maxLen < len) + { + maxLen = len; + *distances++ = len; + *distances++ = delta - 1; + if (len == lenLimit) + return distances; + } + } + } + } + */ + + const Byte *lim = cur + lenLimit; + son[_cyclicBufferPos] = curMatch; + do + { + UInt32 delta = pos - curMatch; + if (delta >= _cyclicBufferSize) + break; + { + ptrdiff_t diff; + curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; + diff = (ptrdiff_t)0 - delta; + if (cur[maxLen] == cur[maxLen + diff]) + { + const Byte *c = cur; + while (*c == c[diff]) + { + if (++c == lim) + { + distances[0] = (UInt32)(lim - cur); + distances[1] = delta - 1; + return distances + 2; + } + } + { + unsigned len = (unsigned)(c - cur); + if (maxLen < len) + { + maxLen = len; + distances[0] = (UInt32)len; + distances[1] = delta - 1; + distances += 2; + } + } + } + } + } + while (--cutValue); + + return distances; +} + + +MY_FORCE_INLINE +UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, + UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, + UInt32 *distances, UInt32 maxLen) +{ + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; + for (;;) + { + UInt32 delta = pos - curMatch; + if (cutValue-- == 0 || delta >= _cyclicBufferSize) + { + *ptr0 = *ptr1 = kEmptyHashValue; + return distances; + } + { + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + const Byte *pb = cur - delta; + unsigned len = (len0 < len1 ? len0 : len1); + UInt32 pair0 = pair[0]; + if (pb[len] == cur[len]) + { + if (++len != lenLimit && pb[len] == cur[len]) + while (++len != lenLimit) + if (pb[len] != cur[len]) + break; + if (maxLen < len) + { + maxLen = (UInt32)len; + *distances++ = (UInt32)len; + *distances++ = delta - 1; + if (len == lenLimit) + { + *ptr1 = pair0; + *ptr0 = pair[1]; + return distances; + } + } + } + if (pb[len] < cur[len]) + { + *ptr1 = curMatch; + ptr1 = pair + 1; + curMatch = *ptr1; + len1 = len; + } + else + { + *ptr0 = curMatch; + ptr0 = pair; + curMatch = *ptr0; + len0 = len; + } + } + } +} + +static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, + UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) +{ + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; + for (;;) + { + UInt32 delta = pos - curMatch; + if (cutValue-- == 0 || delta >= _cyclicBufferSize) + { + *ptr0 = *ptr1 = kEmptyHashValue; + return; + } + { + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + const Byte *pb = cur - delta; + unsigned len = (len0 < len1 ? len0 : len1); + if (pb[len] == cur[len]) + { + while (++len != lenLimit) + if (pb[len] != cur[len]) + break; + { + if (len == lenLimit) + { + *ptr1 = pair[0]; + *ptr0 = pair[1]; + return; + } + } + } + if (pb[len] < cur[len]) + { + *ptr1 = curMatch; + ptr1 = pair + 1; + curMatch = *ptr1; + len1 = len; + } + else + { + *ptr0 = curMatch; + ptr0 = pair; + curMatch = *ptr0; + len0 = len; + } + } + } +} + +#define MOVE_POS \ + ++p->cyclicBufferPos; \ + p->buffer++; \ + if (++p->pos == p->posLimit) MatchFinder_CheckLimits(p); + +#define MOVE_POS_RET MOVE_POS return (UInt32)offset; + +static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } + +#define GET_MATCHES_HEADER2(minLen, ret_op) \ + unsigned lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ + lenLimit = (unsigned)p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ + cur = p->buffer; + +#define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return 0) +#define SKIP_HEADER(minLen) GET_MATCHES_HEADER2(minLen, continue) + +#define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue + +#define GET_MATCHES_FOOTER(offset, maxLen) \ + offset = (unsigned)(GetMatchesSpec1((UInt32)lenLimit, curMatch, MF_PARAMS(p), \ + distances + offset, (UInt32)maxLen) - distances); MOVE_POS_RET; + +#define SKIP_FOOTER \ + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; + +#define UPDATE_maxLen { \ + ptrdiff_t diff = (ptrdiff_t)0 - d2; \ + const Byte *c = cur + maxLen; \ + const Byte *lim = cur + lenLimit; \ + for (; c != lim; c++) if (*(c + diff) != *c) break; \ + maxLen = (unsigned)(c - cur); } + +static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +{ + unsigned offset; + GET_MATCHES_HEADER(2) + HASH2_CALC; + curMatch = p->hash[hv]; + p->hash[hv] = p->pos; + offset = 0; + GET_MATCHES_FOOTER(offset, 1) +} + +UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +{ + unsigned offset; + GET_MATCHES_HEADER(3) + HASH_ZIP_CALC; + curMatch = p->hash[hv]; + p->hash[hv] = p->pos; + offset = 0; + GET_MATCHES_FOOTER(offset, 2) +} + +static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +{ + UInt32 h2, d2, pos; + unsigned maxLen, offset; + UInt32 *hash; + GET_MATCHES_HEADER(3) + + HASH3_CALC; + + hash = p->hash; + pos = p->pos; + + d2 = pos - hash[h2]; + + curMatch = (hash + kFix3HashSize)[hv]; + + hash[h2] = pos; + (hash + kFix3HashSize)[hv] = pos; + + maxLen = 2; + offset = 0; + + if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + { + UPDATE_maxLen + distances[0] = (UInt32)maxLen; + distances[1] = d2 - 1; + offset = 2; + if (maxLen == lenLimit) + { + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); + MOVE_POS_RET; + } + } + + GET_MATCHES_FOOTER(offset, maxLen) +} + +static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +{ + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen, offset; + UInt32 *hash; + GET_MATCHES_HEADER(4) + + HASH4_CALC; + + hash = p->hash; + pos = p->pos; + + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + + curMatch = (hash + kFix4HashSize)[hv]; + + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[hv] = pos; + + maxLen = 0; + offset = 0; + + if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + { + maxLen = 2; + distances[0] = 2; + distances[1] = d2 - 1; + offset = 2; + } + + if (d2 != d3 && d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + { + maxLen = 3; + distances[(size_t)offset + 1] = d3 - 1; + offset += 2; + d2 = d3; + } + + if (offset != 0) + { + UPDATE_maxLen + distances[(size_t)offset - 2] = (UInt32)maxLen; + if (maxLen == lenLimit) + { + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); + MOVE_POS_RET; + } + } + + if (maxLen < 3) + maxLen = 3; + + GET_MATCHES_FOOTER(offset, maxLen) +} + +/* +static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +{ + UInt32 h2, h3, h4, d2, d3, d4, maxLen, offset, pos; + UInt32 *hash; + GET_MATCHES_HEADER(5) + + HASH5_CALC; + + hash = p->hash; + pos = p->pos; + + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + d4 = pos - (hash + kFix4HashSize)[h4]; + + curMatch = (hash + kFix5HashSize)[hv]; + + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[h4] = pos; + (hash + kFix5HashSize)[hv] = pos; + + maxLen = 0; + offset = 0; + + if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + { + distances[0] = maxLen = 2; + distances[1] = d2 - 1; + offset = 2; + if (*(cur - d2 + 2) == cur[2]) + distances[0] = maxLen = 3; + else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + { + distances[2] = maxLen = 3; + distances[3] = d3 - 1; + offset = 4; + d2 = d3; + } + } + else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + { + distances[0] = maxLen = 3; + distances[1] = d3 - 1; + offset = 2; + d2 = d3; + } + + if (d2 != d4 && d4 < p->cyclicBufferSize + && *(cur - d4) == *cur + && *(cur - d4 + 3) == *(cur + 3)) + { + maxLen = 4; + distances[(size_t)offset + 1] = d4 - 1; + offset += 2; + d2 = d4; + } + + if (offset != 0) + { + UPDATE_maxLen + distances[(size_t)offset - 2] = maxLen; + if (maxLen == lenLimit) + { + SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); + MOVE_POS_RET; + } + } + + if (maxLen < 4) + maxLen = 4; + + GET_MATCHES_FOOTER(offset, maxLen) +} +*/ + +static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +{ + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen, offset; + UInt32 *hash; + GET_MATCHES_HEADER(4) + + HASH4_CALC; + + hash = p->hash; + pos = p->pos; + + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + curMatch = (hash + kFix4HashSize)[hv]; + + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[hv] = pos; + + maxLen = 0; + offset = 0; + + if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + { + maxLen = 2; + distances[0] = 2; + distances[1] = d2 - 1; + offset = 2; + } + + if (d2 != d3 && d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + { + maxLen = 3; + distances[(size_t)offset + 1] = d3 - 1; + offset += 2; + d2 = d3; + } + + if (offset != 0) + { + UPDATE_maxLen + distances[(size_t)offset - 2] = (UInt32)maxLen; + if (maxLen == lenLimit) + { + p->son[p->cyclicBufferPos] = curMatch; + MOVE_POS_RET; + } + } + + if (maxLen < 3) + maxLen = 3; + + offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), + distances + offset, maxLen) - (distances)); + MOVE_POS_RET +} + +/* +static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +{ + UInt32 h2, h3, h4, d2, d3, d4, maxLen, offset, pos + UInt32 *hash; + GET_MATCHES_HEADER(5) + + HASH5_CALC; + + hash = p->hash; + pos = p->pos; + + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + d4 = pos - (hash + kFix4HashSize)[h4]; + + curMatch = (hash + kFix5HashSize)[hv]; + + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[h4] = pos; + (hash + kFix5HashSize)[hv] = pos; + + maxLen = 0; + offset = 0; + + if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) + { + distances[0] = maxLen = 2; + distances[1] = d2 - 1; + offset = 2; + if (*(cur - d2 + 2) == cur[2]) + distances[0] = maxLen = 3; + else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + { + distances[2] = maxLen = 3; + distances[3] = d3 - 1; + offset = 4; + d2 = d3; + } + } + else if (d3 < p->cyclicBufferSize && *(cur - d3) == *cur) + { + distances[0] = maxLen = 3; + distances[1] = d3 - 1; + offset = 2; + d2 = d3; + } + + if (d2 != d4 && d4 < p->cyclicBufferSize + && *(cur - d4) == *cur + && *(cur - d4 + 3) == *(cur + 3)) + { + maxLen = 4; + distances[(size_t)offset + 1] = d4 - 1; + offset += 2; + d2 = d4; + } + + if (offset != 0) + { + UPDATE_maxLen + distances[(size_t)offset - 2] = maxLen; + if (maxLen == lenLimit) + { + p->son[p->cyclicBufferPos] = curMatch; + MOVE_POS_RET; + } + } + + if (maxLen < 4) + maxLen = 4; + + offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), + distances + offset, maxLen) - (distances)); + MOVE_POS_RET +} +*/ + +UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +{ + unsigned offset; + GET_MATCHES_HEADER(3) + HASH_ZIP_CALC; + curMatch = p->hash[hv]; + p->hash[hv] = p->pos; + offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), + distances, 2) - (distances)); + MOVE_POS_RET +} + +static void Bt2_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +{ + do + { + SKIP_HEADER(2) + HASH2_CALC; + curMatch = p->hash[hv]; + p->hash[hv] = p->pos; + SKIP_FOOTER + } + while (--num != 0); +} + +void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +{ + do + { + SKIP_HEADER(3) + HASH_ZIP_CALC; + curMatch = p->hash[hv]; + p->hash[hv] = p->pos; + SKIP_FOOTER + } + while (--num != 0); +} + +static void Bt3_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +{ + do + { + UInt32 h2; + UInt32 *hash; + SKIP_HEADER(3) + HASH3_CALC; + hash = p->hash; + curMatch = (hash + kFix3HashSize)[hv]; + hash[h2] = + (hash + kFix3HashSize)[hv] = p->pos; + SKIP_FOOTER + } + while (--num != 0); +} + +static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +{ + do + { + UInt32 h2, h3; + UInt32 *hash; + SKIP_HEADER(4) + HASH4_CALC; + hash = p->hash; + curMatch = (hash + kFix4HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[hv] = p->pos; + SKIP_FOOTER + } + while (--num != 0); +} + +/* +static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +{ + do + { + UInt32 h2, h3, h4; + UInt32 *hash; + SKIP_HEADER(5) + HASH5_CALC; + hash = p->hash; + curMatch = (hash + kFix5HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[h4] = + (hash + kFix5HashSize)[hv] = p->pos; + SKIP_FOOTER + } + while (--num != 0); +} +*/ + +static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +{ + do + { + UInt32 h2, h3; + UInt32 *hash; + SKIP_HEADER(4) + HASH4_CALC; + hash = p->hash; + curMatch = (hash + kFix4HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[hv] = p->pos; + p->son[p->cyclicBufferPos] = curMatch; + MOVE_POS + } + while (--num != 0); +} + +/* +static void Hc5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +{ + do + { + UInt32 h2, h3, h4; + UInt32 *hash; + SKIP_HEADER(5) + HASH5_CALC; + hash = p->hash; + curMatch = hash + kFix5HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[h4] = + (hash + kFix5HashSize)[hv] = p->pos; + p->son[p->cyclicBufferPos] = curMatch; + MOVE_POS + } + while (--num != 0); +} +*/ + +void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +{ + do + { + SKIP_HEADER(3) + HASH_ZIP_CALC; + curMatch = p->hash[hv]; + p->hash[hv] = p->pos; + p->son[p->cyclicBufferPos] = curMatch; + MOVE_POS + } + while (--num != 0); +} + +void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable) +{ + vTable->Init = (Mf_Init_Func)MatchFinder_Init; + vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinder_GetNumAvailableBytes; + vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinder_GetPointerToCurrentPos; + if (!p->btMode) + { + /* if (p->numHashBytes <= 4) */ + { + vTable->GetMatches = (Mf_GetMatches_Func)Hc4_MatchFinder_GetMatches; + vTable->Skip = (Mf_Skip_Func)Hc4_MatchFinder_Skip; + } + /* + else + { + vTable->GetMatches = (Mf_GetMatches_Func)Hc5_MatchFinder_GetMatches; + vTable->Skip = (Mf_Skip_Func)Hc5_MatchFinder_Skip; + } + */ + } + else if (p->numHashBytes == 2) + { + vTable->GetMatches = (Mf_GetMatches_Func)Bt2_MatchFinder_GetMatches; + vTable->Skip = (Mf_Skip_Func)Bt2_MatchFinder_Skip; + } + else if (p->numHashBytes == 3) + { + vTable->GetMatches = (Mf_GetMatches_Func)Bt3_MatchFinder_GetMatches; + vTable->Skip = (Mf_Skip_Func)Bt3_MatchFinder_Skip; + } + else /* if (p->numHashBytes == 4) */ + { + vTable->GetMatches = (Mf_GetMatches_Func)Bt4_MatchFinder_GetMatches; + vTable->Skip = (Mf_Skip_Func)Bt4_MatchFinder_Skip; + } + /* + else + { + vTable->GetMatches = (Mf_GetMatches_Func)Bt5_MatchFinder_GetMatches; + vTable->Skip = (Mf_Skip_Func)Bt5_MatchFinder_Skip; + } + */ +} diff --git a/code/nel/3rdparty/seven_zip/LzFind.h b/code/nel/3rdparty/seven_zip/LzFind.h new file mode 100644 index 000000000..42c13be15 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzFind.h @@ -0,0 +1,121 @@ +/* LzFind.h -- Match finder for LZ algorithms +2017-06-10 : Igor Pavlov : Public domain */ + +#ifndef __LZ_FIND_H +#define __LZ_FIND_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +typedef UInt32 CLzRef; + +typedef struct _CMatchFinder +{ + Byte *buffer; + UInt32 pos; + UInt32 posLimit; + UInt32 streamPos; + UInt32 lenLimit; + + UInt32 cyclicBufferPos; + UInt32 cyclicBufferSize; /* it must be = (historySize + 1) */ + + Byte streamEndWasReached; + Byte btMode; + Byte bigHash; + Byte directInput; + + UInt32 matchMaxLen; + CLzRef *hash; + CLzRef *son; + UInt32 hashMask; + UInt32 cutValue; + + Byte *bufferBase; + ISeqInStream *stream; + + UInt32 blockSize; + UInt32 keepSizeBefore; + UInt32 keepSizeAfter; + + UInt32 numHashBytes; + size_t directInputRem; + UInt32 historySize; + UInt32 fixedHashSize; + UInt32 hashSizeSum; + SRes result; + UInt32 crc[256]; + size_t numRefs; + + UInt64 expectedDataSize; +} CMatchFinder; + +#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer) + +#define Inline_MatchFinder_GetNumAvailableBytes(p) ((p)->streamPos - (p)->pos) + +#define Inline_MatchFinder_IsFinishedOK(p) \ + ((p)->streamEndWasReached \ + && (p)->streamPos == (p)->pos \ + && (!(p)->directInput || (p)->directInputRem == 0)) + +int MatchFinder_NeedMove(CMatchFinder *p); +Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p); +void MatchFinder_MoveBlock(CMatchFinder *p); +void MatchFinder_ReadIfRequired(CMatchFinder *p); + +void MatchFinder_Construct(CMatchFinder *p); + +/* Conditions: + historySize <= 3 GB + keepAddBufferBefore + matchMaxLen + keepAddBufferAfter < 511MB +*/ +int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, + UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, + ISzAllocPtr alloc); +void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc); +void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems); +void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue); + +UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *buffer, CLzRef *son, + UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, + UInt32 *distances, UInt32 maxLen); + +/* +Conditions: + Mf_GetNumAvailableBytes_Func must be called before each Mf_GetMatchLen_Func. + Mf_GetPointerToCurrentPos_Func's result must be used only before any other function +*/ + +typedef void (*Mf_Init_Func)(void *object); +typedef UInt32 (*Mf_GetNumAvailableBytes_Func)(void *object); +typedef const Byte * (*Mf_GetPointerToCurrentPos_Func)(void *object); +typedef UInt32 (*Mf_GetMatches_Func)(void *object, UInt32 *distances); +typedef void (*Mf_Skip_Func)(void *object, UInt32); + +typedef struct _IMatchFinder +{ + Mf_Init_Func Init; + Mf_GetNumAvailableBytes_Func GetNumAvailableBytes; + Mf_GetPointerToCurrentPos_Func GetPointerToCurrentPos; + Mf_GetMatches_Func GetMatches; + Mf_Skip_Func Skip; +} IMatchFinder; + +void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable); + +void MatchFinder_Init_LowHash(CMatchFinder *p); +void MatchFinder_Init_HighHash(CMatchFinder *p); +void MatchFinder_Init_3(CMatchFinder *p, int readData); +void MatchFinder_Init(CMatchFinder *p); + +UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); +UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); + +void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num); +void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/LzHash.h b/code/nel/3rdparty/seven_zip/LzHash.h new file mode 100644 index 000000000..e7c942303 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzHash.h @@ -0,0 +1,57 @@ +/* LzHash.h -- HASH functions for LZ algorithms +2015-04-12 : Igor Pavlov : Public domain */ + +#ifndef __LZ_HASH_H +#define __LZ_HASH_H + +#define kHash2Size (1 << 10) +#define kHash3Size (1 << 16) +#define kHash4Size (1 << 20) + +#define kFix3HashSize (kHash2Size) +#define kFix4HashSize (kHash2Size + kHash3Size) +#define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) + +#define HASH2_CALC hv = cur[0] | ((UInt32)cur[1] << 8); + +#define HASH3_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + hv = (temp ^ ((UInt32)cur[2] << 8)) & p->hashMask; } + +#define HASH4_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + temp ^= ((UInt32)cur[2] << 8); \ + h3 = temp & (kHash3Size - 1); \ + hv = (temp ^ (p->crc[cur[3]] << 5)) & p->hashMask; } + +#define HASH5_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + temp ^= ((UInt32)cur[2] << 8); \ + h3 = temp & (kHash3Size - 1); \ + temp ^= (p->crc[cur[3]] << 5); \ + h4 = temp & (kHash4Size - 1); \ + hv = (temp ^ (p->crc[cur[4]] << 3)) & p->hashMask; } + +/* #define HASH_ZIP_CALC hv = ((cur[0] | ((UInt32)cur[1] << 8)) ^ p->crc[cur[2]]) & 0xFFFF; */ +#define HASH_ZIP_CALC hv = ((cur[2] | ((UInt32)cur[0] << 8)) ^ p->crc[cur[1]]) & 0xFFFF; + + +#define MT_HASH2_CALC \ + h2 = (p->crc[cur[0]] ^ cur[1]) & (kHash2Size - 1); + +#define MT_HASH3_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + h3 = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); } + +#define MT_HASH4_CALC { \ + UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ + h2 = temp & (kHash2Size - 1); \ + temp ^= ((UInt32)cur[2] << 8); \ + h3 = temp & (kHash3Size - 1); \ + h4 = (temp ^ (p->crc[cur[3]] << 5)) & (kHash4Size - 1); } + +#endif diff --git a/code/nel/3rdparty/seven_zip/Lzma2Dec.c b/code/nel/3rdparty/seven_zip/Lzma2Dec.c new file mode 100644 index 000000000..4e138a4ae --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Lzma2Dec.c @@ -0,0 +1,488 @@ +/* Lzma2Dec.c -- LZMA2 Decoder +2019-02-02 : Igor Pavlov : Public domain */ + +/* #define SHOW_DEBUG_INFO */ + +#include "Precomp.h" + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#include + +#include "Lzma2Dec.h" + +/* +00000000 - End of data +00000001 U U - Uncompressed, reset dic, need reset state and set new prop +00000010 U U - Uncompressed, no reset +100uuuuu U U P P - LZMA, no reset +101uuuuu U U P P - LZMA, reset state +110uuuuu U U P P S - LZMA, reset state + set new prop +111uuuuu U U P P S - LZMA, reset state + set new prop, reset dic + + u, U - Unpack Size + P - Pack Size + S - Props +*/ + +#define LZMA2_CONTROL_COPY_RESET_DIC 1 + +#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & (1 << 7)) == 0) + +#define LZMA2_LCLP_MAX 4 +#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +typedef enum +{ + LZMA2_STATE_CONTROL, + LZMA2_STATE_UNPACK0, + LZMA2_STATE_UNPACK1, + LZMA2_STATE_PACK0, + LZMA2_STATE_PACK1, + LZMA2_STATE_PROP, + LZMA2_STATE_DATA, + LZMA2_STATE_DATA_CONT, + LZMA2_STATE_FINISHED, + LZMA2_STATE_ERROR +} ELzma2State; + +static SRes Lzma2Dec_GetOldProps(Byte prop, Byte *props) +{ + UInt32 dicSize; + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + dicSize = (prop == 40) ? 0xFFFFFFFF : LZMA2_DIC_SIZE_FROM_PROP(prop); + props[0] = (Byte)LZMA2_LCLP_MAX; + props[1] = (Byte)(dicSize); + props[2] = (Byte)(dicSize >> 8); + props[3] = (Byte)(dicSize >> 16); + props[4] = (Byte)(dicSize >> 24); + return SZ_OK; +} + +SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) +{ + Byte props[LZMA_PROPS_SIZE]; + RINOK(Lzma2Dec_GetOldProps(prop, props)); + return LzmaDec_AllocateProbs(&p->decoder, props, LZMA_PROPS_SIZE, alloc); +} + +SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) +{ + Byte props[LZMA_PROPS_SIZE]; + RINOK(Lzma2Dec_GetOldProps(prop, props)); + return LzmaDec_Allocate(&p->decoder, props, LZMA_PROPS_SIZE, alloc); +} + +void Lzma2Dec_Init(CLzma2Dec *p) +{ + p->state = LZMA2_STATE_CONTROL; + p->needInitLevel = 0xE0; + p->isExtraMode = False; + p->unpackSize = 0; + + // p->decoder.dicPos = 0; // we can use it instead of full init + LzmaDec_Init(&p->decoder); +} + +static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) +{ + switch (p->state) + { + case LZMA2_STATE_CONTROL: + p->isExtraMode = False; + p->control = b; + PRF(printf("\n %8X", (unsigned)p->decoder.dicPos)); + PRF(printf(" %02X", (unsigned)b)); + if (b == 0) + return LZMA2_STATE_FINISHED; + if (LZMA2_IS_UNCOMPRESSED_STATE(p)) + { + if (b == LZMA2_CONTROL_COPY_RESET_DIC) + p->needInitLevel = 0xC0; + else if (b > 2 || p->needInitLevel == 0xE0) + return LZMA2_STATE_ERROR; + } + else + { + if (b < p->needInitLevel) + return LZMA2_STATE_ERROR; + p->needInitLevel = 0; + p->unpackSize = (UInt32)(b & 0x1F) << 16; + } + return LZMA2_STATE_UNPACK0; + + case LZMA2_STATE_UNPACK0: + p->unpackSize |= (UInt32)b << 8; + return LZMA2_STATE_UNPACK1; + + case LZMA2_STATE_UNPACK1: + p->unpackSize |= (UInt32)b; + p->unpackSize++; + PRF(printf(" %7u", (unsigned)p->unpackSize)); + return LZMA2_IS_UNCOMPRESSED_STATE(p) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; + + case LZMA2_STATE_PACK0: + p->packSize = (UInt32)b << 8; + return LZMA2_STATE_PACK1; + + case LZMA2_STATE_PACK1: + p->packSize |= (UInt32)b; + p->packSize++; + // if (p->packSize < 5) return LZMA2_STATE_ERROR; + PRF(printf(" %5u", (unsigned)p->packSize)); + return (p->control & 0x40) ? LZMA2_STATE_PROP : LZMA2_STATE_DATA; + + case LZMA2_STATE_PROP: + { + unsigned lc, lp; + if (b >= (9 * 5 * 5)) + return LZMA2_STATE_ERROR; + lc = b % 9; + b /= 9; + p->decoder.prop.pb = (Byte)(b / 5); + lp = b % 5; + if (lc + lp > LZMA2_LCLP_MAX) + return LZMA2_STATE_ERROR; + p->decoder.prop.lc = (Byte)lc; + p->decoder.prop.lp = (Byte)lp; + return LZMA2_STATE_DATA; + } + } + return LZMA2_STATE_ERROR; +} + +static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, const Byte *src, SizeT size) +{ + memcpy(p->dic + p->dicPos, src, size); + p->dicPos += size; + if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= size) + p->checkDicSize = p->prop.dicSize; + p->processedPos += (UInt32)size; +} + +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState); + + +SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, + const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) +{ + SizeT inSize = *srcLen; + *srcLen = 0; + *status = LZMA_STATUS_NOT_SPECIFIED; + + while (p->state != LZMA2_STATE_ERROR) + { + SizeT dicPos; + + if (p->state == LZMA2_STATE_FINISHED) + { + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } + + dicPos = p->decoder.dicPos; + + if (dicPos == dicLimit && finishMode == LZMA_FINISH_ANY) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_OK; + } + + if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT) + { + if (*srcLen == inSize) + { + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + (*srcLen)++; + p->state = Lzma2Dec_UpdateState(p, *src++); + if (dicPos == dicLimit && p->state != LZMA2_STATE_FINISHED) + break; + continue; + } + + { + SizeT inCur = inSize - *srcLen; + SizeT outCur = dicLimit - dicPos; + ELzmaFinishMode curFinishMode = LZMA_FINISH_ANY; + + if (outCur >= p->unpackSize) + { + outCur = (SizeT)p->unpackSize; + curFinishMode = LZMA_FINISH_END; + } + + if (LZMA2_IS_UNCOMPRESSED_STATE(p)) + { + if (inCur == 0) + { + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + + if (p->state == LZMA2_STATE_DATA) + { + BoolInt initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); + LzmaDec_InitDicAndState(&p->decoder, initDic, False); + } + + if (inCur > outCur) + inCur = outCur; + if (inCur == 0) + break; + + LzmaDec_UpdateWithUncompressed(&p->decoder, src, inCur); + + src += inCur; + *srcLen += inCur; + p->unpackSize -= (UInt32)inCur; + p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; + } + else + { + SRes res; + + if (p->state == LZMA2_STATE_DATA) + { + BoolInt initDic = (p->control >= 0xE0); + BoolInt initState = (p->control >= 0xA0); + LzmaDec_InitDicAndState(&p->decoder, initDic, initState); + p->state = LZMA2_STATE_DATA_CONT; + } + + if (inCur > p->packSize) + inCur = (SizeT)p->packSize; + + res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status); + + src += inCur; + *srcLen += inCur; + p->packSize -= (UInt32)inCur; + outCur = p->decoder.dicPos - dicPos; + p->unpackSize -= (UInt32)outCur; + + if (res != 0) + break; + + if (*status == LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (p->packSize == 0) + break; + return SZ_OK; + } + + if (inCur == 0 && outCur == 0) + { + if (*status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + || p->unpackSize != 0 + || p->packSize != 0) + break; + p->state = LZMA2_STATE_CONTROL; + } + + *status = LZMA_STATUS_NOT_SPECIFIED; + } + } + } + + *status = LZMA_STATUS_NOT_SPECIFIED; + p->state = LZMA2_STATE_ERROR; + return SZ_ERROR_DATA; +} + + + + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, + const Byte *src, SizeT *srcLen, + int checkFinishBlock) +{ + SizeT inSize = *srcLen; + *srcLen = 0; + + while (p->state != LZMA2_STATE_ERROR) + { + if (p->state == LZMA2_STATE_FINISHED) + return (ELzma2ParseStatus)LZMA_STATUS_FINISHED_WITH_MARK; + + if (outSize == 0 && !checkFinishBlock) + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + + if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT) + { + if (*srcLen == inSize) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + (*srcLen)++; + + p->state = Lzma2Dec_UpdateState(p, *src++); + + if (p->state == LZMA2_STATE_UNPACK0) + { + // if (p->decoder.dicPos != 0) + if (p->control == LZMA2_CONTROL_COPY_RESET_DIC || p->control >= 0xE0) + return LZMA2_PARSE_STATUS_NEW_BLOCK; + // if (outSize == 0) return LZMA_STATUS_NOT_FINISHED; + } + + // The following code can be commented. + // It's not big problem, if we read additional input bytes. + // It will be stopped later in LZMA2_STATE_DATA / LZMA2_STATE_DATA_CONT state. + + if (outSize == 0 && p->state != LZMA2_STATE_FINISHED) + { + // checkFinishBlock is true. So we expect that block must be finished, + // We can return LZMA_STATUS_NOT_SPECIFIED or LZMA_STATUS_NOT_FINISHED here + // break; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + } + + if (p->state == LZMA2_STATE_DATA) + return LZMA2_PARSE_STATUS_NEW_CHUNK; + + continue; + } + + if (outSize == 0) + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + + { + SizeT inCur = inSize - *srcLen; + + if (LZMA2_IS_UNCOMPRESSED_STATE(p)) + { + if (inCur == 0) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + if (inCur > p->unpackSize) + inCur = p->unpackSize; + if (inCur > outSize) + inCur = outSize; + p->decoder.dicPos += inCur; + src += inCur; + *srcLen += inCur; + outSize -= inCur; + p->unpackSize -= (UInt32)inCur; + p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; + } + else + { + p->isExtraMode = True; + + if (inCur == 0) + { + if (p->packSize != 0) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + } + else if (p->state == LZMA2_STATE_DATA) + { + p->state = LZMA2_STATE_DATA_CONT; + if (*src != 0) + { + // first byte of lzma chunk must be Zero + *srcLen += 1; + p->packSize--; + break; + } + } + + if (inCur > p->packSize) + inCur = (SizeT)p->packSize; + + src += inCur; + *srcLen += inCur; + p->packSize -= (UInt32)inCur; + + if (p->packSize == 0) + { + SizeT rem = outSize; + if (rem > p->unpackSize) + rem = p->unpackSize; + p->decoder.dicPos += rem; + p->unpackSize -= (UInt32)rem; + outSize -= rem; + if (p->unpackSize == 0) + p->state = LZMA2_STATE_CONTROL; + } + } + } + } + + p->state = LZMA2_STATE_ERROR; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; +} + + + + +SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) +{ + SizeT outSize = *destLen, inSize = *srcLen; + *srcLen = *destLen = 0; + + for (;;) + { + SizeT inCur = inSize, outCur, dicPos; + ELzmaFinishMode curFinishMode; + SRes res; + + if (p->decoder.dicPos == p->decoder.dicBufSize) + p->decoder.dicPos = 0; + dicPos = p->decoder.dicPos; + curFinishMode = LZMA_FINISH_ANY; + outCur = p->decoder.dicBufSize - dicPos; + + if (outCur >= outSize) + { + outCur = outSize; + curFinishMode = finishMode; + } + + res = Lzma2Dec_DecodeToDic(p, dicPos + outCur, src, &inCur, curFinishMode, status); + + src += inCur; + inSize -= inCur; + *srcLen += inCur; + outCur = p->decoder.dicPos - dicPos; + memcpy(dest, p->decoder.dic + dicPos, outCur); + dest += outCur; + outSize -= outCur; + *destLen += outCur; + if (res != 0) + return res; + if (outCur == 0 || outSize == 0) + return SZ_OK; + } +} + + +SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAllocPtr alloc) +{ + CLzma2Dec p; + SRes res; + SizeT outSize = *destLen, inSize = *srcLen; + *destLen = *srcLen = 0; + *status = LZMA_STATUS_NOT_SPECIFIED; + Lzma2Dec_Construct(&p); + RINOK(Lzma2Dec_AllocateProbs(&p, prop, alloc)); + p.decoder.dic = dest; + p.decoder.dicBufSize = outSize; + Lzma2Dec_Init(&p); + *srcLen = inSize; + res = Lzma2Dec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status); + *destLen = p.decoder.dicPos; + if (res == SZ_OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT) + res = SZ_ERROR_INPUT_EOF; + Lzma2Dec_FreeProbs(&p, alloc); + return res; +} diff --git a/code/nel/3rdparty/seven_zip/Lzma2Dec.h b/code/nel/3rdparty/seven_zip/Lzma2Dec.h new file mode 100644 index 000000000..b8ddeac89 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Lzma2Dec.h @@ -0,0 +1,120 @@ +/* Lzma2Dec.h -- LZMA2 Decoder +2018-02-19 : Igor Pavlov : Public domain */ + +#ifndef __LZMA2_DEC_H +#define __LZMA2_DEC_H + +#include "LzmaDec.h" + +EXTERN_C_BEGIN + +/* ---------- State Interface ---------- */ + +typedef struct +{ + unsigned state; + Byte control; + Byte needInitLevel; + Byte isExtraMode; + Byte _pad_; + UInt32 packSize; + UInt32 unpackSize; + CLzmaDec decoder; +} CLzma2Dec; + +#define Lzma2Dec_Construct(p) LzmaDec_Construct(&(p)->decoder) +#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc) +#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc) + +SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); +SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); +void Lzma2Dec_Init(CLzma2Dec *p); + +/* +finishMode: + It has meaning only if the decoding reaches output limit (*destLen or dicLimit). + LZMA_FINISH_ANY - use smallest number of input bytes + LZMA_FINISH_END - read EndOfStream marker after decoding + +Returns: + SZ_OK + status: + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED + LZMA_STATUS_NEEDS_MORE_INPUT + SZ_ERROR_DATA - Data error +*/ + +SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, + const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); + +SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); + + +/* ---------- LZMA2 block and chunk parsing ---------- */ + +/* +Lzma2Dec_Parse() parses compressed data stream up to next independent block or next chunk data. +It can return LZMA_STATUS_* code or LZMA2_PARSE_STATUS_* code: + - LZMA2_PARSE_STATUS_NEW_BLOCK - there is new block, and 1 additional byte (control byte of next block header) was read from input. + - LZMA2_PARSE_STATUS_NEW_CHUNK - there is new chunk, and only lzma2 header of new chunk was read. + CLzma2Dec::unpackSize contains unpack size of that chunk +*/ + +typedef enum +{ +/* + LZMA_STATUS_NOT_SPECIFIED // data error + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED // + LZMA_STATUS_NEEDS_MORE_INPUT + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK // unused +*/ + LZMA2_PARSE_STATUS_NEW_BLOCK = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + 1, + LZMA2_PARSE_STATUS_NEW_CHUNK +} ELzma2ParseStatus; + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, // output size + const Byte *src, SizeT *srcLen, + int checkFinishBlock // set (checkFinishBlock = 1), if it must read full input data, if decoder.dicPos reaches blockMax position. + ); + +/* +LZMA2 parser doesn't decode LZMA chunks, so we must read + full input LZMA chunk to decode some part of LZMA chunk. + +Lzma2Dec_GetUnpackExtra() returns the value that shows + max possible number of output bytes that can be output by decoder + at current input positon. +*/ + +#define Lzma2Dec_GetUnpackExtra(p) ((p)->isExtraMode ? (p)->unpackSize : 0); + + +/* ---------- One Call Interface ---------- */ + +/* +finishMode: + It has meaning only if the decoding reaches output limit (*destLen). + LZMA_FINISH_ANY - use smallest number of input bytes + LZMA_FINISH_END - read EndOfStream marker after decoding + +Returns: + SZ_OK + status: + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED + SZ_ERROR_DATA - Data error + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_UNSUPPORTED - Unsupported properties + SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src). +*/ + +SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAllocPtr alloc); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/Lzma2DecMt.c b/code/nel/3rdparty/seven_zip/Lzma2DecMt.c new file mode 100644 index 000000000..988643d95 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Lzma2DecMt.c @@ -0,0 +1,1082 @@ +/* Lzma2DecMt.c -- LZMA2 Decoder Multi-thread +2019-02-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) +#define PRF_STR_INT_2(s, d1, d2) PRF(printf("\n" s " %d %d\n", (unsigned)d1, (unsigned)d2)) + +// #define _7ZIP_ST + +#include "Alloc.h" + +#include "Lzma2Dec.h" +#include "Lzma2DecMt.h" + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +#define LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT (1 << 28) + +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p) +{ + p->inBufSize_ST = 1 << 20; + p->outStep_ST = 1 << 20; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->outBlockMax = LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT; + p->inBlockMax = p->outBlockMax + p->outBlockMax / 16; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CLzma2DecMtThread ---------- */ + +typedef struct +{ + CLzma2Dec dec; + Byte dec_created; + Byte needInit; + + Byte *outBuf; + size_t outBufSize; + + EMtDecParseState state; + ELzma2ParseStatus parseStatus; + + size_t inPreSize; + size_t outPreSize; + + size_t inCodeSize; + size_t outCodeSize; + SRes codeRes; + + CAlignOffsetAlloc alloc; + + Byte mtPad[1 << 7]; +} CLzma2DecMtThread; + +#endif + + +/* ---------- CLzma2DecMt ---------- */ + +typedef struct +{ + // ISzAllocPtr alloc; + ISzAllocPtr allocMid; + + CAlignOffsetAlloc alignOffsetAlloc; + CLzma2DecMtProps props; + Byte prop; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + + BoolInt finishMode; + BoolInt outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + BoolInt readWasFinished; + SRes readRes; + + Byte *inBuf; + size_t inBufSize; + Byte dec_created; + CLzma2Dec dec; + + size_t inPos; + size_t inLim; + + #ifndef _7ZIP_ST + UInt64 outProcessed_Parse; + BoolInt mtc_WasConstructed; + CMtDec mtc; + CLzma2DecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CLzma2DecMt; + + + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CLzma2DecMt *p = (CLzma2DecMt *)ISzAlloc_Alloc(alloc, sizeof(CLzma2DecMt)); + if (!p) + return NULL; + + // p->alloc = alloc; + p->allocMid = allocMid; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + p->alignOffsetAlloc.baseAlloc = alloc; + + p->inBuf = NULL; + p->inBufSize = 0; + p->dec_created = False; + + // Lzma2DecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + t->dec_created = False; + t->outBuf = NULL; + t->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_FreeOutBufs(CLzma2DecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->outBuf) + { + ISzAlloc_Free(p->allocMid, t->outBuf); + t->outBuf = NULL; + t->outBufSize = 0; + } + } +} + +#endif + + +static void Lzma2DecMt_FreeSt(CLzma2DecMt *p) +{ + if (p->dec_created) + { + Lzma2Dec_Free(&p->dec, &p->alignOffsetAlloc.vt); + p->dec_created = False; + } + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void Lzma2DecMt_Destroy(CLzma2DecMtHandle pp) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + Lzma2DecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + Lzma2Dec_FreeProbs(&t->dec, &t->alloc.vt); // p->alloc !!! + t->dec_created = False; + } + } + } + Lzma2DecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CLzma2DecMt *me = (CLzma2DecMt *)obj; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + PRF_STR_INT_2("Parse", coderIndex, cc->srcSize); + + cc->state = MTDEC_PARSE_CONTINUE; + + if (cc->startCall) + { + if (!t->dec_created) + { + Lzma2Dec_Construct(&t->dec); + t->dec_created = True; + AlignOffsetAlloc_CreateVTable(&t->alloc); + { + /* (1 << 12) is expected size of one way in data cache. + We optimize alignment for cache line size of 128 bytes and smaller */ + const unsigned kNumAlignBits = 12; + const unsigned kNumCacheLineBits = 7; /* <= kNumAlignBits */ + t->alloc.numAlignBits = kNumAlignBits; + t->alloc.offset = ((UInt32)coderIndex * ((1 << 11) + (1 << 8) + (1 << 6))) & ((1 << kNumAlignBits) - (1 << kNumCacheLineBits)); + t->alloc.baseAlloc = me->alignOffsetAlloc.baseAlloc; + } + } + Lzma2Dec_Init(&t->dec); + + t->inPreSize = 0; + t->outPreSize = 0; + // t->blockWasFinished = False; + // t->finishedWithMark = False; + t->parseStatus = (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; + t->state = MTDEC_PARSE_CONTINUE; + + t->inCodeSize = 0; + t->outCodeSize = 0; + t->codeRes = SZ_OK; + + // (cc->srcSize == 0) is allowed + } + + { + ELzma2ParseStatus status; + BoolInt overflow; + UInt32 unpackRem = 0; + + int checkFinishBlock = True; + size_t limit = me->props.outBlockMax; + if (me->outSize_Defined) + { + UInt64 rem = me->outSize - me->outProcessed_Parse; + if (limit >= rem) + { + limit = (size_t)rem; + if (!me->finishMode) + checkFinishBlock = False; + } + } + + // checkFinishBlock = False, if we want to decode partial data + // that must be finished at position <= outBlockMax. + + { + const SizeT srcOrig = cc->srcSize; + SizeT srcSize_Point = 0; + SizeT dicPos_Point = 0; + + cc->srcSize = 0; + overflow = False; + + for (;;) + { + SizeT srcCur = srcOrig - cc->srcSize; + + status = Lzma2Dec_Parse(&t->dec, + limit - t->dec.decoder.dicPos, + cc->src + cc->srcSize, &srcCur, + checkFinishBlock); + + cc->srcSize += srcCur; + + if (status == LZMA2_PARSE_STATUS_NEW_CHUNK) + { + if (t->dec.unpackSize > me->props.outBlockMax - t->dec.decoder.dicPos) + { + overflow = True; + break; + } + continue; + } + + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + if (t->dec.decoder.dicPos == 0) + continue; + // we decode small blocks in one thread + if (t->dec.decoder.dicPos >= (1 << 14)) + break; + dicPos_Point = t->dec.decoder.dicPos; + srcSize_Point = cc->srcSize; + continue; + } + + if ((int)status == LZMA_STATUS_NOT_FINISHED && checkFinishBlock + // && limit == t->dec.decoder.dicPos + // && limit == me->props.outBlockMax + ) + { + overflow = True; + break; + } + + unpackRem = Lzma2Dec_GetUnpackExtra(&t->dec); + break; + } + + if (dicPos_Point != 0 + && (int)status != LZMA2_PARSE_STATUS_NEW_BLOCK + && (int)status != LZMA_STATUS_FINISHED_WITH_MARK + && (int)status != LZMA_STATUS_NOT_SPECIFIED) + { + // we revert to latest newBlock state + status = LZMA2_PARSE_STATUS_NEW_BLOCK; + unpackRem = 0; + t->dec.decoder.dicPos = dicPos_Point; + cc->srcSize = srcSize_Point; + overflow = False; + } + } + + t->inPreSize += cc->srcSize; + t->parseStatus = status; + + if (overflow) + cc->state = MTDEC_PARSE_OVERFLOW; + else + { + size_t dicPos = t->dec.decoder.dicPos; + + if ((int)status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + cc->state = MTDEC_PARSE_NEW; + cc->srcSize--; // we don't need control byte of next block + t->inPreSize--; + } + else + { + cc->state = MTDEC_PARSE_END; + if ((int)status != LZMA_STATUS_FINISHED_WITH_MARK) + { + // (status == LZMA_STATUS_NOT_SPECIFIED) + // (status == LZMA_STATUS_NOT_FINISHED) + if (unpackRem != 0) + { + /* we also reserve space for max possible number of output bytes of current LZMA chunk */ + SizeT rem = limit - dicPos; + if (rem > unpackRem) + rem = unpackRem; + dicPos += rem; + } + } + } + + me->outProcessed_Parse += dicPos; + } + + cc->outPos = dicPos; + t->outPreSize = (size_t)dicPos; + } + + t->state = cc->state; + return; + } +} + + +static SRes Lzma2DecMt_MtCallback_PreCode(void *pp, unsigned coderIndex) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + Byte *dest = t->outBuf; + + if (t->inPreSize == 0) + { + t->codeRes = SZ_ERROR_DATA; + return t->codeRes; + } + + if (!dest || t->outBufSize < t->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + t->outBuf = NULL; + t->outBufSize = 0; + } + + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, t->outPreSize + // + (1 << 28) + ); + // Sleep(200); + if (!dest) + return SZ_ERROR_MEM; + t->outBuf = dest; + t->outBufSize = t->outPreSize; + } + + t->dec.decoder.dic = dest; + t->dec.decoder.dicBufSize = t->outPreSize; + + t->needInit = True; + + return Lzma2Dec_AllocateProbs(&t->dec, me->prop, &t->alloc.vt); // alloc.vt +} + + +static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + UNUSED_VAR(srcFinished) + + PRF_STR_INT_2("Code", coderIndex, srcSize); + + *inCodePos = t->inCodeSize; + *outCodePos = 0; + *stop = True; + + if (t->needInit) + { + Lzma2Dec_Init(&t->dec); + t->needInit = False; + } + + { + ELzmaStatus status; + size_t srcProcessed = srcSize; + BoolInt blockWasFinished = + ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK); + + SRes res = Lzma2Dec_DecodeToDic(&t->dec, + t->outPreSize, + src, &srcProcessed, + blockWasFinished ? LZMA_FINISH_END : LZMA_FINISH_ANY, + &status); + + t->codeRes = res; + + t->inCodeSize += srcProcessed; + *inCodePos = t->inCodeSize; + t->outCodeSize = t->dec.decoder.dicPos; + *outCodePos = t->dec.decoder.dicPos; + + if (res != SZ_OK) + return res; + + if (srcProcessed == srcSize) + *stop = False; + + if (blockWasFinished) + { + if (srcSize != srcProcessed) + return SZ_ERROR_FAIL; + + if (t->inPreSize == t->inCodeSize) + { + if (t->outPreSize != t->outCodeSize) + return SZ_ERROR_FAIL; + *stop = True; + } + } + else + { + if (t->outPreSize == t->outCodeSize) + *stop = True; + } + + return SZ_OK; + } +} + + +#define LZMA2DECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, + BoolInt *needContinue, BoolInt *canRecode) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + const CLzma2DecMtThread *t = &me->coders[coderIndex]; + size_t size = t->outCodeSize; + const Byte *data = t->outBuf; + BoolInt needContinue2 = True; + + PRF_STR_INT_2("Write", coderIndex, srcSize); + + *needContinue = False; + *canRecode = True; + UNUSED_VAR(src) + UNUSED_VAR(srcSize) + + if ( + // t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + t->state == MTDEC_PARSE_OVERFLOW + || t->state == MTDEC_PARSE_END) + needContinue2 = False; + + + if (!needWriteToStream) + return SZ_OK; + + me->mtc.inProcessed += t->inCodeSize; + + if (t->codeRes == SZ_OK) + if ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK) + if (t->outPreSize != t->outCodeSize + || t->inPreSize != t->inCodeSize) + return SZ_ERROR_FAIL; + + *canRecode = False; + + if (me->outStream) + { + for (;;) + { + size_t cur = size; + size_t written; + if (cur > LZMA2DECMT_STREAM_WRITE_STEP) + cur = LZMA2DECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + me->outProcessed += written; + // me->mtc.writtenTotal += written; + if (written != cur) + return SZ_ERROR_WRITE; + data += cur; + size -= cur; + if (size == 0) + { + *needContinue = needContinue2; + return SZ_OK; + } + RINOK(MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0)); + } + } + + return SZ_ERROR_FAIL; + /* + if (size > me->outBufSize) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBufSize -= size; + me->outBuf += size; + *needContinue = needContinue2; + return SZ_OK; + */ +} + +#endif + + +static SRes Lzma2Dec_Prepare_ST(CLzma2DecMt *p) +{ + if (!p->dec_created) + { + Lzma2Dec_Construct(&p->dec); + p->dec_created = True; + } + + RINOK(Lzma2Dec_Allocate(&p->dec, p->prop, &p->alignOffsetAlloc.vt)); + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + Lzma2Dec_Init(&p->dec); + + return SZ_OK; +} + + +static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p + #ifndef _7ZIP_ST + , BoolInt tMode + #endif + ) +{ + SizeT wrPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CLzma2Dec *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + Lzma2DecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + RINOK(Lzma2Dec_Prepare_ST(p)); + + dec = &p->dec; + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + wrPos = dec->decoder.dicPos; + + for (;;) + { + SizeT dicPos; + SizeT size; + ELzmaFinishMode finishMode; + SizeT inProcessed; + ELzmaStatus status; + SRes res; + + SizeT outProcessed; + BoolInt outFinished; + BoolInt needStop; + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + // p->readProcessed += inLim; + // inLim -= 5; p->readWasFinished = True; // for test + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + dicPos = dec->decoder.dicPos; + { + SizeT next = dec->decoder.dicBufSize; + if (next - wrPos > p->props.outStep_ST) + next = wrPos + p->props.outStep_ST; + size = next - dicPos; + } + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (SizeT)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + inProcessed = inLim - inPos; + + res = Lzma2Dec_DecodeToDic(dec, dicPos + size, inData + inPos, &inProcessed, finishMode, &status); + + inPos += inProcessed; + p->inProcessed += inProcessed; + outProcessed = dec->decoder.dicPos - dicPos; + p->outProcessed += outProcessed; + + outFinished = (p->outSize_Defined && p->outSize <= p->outProcessed); + + needStop = (res != SZ_OK + || (inProcessed == 0 && outProcessed == 0) + || status == LZMA_STATUS_FINISHED_WITH_MARK + || (!p->finishMode && outFinished)); + + if (needStop || outProcessed >= size) + { + SRes res2; + { + size_t writeSize = dec->decoder.dicPos - wrPos; + size_t written = ISeqOutStream_Write(p->outStream, dec->decoder.dic + wrPos, writeSize); + res2 = (written == writeSize) ? SZ_OK : SZ_ERROR_WRITE; + } + + if (dec->decoder.dicPos == dec->decoder.dicBufSize) + dec->decoder.dicPos = 0; + wrPos = dec->decoder.dicPos; + + RINOK(res2); + + if (needStop) + { + if (res != SZ_OK) + return res; + + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + { + if (p->finishMode) + { + if (p->outSize_Defined && p->outSize != p->outProcessed) + return SZ_ERROR_DATA; + } + return SZ_OK; + } + + if (!p->finishMode && outFinished) + return SZ_OK; + + if (status == LZMA_STATUS_NEEDS_MORE_INPUT) + return SZ_ERROR_INPUT_EOF; + + return SZ_ERROR_DATA; + } + } + + if (p->progress) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + RINOK(ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed)); + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + } +} + + + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, const UInt64 *outDataSize, int finishMode, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + UInt64 *inProcessed, + // UInt64 *outProcessed, + int *isMT, + ICompressProgress *progress) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + #ifndef _7ZIP_ST + BoolInt tMode; + #endif + + *inProcessed = 0; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->readWasFinished = False; + + *isMT = False; + + + #ifndef _7ZIP_ST + + tMode = False; + + // p->mtc.parseRes = SZ_OK; + + // p->mtc.numFilledThreads = 0; + // p->mtc.crossStart = 0; + // p->mtc.crossEnd = 0; + // p->mtc.allocError_for_Read_BlockIndex = 0; + // p->mtc.isAllocError = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + Lzma2DecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + + // p->outBuf = NULL; + // p->outBufSize = 0; + /* + if (!outStream) + { + // p->outBuf = outBuf; + // p->outBufSize = *outBufSize; + // *outBufSize = 0; + return SZ_ERROR_PARAM; + } + */ + + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->alignOffsetAlloc.baseAlloc; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.inBufSize = p->props.inBufSize_MT; + + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = Lzma2DecMt_MtCallback_Parse; + vt.PreCode = Lzma2DecMt_MtCallback_PreCode; + vt.Code = Lzma2DecMt_MtCallback_Code; + vt.Write = Lzma2DecMt_MtCallback_Write; + + { + BoolInt needContinue = False; + + SRes res = MtDec_Code(&p->mtc); + + /* + if (!outStream) + *outBufSize = p->outBuf - outBuf; + */ + + *inProcessed = p->mtc.inProcessed; + + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + res = p->mtc.mtProgress.res; + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + if (res == SZ_OK) + return p->mtc.readRes; + return res; + } + + tMode = True; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->inProcessed = p->mtc.inProcessed; + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = Lzma2Dec_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + ); + + *inProcessed = p->inProcessed; + + // res = SZ_OK; // for test + if (res == SZ_OK && p->readRes != SZ_OK) + res = p->readRes; + + /* + #ifndef _7ZIP_ST + if (res == SZ_OK && tMode && p->mtc.parseRes != SZ_OK) + res = p->mtc.parseRes; + #endif + */ + + return res; + } +} + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->inPos = 0; + p->inLim = 0; + + return Lzma2Dec_Prepare_ST(p); +} + + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + ELzmaFinishMode finishMode; + SRes readRes; + size_t size = *outSize; + + *outSize = 0; + *inStreamProcessed = 0; + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (size_t)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + readRes = SZ_OK; + + for (;;) + { + SizeT inCur; + SizeT outCur; + ELzmaStatus status; + SRes res; + + if (p->inPos == p->inLim && readRes == SZ_OK) + { + p->inPos = 0; + p->inLim = p->props.inBufSize_ST; + readRes = ISeqInStream_Read(p->inStream, p->inBuf, &p->inLim); + } + + inCur = p->inLim - p->inPos; + outCur = size; + + res = Lzma2Dec_DecodeToBuf(&p->dec, data, &outCur, + p->inBuf + p->inPos, &inCur, finishMode, &status); + + p->inPos += inCur; + p->inProcessed += inCur; + *inStreamProcessed += inCur; + p->outProcessed += outCur; + *outSize += outCur; + size -= outCur; + data += outCur; + + if (res != 0) + return res; + + /* + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + return readRes; + + if (size == 0 && status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (p->finishMode && p->outSize_Defined && p->outProcessed >= p->outSize) + return SZ_ERROR_DATA; + return readRes; + } + */ + + if (inCur == 0 && outCur == 0) + return readRes; + } +} diff --git a/code/nel/3rdparty/seven_zip/Lzma2DecMt.h b/code/nel/3rdparty/seven_zip/Lzma2DecMt.h new file mode 100644 index 000000000..7791c310b --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Lzma2DecMt.h @@ -0,0 +1,79 @@ +/* Lzma2DecMt.h -- LZMA2 Decoder Multi-thread +2018-02-17 : Igor Pavlov : Public domain */ + +#ifndef __LZMA2_DEC_MT_H +#define __LZMA2_DEC_MT_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t outBlockMax; + size_t inBlockMax; + #endif +} CLzma2DecMtProps; + +/* init to single-thread mode */ +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p); + + +/* ---------- CLzma2DecMtHandle Interface ---------- */ + +/* Lzma2DecMt_ * functions can return the following exit codes: +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater in props + SZ_ERROR_WRITE - ISeqOutStream write callback error + // SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output + SZ_ERROR_PROGRESS - some break from progress callback + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) +*/ + +typedef void * CLzma2DecMtHandle; + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void Lzma2DecMt_Destroy(CLzma2DecMtHandle p); + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - if lzma2 stream must be finished + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + + // out variables: + UInt64 *inProcessed, + int *isMT, /* out: (*isMT == 0), if single thread decoding was used */ + + // UInt64 *outProcessed, + ICompressProgress *progress); + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream); + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed); + + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/Lzma2Enc.c b/code/nel/3rdparty/seven_zip/Lzma2Enc.c new file mode 100644 index 000000000..5c1ad4931 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Lzma2Enc.c @@ -0,0 +1,803 @@ +/* Lzma2Enc.c -- LZMA2 Encoder +2018-07-04 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +/* #define _7ZIP_ST */ + +#include "Lzma2Enc.h" + +#ifndef _7ZIP_ST +#include "MtCoder.h" +#else +#define MTCODER__THREADS_MAX 1 +#endif + +#define LZMA2_CONTROL_LZMA (1 << 7) +#define LZMA2_CONTROL_COPY_NO_RESET 2 +#define LZMA2_CONTROL_COPY_RESET_DIC 1 +#define LZMA2_CONTROL_EOF 0 + +#define LZMA2_LCLP_MAX 4 + +#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) + +#define LZMA2_PACK_SIZE_MAX (1 << 16) +#define LZMA2_COPY_CHUNK_SIZE LZMA2_PACK_SIZE_MAX +#define LZMA2_UNPACK_SIZE_MAX (1 << 21) +#define LZMA2_KEEP_WINDOW_SIZE LZMA2_UNPACK_SIZE_MAX + +#define LZMA2_CHUNK_SIZE_COMPRESSED_MAX ((1 << 16) + 16) + + +#define PRF(x) /* x */ + + +/* ---------- CLimitedSeqInStream ---------- */ + +typedef struct +{ + ISeqInStream vt; + ISeqInStream *realStream; + UInt64 limit; + UInt64 processed; + int finished; +} CLimitedSeqInStream; + +static void LimitedSeqInStream_Init(CLimitedSeqInStream *p) +{ + p->limit = (UInt64)(Int64)-1; + p->processed = 0; + p->finished = 0; +} + +static SRes LimitedSeqInStream_Read(const ISeqInStream *pp, void *data, size_t *size) +{ + CLimitedSeqInStream *p = CONTAINER_FROM_VTBL(pp, CLimitedSeqInStream, vt); + size_t size2 = *size; + SRes res = SZ_OK; + + if (p->limit != (UInt64)(Int64)-1) + { + UInt64 rem = p->limit - p->processed; + if (size2 > rem) + size2 = (size_t)rem; + } + if (size2 != 0) + { + res = ISeqInStream_Read(p->realStream, data, &size2); + p->finished = (size2 == 0 ? 1 : 0); + p->processed += size2; + } + *size = size2; + return res; +} + + +/* ---------- CLzma2EncInt ---------- */ + +typedef struct +{ + CLzmaEncHandle enc; + Byte propsAreSet; + Byte propsByte; + Byte needInitState; + Byte needInitProp; + UInt64 srcPos; +} CLzma2EncInt; + + +static SRes Lzma2EncInt_InitStream(CLzma2EncInt *p, const CLzma2EncProps *props) +{ + if (!p->propsAreSet) + { + SizeT propsSize = LZMA_PROPS_SIZE; + Byte propsEncoded[LZMA_PROPS_SIZE]; + RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); + RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); + p->propsByte = propsEncoded[0]; + p->propsAreSet = True; + } + return SZ_OK; +} + +static void Lzma2EncInt_InitBlock(CLzma2EncInt *p) +{ + p->srcPos = 0; + p->needInitState = True; + p->needInitProp = True; +} + + +SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, + ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, + Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); +const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); +void LzmaEnc_Finish(CLzmaEncHandle pp); +void LzmaEnc_SaveState(CLzmaEncHandle pp); +void LzmaEnc_RestoreState(CLzmaEncHandle pp); + +/* +UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp); +*/ + +static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, + size_t *packSizeRes, ISeqOutStream *outStream) +{ + size_t packSizeLimit = *packSizeRes; + size_t packSize = packSizeLimit; + UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX; + unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0); + BoolInt useCopyBlock; + SRes res; + + *packSizeRes = 0; + if (packSize < lzHeaderSize) + return SZ_ERROR_OUTPUT_EOF; + packSize -= lzHeaderSize; + + LzmaEnc_SaveState(p->enc); + res = LzmaEnc_CodeOneMemBlock(p->enc, p->needInitState, + outBuf + lzHeaderSize, &packSize, LZMA2_PACK_SIZE_MAX, &unpackSize); + + PRF(printf("\npackSize = %7d unpackSize = %7d ", packSize, unpackSize)); + + if (unpackSize == 0) + return res; + + if (res == SZ_OK) + useCopyBlock = (packSize + 2 >= unpackSize || packSize > (1 << 16)); + else + { + if (res != SZ_ERROR_OUTPUT_EOF) + return res; + res = SZ_OK; + useCopyBlock = True; + } + + if (useCopyBlock) + { + size_t destPos = 0; + PRF(printf("################# COPY ")); + + while (unpackSize > 0) + { + UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE; + if (packSizeLimit - destPos < u + 3) + return SZ_ERROR_OUTPUT_EOF; + outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET); + outBuf[destPos++] = (Byte)((u - 1) >> 8); + outBuf[destPos++] = (Byte)(u - 1); + memcpy(outBuf + destPos, LzmaEnc_GetCurBuf(p->enc) - unpackSize, u); + unpackSize -= u; + destPos += u; + p->srcPos += u; + + if (outStream) + { + *packSizeRes += destPos; + if (ISeqOutStream_Write(outStream, outBuf, destPos) != destPos) + return SZ_ERROR_WRITE; + destPos = 0; + } + else + *packSizeRes = destPos; + /* needInitState = True; */ + } + + LzmaEnc_RestoreState(p->enc); + return SZ_OK; + } + + { + size_t destPos = 0; + UInt32 u = unpackSize - 1; + UInt32 pm = (UInt32)(packSize - 1); + unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0); + + PRF(printf(" ")); + + outBuf[destPos++] = (Byte)(LZMA2_CONTROL_LZMA | (mode << 5) | ((u >> 16) & 0x1F)); + outBuf[destPos++] = (Byte)(u >> 8); + outBuf[destPos++] = (Byte)u; + outBuf[destPos++] = (Byte)(pm >> 8); + outBuf[destPos++] = (Byte)pm; + + if (p->needInitProp) + outBuf[destPos++] = p->propsByte; + + p->needInitProp = False; + p->needInitState = False; + destPos += packSize; + p->srcPos += unpackSize; + + if (outStream) + if (ISeqOutStream_Write(outStream, outBuf, destPos) != destPos) + return SZ_ERROR_WRITE; + + *packSizeRes = destPos; + return SZ_OK; + } +} + + +/* ---------- Lzma2 Props ---------- */ + +void Lzma2EncProps_Init(CLzma2EncProps *p) +{ + LzmaEncProps_Init(&p->lzmaProps); + p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO; + p->numBlockThreads_Reduced = -1; + p->numBlockThreads_Max = -1; + p->numTotalThreads = -1; +} + +void Lzma2EncProps_Normalize(CLzma2EncProps *p) +{ + UInt64 fileSize; + int t1, t1n, t2, t2r, t3; + { + CLzmaEncProps lzmaProps = p->lzmaProps; + LzmaEncProps_Normalize(&lzmaProps); + t1n = lzmaProps.numThreads; + } + + t1 = p->lzmaProps.numThreads; + t2 = p->numBlockThreads_Max; + t3 = p->numTotalThreads; + + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; + + if (t3 <= 0) + { + if (t2 <= 0) + t2 = 1; + t3 = t1n * t2; + } + else if (t2 <= 0) + { + t2 = t3 / t1n; + if (t2 == 0) + { + t1 = 1; + t2 = t3; + } + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; + } + else if (t1 <= 0) + { + t1 = t3 / t2; + if (t1 == 0) + t1 = 1; + } + else + t3 = t1n * t2; + + p->lzmaProps.numThreads = t1; + + t2r = t2; + + fileSize = p->lzmaProps.reduceSize; + + if ( p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO + && (p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) + p->lzmaProps.reduceSize = p->blockSize; + + LzmaEncProps_Normalize(&p->lzmaProps); + + p->lzmaProps.reduceSize = fileSize; + + t1 = p->lzmaProps.numThreads; + + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + { + t2r = t2 = 1; + t3 = t1; + } + else if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO && t2 <= 1) + { + /* if there is no block multi-threading, we use SOLID block */ + p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; + } + else + { + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + { + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + const UInt32 dictSize = p->lzmaProps.dictSize; + UInt64 blockSize = (UInt64)dictSize << 2; + if (blockSize < kMinSize) blockSize = kMinSize; + if (blockSize > kMaxSize) blockSize = kMaxSize; + if (blockSize < dictSize) blockSize = dictSize; + blockSize += (kMinSize - 1); + blockSize &= ~(UInt64)(kMinSize - 1); + p->blockSize = blockSize; + } + + if (t2 > 1 && fileSize != (UInt64)(Int64)-1) + { + UInt64 numBlocks = fileSize / p->blockSize; + if (numBlocks * p->blockSize != fileSize) + numBlocks++; + if (numBlocks < (unsigned)t2) + { + t2r = (unsigned)numBlocks; + if (t2r == 0) + t2r = 1; + t3 = t1 * t2r; + } + } + } + + p->numBlockThreads_Max = t2; + p->numBlockThreads_Reduced = t2r; + p->numTotalThreads = t3; +} + + +static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) +{ + return (p && ICompressProgress_Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; +} + + +/* ---------- Lzma2 ---------- */ + +typedef struct +{ + Byte propEncoded; + CLzma2EncProps props; + UInt64 expectedDataSize; + + Byte *tempBufLzma; + + ISzAllocPtr alloc; + ISzAllocPtr allocBig; + + CLzma2EncInt coders[MTCODER__THREADS_MAX]; + + #ifndef _7ZIP_ST + + ISeqOutStream *outStream; + Byte *outBuf; + size_t outBuf_Rem; /* remainder in outBuf */ + + size_t outBufSize; /* size of allocated outBufs[i] */ + size_t outBufsDataSizes[MTCODER__BLOCKS_MAX]; + BoolInt mtCoder_WasConstructed; + CMtCoder mtCoder; + Byte *outBufs[MTCODER__BLOCKS_MAX]; + + #endif + +} CLzma2Enc; + + + +CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CLzma2Enc *p = (CLzma2Enc *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Enc)); + if (!p) + return NULL; + Lzma2EncProps_Init(&p->props); + Lzma2EncProps_Normalize(&p->props); + p->expectedDataSize = (UInt64)(Int64)-1; + p->tempBufLzma = NULL; + p->alloc = alloc; + p->allocBig = allocBig; + { + unsigned i; + for (i = 0; i < MTCODER__THREADS_MAX; i++) + p->coders[i].enc = NULL; + } + + #ifndef _7ZIP_ST + p->mtCoder_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->outBufs[i] = NULL; + p->outBufSize = 0; + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void Lzma2Enc_FreeOutBufs(CLzma2Enc *p) +{ + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + if (p->outBufs[i]) + { + ISzAlloc_Free(p->alloc, p->outBufs[i]); + p->outBufs[i] = NULL; + } + p->outBufSize = 0; +} + +#endif + + +void Lzma2Enc_Destroy(CLzma2EncHandle pp) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + unsigned i; + for (i = 0; i < MTCODER__THREADS_MAX; i++) + { + CLzma2EncInt *t = &p->coders[i]; + if (t->enc) + { + LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig); + t->enc = NULL; + } + } + + + #ifndef _7ZIP_ST + if (p->mtCoder_WasConstructed) + { + MtCoder_Destruct(&p->mtCoder); + p->mtCoder_WasConstructed = False; + } + Lzma2Enc_FreeOutBufs(p); + #endif + + ISzAlloc_Free(p->alloc, p->tempBufLzma); + p->tempBufLzma = NULL; + + ISzAlloc_Free(p->alloc, pp); +} + + +SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + CLzmaEncProps lzmaProps = props->lzmaProps; + LzmaEncProps_Normalize(&lzmaProps); + if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX) + return SZ_ERROR_PARAM; + p->props = *props; + Lzma2EncProps_Normalize(&p->props); + return SZ_OK; +} + + +void Lzma2Enc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + p->expectedDataSize = expectedDataSiize; +} + + +Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + unsigned i; + UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps); + for (i = 0; i < 40; i++) + if (dicSize <= LZMA2_DIC_SIZE_FROM_PROP(i)) + break; + return (Byte)i; +} + + +static SRes Lzma2Enc_EncodeMt1( + CLzma2Enc *me, + CLzma2EncInt *p, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + int finished, + ICompressProgress *progress) +{ + UInt64 unpackTotal = 0; + UInt64 packTotal = 0; + size_t outLim = 0; + CLimitedSeqInStream limitedInStream; + + if (outBuf) + { + outLim = *outBufSize; + *outBufSize = 0; + } + + if (!p->enc) + { + p->propsAreSet = False; + p->enc = LzmaEnc_Create(me->alloc); + if (!p->enc) + return SZ_ERROR_MEM; + } + + limitedInStream.realStream = inStream; + if (inStream) + { + limitedInStream.vt.Read = LimitedSeqInStream_Read; + } + + if (!outBuf) + { + // outStream version works only in one thread. So we use CLzma2Enc::tempBufLzma + if (!me->tempBufLzma) + { + me->tempBufLzma = (Byte *)ISzAlloc_Alloc(me->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); + if (!me->tempBufLzma) + return SZ_ERROR_MEM; + } + } + + RINOK(Lzma2EncInt_InitStream(p, &me->props)); + + for (;;) + { + SRes res = SZ_OK; + size_t inSizeCur = 0; + + Lzma2EncInt_InitBlock(p); + + LimitedSeqInStream_Init(&limitedInStream); + limitedInStream.limit = me->props.blockSize; + + if (inStream) + { + UInt64 expected = (UInt64)(Int64)-1; + // inStream version works only in one thread. So we use CLzma2Enc::expectedDataSize + if (me->expectedDataSize != (UInt64)(Int64)-1 + && me->expectedDataSize >= unpackTotal) + expected = me->expectedDataSize - unpackTotal; + if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && expected > me->props.blockSize) + expected = (size_t)me->props.blockSize; + + LzmaEnc_SetDataSize(p->enc, expected); + + RINOK(LzmaEnc_PrepareForLzma2(p->enc, + &limitedInStream.vt, + LZMA2_KEEP_WINDOW_SIZE, + me->alloc, + me->allocBig)); + } + else + { + inSizeCur = inDataSize - (size_t)unpackTotal; + if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && inSizeCur > me->props.blockSize) + inSizeCur = (size_t)me->props.blockSize; + + // LzmaEnc_SetDataSize(p->enc, inSizeCur); + + RINOK(LzmaEnc_MemPrepare(p->enc, + inData + (size_t)unpackTotal, inSizeCur, + LZMA2_KEEP_WINDOW_SIZE, + me->alloc, + me->allocBig)); + } + + for (;;) + { + size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; + if (outBuf) + packSize = outLim - (size_t)packTotal; + + res = Lzma2EncInt_EncodeSubblock(p, + outBuf ? outBuf + (size_t)packTotal : me->tempBufLzma, &packSize, + outBuf ? NULL : outStream); + + if (res != SZ_OK) + break; + + packTotal += packSize; + if (outBuf) + *outBufSize = (size_t)packTotal; + + res = Progress(progress, unpackTotal + p->srcPos, packTotal); + if (res != SZ_OK) + break; + + /* + if (LzmaEnc_GetNumAvailableBytes(p->enc) == 0) + break; + */ + + if (packSize == 0) + break; + } + + LzmaEnc_Finish(p->enc); + + unpackTotal += p->srcPos; + + RINOK(res); + + if (p->srcPos != (inStream ? limitedInStream.processed : inSizeCur)) + return SZ_ERROR_FAIL; + + if (inStream ? limitedInStream.finished : (unpackTotal == inDataSize)) + { + if (finished) + { + if (outBuf) + { + size_t destPos = *outBufSize; + if (destPos >= outLim) + return SZ_ERROR_OUTPUT_EOF; + outBuf[destPos] = 0; + *outBufSize = destPos + 1; + } + else + { + Byte b = 0; + if (ISeqOutStream_Write(outStream, &b, 1) != 1) + return SZ_ERROR_WRITE; + } + } + return SZ_OK; + } + } +} + + + +#ifndef _7ZIP_ST + +static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, + const Byte *src, size_t srcSize, int finished) +{ + CLzma2Enc *me = (CLzma2Enc *)pp; + size_t destSize = me->outBufSize; + SRes res; + CMtProgressThunk progressThunk; + + Byte *dest = me->outBufs[outBufIndex]; + + me->outBufsDataSizes[outBufIndex] = 0; + + if (!dest) + { + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); + if (!dest) + return SZ_ERROR_MEM; + me->outBufs[outBufIndex] = dest; + } + + MtProgressThunk_CreateVTable(&progressThunk); + progressThunk.mtProgress = &me->mtCoder.mtProgress; + progressThunk.inSize = 0; + progressThunk.outSize = 0; + + res = Lzma2Enc_EncodeMt1(me, + &me->coders[coderIndex], + NULL, dest, &destSize, + NULL, src, srcSize, + finished, + &progressThunk.vt); + + me->outBufsDataSizes[outBufIndex] = destSize; + + return res; +} + + +static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex) +{ + CLzma2Enc *me = (CLzma2Enc *)pp; + size_t size = me->outBufsDataSizes[outBufIndex]; + const Byte *data = me->outBufs[outBufIndex]; + + if (me->outStream) + return ISeqOutStream_Write(me->outStream, data, size) == size ? SZ_OK : SZ_ERROR_WRITE; + + if (size > me->outBuf_Rem) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBuf_Rem -= size; + me->outBuf += size; + return SZ_OK; +} + +#endif + + + +SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + ICompressProgress *progress) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + + if (inStream && inData) + return SZ_ERROR_PARAM; + + if (outStream && outBuf) + return SZ_ERROR_PARAM; + + { + unsigned i; + for (i = 0; i < MTCODER__THREADS_MAX; i++) + p->coders[i].propsAreSet = False; + } + + #ifndef _7ZIP_ST + + if (p->props.numBlockThreads_Reduced > 1) + { + IMtCoderCallback2 vt; + + if (!p->mtCoder_WasConstructed) + { + p->mtCoder_WasConstructed = True; + MtCoder_Construct(&p->mtCoder); + } + + vt.Code = Lzma2Enc_MtCallback_Code; + vt.Write = Lzma2Enc_MtCallback_Write; + + p->outStream = outStream; + p->outBuf = NULL; + p->outBuf_Rem = 0; + if (!outStream) + { + p->outBuf = outBuf; + p->outBuf_Rem = *outBufSize; + *outBufSize = 0; + } + + p->mtCoder.allocBig = p->allocBig; + p->mtCoder.progress = progress; + p->mtCoder.inStream = inStream; + p->mtCoder.inData = inData; + p->mtCoder.inDataSize = inDataSize; + p->mtCoder.mtCallback = &vt; + p->mtCoder.mtCallbackObject = p; + + p->mtCoder.blockSize = (size_t)p->props.blockSize; + if (p->mtCoder.blockSize != p->props.blockSize) + return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ + + { + size_t destBlockSize = p->mtCoder.blockSize + (p->mtCoder.blockSize >> 10) + 16; + if (destBlockSize < p->mtCoder.blockSize) + return SZ_ERROR_PARAM; + if (p->outBufSize != destBlockSize) + Lzma2Enc_FreeOutBufs(p); + p->outBufSize = destBlockSize; + } + + p->mtCoder.numThreadsMax = p->props.numBlockThreads_Max; + p->mtCoder.expectedDataSize = p->expectedDataSize; + + { + SRes res = MtCoder_Code(&p->mtCoder); + if (!outStream) + *outBufSize = p->outBuf - outBuf; + return res; + } + } + + #endif + + + return Lzma2Enc_EncodeMt1(p, + &p->coders[0], + outStream, outBuf, outBufSize, + inStream, inData, inDataSize, + True, /* finished */ + progress); +} diff --git a/code/nel/3rdparty/seven_zip/Lzma2Enc.h b/code/nel/3rdparty/seven_zip/Lzma2Enc.h new file mode 100644 index 000000000..6a6110ff7 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Lzma2Enc.h @@ -0,0 +1,55 @@ +/* Lzma2Enc.h -- LZMA2 Encoder +2017-07-27 : Igor Pavlov : Public domain */ + +#ifndef __LZMA2_ENC_H +#define __LZMA2_ENC_H + +#include "LzmaEnc.h" + +EXTERN_C_BEGIN + +#define LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO 0 +#define LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID ((UInt64)(Int64)-1) + +typedef struct +{ + CLzmaEncProps lzmaProps; + UInt64 blockSize; + int numBlockThreads_Reduced; + int numBlockThreads_Max; + int numTotalThreads; +} CLzma2EncProps; + +void Lzma2EncProps_Init(CLzma2EncProps *p); +void Lzma2EncProps_Normalize(CLzma2EncProps *p); + +/* ---------- CLzmaEnc2Handle Interface ---------- */ + +/* Lzma2Enc_* functions can return the following exit codes: +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater in props + SZ_ERROR_WRITE - ISeqOutStream write callback error + SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output + SZ_ERROR_PROGRESS - some break from progress callback + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) +*/ + +typedef void * CLzma2EncHandle; + +CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); +void Lzma2Enc_Destroy(CLzma2EncHandle p); +SRes Lzma2Enc_SetProps(CLzma2EncHandle p, const CLzma2EncProps *props); +void Lzma2Enc_SetDataSize(CLzma2EncHandle p, UInt64 expectedDataSiize); +Byte Lzma2Enc_WriteProperties(CLzma2EncHandle p); +SRes Lzma2Enc_Encode2(CLzma2EncHandle p, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + ICompressProgress *progress); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/Lzma86.h b/code/nel/3rdparty/seven_zip/Lzma86.h new file mode 100644 index 000000000..bebed5cb7 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Lzma86.h @@ -0,0 +1,111 @@ +/* Lzma86.h -- LZMA + x86 (BCJ) Filter +2013-01-18 : Igor Pavlov : Public domain */ + +#ifndef __LZMA86_H +#define __LZMA86_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define LZMA86_SIZE_OFFSET (1 + 5) +#define LZMA86_HEADER_SIZE (LZMA86_SIZE_OFFSET + 8) + +/* +It's an example for LZMA + x86 Filter use. +You can use .lzma86 extension, if you write that stream to file. +.lzma86 header adds one additional byte to standard .lzma header. +.lzma86 header (14 bytes): + Offset Size Description + 0 1 = 0 - no filter, pure LZMA + = 1 - x86 filter + LZMA + 1 1 lc, lp and pb in encoded form + 2 4 dictSize (little endian) + 6 8 uncompressed size (little endian) + + +Lzma86_Encode +------------- +level - compression level: 0 <= level <= 9, the default value for "level" is 5. + +dictSize - The dictionary size in bytes. The maximum value is + 128 MB = (1 << 27) bytes for 32-bit version + 1 GB = (1 << 30) bytes for 64-bit version + The default value is 16 MB = (1 << 24) bytes, for level = 5. + It's recommended to use the dictionary that is larger than 4 KB and + that can be calculated as (1 << N) or (3 << N) sizes. + For better compression ratio dictSize must be >= inSize. + +filterMode: + SZ_FILTER_NO - no Filter + SZ_FILTER_YES - x86 Filter + SZ_FILTER_AUTO - it tries both alternatives to select best. + Encoder will use 2 or 3 passes: + 2 passes when FILTER_NO provides better compression. + 3 passes when FILTER_YES provides better compression. + +Lzma86Encode allocates Data with MyAlloc functions. +RAM Requirements for compressing: + RamSize = dictionarySize * 11.5 + 6MB + FilterBlockSize + filterMode FilterBlockSize + SZ_FILTER_NO 0 + SZ_FILTER_YES inSize + SZ_FILTER_AUTO inSize + + +Return code: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater + SZ_ERROR_OUTPUT_EOF - output buffer overflow + SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) +*/ + +enum ESzFilterMode +{ + SZ_FILTER_NO, + SZ_FILTER_YES, + SZ_FILTER_AUTO +}; + +SRes Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, + int level, UInt32 dictSize, int filterMode); + + +/* +Lzma86_GetUnpackSize: + In: + src - input data + srcLen - input data size + Out: + unpackSize - size of uncompressed stream + Return code: + SZ_OK - OK + SZ_ERROR_INPUT_EOF - Error in headers +*/ + +SRes Lzma86_GetUnpackSize(const Byte *src, SizeT srcLen, UInt64 *unpackSize); + +/* +Lzma86_Decode: + In: + dest - output data + destLen - output data size + src - input data + srcLen - input data size + Out: + destLen - processed output size + srcLen - processed input size + Return code: + SZ_OK - OK + SZ_ERROR_DATA - Data error + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_UNSUPPORTED - unsupported file + SZ_ERROR_INPUT_EOF - it needs more bytes in input buffer +*/ + +SRes Lzma86_Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/Lzma86Dec.c b/code/nel/3rdparty/seven_zip/Lzma86Dec.c new file mode 100644 index 000000000..21031745c --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Lzma86Dec.c @@ -0,0 +1,54 @@ +/* Lzma86Dec.c -- LZMA + x86 (BCJ) Filter Decoder +2016-05-16 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "Lzma86.h" + +#include "Alloc.h" +#include "Bra.h" +#include "LzmaDec.h" + +SRes Lzma86_GetUnpackSize(const Byte *src, SizeT srcLen, UInt64 *unpackSize) +{ + unsigned i; + if (srcLen < LZMA86_HEADER_SIZE) + return SZ_ERROR_INPUT_EOF; + *unpackSize = 0; + for (i = 0; i < sizeof(UInt64); i++) + *unpackSize += ((UInt64)src[LZMA86_SIZE_OFFSET + i]) << (8 * i); + return SZ_OK; +} + +SRes Lzma86_Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen) +{ + SRes res; + int useFilter; + SizeT inSizePure; + ELzmaStatus status; + + if (*srcLen < LZMA86_HEADER_SIZE) + return SZ_ERROR_INPUT_EOF; + + useFilter = src[0]; + + if (useFilter > 1) + { + *destLen = 0; + return SZ_ERROR_UNSUPPORTED; + } + + inSizePure = *srcLen - LZMA86_HEADER_SIZE; + res = LzmaDecode(dest, destLen, src + LZMA86_HEADER_SIZE, &inSizePure, + src + 1, LZMA_PROPS_SIZE, LZMA_FINISH_ANY, &status, &g_Alloc); + *srcLen = inSizePure + LZMA86_HEADER_SIZE; + if (res != SZ_OK) + return res; + if (useFilter == 1) + { + UInt32 x86State; + x86_Convert_Init(x86State); + x86_Convert(dest, *destLen, 0, &x86State, 0); + } + return SZ_OK; +} diff --git a/code/nel/3rdparty/seven_zip/Lzma86Enc.c b/code/nel/3rdparty/seven_zip/Lzma86Enc.c new file mode 100644 index 000000000..2617bab8e --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Lzma86Enc.c @@ -0,0 +1,106 @@ +/* Lzma86Enc.c -- LZMA + x86 (BCJ) Filter Encoder +2018-07-04 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +#include "Lzma86.h" + +#include "Alloc.h" +#include "Bra.h" +#include "LzmaEnc.h" + +#define SZE_OUT_OVERFLOW SZE_DATA_ERROR + +int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, + int level, UInt32 dictSize, int filterMode) +{ + size_t outSize2 = *destLen; + Byte *filteredStream; + BoolInt useFilter; + int mainResult = SZ_ERROR_OUTPUT_EOF; + CLzmaEncProps props; + LzmaEncProps_Init(&props); + props.level = level; + props.dictSize = dictSize; + + *destLen = 0; + if (outSize2 < LZMA86_HEADER_SIZE) + return SZ_ERROR_OUTPUT_EOF; + + { + int i; + UInt64 t = srcLen; + for (i = 0; i < 8; i++, t >>= 8) + dest[LZMA86_SIZE_OFFSET + i] = (Byte)t; + } + + filteredStream = 0; + useFilter = (filterMode != SZ_FILTER_NO); + if (useFilter) + { + if (srcLen != 0) + { + filteredStream = (Byte *)MyAlloc(srcLen); + if (filteredStream == 0) + return SZ_ERROR_MEM; + memcpy(filteredStream, src, srcLen); + } + { + UInt32 x86State; + x86_Convert_Init(x86State); + x86_Convert(filteredStream, srcLen, 0, &x86State, 1); + } + } + + { + size_t minSize = 0; + BoolInt bestIsFiltered = False; + + /* passes for SZ_FILTER_AUTO: + 0 - BCJ + LZMA + 1 - LZMA + 2 - BCJ + LZMA agaian, if pass 0 (BCJ + LZMA) is better. + */ + int numPasses = (filterMode == SZ_FILTER_AUTO) ? 3 : 1; + + int i; + for (i = 0; i < numPasses; i++) + { + size_t outSizeProcessed = outSize2 - LZMA86_HEADER_SIZE; + size_t outPropsSize = 5; + SRes curRes; + BoolInt curModeIsFiltered = (numPasses > 1 && i == numPasses - 1); + if (curModeIsFiltered && !bestIsFiltered) + break; + if (useFilter && i == 0) + curModeIsFiltered = True; + + curRes = LzmaEncode(dest + LZMA86_HEADER_SIZE, &outSizeProcessed, + curModeIsFiltered ? filteredStream : src, srcLen, + &props, dest + 1, &outPropsSize, 0, + NULL, &g_Alloc, &g_Alloc); + + if (curRes != SZ_ERROR_OUTPUT_EOF) + { + if (curRes != SZ_OK) + { + mainResult = curRes; + break; + } + if (outSizeProcessed <= minSize || mainResult != SZ_OK) + { + minSize = outSizeProcessed; + bestIsFiltered = curModeIsFiltered; + mainResult = SZ_OK; + } + } + } + dest[0] = (Byte)(bestIsFiltered ? 1 : 0); + *destLen = LZMA86_HEADER_SIZE + minSize; + } + if (useFilter) + MyFree(filteredStream); + return mainResult; +} diff --git a/code/nel/3rdparty/seven_zip/LzmaDec.c b/code/nel/3rdparty/seven_zip/LzmaDec.c new file mode 100644 index 000000000..ba3e1dd50 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaDec.c @@ -0,0 +1,1185 @@ +/* LzmaDec.c -- LZMA Decoder +2018-07-04 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +/* #include "CpuArch.h" */ +#include "LzmaDec.h" + +#define kNumTopBits 24 +#define kTopValue ((UInt32)1 << kNumTopBits) + +#define kNumBitModelTotalBits 11 +#define kBitModelTotal (1 << kNumBitModelTotalBits) +#define kNumMoveBits 5 + +#define RC_INIT_SIZE 5 + +#define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } + +#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) +#define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); +#define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); +#define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ + { UPDATE_0(p); i = (i + i); A0; } else \ + { UPDATE_1(p); i = (i + i) + 1; A1; } + +#define TREE_GET_BIT(probs, i) { GET_BIT2(probs + i, i, ;, ;); } + +#define REV_BIT(p, i, A0, A1) IF_BIT_0(p + i) \ + { UPDATE_0(p + i); A0; } else \ + { UPDATE_1(p + i); A1; } +#define REV_BIT_VAR( p, i, m) REV_BIT(p, i, i += m; m += m, m += m; i += m; ) +#define REV_BIT_CONST(p, i, m) REV_BIT(p, i, i += m; , i += m * 2; ) +#define REV_BIT_LAST( p, i, m) REV_BIT(p, i, i -= m , ; ) + +#define TREE_DECODE(probs, limit, i) \ + { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } + +/* #define _LZMA_SIZE_OPT */ + +#ifdef _LZMA_SIZE_OPT +#define TREE_6_DECODE(probs, i) TREE_DECODE(probs, (1 << 6), i) +#else +#define TREE_6_DECODE(probs, i) \ + { i = 1; \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + i -= 0x40; } +#endif + +#define NORMAL_LITER_DEC TREE_GET_BIT(prob, symbol) +#define MATCHED_LITER_DEC \ + matchByte += matchByte; \ + bit = offs; \ + offs &= matchByte; \ + probLit = prob + (offs + bit + symbol); \ + GET_BIT2(probLit, symbol, offs ^= bit; , ;) + + + +#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } + +#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) +#define UPDATE_0_CHECK range = bound; +#define UPDATE_1_CHECK range -= bound; code -= bound; +#define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \ + { UPDATE_0_CHECK; i = (i + i); A0; } else \ + { UPDATE_1_CHECK; i = (i + i) + 1; A1; } +#define GET_BIT_CHECK(p, i) GET_BIT2_CHECK(p, i, ; , ;) +#define TREE_DECODE_CHECK(probs, limit, i) \ + { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } + + +#define REV_BIT_CHECK(p, i, m) IF_BIT_0_CHECK(p + i) \ + { UPDATE_0_CHECK; i += m; m += m; } else \ + { UPDATE_1_CHECK; m += m; i += m; } + + +#define kNumPosBitsMax 4 +#define kNumPosStatesMax (1 << kNumPosBitsMax) + +#define kLenNumLowBits 3 +#define kLenNumLowSymbols (1 << kLenNumLowBits) +#define kLenNumHighBits 8 +#define kLenNumHighSymbols (1 << kLenNumHighBits) + +#define LenLow 0 +#define LenHigh (LenLow + 2 * (kNumPosStatesMax << kLenNumLowBits)) +#define kNumLenProbs (LenHigh + kLenNumHighSymbols) + +#define LenChoice LenLow +#define LenChoice2 (LenLow + (1 << kLenNumLowBits)) + +#define kNumStates 12 +#define kNumStates2 16 +#define kNumLitStates 7 + +#define kStartPosModelIndex 4 +#define kEndPosModelIndex 14 +#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) + +#define kNumPosSlotBits 6 +#define kNumLenToPosStates 4 + +#define kNumAlignBits 4 +#define kAlignTableSize (1 << kNumAlignBits) + +#define kMatchMinLen 2 +#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) + +/* External ASM code needs same CLzmaProb array layout. So don't change it. */ + +/* (probs_1664) is faster and better for code size at some platforms */ +/* +#ifdef MY_CPU_X86_OR_AMD64 +*/ +#define kStartOffset 1664 +#define GET_PROBS p->probs_1664 +/* +#define GET_PROBS p->probs + kStartOffset +#else +#define kStartOffset 0 +#define GET_PROBS p->probs +#endif +*/ + +#define SpecPos (-kStartOffset) +#define IsRep0Long (SpecPos + kNumFullDistances) +#define RepLenCoder (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) +#define LenCoder (RepLenCoder + kNumLenProbs) +#define IsMatch (LenCoder + kNumLenProbs) +#define Align (IsMatch + (kNumStates2 << kNumPosBitsMax)) +#define IsRep (Align + kAlignTableSize) +#define IsRepG0 (IsRep + kNumStates) +#define IsRepG1 (IsRepG0 + kNumStates) +#define IsRepG2 (IsRepG1 + kNumStates) +#define PosSlot (IsRepG2 + kNumStates) +#define Literal (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define NUM_BASE_PROBS (Literal + kStartOffset) + +#if Align != 0 && kStartOffset != 0 + #error Stop_Compiling_Bad_LZMA_kAlign +#endif + +#if NUM_BASE_PROBS != 1984 + #error Stop_Compiling_Bad_LZMA_PROBS +#endif + + +#define LZMA_LIT_SIZE 0x300 + +#define LzmaProps_GetNumProbs(p) (NUM_BASE_PROBS + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) + + +#define CALC_POS_STATE(processedPos, pbMask) (((processedPos) & (pbMask)) << 4) +#define COMBINED_PS_STATE (posState + state) +#define GET_LEN_STATE (posState) + +#define LZMA_DIC_MIN (1 << 12) + +/* +p->remainLen : shows status of LZMA decoder: + < kMatchSpecLenStart : normal remain + = kMatchSpecLenStart : finished + = kMatchSpecLenStart + 1 : need init range coder + = kMatchSpecLenStart + 2 : need init range coder and state +*/ + +/* ---------- LZMA_DECODE_REAL ---------- */ +/* +LzmaDec_DecodeReal_3() can be implemented in external ASM file. +3 - is the code compatibility version of that function for check at link time. +*/ + +#define LZMA_DECODE_REAL LzmaDec_DecodeReal_3 + +/* +LZMA_DECODE_REAL() +In: + RangeCoder is normalized + if (p->dicPos == limit) + { + LzmaDec_TryDummy() was called before to exclude LITERAL and MATCH-REP cases. + So first symbol can be only MATCH-NON-REP. And if that MATCH-NON-REP symbol + is not END_OF_PAYALOAD_MARKER, then function returns error code. + } + +Processing: + first LZMA symbol will be decoded in any case + All checks for limits are at the end of main loop, + It will decode new LZMA-symbols while (p->buf < bufLimit && dicPos < limit), + RangeCoder is still without last normalization when (p->buf < bufLimit) is being checked. + +Out: + RangeCoder is normalized + Result: + SZ_OK - OK + SZ_ERROR_DATA - Error + p->remainLen: + < kMatchSpecLenStart : normal remain + = kMatchSpecLenStart : finished +*/ + + +#ifdef _LZMA_DEC_OPT + +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit); + +#else + +static +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +{ + CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; + UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; + unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; + unsigned lc = p->prop.lc; + unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); + + Byte *dic = p->dic; + SizeT dicBufSize = p->dicBufSize; + SizeT dicPos = p->dicPos; + + UInt32 processedPos = p->processedPos; + UInt32 checkDicSize = p->checkDicSize; + unsigned len = 0; + + const Byte *buf = p->buf; + UInt32 range = p->range; + UInt32 code = p->code; + + do + { + CLzmaProb *prob; + UInt32 bound; + unsigned ttt; + unsigned posState = CALC_POS_STATE(processedPos, pbMask); + + prob = probs + IsMatch + COMBINED_PS_STATE; + IF_BIT_0(prob) + { + unsigned symbol; + UPDATE_0(prob); + prob = probs + Literal; + if (processedPos != 0 || checkDicSize != 0) + prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); + processedPos++; + + if (state < kNumLitStates) + { + state -= (state < 4) ? state : 3; + symbol = 1; + #ifdef _LZMA_SIZE_OPT + do { NORMAL_LITER_DEC } while (symbol < 0x100); + #else + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + #endif + } + else + { + unsigned matchByte = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + unsigned offs = 0x100; + state -= (state < 10) ? 3 : 6; + symbol = 1; + #ifdef _LZMA_SIZE_OPT + do + { + unsigned bit; + CLzmaProb *probLit; + MATCHED_LITER_DEC + } + while (symbol < 0x100); + #else + { + unsigned bit; + CLzmaProb *probLit; + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + } + #endif + } + + dic[dicPos++] = (Byte)symbol; + continue; + } + + { + UPDATE_1(prob); + prob = probs + IsRep + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + state += kNumStates; + prob = probs + LenCoder; + } + else + { + UPDATE_1(prob); + /* + // that case was checked before with kBadRepCode + if (checkDicSize == 0 && processedPos == 0) + return SZ_ERROR_DATA; + */ + prob = probs + IsRepG0 + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + prob = probs + IsRep0Long + COMBINED_PS_STATE; + IF_BIT_0(prob) + { + UPDATE_0(prob); + dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + dicPos++; + processedPos++; + state = state < kNumLitStates ? 9 : 11; + continue; + } + UPDATE_1(prob); + } + else + { + UInt32 distance; + UPDATE_1(prob); + prob = probs + IsRepG1 + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + distance = rep1; + } + else + { + UPDATE_1(prob); + prob = probs + IsRepG2 + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + distance = rep2; + } + else + { + UPDATE_1(prob); + distance = rep3; + rep3 = rep2; + } + rep2 = rep1; + } + rep1 = rep0; + rep0 = distance; + } + state = state < kNumLitStates ? 8 : 11; + prob = probs + RepLenCoder; + } + + #ifdef _LZMA_SIZE_OPT + { + unsigned lim, offset; + CLzmaProb *probLen = prob + LenChoice; + IF_BIT_0(probLen) + { + UPDATE_0(probLen); + probLen = prob + LenLow + GET_LEN_STATE; + offset = 0; + lim = (1 << kLenNumLowBits); + } + else + { + UPDATE_1(probLen); + probLen = prob + LenChoice2; + IF_BIT_0(probLen) + { + UPDATE_0(probLen); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); + offset = kLenNumLowSymbols; + lim = (1 << kLenNumLowBits); + } + else + { + UPDATE_1(probLen); + probLen = prob + LenHigh; + offset = kLenNumLowSymbols * 2; + lim = (1 << kLenNumHighBits); + } + } + TREE_DECODE(probLen, lim, len); + len += offset; + } + #else + { + CLzmaProb *probLen = prob + LenChoice; + IF_BIT_0(probLen) + { + UPDATE_0(probLen); + probLen = prob + LenLow + GET_LEN_STATE; + len = 1; + TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len); + len -= 8; + } + else + { + UPDATE_1(probLen); + probLen = prob + LenChoice2; + IF_BIT_0(probLen) + { + UPDATE_0(probLen); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); + len = 1; + TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len); + TREE_GET_BIT(probLen, len); + } + else + { + UPDATE_1(probLen); + probLen = prob + LenHigh; + TREE_DECODE(probLen, (1 << kLenNumHighBits), len); + len += kLenNumLowSymbols * 2; + } + } + } + #endif + + if (state >= kNumStates) + { + UInt32 distance; + prob = probs + PosSlot + + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); + TREE_6_DECODE(prob, distance); + if (distance >= kStartPosModelIndex) + { + unsigned posSlot = (unsigned)distance; + unsigned numDirectBits = (unsigned)(((distance >> 1) - 1)); + distance = (2 | (distance & 1)); + if (posSlot < kEndPosModelIndex) + { + distance <<= numDirectBits; + prob = probs + SpecPos; + { + UInt32 m = 1; + distance++; + do + { + REV_BIT_VAR(prob, distance, m); + } + while (--numDirectBits); + distance -= m; + } + } + else + { + numDirectBits -= kNumAlignBits; + do + { + NORMALIZE + range >>= 1; + + { + UInt32 t; + code -= range; + t = (0 - ((UInt32)code >> 31)); /* (UInt32)((Int32)code >> 31) */ + distance = (distance << 1) + (t + 1); + code += range & t; + } + /* + distance <<= 1; + if (code >= range) + { + code -= range; + distance |= 1; + } + */ + } + while (--numDirectBits); + prob = probs + Align; + distance <<= kNumAlignBits; + { + unsigned i = 1; + REV_BIT_CONST(prob, i, 1); + REV_BIT_CONST(prob, i, 2); + REV_BIT_CONST(prob, i, 4); + REV_BIT_LAST (prob, i, 8); + distance |= i; + } + if (distance == (UInt32)0xFFFFFFFF) + { + len = kMatchSpecLenStart; + state -= kNumStates; + break; + } + } + } + + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance + 1; + state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) + { + p->dicPos = dicPos; + return SZ_ERROR_DATA; + } + } + + len += kMatchMinLen; + + { + SizeT rem; + unsigned curLen; + SizeT pos; + + if ((rem = limit - dicPos) == 0) + { + p->dicPos = dicPos; + return SZ_ERROR_DATA; + } + + curLen = ((rem < len) ? (unsigned)rem : len); + pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); + + processedPos += (UInt32)curLen; + + len -= curLen; + if (curLen <= dicBufSize - pos) + { + Byte *dest = dic + dicPos; + ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; + const Byte *lim = dest + curLen; + dicPos += (SizeT)curLen; + do + *(dest) = (Byte)*(dest + src); + while (++dest != lim); + } + else + { + do + { + dic[dicPos++] = dic[pos]; + if (++pos == dicBufSize) + pos = 0; + } + while (--curLen != 0); + } + } + } + } + while (dicPos < limit && buf < bufLimit); + + NORMALIZE; + + p->buf = buf; + p->range = range; + p->code = code; + p->remainLen = (UInt32)len; + p->dicPos = dicPos; + p->processedPos = processedPos; + p->reps[0] = rep0; + p->reps[1] = rep1; + p->reps[2] = rep2; + p->reps[3] = rep3; + p->state = (UInt32)state; + + return SZ_OK; +} +#endif + +static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) +{ + if (p->remainLen != 0 && p->remainLen < kMatchSpecLenStart) + { + Byte *dic = p->dic; + SizeT dicPos = p->dicPos; + SizeT dicBufSize = p->dicBufSize; + unsigned len = (unsigned)p->remainLen; + SizeT rep0 = p->reps[0]; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ + SizeT rem = limit - dicPos; + if (rem < len) + len = (unsigned)(rem); + + if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) + p->checkDicSize = p->prop.dicSize; + + p->processedPos += (UInt32)len; + p->remainLen -= (UInt32)len; + while (len != 0) + { + len--; + dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + dicPos++; + } + p->dicPos = dicPos; + } +} + + +#define kRange0 0xFFFFFFFF +#define kBound0 ((kRange0 >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1)) +#define kBadRepCode (kBound0 + (((kRange0 - kBound0) >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1))) +#if kBadRepCode != (0xC0000000 - 0x400) + #error Stop_Compiling_Bad_LZMA_Check +#endif + +static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +{ + do + { + SizeT limit2 = limit; + if (p->checkDicSize == 0) + { + UInt32 rem = p->prop.dicSize - p->processedPos; + if (limit - p->dicPos > rem) + limit2 = p->dicPos + rem; + + if (p->processedPos == 0) + if (p->code >= kBadRepCode) + return SZ_ERROR_DATA; + } + + RINOK(LZMA_DECODE_REAL(p, limit2, bufLimit)); + + if (p->checkDicSize == 0 && p->processedPos >= p->prop.dicSize) + p->checkDicSize = p->prop.dicSize; + + LzmaDec_WriteRem(p, limit); + } + while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); + + return 0; +} + +typedef enum +{ + DUMMY_ERROR, /* unexpected end of input stream */ + DUMMY_LIT, + DUMMY_MATCH, + DUMMY_REP +} ELzmaDummy; + +static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inSize) +{ + UInt32 range = p->range; + UInt32 code = p->code; + const Byte *bufLimit = buf + inSize; + const CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; + ELzmaDummy res; + + { + const CLzmaProb *prob; + UInt32 bound; + unsigned ttt; + unsigned posState = CALC_POS_STATE(p->processedPos, (1 << p->prop.pb) - 1); + + prob = probs + IsMatch + COMBINED_PS_STATE; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK + + /* if (bufLimit - buf >= 7) return DUMMY_LIT; */ + + prob = probs + Literal; + if (p->checkDicSize != 0 || p->processedPos != 0) + prob += ((UInt32)LZMA_LIT_SIZE * + ((((p->processedPos) & ((1 << (p->prop.lp)) - 1)) << p->prop.lc) + + (p->dic[(p->dicPos == 0 ? p->dicBufSize : p->dicPos) - 1] >> (8 - p->prop.lc)))); + + if (state < kNumLitStates) + { + unsigned symbol = 1; + do { GET_BIT_CHECK(prob + symbol, symbol) } while (symbol < 0x100); + } + else + { + unsigned matchByte = p->dic[p->dicPos - p->reps[0] + + (p->dicPos < p->reps[0] ? p->dicBufSize : 0)]; + unsigned offs = 0x100; + unsigned symbol = 1; + do + { + unsigned bit; + const CLzmaProb *probLit; + matchByte += matchByte; + bit = offs; + offs &= matchByte; + probLit = prob + (offs + bit + symbol); + GET_BIT2_CHECK(probLit, symbol, offs ^= bit; , ; ) + } + while (symbol < 0x100); + } + res = DUMMY_LIT; + } + else + { + unsigned len; + UPDATE_1_CHECK; + + prob = probs + IsRep + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + state = 0; + prob = probs + LenCoder; + res = DUMMY_MATCH; + } + else + { + UPDATE_1_CHECK; + res = DUMMY_REP; + prob = probs + IsRepG0 + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + prob = probs + IsRep0Long + COMBINED_PS_STATE; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + NORMALIZE_CHECK; + return DUMMY_REP; + } + else + { + UPDATE_1_CHECK; + } + } + else + { + UPDATE_1_CHECK; + prob = probs + IsRepG1 + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + } + else + { + UPDATE_1_CHECK; + prob = probs + IsRepG2 + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + } + else + { + UPDATE_1_CHECK; + } + } + } + state = kNumStates; + prob = probs + RepLenCoder; + } + { + unsigned limit, offset; + const CLzmaProb *probLen = prob + LenChoice; + IF_BIT_0_CHECK(probLen) + { + UPDATE_0_CHECK; + probLen = prob + LenLow + GET_LEN_STATE; + offset = 0; + limit = 1 << kLenNumLowBits; + } + else + { + UPDATE_1_CHECK; + probLen = prob + LenChoice2; + IF_BIT_0_CHECK(probLen) + { + UPDATE_0_CHECK; + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); + offset = kLenNumLowSymbols; + limit = 1 << kLenNumLowBits; + } + else + { + UPDATE_1_CHECK; + probLen = prob + LenHigh; + offset = kLenNumLowSymbols * 2; + limit = 1 << kLenNumHighBits; + } + } + TREE_DECODE_CHECK(probLen, limit, len); + len += offset; + } + + if (state < 4) + { + unsigned posSlot; + prob = probs + PosSlot + + ((len < kNumLenToPosStates - 1 ? len : kNumLenToPosStates - 1) << + kNumPosSlotBits); + TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); + if (posSlot >= kStartPosModelIndex) + { + unsigned numDirectBits = ((posSlot >> 1) - 1); + + /* if (bufLimit - buf >= 8) return DUMMY_MATCH; */ + + if (posSlot < kEndPosModelIndex) + { + prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits); + } + else + { + numDirectBits -= kNumAlignBits; + do + { + NORMALIZE_CHECK + range >>= 1; + code -= range & (((code - range) >> 31) - 1); + /* if (code >= range) code -= range; */ + } + while (--numDirectBits); + prob = probs + Align; + numDirectBits = kNumAlignBits; + } + { + unsigned i = 1; + unsigned m = 1; + do + { + REV_BIT_CHECK(prob, i, m); + } + while (--numDirectBits); + } + } + } + } + } + NORMALIZE_CHECK; + return res; +} + + +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState) +{ + p->remainLen = kMatchSpecLenStart + 1; + p->tempBufSize = 0; + + if (initDic) + { + p->processedPos = 0; + p->checkDicSize = 0; + p->remainLen = kMatchSpecLenStart + 2; + } + if (initState) + p->remainLen = kMatchSpecLenStart + 2; +} + +void LzmaDec_Init(CLzmaDec *p) +{ + p->dicPos = 0; + LzmaDec_InitDicAndState(p, True, True); +} + + +SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, + ELzmaFinishMode finishMode, ELzmaStatus *status) +{ + SizeT inSize = *srcLen; + (*srcLen) = 0; + + *status = LZMA_STATUS_NOT_SPECIFIED; + + if (p->remainLen > kMatchSpecLenStart) + { + for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) + p->tempBuf[p->tempBufSize++] = *src++; + if (p->tempBufSize != 0 && p->tempBuf[0] != 0) + return SZ_ERROR_DATA; + if (p->tempBufSize < RC_INIT_SIZE) + { + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + p->code = + ((UInt32)p->tempBuf[1] << 24) + | ((UInt32)p->tempBuf[2] << 16) + | ((UInt32)p->tempBuf[3] << 8) + | ((UInt32)p->tempBuf[4]); + p->range = 0xFFFFFFFF; + p->tempBufSize = 0; + + if (p->remainLen > kMatchSpecLenStart + 1) + { + SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); + SizeT i; + CLzmaProb *probs = p->probs; + for (i = 0; i < numProbs; i++) + probs[i] = kBitModelTotal >> 1; + p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; + p->state = 0; + } + + p->remainLen = 0; + } + + LzmaDec_WriteRem(p, dicLimit); + + while (p->remainLen != kMatchSpecLenStart) + { + int checkEndMarkNow = 0; + + if (p->dicPos >= dicLimit) + { + if (p->remainLen == 0 && p->code == 0) + { + *status = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK; + return SZ_OK; + } + if (finishMode == LZMA_FINISH_ANY) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_OK; + } + if (p->remainLen != 0) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_ERROR_DATA; + } + checkEndMarkNow = 1; + } + + if (p->tempBufSize == 0) + { + SizeT processed; + const Byte *bufLimit; + if (inSize < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) + { + int dummyRes = LzmaDec_TryDummy(p, src, inSize); + if (dummyRes == DUMMY_ERROR) + { + memcpy(p->tempBuf, src, inSize); + p->tempBufSize = (unsigned)inSize; + (*srcLen) += inSize; + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + if (checkEndMarkNow && dummyRes != DUMMY_MATCH) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_ERROR_DATA; + } + bufLimit = src; + } + else + bufLimit = src + inSize - LZMA_REQUIRED_INPUT_MAX; + p->buf = src; + if (LzmaDec_DecodeReal2(p, dicLimit, bufLimit) != 0) + return SZ_ERROR_DATA; + processed = (SizeT)(p->buf - src); + (*srcLen) += processed; + src += processed; + inSize -= processed; + } + else + { + unsigned rem = p->tempBufSize, lookAhead = 0; + while (rem < LZMA_REQUIRED_INPUT_MAX && lookAhead < inSize) + p->tempBuf[rem++] = src[lookAhead++]; + p->tempBufSize = rem; + if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) + { + int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, (SizeT)rem); + if (dummyRes == DUMMY_ERROR) + { + (*srcLen) += (SizeT)lookAhead; + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + if (checkEndMarkNow && dummyRes != DUMMY_MATCH) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_ERROR_DATA; + } + } + p->buf = p->tempBuf; + if (LzmaDec_DecodeReal2(p, dicLimit, p->buf) != 0) + return SZ_ERROR_DATA; + + { + unsigned kkk = (unsigned)(p->buf - p->tempBuf); + if (rem < kkk) + return SZ_ERROR_FAIL; /* some internal error */ + rem -= kkk; + if (lookAhead < rem) + return SZ_ERROR_FAIL; /* some internal error */ + lookAhead -= rem; + } + (*srcLen) += (SizeT)lookAhead; + src += lookAhead; + inSize -= (SizeT)lookAhead; + p->tempBufSize = 0; + } + } + + if (p->code != 0) + return SZ_ERROR_DATA; + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; +} + + +SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) +{ + SizeT outSize = *destLen; + SizeT inSize = *srcLen; + *srcLen = *destLen = 0; + for (;;) + { + SizeT inSizeCur = inSize, outSizeCur, dicPos; + ELzmaFinishMode curFinishMode; + SRes res; + if (p->dicPos == p->dicBufSize) + p->dicPos = 0; + dicPos = p->dicPos; + if (outSize > p->dicBufSize - dicPos) + { + outSizeCur = p->dicBufSize; + curFinishMode = LZMA_FINISH_ANY; + } + else + { + outSizeCur = dicPos + outSize; + curFinishMode = finishMode; + } + + res = LzmaDec_DecodeToDic(p, outSizeCur, src, &inSizeCur, curFinishMode, status); + src += inSizeCur; + inSize -= inSizeCur; + *srcLen += inSizeCur; + outSizeCur = p->dicPos - dicPos; + memcpy(dest, p->dic + dicPos, outSizeCur); + dest += outSizeCur; + outSize -= outSizeCur; + *destLen += outSizeCur; + if (res != 0) + return res; + if (outSizeCur == 0 || outSize == 0) + return SZ_OK; + } +} + +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->probs); + p->probs = NULL; +} + +static void LzmaDec_FreeDict(CLzmaDec *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->dic); + p->dic = NULL; +} + +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc) +{ + LzmaDec_FreeProbs(p, alloc); + LzmaDec_FreeDict(p, alloc); +} + +SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size) +{ + UInt32 dicSize; + Byte d; + + if (size < LZMA_PROPS_SIZE) + return SZ_ERROR_UNSUPPORTED; + else + dicSize = data[1] | ((UInt32)data[2] << 8) | ((UInt32)data[3] << 16) | ((UInt32)data[4] << 24); + + if (dicSize < LZMA_DIC_MIN) + dicSize = LZMA_DIC_MIN; + p->dicSize = dicSize; + + d = data[0]; + if (d >= (9 * 5 * 5)) + return SZ_ERROR_UNSUPPORTED; + + p->lc = (Byte)(d % 9); + d /= 9; + p->pb = (Byte)(d / 5); + p->lp = (Byte)(d % 5); + + return SZ_OK; +} + +static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAllocPtr alloc) +{ + UInt32 numProbs = LzmaProps_GetNumProbs(propNew); + if (!p->probs || numProbs != p->numProbs) + { + LzmaDec_FreeProbs(p, alloc); + p->probs = (CLzmaProb *)ISzAlloc_Alloc(alloc, numProbs * sizeof(CLzmaProb)); + if (!p->probs) + return SZ_ERROR_MEM; + p->probs_1664 = p->probs + 1664; + p->numProbs = numProbs; + } + return SZ_OK; +} + +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) +{ + CLzmaProps propNew; + RINOK(LzmaProps_Decode(&propNew, props, propsSize)); + RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); + p->prop = propNew; + return SZ_OK; +} + +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) +{ + CLzmaProps propNew; + SizeT dicBufSize; + RINOK(LzmaProps_Decode(&propNew, props, propsSize)); + RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); + + { + UInt32 dictSize = propNew.dicSize; + SizeT mask = ((UInt32)1 << 12) - 1; + if (dictSize >= ((UInt32)1 << 30)) mask = ((UInt32)1 << 22) - 1; + else if (dictSize >= ((UInt32)1 << 22)) mask = ((UInt32)1 << 20) - 1;; + dicBufSize = ((SizeT)dictSize + mask) & ~mask; + if (dicBufSize < dictSize) + dicBufSize = dictSize; + } + + if (!p->dic || dicBufSize != p->dicBufSize) + { + LzmaDec_FreeDict(p, alloc); + p->dic = (Byte *)ISzAlloc_Alloc(alloc, dicBufSize); + if (!p->dic) + { + LzmaDec_FreeProbs(p, alloc); + return SZ_ERROR_MEM; + } + } + p->dicBufSize = dicBufSize; + p->prop = propNew; + return SZ_OK; +} + +SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, + ELzmaStatus *status, ISzAllocPtr alloc) +{ + CLzmaDec p; + SRes res; + SizeT outSize = *destLen, inSize = *srcLen; + *destLen = *srcLen = 0; + *status = LZMA_STATUS_NOT_SPECIFIED; + if (inSize < RC_INIT_SIZE) + return SZ_ERROR_INPUT_EOF; + LzmaDec_Construct(&p); + RINOK(LzmaDec_AllocateProbs(&p, propData, propSize, alloc)); + p.dic = dest; + p.dicBufSize = outSize; + LzmaDec_Init(&p); + *srcLen = inSize; + res = LzmaDec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status); + *destLen = p.dicPos; + if (res == SZ_OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT) + res = SZ_ERROR_INPUT_EOF; + LzmaDec_FreeProbs(&p, alloc); + return res; +} diff --git a/code/nel/3rdparty/seven_zip/LzmaDec.h b/code/nel/3rdparty/seven_zip/LzmaDec.h new file mode 100644 index 000000000..1f0927ab1 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaDec.h @@ -0,0 +1,234 @@ +/* LzmaDec.h -- LZMA Decoder +2018-04-21 : Igor Pavlov : Public domain */ + +#ifndef __LZMA_DEC_H +#define __LZMA_DEC_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +/* #define _LZMA_PROB32 */ +/* _LZMA_PROB32 can increase the speed on some CPUs, + but memory usage for CLzmaDec::probs will be doubled in that case */ + +typedef +#ifdef _LZMA_PROB32 + UInt32 +#else + UInt16 +#endif + CLzmaProb; + + +/* ---------- LZMA Properties ---------- */ + +#define LZMA_PROPS_SIZE 5 + +typedef struct _CLzmaProps +{ + Byte lc; + Byte lp; + Byte pb; + Byte _pad_; + UInt32 dicSize; +} CLzmaProps; + +/* LzmaProps_Decode - decodes properties +Returns: + SZ_OK + SZ_ERROR_UNSUPPORTED - Unsupported properties +*/ + +SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size); + + +/* ---------- LZMA Decoder state ---------- */ + +/* LZMA_REQUIRED_INPUT_MAX = number of required input bytes for worst case. + Num bits = log2((2^11 / 31) ^ 22) + 26 < 134 + 26 = 160; */ + +#define LZMA_REQUIRED_INPUT_MAX 20 + +typedef struct +{ + /* Don't change this structure. ASM code can use it. */ + CLzmaProps prop; + CLzmaProb *probs; + CLzmaProb *probs_1664; + Byte *dic; + SizeT dicBufSize; + SizeT dicPos; + const Byte *buf; + UInt32 range; + UInt32 code; + UInt32 processedPos; + UInt32 checkDicSize; + UInt32 reps[4]; + UInt32 state; + UInt32 remainLen; + + UInt32 numProbs; + unsigned tempBufSize; + Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; +} CLzmaDec; + +#define LzmaDec_Construct(p) { (p)->dic = NULL; (p)->probs = NULL; } + +void LzmaDec_Init(CLzmaDec *p); + +/* There are two types of LZMA streams: + - Stream with end mark. That end mark adds about 6 bytes to compressed size. + - Stream without end mark. You must know exact uncompressed size to decompress such stream. */ + +typedef enum +{ + LZMA_FINISH_ANY, /* finish at any point */ + LZMA_FINISH_END /* block must be finished at the end */ +} ELzmaFinishMode; + +/* ELzmaFinishMode has meaning only if the decoding reaches output limit !!! + + You must use LZMA_FINISH_END, when you know that current output buffer + covers last bytes of block. In other cases you must use LZMA_FINISH_ANY. + + If LZMA decoder sees end marker before reaching output limit, it returns SZ_OK, + and output value of destLen will be less than output buffer size limit. + You can check status result also. + + You can use multiple checks to test data integrity after full decompression: + 1) Check Result and "status" variable. + 2) Check that output(destLen) = uncompressedSize, if you know real uncompressedSize. + 3) Check that output(srcLen) = compressedSize, if you know real compressedSize. + You must use correct finish mode in that case. */ + +typedef enum +{ + LZMA_STATUS_NOT_SPECIFIED, /* use main error code instead */ + LZMA_STATUS_FINISHED_WITH_MARK, /* stream was finished with end mark. */ + LZMA_STATUS_NOT_FINISHED, /* stream was not finished */ + LZMA_STATUS_NEEDS_MORE_INPUT, /* you must provide more input bytes */ + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK /* there is probability that stream was finished without end mark */ +} ELzmaStatus; + +/* ELzmaStatus is used only as output value for function call */ + + +/* ---------- Interfaces ---------- */ + +/* There are 3 levels of interfaces: + 1) Dictionary Interface + 2) Buffer Interface + 3) One Call Interface + You can select any of these interfaces, but don't mix functions from different + groups for same object. */ + + +/* There are two variants to allocate state for Dictionary Interface: + 1) LzmaDec_Allocate / LzmaDec_Free + 2) LzmaDec_AllocateProbs / LzmaDec_FreeProbs + You can use variant 2, if you set dictionary buffer manually. + For Buffer Interface you must always use variant 1. + +LzmaDec_Allocate* can return: + SZ_OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_UNSUPPORTED - Unsupported properties +*/ + +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc); + +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc); + +/* ---------- Dictionary Interface ---------- */ + +/* You can use it, if you want to eliminate the overhead for data copying from + dictionary to some other external buffer. + You must work with CLzmaDec variables directly in this interface. + + STEPS: + LzmaDec_Construct() + LzmaDec_Allocate() + for (each new stream) + { + LzmaDec_Init() + while (it needs more decompression) + { + LzmaDec_DecodeToDic() + use data from CLzmaDec::dic and update CLzmaDec::dicPos + } + } + LzmaDec_Free() +*/ + +/* LzmaDec_DecodeToDic + + The decoding to internal dictionary buffer (CLzmaDec::dic). + You must manually update CLzmaDec::dicPos, if it reaches CLzmaDec::dicBufSize !!! + +finishMode: + It has meaning only if the decoding reaches output limit (dicLimit). + LZMA_FINISH_ANY - Decode just dicLimit bytes. + LZMA_FINISH_END - Stream must be finished after dicLimit. + +Returns: + SZ_OK + status: + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED + LZMA_STATUS_NEEDS_MORE_INPUT + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + SZ_ERROR_DATA - Data error +*/ + +SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, + const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); + + +/* ---------- Buffer Interface ---------- */ + +/* It's zlib-like interface. + See LzmaDec_DecodeToDic description for information about STEPS and return results, + but you must use LzmaDec_DecodeToBuf instead of LzmaDec_DecodeToDic and you don't need + to work with CLzmaDec variables manually. + +finishMode: + It has meaning only if the decoding reaches output limit (*destLen). + LZMA_FINISH_ANY - Decode just destLen bytes. + LZMA_FINISH_END - Stream must be finished after (*destLen). +*/ + +SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); + + +/* ---------- One Call Interface ---------- */ + +/* LzmaDecode + +finishMode: + It has meaning only if the decoding reaches output limit (*destLen). + LZMA_FINISH_ANY - Decode just destLen bytes. + LZMA_FINISH_END - Stream must be finished after (*destLen). + +Returns: + SZ_OK + status: + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + SZ_ERROR_DATA - Data error + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_UNSUPPORTED - Unsupported properties + SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src). +*/ + +SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, + ELzmaStatus *status, ISzAllocPtr alloc); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/LzmaEnc.c b/code/nel/3rdparty/seven_zip/LzmaEnc.c new file mode 100644 index 000000000..46a0db000 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaEnc.c @@ -0,0 +1,2976 @@ +/* LzmaEnc.c -- LZMA Encoder +2019-01-10: Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +/* #define SHOW_STAT */ +/* #define SHOW_STAT2 */ + +#if defined(SHOW_STAT) || defined(SHOW_STAT2) +#include +#endif + +#include "LzmaEnc.h" + +#include "LzFind.h" +#ifndef _7ZIP_ST +#include "LzFindMt.h" +#endif + +#ifdef SHOW_STAT +static unsigned g_STAT_OFFSET = 0; +#endif + +#define kLzmaMaxHistorySize ((UInt32)3 << 29) +/* #define kLzmaMaxHistorySize ((UInt32)7 << 29) */ + +#define kNumTopBits 24 +#define kTopValue ((UInt32)1 << kNumTopBits) + +#define kNumBitModelTotalBits 11 +#define kBitModelTotal (1 << kNumBitModelTotalBits) +#define kNumMoveBits 5 +#define kProbInitValue (kBitModelTotal >> 1) + +#define kNumMoveReducingBits 4 +#define kNumBitPriceShiftBits 4 +#define kBitPrice (1 << kNumBitPriceShiftBits) + +#define REP_LEN_COUNT 64 + +void LzmaEncProps_Init(CLzmaEncProps *p) +{ + p->level = 5; + p->dictSize = p->mc = 0; + p->reduceSize = (UInt64)(Int64)-1; + p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1; + p->writeEndMark = 0; +} + +void LzmaEncProps_Normalize(CLzmaEncProps *p) +{ + int level = p->level; + if (level < 0) level = 5; + p->level = level; + + if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level <= 7 ? (1 << 25) : (1 << 26))); + if (p->dictSize > p->reduceSize) + { + unsigned i; + UInt32 reduceSize = (UInt32)p->reduceSize; + for (i = 11; i <= 30; i++) + { + if (reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; } + if (reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; } + } + } + + if (p->lc < 0) p->lc = 3; + if (p->lp < 0) p->lp = 0; + if (p->pb < 0) p->pb = 2; + + if (p->algo < 0) p->algo = (level < 5 ? 0 : 1); + if (p->fb < 0) p->fb = (level < 7 ? 32 : 64); + if (p->btMode < 0) p->btMode = (p->algo == 0 ? 0 : 1); + if (p->numHashBytes < 0) p->numHashBytes = 4; + if (p->mc == 0) p->mc = (16 + (p->fb >> 1)) >> (p->btMode ? 0 : 1); + + if (p->numThreads < 0) + p->numThreads = + #ifndef _7ZIP_ST + ((p->btMode && p->algo) ? 2 : 1); + #else + 1; + #endif +} + +UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2) +{ + CLzmaEncProps props = *props2; + LzmaEncProps_Normalize(&props); + return props.dictSize; +} + +#if (_MSC_VER >= 1400) +/* BSR code is fast for some new CPUs */ +/* #define LZMA_LOG_BSR */ +#endif + +#ifdef LZMA_LOG_BSR + +#define kDicLogSizeMaxCompress 32 + +#define BSR2_RET(pos, res) { unsigned long zz; _BitScanReverse(&zz, (pos)); res = (zz + zz) + ((pos >> (zz - 1)) & 1); } + +static unsigned GetPosSlot1(UInt32 pos) +{ + unsigned res; + BSR2_RET(pos, res); + return res; +} +#define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } +#define GetPosSlot(pos, res) { if (pos < 2) res = pos; else BSR2_RET(pos, res); } + +#else + +#define kNumLogBits (9 + sizeof(size_t) / 2) +/* #define kNumLogBits (11 + sizeof(size_t) / 8 * 3) */ + +#define kDicLogSizeMaxCompress ((kNumLogBits - 1) * 2 + 7) + +static void LzmaEnc_FastPosInit(Byte *g_FastPos) +{ + unsigned slot; + g_FastPos[0] = 0; + g_FastPos[1] = 1; + g_FastPos += 2; + + for (slot = 2; slot < kNumLogBits * 2; slot++) + { + size_t k = ((size_t)1 << ((slot >> 1) - 1)); + size_t j; + for (j = 0; j < k; j++) + g_FastPos[j] = (Byte)slot; + g_FastPos += k; + } +} + +/* we can use ((limit - pos) >> 31) only if (pos < ((UInt32)1 << 31)) */ +/* +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ + (0 - (((((UInt32)1 << (kNumLogBits + 6)) - 1) - pos) >> 31))); \ + res = p->g_FastPos[pos >> zz] + (zz * 2); } +*/ + +/* +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ + (0 - (((((UInt32)1 << (kNumLogBits)) - 1) - (pos >> 6)) >> 31))); \ + res = p->g_FastPos[pos >> zz] + (zz * 2); } +*/ + +#define BSR2_RET(pos, res) { unsigned zz = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ + res = p->g_FastPos[pos >> zz] + (zz * 2); } + +/* +#define BSR2_RET(pos, res) { res = (pos < (1 << (kNumLogBits + 6))) ? \ + p->g_FastPos[pos >> 6] + 12 : \ + p->g_FastPos[pos >> (6 + kNumLogBits - 1)] + (6 + (kNumLogBits - 1)) * 2; } +*/ + +#define GetPosSlot1(pos) p->g_FastPos[pos] +#define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } +#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos & (kNumFullDistances - 1)]; else BSR2_RET(pos, res); } + +#endif + + +#define LZMA_NUM_REPS 4 + +typedef UInt16 CState; +typedef UInt16 CExtra; + +typedef struct +{ + UInt32 price; + CState state; + CExtra extra; + // 0 : normal + // 1 : LIT : MATCH + // > 1 : MATCH (extra-1) : LIT : REP0 (len) + UInt32 len; + UInt32 dist; + UInt32 reps[LZMA_NUM_REPS]; +} COptimal; + + +// 18.06 +#define kNumOpts (1 << 11) +#define kPackReserve (kNumOpts * 8) +// #define kNumOpts (1 << 12) +// #define kPackReserve (1 + kNumOpts * 2) + +#define kNumLenToPosStates 4 +#define kNumPosSlotBits 6 +#define kDicLogSizeMin 0 +#define kDicLogSizeMax 32 +#define kDistTableSizeMax (kDicLogSizeMax * 2) + +#define kNumAlignBits 4 +#define kAlignTableSize (1 << kNumAlignBits) +#define kAlignMask (kAlignTableSize - 1) + +#define kStartPosModelIndex 4 +#define kEndPosModelIndex 14 +#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) + +typedef +#ifdef _LZMA_PROB32 + UInt32 +#else + UInt16 +#endif + CLzmaProb; + +#define LZMA_PB_MAX 4 +#define LZMA_LC_MAX 8 +#define LZMA_LP_MAX 4 + +#define LZMA_NUM_PB_STATES_MAX (1 << LZMA_PB_MAX) + +#define kLenNumLowBits 3 +#define kLenNumLowSymbols (1 << kLenNumLowBits) +#define kLenNumHighBits 8 +#define kLenNumHighSymbols (1 << kLenNumHighBits) +#define kLenNumSymbolsTotal (kLenNumLowSymbols * 2 + kLenNumHighSymbols) + +#define LZMA_MATCH_LEN_MIN 2 +#define LZMA_MATCH_LEN_MAX (LZMA_MATCH_LEN_MIN + kLenNumSymbolsTotal - 1) + +#define kNumStates 12 + + +typedef struct +{ + CLzmaProb low[LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)]; + CLzmaProb high[kLenNumHighSymbols]; +} CLenEnc; + + +typedef struct +{ + unsigned tableSize; + UInt32 prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal]; + // UInt32 prices1[LZMA_NUM_PB_STATES_MAX][kLenNumLowSymbols * 2]; + // UInt32 prices2[kLenNumSymbolsTotal]; +} CLenPriceEnc; + +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices[posState][(size_t)(len) - LZMA_MATCH_LEN_MIN]) + +/* +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices2[(size_t)(len) - 2] + ((p)->prices1[posState][((len) - 2) & (kLenNumLowSymbols * 2 - 1)] & (((len) - 2 - kLenNumLowSymbols * 2) >> 9))) +*/ + +typedef struct +{ + UInt32 range; + unsigned cache; + UInt64 low; + UInt64 cacheSize; + Byte *buf; + Byte *bufLim; + Byte *bufBase; + ISeqOutStream *outStream; + UInt64 processed; + SRes res; +} CRangeEnc; + + +typedef struct +{ + CLzmaProb *litProbs; + + unsigned state; + UInt32 reps[LZMA_NUM_REPS]; + + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb isRep[kNumStates]; + CLzmaProb isRepG0[kNumStates]; + CLzmaProb isRepG1[kNumStates]; + CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; + + CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; + CLzmaProb posEncoders[kNumFullDistances]; + + CLenEnc lenProbs; + CLenEnc repLenProbs; + +} CSaveState; + + +typedef UInt32 CProbPrice; + + +typedef struct +{ + void *matchFinderObj; + IMatchFinder matchFinder; + + unsigned optCur; + unsigned optEnd; + + unsigned longestMatchLen; + unsigned numPairs; + UInt32 numAvail; + + unsigned state; + unsigned numFastBytes; + unsigned additionalOffset; + UInt32 reps[LZMA_NUM_REPS]; + unsigned lpMask, pbMask; + CLzmaProb *litProbs; + CRangeEnc rc; + + UInt32 backRes; + + unsigned lc, lp, pb; + unsigned lclp; + + BoolInt fastMode; + BoolInt writeEndMark; + BoolInt finished; + BoolInt multiThread; + BoolInt needInit; + // BoolInt _maxMode; + + UInt64 nowPos64; + + unsigned matchPriceCount; + // unsigned alignPriceCount; + int repLenEncCounter; + + unsigned distTableSize; + + UInt32 dictSize; + SRes result; + + #ifndef _7ZIP_ST + BoolInt mtMode; + // begin of CMatchFinderMt is used in LZ thread + CMatchFinderMt matchFinderMt; + // end of CMatchFinderMt is used in BT and HASH threads + #endif + + CMatchFinder matchFinderBase; + + #ifndef _7ZIP_ST + Byte pad[128]; + #endif + + // LZ thread + CProbPrice ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; + + UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2 + 1]; + + UInt32 alignPrices[kAlignTableSize]; + UInt32 posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; + UInt32 distancesPrices[kNumLenToPosStates][kNumFullDistances]; + + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb isRep[kNumStates]; + CLzmaProb isRepG0[kNumStates]; + CLzmaProb isRepG1[kNumStates]; + CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; + CLzmaProb posEncoders[kNumFullDistances]; + + CLenEnc lenProbs; + CLenEnc repLenProbs; + + #ifndef LZMA_LOG_BSR + Byte g_FastPos[1 << kNumLogBits]; + #endif + + CLenPriceEnc lenEnc; + CLenPriceEnc repLenEnc; + + COptimal opt[kNumOpts]; + + CSaveState saveState; + + #ifndef _7ZIP_ST + Byte pad2[128]; + #endif +} CLzmaEnc; + + + +#define COPY_ARR(dest, src, arr) memcpy(dest->arr, src->arr, sizeof(src->arr)); + +void LzmaEnc_SaveState(CLzmaEncHandle pp) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + CSaveState *dest = &p->saveState; + + dest->state = p->state; + + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; + + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); + + memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << p->lclp) * sizeof(CLzmaProb)); +} + + +void LzmaEnc_RestoreState(CLzmaEncHandle pp) +{ + CLzmaEnc *dest = (CLzmaEnc *)pp; + const CSaveState *p = &dest->saveState; + + dest->state = p->state; + + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; + + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); + + memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << dest->lclp) * sizeof(CLzmaProb)); +} + + + +SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + CLzmaEncProps props = *props2; + LzmaEncProps_Normalize(&props); + + if (props.lc > LZMA_LC_MAX + || props.lp > LZMA_LP_MAX + || props.pb > LZMA_PB_MAX + || props.dictSize > ((UInt64)1 << kDicLogSizeMaxCompress) + || props.dictSize > kLzmaMaxHistorySize) + return SZ_ERROR_PARAM; + + p->dictSize = props.dictSize; + { + unsigned fb = props.fb; + if (fb < 5) + fb = 5; + if (fb > LZMA_MATCH_LEN_MAX) + fb = LZMA_MATCH_LEN_MAX; + p->numFastBytes = fb; + } + p->lc = props.lc; + p->lp = props.lp; + p->pb = props.pb; + p->fastMode = (props.algo == 0); + // p->_maxMode = True; + p->matchFinderBase.btMode = (Byte)(props.btMode ? 1 : 0); + { + unsigned numHashBytes = 4; + if (props.btMode) + { + if (props.numHashBytes < 2) + numHashBytes = 2; + else if (props.numHashBytes < 4) + numHashBytes = props.numHashBytes; + } + p->matchFinderBase.numHashBytes = numHashBytes; + } + + p->matchFinderBase.cutValue = props.mc; + + p->writeEndMark = props.writeEndMark; + + #ifndef _7ZIP_ST + /* + if (newMultiThread != _multiThread) + { + ReleaseMatchFinder(); + _multiThread = newMultiThread; + } + */ + p->multiThread = (props.numThreads > 1); + #endif + + return SZ_OK; +} + + +void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + p->matchFinderBase.expectedDataSize = expectedDataSiize; +} + + +#define kState_Start 0 +#define kState_LitAfterMatch 4 +#define kState_LitAfterRep 5 +#define kState_MatchAfterLit 7 +#define kState_RepAfterLit 8 + +static const Byte kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; +static const Byte kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; +static const Byte kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; +static const Byte kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; + +#define IsLitState(s) ((s) < 7) +#define GetLenToPosState2(len) (((len) < kNumLenToPosStates - 1) ? (len) : kNumLenToPosStates - 1) +#define GetLenToPosState(len) (((len) < kNumLenToPosStates + 1) ? (len) - 2 : kNumLenToPosStates - 1) + +#define kInfinityPrice (1 << 30) + +static void RangeEnc_Construct(CRangeEnc *p) +{ + p->outStream = NULL; + p->bufBase = NULL; +} + +#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed_sizet(p) ((size_t)(p)->processed + ((p)->buf - (p)->bufBase) + (size_t)(p)->cacheSize) + +#define RC_BUF_SIZE (1 << 16) + +static int RangeEnc_Alloc(CRangeEnc *p, ISzAllocPtr alloc) +{ + if (!p->bufBase) + { + p->bufBase = (Byte *)ISzAlloc_Alloc(alloc, RC_BUF_SIZE); + if (!p->bufBase) + return 0; + p->bufLim = p->bufBase + RC_BUF_SIZE; + } + return 1; +} + +static void RangeEnc_Free(CRangeEnc *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->bufBase); + p->bufBase = 0; +} + +static void RangeEnc_Init(CRangeEnc *p) +{ + /* Stream.Init(); */ + p->range = 0xFFFFFFFF; + p->cache = 0; + p->low = 0; + p->cacheSize = 0; + + p->buf = p->bufBase; + + p->processed = 0; + p->res = SZ_OK; +} + +MY_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) +{ + size_t num; + if (p->res != SZ_OK) + return; + num = p->buf - p->bufBase; + if (num != ISeqOutStream_Write(p->outStream, p->bufBase, num)) + p->res = SZ_ERROR_WRITE; + p->processed += num; + p->buf = p->bufBase; +} + +MY_NO_INLINE static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) +{ + UInt32 low = (UInt32)p->low; + unsigned high = (unsigned)(p->low >> 32); + p->low = (UInt32)(low << 8); + if (low < (UInt32)0xFF000000 || high != 0) + { + { + Byte *buf = p->buf; + *buf++ = (Byte)(p->cache + high); + p->cache = (unsigned)(low >> 24); + p->buf = buf; + if (buf == p->bufLim) + RangeEnc_FlushStream(p); + if (p->cacheSize == 0) + return; + } + high += 0xFF; + for (;;) + { + Byte *buf = p->buf; + *buf++ = (Byte)(high); + p->buf = buf; + if (buf == p->bufLim) + RangeEnc_FlushStream(p); + if (--p->cacheSize == 0) + return; + } + } + p->cacheSize++; +} + +static void RangeEnc_FlushData(CRangeEnc *p) +{ + int i; + for (i = 0; i < 5; i++) + RangeEnc_ShiftLow(p); +} + +#define RC_NORM(p) if (range < kTopValue) { range <<= 8; RangeEnc_ShiftLow(p); } + +#define RC_BIT_PRE(p, prob) \ + ttt = *(prob); \ + newBound = (range >> kNumBitModelTotalBits) * ttt; + +// #define _LZMA_ENC_USE_BRANCH + +#ifdef _LZMA_ENC_USE_BRANCH + +#define RC_BIT(p, prob, bit) { \ + RC_BIT_PRE(p, prob) \ + if (bit == 0) { range = newBound; ttt += (kBitModelTotal - ttt) >> kNumMoveBits; } \ + else { (p)->low += newBound; range -= newBound; ttt -= ttt >> kNumMoveBits; } \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ + } + +#else + +#define RC_BIT(p, prob, bit) { \ + UInt32 mask; \ + RC_BIT_PRE(p, prob) \ + mask = 0 - (UInt32)bit; \ + range &= mask; \ + mask &= newBound; \ + range -= mask; \ + (p)->low += mask; \ + mask = (UInt32)bit - 1; \ + range += newBound & mask; \ + mask &= (kBitModelTotal - ((1 << kNumMoveBits) - 1)); \ + mask += ((1 << kNumMoveBits) - 1); \ + ttt += (Int32)(mask - ttt) >> kNumMoveBits; \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ + } + +#endif + + + + +#define RC_BIT_0_BASE(p, prob) \ + range = newBound; *(prob) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); + +#define RC_BIT_1_BASE(p, prob) \ + range -= newBound; (p)->low += newBound; *(prob) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); \ + +#define RC_BIT_0(p, prob) \ + RC_BIT_0_BASE(p, prob) \ + RC_NORM(p) + +#define RC_BIT_1(p, prob) \ + RC_BIT_1_BASE(p, prob) \ + RC_NORM(p) + +static void RangeEnc_EncodeBit_0(CRangeEnc *p, CLzmaProb *prob) +{ + UInt32 range, ttt, newBound; + range = p->range; + RC_BIT_PRE(p, prob) + RC_BIT_0(p, prob) + p->range = range; +} + +static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 sym) +{ + UInt32 range = p->range; + sym |= 0x100; + do + { + UInt32 ttt, newBound; + // RangeEnc_EncodeBit(p, probs + (sym >> 8), (sym >> 7) & 1); + CLzmaProb *prob = probs + (sym >> 8); + UInt32 bit = (sym >> 7) & 1; + sym <<= 1; + RC_BIT(p, prob, bit); + } + while (sym < 0x10000); + p->range = range; +} + +static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 sym, UInt32 matchByte) +{ + UInt32 range = p->range; + UInt32 offs = 0x100; + sym |= 0x100; + do + { + UInt32 ttt, newBound; + CLzmaProb *prob; + UInt32 bit; + matchByte <<= 1; + // RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (sym >> 8)), (sym >> 7) & 1); + prob = probs + (offs + (matchByte & offs) + (sym >> 8)); + bit = (sym >> 7) & 1; + sym <<= 1; + offs &= ~(matchByte ^ sym); + RC_BIT(p, prob, bit); + } + while (sym < 0x10000); + p->range = range; +} + + + +static void LzmaEnc_InitPriceTables(CProbPrice *ProbPrices) +{ + UInt32 i; + for (i = 0; i < (kBitModelTotal >> kNumMoveReducingBits); i++) + { + const unsigned kCyclesBits = kNumBitPriceShiftBits; + UInt32 w = (i << kNumMoveReducingBits) + (1 << (kNumMoveReducingBits - 1)); + unsigned bitCount = 0; + unsigned j; + for (j = 0; j < kCyclesBits; j++) + { + w = w * w; + bitCount <<= 1; + while (w >= ((UInt32)1 << 16)) + { + w >>= 1; + bitCount++; + } + } + ProbPrices[i] = (CProbPrice)((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + // printf("\n%3d: %5d", i, ProbPrices[i]); + } +} + + +#define GET_PRICE(prob, bit) \ + p->ProbPrices[((prob) ^ (unsigned)(((-(int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; + +#define GET_PRICEa(prob, bit) \ + ProbPrices[((prob) ^ (unsigned)((-((int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; + +#define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits] +#define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] + +#define GET_PRICEa_0(prob) ProbPrices[(prob) >> kNumMoveReducingBits] +#define GET_PRICEa_1(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] + + +static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 sym, const CProbPrice *ProbPrices) +{ + UInt32 price = 0; + sym |= 0x100; + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); + } + while (sym >= 2); + return price; +} + + +static UInt32 LitEnc_Matched_GetPrice(const CLzmaProb *probs, UInt32 sym, UInt32 matchByte, const CProbPrice *ProbPrices) +{ + UInt32 price = 0; + UInt32 offs = 0x100; + sym |= 0x100; + do + { + matchByte <<= 1; + price += GET_PRICEa(probs[offs + (matchByte & offs) + (sym >> 8)], (sym >> 7) & 1); + sym <<= 1; + offs &= ~(matchByte ^ sym); + } + while (sym < 0x10000); + return price; +} + + +static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, unsigned numBits, unsigned sym) +{ + UInt32 range = rc->range; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + unsigned bit = sym & 1; + // RangeEnc_EncodeBit(rc, probs + m, bit); + sym >>= 1; + RC_BIT(rc, probs + m, bit); + m = (m << 1) | bit; + } + while (--numBits); + rc->range = range; +} + + + +static void LenEnc_Init(CLenEnc *p) +{ + unsigned i; + for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)); i++) + p->low[i] = kProbInitValue; + for (i = 0; i < kLenNumHighSymbols; i++) + p->high[i] = kProbInitValue; +} + +static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, unsigned sym, unsigned posState) +{ + UInt32 range, ttt, newBound; + CLzmaProb *probs = p->low; + range = rc->range; + RC_BIT_PRE(rc, probs); + if (sym >= kLenNumLowSymbols) + { + RC_BIT_1(rc, probs); + probs += kLenNumLowSymbols; + RC_BIT_PRE(rc, probs); + if (sym >= kLenNumLowSymbols * 2) + { + RC_BIT_1(rc, probs); + rc->range = range; + // RcTree_Encode(rc, p->high, kLenNumHighBits, sym - kLenNumLowSymbols * 2); + LitEnc_Encode(rc, p->high, sym - kLenNumLowSymbols * 2); + return; + } + sym -= kLenNumLowSymbols; + } + + // RcTree_Encode(rc, probs + (posState << kLenNumLowBits), kLenNumLowBits, sym); + { + unsigned m; + unsigned bit; + RC_BIT_0(rc, probs); + probs += (posState << (1 + kLenNumLowBits)); + bit = (sym >> 2) ; RC_BIT(rc, probs + 1, bit); m = (1 << 1) + bit; + bit = (sym >> 1) & 1; RC_BIT(rc, probs + m, bit); m = (m << 1) + bit; + bit = sym & 1; RC_BIT(rc, probs + m, bit); + rc->range = range; + } +} + +static void SetPrices_3(const CLzmaProb *probs, UInt32 startPrice, UInt32 *prices, const CProbPrice *ProbPrices) +{ + unsigned i; + for (i = 0; i < 8; i += 2) + { + UInt32 price = startPrice; + UInt32 prob; + price += GET_PRICEa(probs[1 ], (i >> 2)); + price += GET_PRICEa(probs[2 + (i >> 2)], (i >> 1) & 1); + prob = probs[4 + (i >> 1)]; + prices[i ] = price + GET_PRICEa_0(prob); + prices[i + 1] = price + GET_PRICEa_1(prob); + } +} + + +MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTables( + CLenPriceEnc *p, + unsigned numPosStates, + const CLenEnc *enc, + const CProbPrice *ProbPrices) +{ + UInt32 b; + + { + unsigned prob = enc->low[0]; + UInt32 a, c; + unsigned posState; + b = GET_PRICEa_1(prob); + a = GET_PRICEa_0(prob); + c = b + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (posState = 0; posState < numPosStates; posState++) + { + UInt32 *prices = p->prices[posState]; + const CLzmaProb *probs = enc->low + (posState << (1 + kLenNumLowBits)); + SetPrices_3(probs, a, prices, ProbPrices); + SetPrices_3(probs + kLenNumLowSymbols, c, prices + kLenNumLowSymbols, ProbPrices); + } + } + + /* + { + unsigned i; + UInt32 b; + a = GET_PRICEa_0(enc->low[0]); + for (i = 0; i < kLenNumLowSymbols; i++) + p->prices2[i] = a; + a = GET_PRICEa_1(enc->low[0]); + b = a + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (i = kLenNumLowSymbols; i < kLenNumLowSymbols * 2; i++) + p->prices2[i] = b; + a += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + } + */ + + // p->counter = numSymbols; + // p->counter = 64; + + { + unsigned i = p->tableSize; + + if (i > kLenNumLowSymbols * 2) + { + const CLzmaProb *probs = enc->high; + UInt32 *prices = p->prices[0] + kLenNumLowSymbols * 2; + i -= kLenNumLowSymbols * 2 - 1; + i >>= 1; + b += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + do + { + /* + p->prices2[i] = a + + // RcTree_GetPrice(enc->high, kLenNumHighBits, i - kLenNumLowSymbols * 2, ProbPrices); + LitEnc_GetPrice(probs, i - kLenNumLowSymbols * 2, ProbPrices); + */ + // UInt32 price = a + RcTree_GetPrice(probs, kLenNumHighBits - 1, sym, ProbPrices); + unsigned sym = --i + (1 << (kLenNumHighBits - 1)); + UInt32 price = b; + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); + } + while (sym >= 2); + + { + unsigned prob = probs[(size_t)i + (1 << (kLenNumHighBits - 1))]; + prices[(size_t)i * 2 ] = price + GET_PRICEa_0(prob); + prices[(size_t)i * 2 + 1] = price + GET_PRICEa_1(prob); + } + } + while (i); + + { + unsigned posState; + size_t num = (p->tableSize - kLenNumLowSymbols * 2) * sizeof(p->prices[0][0]); + for (posState = 1; posState < numPosStates; posState++) + memcpy(p->prices[posState] + kLenNumLowSymbols * 2, p->prices[0] + kLenNumLowSymbols * 2, num); + } + } + } +} + +/* + #ifdef SHOW_STAT + g_STAT_OFFSET += num; + printf("\n MovePos %u", num); + #endif +*/ + +#define MOVE_POS(p, num) { \ + p->additionalOffset += (num); \ + p->matchFinder.Skip(p->matchFinderObj, (UInt32)(num)); } + + +static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) +{ + unsigned numPairs; + + p->additionalOffset++; + p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); + numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); + *numPairsRes = numPairs; + + #ifdef SHOW_STAT + printf("\n i = %u numPairs = %u ", g_STAT_OFFSET, numPairs / 2); + g_STAT_OFFSET++; + { + unsigned i; + for (i = 0; i < numPairs; i += 2) + printf("%2u %6u | ", p->matches[i], p->matches[i + 1]); + } + #endif + + if (numPairs == 0) + return 0; + { + unsigned len = p->matches[(size_t)numPairs - 2]; + if (len != p->numFastBytes) + return len; + { + UInt32 numAvail = p->numAvail; + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + { + const Byte *p1 = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + const Byte *p2 = p1 + len; + ptrdiff_t dif = (ptrdiff_t)-1 - p->matches[(size_t)numPairs - 1]; + const Byte *lim = p1 + numAvail; + for (; p2 != lim && *p2 == p2[dif]; p2++) + {} + return (unsigned)(p2 - p1); + } + } + } +} + +#define MARK_LIT ((UInt32)(Int32)-1) + +#define MakeAs_Lit(p) { (p)->dist = MARK_LIT; (p)->extra = 0; } +#define MakeAs_ShortRep(p) { (p)->dist = 0; (p)->extra = 0; } +#define IsShortRep(p) ((p)->dist == 0) + + +#define GetPrice_ShortRep(p, state, posState) \ + ( GET_PRICE_0(p->isRepG0[state]) + GET_PRICE_0(p->isRep0Long[state][posState])) + +#define GetPrice_Rep_0(p, state, posState) ( \ + GET_PRICE_1(p->isMatch[state][posState]) \ + + GET_PRICE_1(p->isRep0Long[state][posState])) \ + + GET_PRICE_1(p->isRep[state]) \ + + GET_PRICE_0(p->isRepG0[state]) + +MY_FORCE_INLINE +static UInt32 GetPrice_PureRep(const CLzmaEnc *p, unsigned repIndex, size_t state, size_t posState) +{ + UInt32 price; + UInt32 prob = p->isRepG0[state]; + if (repIndex == 0) + { + price = GET_PRICE_0(prob); + price += GET_PRICE_1(p->isRep0Long[state][posState]); + } + else + { + price = GET_PRICE_1(prob); + prob = p->isRepG1[state]; + if (repIndex == 1) + price += GET_PRICE_0(prob); + else + { + price += GET_PRICE_1(prob); + price += GET_PRICE(p->isRepG2[state], repIndex - 2); + } + } + return price; +} + + +static unsigned Backward(CLzmaEnc *p, unsigned cur) +{ + unsigned wr = cur + 1; + p->optEnd = wr; + + for (;;) + { + UInt32 dist = p->opt[cur].dist; + unsigned len = (unsigned)p->opt[cur].len; + unsigned extra = (unsigned)p->opt[cur].extra; + cur -= len; + + if (extra) + { + wr--; + p->opt[wr].len = (UInt32)len; + cur -= extra; + len = extra; + if (extra == 1) + { + p->opt[wr].dist = dist; + dist = MARK_LIT; + } + else + { + p->opt[wr].dist = 0; + len--; + wr--; + p->opt[wr].dist = MARK_LIT; + p->opt[wr].len = 1; + } + } + + if (cur == 0) + { + p->backRes = dist; + p->optCur = wr; + return len; + } + + wr--; + p->opt[wr].dist = dist; + p->opt[wr].len = (UInt32)len; + } +} + + + +#define LIT_PROBS(pos, prevByte) \ + (p->litProbs + (UInt32)3 * (((((pos) << 8) + (prevByte)) & p->lpMask) << p->lc)) + + +static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) +{ + unsigned last, cur; + UInt32 reps[LZMA_NUM_REPS]; + unsigned repLens[LZMA_NUM_REPS]; + UInt32 *matches; + + { + UInt32 numAvail; + unsigned numPairs, mainLen, repMaxIndex, i, posState; + UInt32 matchPrice, repMatchPrice; + const Byte *data; + Byte curByte, matchByte; + + p->optCur = p->optEnd = 0; + + if (p->additionalOffset == 0) + mainLen = ReadMatchDistances(p, &numPairs); + else + { + mainLen = p->longestMatchLen; + numPairs = p->numPairs; + } + + numAvail = p->numAvail; + if (numAvail < 2) + { + p->backRes = MARK_LIT; + return 1; + } + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + repMaxIndex = 0; + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned len; + const Byte *data2; + reps[i] = p->reps[i]; + data2 = data - reps[i]; + if (data[0] != data2[0] || data[1] != data2[1]) + { + repLens[i] = 0; + continue; + } + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + repLens[i] = len; + if (len > repLens[repMaxIndex]) + repMaxIndex = i; + } + + if (repLens[repMaxIndex] >= p->numFastBytes) + { + unsigned len; + p->backRes = (UInt32)repMaxIndex; + len = repLens[repMaxIndex]; + MOVE_POS(p, len - 1) + return len; + } + + matches = p->matches; + + if (mainLen >= p->numFastBytes) + { + p->backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) + return mainLen; + } + + curByte = *data; + matchByte = *(data - reps[0]); + + last = repLens[repMaxIndex]; + if (last <= mainLen) + last = mainLen; + + if (last < 2 && curByte != matchByte) + { + p->backRes = MARK_LIT; + return 1; + } + + p->opt[0].state = (CState)p->state; + + posState = (position & p->pbMask); + + { + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + + (!IsLitState(p->state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + } + + MakeAs_Lit(&p->opt[1]); + + matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); + repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); + + // 18.06 + if (matchByte == curByte && repLens[0] == 0) + { + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, p->state, posState); + if (shortRepPrice < p->opt[1].price) + { + p->opt[1].price = shortRepPrice; + MakeAs_ShortRep(&p->opt[1]); + } + if (last < 2) + { + p->backRes = p->opt[1].dist; + return 1; + } + } + + p->opt[1].len = 1; + + p->opt[0].reps[0] = reps[0]; + p->opt[0].reps[1] = reps[1]; + p->opt[0].reps[2] = reps[2]; + p->opt[0].reps[3] = reps[3]; + + // ---------- REP ---------- + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned repLen = repLens[i]; + UInt32 price; + if (repLen < 2) + continue; + price = repMatchPrice + GetPrice_PureRep(p, i, p->state, posState); + do + { + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, repLen); + COptimal *opt = &p->opt[repLen]; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)repLen; + opt->dist = (UInt32)i; + opt->extra = 0; + } + } + while (--repLen >= 2); + } + + + // ---------- MATCH ---------- + { + unsigned len = repLens[0] + 1; + if (len <= mainLen) + { + unsigned offs = 0; + UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); + + if (len < 2) + len = 2; + else + while (len > matches[offs]) + offs += 2; + + for (; ; len++) + { + COptimal *opt; + UInt32 dist = matches[(size_t)offs + 1]; + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); + unsigned lenToPosState = GetLenToPosState(len); + + if (dist < kNumFullDistances) + price += p->distancesPrices[lenToPosState][dist & (kNumFullDistances - 1)]; + else + { + unsigned slot; + GetPosSlot2(dist, slot); + price += p->alignPrices[dist & kAlignMask]; + price += p->posSlotPrices[lenToPosState][slot]; + } + + opt = &p->opt[len]; + + if (price < opt->price) + { + opt->price = price; + opt->len = (UInt32)len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } + + if (len == matches[offs]) + { + offs += 2; + if (offs == numPairs) + break; + } + } + } + } + + + cur = 0; + + #ifdef SHOW_STAT2 + /* if (position >= 0) */ + { + unsigned i; + printf("\n pos = %4X", position); + for (i = cur; i <= last; i++) + printf("\nprice[%4X] = %u", position - cur + i, p->opt[i].price); + } + #endif + } + + + + // ---------- Optimal Parsing ---------- + + for (;;) + { + unsigned numAvail; + UInt32 numAvailFull; + unsigned newLen, numPairs, prev, state, posState, startLen; + UInt32 litPrice, matchPrice, repMatchPrice; + BoolInt nextIsLit; + Byte curByte, matchByte; + const Byte *data; + COptimal *curOpt, *nextOpt; + + if (++cur == last) + break; + + // 18.06 + if (cur >= kNumOpts - 64) + { + unsigned j, best; + UInt32 price = p->opt[cur].price; + best = cur; + for (j = cur + 1; j <= last; j++) + { + UInt32 price2 = p->opt[j].price; + if (price >= price2) + { + price = price2; + best = j; + } + } + { + unsigned delta = best - cur; + if (delta != 0) + { + MOVE_POS(p, delta); + } + } + cur = best; + break; + } + + newLen = ReadMatchDistances(p, &numPairs); + + if (newLen >= p->numFastBytes) + { + p->numPairs = numPairs; + p->longestMatchLen = newLen; + break; + } + + curOpt = &p->opt[cur]; + + position++; + + // we need that check here, if skip_items in p->opt are possible + /* + if (curOpt->price >= kInfinityPrice) + continue; + */ + + prev = cur - curOpt->len; + + if (curOpt->len == 1) + { + state = (unsigned)p->opt[prev].state; + if (IsShortRep(curOpt)) + state = kShortRepNextStates[state]; + else + state = kLiteralNextStates[state]; + } + else + { + const COptimal *prevOpt; + UInt32 b0; + UInt32 dist = curOpt->dist; + + if (curOpt->extra) + { + prev -= (unsigned)curOpt->extra; + state = kState_RepAfterLit; + if (curOpt->extra == 1) + state = (dist < LZMA_NUM_REPS ? kState_RepAfterLit : kState_MatchAfterLit); + } + else + { + state = (unsigned)p->opt[prev].state; + if (dist < LZMA_NUM_REPS) + state = kRepNextStates[state]; + else + state = kMatchNextStates[state]; + } + + prevOpt = &p->opt[prev]; + b0 = prevOpt->reps[0]; + + if (dist < LZMA_NUM_REPS) + { + if (dist == 0) + { + reps[0] = b0; + reps[1] = prevOpt->reps[1]; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[1] = b0; + b0 = prevOpt->reps[1]; + if (dist == 1) + { + reps[0] = b0; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[2] = b0; + reps[0] = prevOpt->reps[dist]; + reps[3] = prevOpt->reps[dist ^ 1]; + } + } + } + else + { + reps[0] = (dist - LZMA_NUM_REPS + 1); + reps[1] = b0; + reps[2] = prevOpt->reps[1]; + reps[3] = prevOpt->reps[2]; + } + } + + curOpt->state = (CState)state; + curOpt->reps[0] = reps[0]; + curOpt->reps[1] = reps[1]; + curOpt->reps[2] = reps[2]; + curOpt->reps[3] = reps[3]; + + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + curByte = *data; + matchByte = *(data - reps[0]); + + posState = (position & p->pbMask); + + /* + The order of Price checks: + < LIT + <= SHORT_REP + < LIT : REP_0 + < REP [ : LIT : REP_0 ] + < MATCH [ : LIT : REP_0 ] + */ + + { + UInt32 curPrice = curOpt->price; + unsigned prob = p->isMatch[state][posState]; + matchPrice = curPrice + GET_PRICE_1(prob); + litPrice = curPrice + GET_PRICE_0(prob); + } + + nextOpt = &p->opt[(size_t)cur + 1]; + nextIsLit = False; + + // here we can allow skip_items in p->opt, if we don't check (nextOpt->price < kInfinityPrice) + // 18.new.06 + if ((nextOpt->price < kInfinityPrice + // && !IsLitState(state) + && matchByte == curByte) + || litPrice > nextOpt->price + ) + litPrice = 0; + else + { + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + litPrice += (!IsLitState(state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + + if (litPrice < nextOpt->price) + { + nextOpt->price = litPrice; + nextOpt->len = 1; + MakeAs_Lit(nextOpt); + nextIsLit = True; + } + } + + repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); + + numAvailFull = p->numAvail; + { + unsigned temp = kNumOpts - 1 - cur; + if (numAvailFull > temp) + numAvailFull = (UInt32)temp; + } + + // 18.06 + // ---------- SHORT_REP ---------- + if (IsLitState(state)) // 18.new + if (matchByte == curByte) + if (repMatchPrice < nextOpt->price) // 18.new + // if (numAvailFull < 2 || data[1] != *(data - reps[0] + 1)) + if ( + // nextOpt->price >= kInfinityPrice || + nextOpt->len < 2 // we can check nextOpt->len, if skip items are not allowed in p->opt + || (nextOpt->dist != 0 + // && nextOpt->extra <= 1 // 17.old + ) + ) + { + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, state, posState); + // if (shortRepPrice <= nextOpt->price) // 17.old + if (shortRepPrice < nextOpt->price) // 18.new + { + nextOpt->price = shortRepPrice; + nextOpt->len = 1; + MakeAs_ShortRep(nextOpt); + nextIsLit = False; + } + } + + if (numAvailFull < 2) + continue; + numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes); + + // numAvail <= p->numFastBytes + + // ---------- LIT : REP_0 ---------- + + if (!nextIsLit + && litPrice != 0 // 18.new + && matchByte != curByte + && numAvailFull > 2) + { + const Byte *data2 = data - reps[0]; + if (data[1] == data2[1] && data[2] == data2[2]) + { + unsigned len; + unsigned limit = p->numFastBytes + 1; + if (limit > numAvailFull) + limit = numAvailFull; + for (len = 3; len < limit && data[len] == data2[len]; len++) + {} + + { + unsigned state2 = kLiteralNextStates[state]; + unsigned posState2 = (position + 1) & p->pbMask; + UInt32 price = litPrice + GetPrice_Rep_0(p, state2, posState2); + { + unsigned offset = cur + len; + + if (last < offset) + last = offset; + + // do + { + UInt32 price2; + COptimal *opt; + len--; + // price2 = price + GetPrice_Len_Rep_0(p, len, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len); + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len; + opt->dist = 0; + opt->extra = 1; + } + } + // while (len >= 3); + } + } + } + } + + startLen = 2; /* speed optimization */ + + { + // ---------- REP ---------- + unsigned repIndex = 0; // 17.old + // unsigned repIndex = IsLitState(state) ? 0 : 1; // 18.notused + for (; repIndex < LZMA_NUM_REPS; repIndex++) + { + unsigned len; + UInt32 price; + const Byte *data2 = data - reps[repIndex]; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; + + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + + // if (len < startLen) continue; // 18.new: speed optimization + + { + unsigned offset = cur + len; + if (last < offset) + last = offset; + } + { + unsigned len2 = len; + price = repMatchPrice + GetPrice_PureRep(p, repIndex, state, posState); + do + { + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, len2); + COptimal *opt = &p->opt[cur + len2]; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len2; + opt->dist = (UInt32)repIndex; + opt->extra = 0; + } + } + while (--len2 >= 2); + } + + if (repIndex == 0) startLen = len + 1; // 17.old + // startLen = len + 1; // 18.new + + /* if (_maxMode) */ + { + // ---------- REP : LIT : REP_0 ---------- + // numFastBytes + 1 + numFastBytes + + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; + if (limit > numAvailFull) + limit = numAvailFull; + + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) + { + unsigned state2 = kRepNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + price += GET_PRICE_LEN(&p->repLenEnc, posState, len) + + GET_PRICE_0(p->isMatch[state2][posState2]) + + LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); + + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterRep; + posState2 = (posState2 + 1) & p->pbMask; + + + price += GetPrice_Rep_0(p, state2, posState2); + + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + // if (len2 >= 3) + { + { + unsigned offset = cur + len + len2; + + if (last < offset) + last = offset; + // do + { + UInt32 price2; + COptimal *opt; + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len2; + opt->extra = (CExtra)(len + 1); + opt->dist = (UInt32)repIndex; + } + } + // while (len2 >= 3); + } + } + } + } + } + } + + + // ---------- MATCH ---------- + /* for (unsigned len = 2; len <= newLen; len++) */ + if (newLen > numAvail) + { + newLen = numAvail; + for (numPairs = 0; newLen > matches[numPairs]; numPairs += 2); + matches[numPairs] = (UInt32)newLen; + numPairs += 2; + } + + // startLen = 2; /* speed optimization */ + + if (newLen >= startLen) + { + UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]); + UInt32 dist; + unsigned offs, posSlot, len; + + { + unsigned offset = cur + newLen; + if (last < offset) + last = offset; + } + + offs = 0; + while (startLen > matches[offs]) + offs += 2; + dist = matches[(size_t)offs + 1]; + + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); + + for (len = /*2*/ startLen; ; len++) + { + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); + { + COptimal *opt; + unsigned lenNorm = len - 2; + lenNorm = GetLenToPosState2(lenNorm); + if (dist < kNumFullDistances) + price += p->distancesPrices[lenNorm][dist & (kNumFullDistances - 1)]; + else + price += p->posSlotPrices[lenNorm][posSlot] + p->alignPrices[dist & kAlignMask]; + + opt = &p->opt[cur + len]; + if (price < opt->price) + { + opt->price = price; + opt->len = (UInt32)len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } + } + + if (len == matches[offs]) + { + // if (p->_maxMode) { + // MATCH : LIT : REP_0 + + const Byte *data2 = data - dist - 1; + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; + if (limit > numAvailFull) + limit = numAvailFull; + + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) + { + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + + // if (len2 >= 3) + { + unsigned state2 = kMatchNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + unsigned offset; + price += GET_PRICE_0(p->isMatch[state2][posState2]); + price += LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); + + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterMatch; + + posState2 = (posState2 + 1) & p->pbMask; + price += GetPrice_Rep_0(p, state2, posState2); + + offset = cur + len + len2; + + if (last < offset) + last = offset; + // do + { + UInt32 price2; + COptimal *opt; + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len2; + opt->extra = (CExtra)(len + 1); + opt->dist = dist + LZMA_NUM_REPS; + } + } + // while (len2 >= 3); + } + + } + + offs += 2; + if (offs == numPairs) + break; + dist = matches[(size_t)offs + 1]; + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); + } + } + } + } + + do + p->opt[last].price = kInfinityPrice; + while (--last); + + return Backward(p, cur); +} + + + +#define ChangePair(smallDist, bigDist) (((bigDist) >> 7) > (smallDist)) + + + +static unsigned GetOptimumFast(CLzmaEnc *p) +{ + UInt32 numAvail, mainDist; + unsigned mainLen, numPairs, repIndex, repLen, i; + const Byte *data; + + if (p->additionalOffset == 0) + mainLen = ReadMatchDistances(p, &numPairs); + else + { + mainLen = p->longestMatchLen; + numPairs = p->numPairs; + } + + numAvail = p->numAvail; + p->backRes = MARK_LIT; + if (numAvail < 2) + return 1; + // if (mainLen < 2 && p->state == 0) return 1; // 18.06.notused + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + repLen = repIndex = 0; + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned len; + const Byte *data2 = data - p->reps[i]; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + if (len >= p->numFastBytes) + { + p->backRes = (UInt32)i; + MOVE_POS(p, len - 1) + return len; + } + if (len > repLen) + { + repIndex = i; + repLen = len; + } + } + + if (mainLen >= p->numFastBytes) + { + p->backRes = p->matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) + return mainLen; + } + + mainDist = 0; /* for GCC */ + + if (mainLen >= 2) + { + mainDist = p->matches[(size_t)numPairs - 1]; + while (numPairs > 2) + { + UInt32 dist2; + if (mainLen != p->matches[(size_t)numPairs - 4] + 1) + break; + dist2 = p->matches[(size_t)numPairs - 3]; + if (!ChangePair(dist2, mainDist)) + break; + numPairs -= 2; + mainLen--; + mainDist = dist2; + } + if (mainLen == 2 && mainDist >= 0x80) + mainLen = 1; + } + + if (repLen >= 2) + if ( repLen + 1 >= mainLen + || (repLen + 2 >= mainLen && mainDist >= (1 << 9)) + || (repLen + 3 >= mainLen && mainDist >= (1 << 15))) + { + p->backRes = (UInt32)repIndex; + MOVE_POS(p, repLen - 1) + return repLen; + } + + if (mainLen < 2 || numAvail <= 2) + return 1; + + { + unsigned len1 = ReadMatchDistances(p, &p->numPairs); + p->longestMatchLen = len1; + + if (len1 >= 2) + { + UInt32 newDist = p->matches[(size_t)p->numPairs - 1]; + if ( (len1 >= mainLen && newDist < mainDist) + || (len1 == mainLen + 1 && !ChangePair(mainDist, newDist)) + || (len1 > mainLen + 1) + || (len1 + 1 >= mainLen && mainLen >= 3 && ChangePair(newDist, mainDist))) + return 1; + } + } + + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned len, limit; + const Byte *data2 = data - p->reps[i]; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; + limit = mainLen - 1; + for (len = 2;; len++) + { + if (len >= limit) + return 1; + if (data[len] != data2[len]) + break; + } + } + + p->backRes = mainDist + LZMA_NUM_REPS; + if (mainLen != 2) + { + MOVE_POS(p, mainLen - 2) + } + return mainLen; +} + + + + +static void WriteEndMarker(CLzmaEnc *p, unsigned posState) +{ + UInt32 range; + range = p->rc.range; + { + UInt32 ttt, newBound; + CLzmaProb *prob = &p->isMatch[p->state][posState]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_1(&p->rc, prob) + prob = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_0(&p->rc, prob) + } + p->state = kMatchNextStates[p->state]; + + p->rc.range = range; + LenEnc_Encode(&p->lenProbs, &p->rc, 0, posState); + range = p->rc.range; + + { + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[0], (1 << kNumPosSlotBits) - 1); + CLzmaProb *probs = p->posSlotEncoder[0]; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < (1 << kNumPosSlotBits)); + } + { + // RangeEnc_EncodeDirectBits(&p->rc, ((UInt32)1 << (30 - kNumAlignBits)) - 1, 30 - kNumAlignBits); UInt32 range = p->range; + unsigned numBits = 30 - kNumAlignBits; + do + { + range >>= 1; + p->rc.low += range; + RC_NORM(&p->rc) + } + while (--numBits); + } + + { + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + CLzmaProb *probs = p->posAlignEncoder; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < kAlignTableSize); + } + p->rc.range = range; +} + + +static SRes CheckErrors(CLzmaEnc *p) +{ + if (p->result != SZ_OK) + return p->result; + if (p->rc.res != SZ_OK) + p->result = SZ_ERROR_WRITE; + if (p->matchFinderBase.result != SZ_OK) + p->result = SZ_ERROR_READ; + if (p->result != SZ_OK) + p->finished = True; + return p->result; +} + + +MY_NO_INLINE static SRes Flush(CLzmaEnc *p, UInt32 nowPos) +{ + /* ReleaseMFStream(); */ + p->finished = True; + if (p->writeEndMark) + WriteEndMarker(p, nowPos & p->pbMask); + RangeEnc_FlushData(&p->rc); + RangeEnc_FlushStream(&p->rc); + return CheckErrors(p); +} + + +MY_NO_INLINE static void FillAlignPrices(CLzmaEnc *p) +{ + unsigned i; + const CProbPrice *ProbPrices = p->ProbPrices; + const CLzmaProb *probs = p->posAlignEncoder; + // p->alignPriceCount = 0; + for (i = 0; i < kAlignTableSize / 2; i++) + { + UInt32 price = 0; + unsigned sym = i; + unsigned m = 1; + unsigned bit; + UInt32 prob; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + prob = probs[m]; + p->alignPrices[i ] = price + GET_PRICEa_0(prob); + p->alignPrices[i + 8] = price + GET_PRICEa_1(prob); + // p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); + } +} + + +MY_NO_INLINE static void FillDistancesPrices(CLzmaEnc *p) +{ + // int y; for (y = 0; y < 100; y++) { + + UInt32 tempPrices[kNumFullDistances]; + unsigned i, lps; + + const CProbPrice *ProbPrices = p->ProbPrices; + p->matchPriceCount = 0; + + for (i = kStartPosModelIndex / 2; i < kNumFullDistances / 2; i++) + { + unsigned posSlot = GetPosSlot1(i); + unsigned footerBits = (posSlot >> 1) - 1; + unsigned base = ((2 | (posSlot & 1)) << footerBits); + const CLzmaProb *probs = p->posEncoders + (size_t)base * 2; + // tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base, footerBits, i - base, p->ProbPrices); + UInt32 price = 0; + unsigned m = 1; + unsigned sym = i; + unsigned offset = (unsigned)1 << footerBits; + base += i; + + if (footerBits) + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[m], bit); + m = (m << 1) + bit; + } + while (--footerBits); + + { + unsigned prob = probs[m]; + tempPrices[base ] = price + GET_PRICEa_0(prob); + tempPrices[base + offset] = price + GET_PRICEa_1(prob); + } + } + + for (lps = 0; lps < kNumLenToPosStates; lps++) + { + unsigned slot; + unsigned distTableSize2 = (p->distTableSize + 1) >> 1; + UInt32 *posSlotPrices = p->posSlotPrices[lps]; + const CLzmaProb *probs = p->posSlotEncoder[lps]; + + for (slot = 0; slot < distTableSize2; slot++) + { + // posSlotPrices[slot] = RcTree_GetPrice(encoder, kNumPosSlotBits, slot, p->ProbPrices); + UInt32 price; + unsigned bit; + unsigned sym = slot + (1 << (kNumPosSlotBits - 1)); + unsigned prob; + bit = sym & 1; sym >>= 1; price = GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + prob = probs[(size_t)slot + (1 << (kNumPosSlotBits - 1))]; + posSlotPrices[(size_t)slot * 2 ] = price + GET_PRICEa_0(prob); + posSlotPrices[(size_t)slot * 2 + 1] = price + GET_PRICEa_1(prob); + } + + { + UInt32 delta = ((UInt32)((kEndPosModelIndex / 2 - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + for (slot = kEndPosModelIndex / 2; slot < distTableSize2; slot++) + { + posSlotPrices[(size_t)slot * 2 ] += delta; + posSlotPrices[(size_t)slot * 2 + 1] += delta; + delta += ((UInt32)1 << kNumBitPriceShiftBits); + } + } + + { + UInt32 *dp = p->distancesPrices[lps]; + + dp[0] = posSlotPrices[0]; + dp[1] = posSlotPrices[1]; + dp[2] = posSlotPrices[2]; + dp[3] = posSlotPrices[3]; + + for (i = 4; i < kNumFullDistances; i += 2) + { + UInt32 slotPrice = posSlotPrices[GetPosSlot1(i)]; + dp[i ] = slotPrice + tempPrices[i]; + dp[i + 1] = slotPrice + tempPrices[i + 1]; + } + } + } + // } +} + + + +void LzmaEnc_Construct(CLzmaEnc *p) +{ + RangeEnc_Construct(&p->rc); + MatchFinder_Construct(&p->matchFinderBase); + + #ifndef _7ZIP_ST + MatchFinderMt_Construct(&p->matchFinderMt); + p->matchFinderMt.MatchFinder = &p->matchFinderBase; + #endif + + { + CLzmaEncProps props; + LzmaEncProps_Init(&props); + LzmaEnc_SetProps(p, &props); + } + + #ifndef LZMA_LOG_BSR + LzmaEnc_FastPosInit(p->g_FastPos); + #endif + + LzmaEnc_InitPriceTables(p->ProbPrices); + p->litProbs = NULL; + p->saveState.litProbs = NULL; + +} + +CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc) +{ + void *p; + p = ISzAlloc_Alloc(alloc, sizeof(CLzmaEnc)); + if (p) + LzmaEnc_Construct((CLzmaEnc *)p); + return p; +} + +void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->litProbs); + ISzAlloc_Free(alloc, p->saveState.litProbs); + p->litProbs = NULL; + p->saveState.litProbs = NULL; +} + +void LzmaEnc_Destruct(CLzmaEnc *p, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + #ifndef _7ZIP_ST + MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); + #endif + + MatchFinder_Free(&p->matchFinderBase, allocBig); + LzmaEnc_FreeLits(p, alloc); + RangeEnc_Free(&p->rc, alloc); +} + +void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + LzmaEnc_Destruct((CLzmaEnc *)p, alloc, allocBig); + ISzAlloc_Free(alloc, p); +} + + +static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpackSize) +{ + UInt32 nowPos32, startPos32; + if (p->needInit) + { + p->matchFinder.Init(p->matchFinderObj); + p->needInit = 0; + } + + if (p->finished) + return p->result; + RINOK(CheckErrors(p)); + + nowPos32 = (UInt32)p->nowPos64; + startPos32 = nowPos32; + + if (p->nowPos64 == 0) + { + unsigned numPairs; + Byte curByte; + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) + return Flush(p, nowPos32); + ReadMatchDistances(p, &numPairs); + RangeEnc_EncodeBit_0(&p->rc, &p->isMatch[kState_Start][0]); + // p->state = kLiteralNextStates[p->state]; + curByte = *(p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset); + LitEnc_Encode(&p->rc, p->litProbs, curByte); + p->additionalOffset--; + nowPos32++; + } + + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) != 0) + + for (;;) + { + UInt32 dist; + unsigned len, posState; + UInt32 range, ttt, newBound; + CLzmaProb *probs; + + if (p->fastMode) + len = GetOptimumFast(p); + else + { + unsigned oci = p->optCur; + if (p->optEnd == oci) + len = GetOptimum(p, nowPos32); + else + { + const COptimal *opt = &p->opt[oci]; + len = opt->len; + p->backRes = opt->dist; + p->optCur = oci + 1; + } + } + + posState = (unsigned)nowPos32 & p->pbMask; + range = p->rc.range; + probs = &p->isMatch[p->state][posState]; + + RC_BIT_PRE(&p->rc, probs) + + dist = p->backRes; + + #ifdef SHOW_STAT2 + printf("\n pos = %6X, len = %3u pos = %6u", nowPos32, len, dist); + #endif + + if (dist == MARK_LIT) + { + Byte curByte; + const Byte *data; + unsigned state; + + RC_BIT_0(&p->rc, probs); + p->rc.range = range; + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; + probs = LIT_PROBS(nowPos32, *(data - 1)); + curByte = *data; + state = p->state; + p->state = kLiteralNextStates[state]; + if (IsLitState(state)) + LitEnc_Encode(&p->rc, probs, curByte); + else + LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0])); + } + else + { + RC_BIT_1(&p->rc, probs); + probs = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, probs) + + if (dist < LZMA_NUM_REPS) + { + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG0[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 0) + { + RC_BIT_0(&p->rc, probs); + probs = &p->isRep0Long[p->state][posState]; + RC_BIT_PRE(&p->rc, probs) + if (len != 1) + { + RC_BIT_1_BASE(&p->rc, probs); + } + else + { + RC_BIT_0_BASE(&p->rc, probs); + p->state = kShortRepNextStates[p->state]; + } + } + else + { + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG1[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 1) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[1]; + } + else + { + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG2[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 2) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[2]; + } + else + { + RC_BIT_1_BASE(&p->rc, probs); + dist = p->reps[3]; + p->reps[3] = p->reps[2]; + } + p->reps[2] = p->reps[1]; + } + p->reps[1] = p->reps[0]; + p->reps[0] = dist; + } + + RC_NORM(&p->rc) + + p->rc.range = range; + + if (len != 1) + { + LenEnc_Encode(&p->repLenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + --p->repLenEncCounter; + p->state = kRepNextStates[p->state]; + } + } + else + { + unsigned posSlot; + RC_BIT_0(&p->rc, probs); + p->rc.range = range; + p->state = kMatchNextStates[p->state]; + + LenEnc_Encode(&p->lenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + // --p->lenEnc.counter; + + dist -= LZMA_NUM_REPS; + p->reps[3] = p->reps[2]; + p->reps[2] = p->reps[1]; + p->reps[1] = p->reps[0]; + p->reps[0] = dist + 1; + + p->matchPriceCount++; + GetPosSlot(dist, posSlot); + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], posSlot); + { + UInt32 sym = (UInt32)posSlot + (1 << kNumPosSlotBits); + range = p->rc.range; + probs = p->posSlotEncoder[GetLenToPosState(len)]; + do + { + CLzmaProb *prob = probs + (sym >> kNumPosSlotBits); + UInt32 bit = (sym >> (kNumPosSlotBits - 1)) & 1; + sym <<= 1; + RC_BIT(&p->rc, prob, bit); + } + while (sym < (1 << kNumPosSlotBits * 2)); + p->rc.range = range; + } + + if (dist >= kStartPosModelIndex) + { + unsigned footerBits = ((posSlot >> 1) - 1); + + if (dist < kNumFullDistances) + { + unsigned base = ((2 | (posSlot & 1)) << footerBits); + RcTree_ReverseEncode(&p->rc, p->posEncoders + base, footerBits, (unsigned)(dist /* - base */)); + } + else + { + UInt32 pos2 = (dist | 0xF) << (32 - footerBits); + range = p->rc.range; + // RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); + /* + do + { + range >>= 1; + p->rc.low += range & (0 - ((dist >> --footerBits) & 1)); + RC_NORM(&p->rc) + } + while (footerBits > kNumAlignBits); + */ + do + { + range >>= 1; + p->rc.low += range & (0 - (pos2 >> 31)); + pos2 += pos2; + RC_NORM(&p->rc) + } + while (pos2 != 0xF0000000); + + + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); + + { + unsigned m = 1; + unsigned bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); + p->rc.range = range; + // p->alignPriceCount++; + } + } + } + } + } + + nowPos32 += (UInt32)len; + p->additionalOffset -= len; + + if (p->additionalOffset == 0) + { + UInt32 processed; + + if (!p->fastMode) + { + /* + if (p->alignPriceCount >= 16) // kAlignTableSize + FillAlignPrices(p); + if (p->matchPriceCount >= 128) + FillDistancesPrices(p); + if (p->lenEnc.counter <= 0) + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + */ + if (p->matchPriceCount >= 64) + { + FillAlignPrices(p); + // { int y; for (y = 0; y < 100; y++) { + FillDistancesPrices(p); + // }} + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + } + if (p->repLenEncCounter <= 0) + { + p->repLenEncCounter = REP_LEN_COUNT; + LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); + } + } + + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) + break; + processed = nowPos32 - startPos32; + + if (maxPackSize) + { + if (processed + kNumOpts + 300 >= maxUnpackSize + || RangeEnc_GetProcessed_sizet(&p->rc) + kPackReserve >= maxPackSize) + break; + } + else if (processed >= (1 << 17)) + { + p->nowPos64 += nowPos32 - startPos32; + return CheckErrors(p); + } + } + } + + p->nowPos64 += nowPos32 - startPos32; + return Flush(p, nowPos32); +} + + + +#define kBigHashDicLimit ((UInt32)1 << 24) + +static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + UInt32 beforeSize = kNumOpts; + if (!RangeEnc_Alloc(&p->rc, alloc)) + return SZ_ERROR_MEM; + + #ifndef _7ZIP_ST + p->mtMode = (p->multiThread && !p->fastMode && (p->matchFinderBase.btMode != 0)); + #endif + + { + unsigned lclp = p->lc + p->lp; + if (!p->litProbs || !p->saveState.litProbs || p->lclp != lclp) + { + LzmaEnc_FreeLits(p, alloc); + p->litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + p->saveState.litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + if (!p->litProbs || !p->saveState.litProbs) + { + LzmaEnc_FreeLits(p, alloc); + return SZ_ERROR_MEM; + } + p->lclp = lclp; + } + } + + p->matchFinderBase.bigHash = (Byte)(p->dictSize > kBigHashDicLimit ? 1 : 0); + + if (beforeSize + p->dictSize < keepWindowSize) + beforeSize = keepWindowSize - p->dictSize; + + #ifndef _7ZIP_ST + if (p->mtMode) + { + RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, + LZMA_MATCH_LEN_MAX + + 1 /* 18.04 */ + , allocBig)); + p->matchFinderObj = &p->matchFinderMt; + p->matchFinderBase.bigHash = (Byte)( + (p->dictSize > kBigHashDicLimit && p->matchFinderBase.hashMask >= 0xFFFFFF) ? 1 : 0); + MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder); + } + else + #endif + { + if (!MatchFinder_Create(&p->matchFinderBase, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)) + return SZ_ERROR_MEM; + p->matchFinderObj = &p->matchFinderBase; + MatchFinder_CreateVTable(&p->matchFinderBase, &p->matchFinder); + } + + return SZ_OK; +} + +void LzmaEnc_Init(CLzmaEnc *p) +{ + unsigned i; + p->state = 0; + p->reps[0] = + p->reps[1] = + p->reps[2] = + p->reps[3] = 1; + + RangeEnc_Init(&p->rc); + + for (i = 0; i < (1 << kNumAlignBits); i++) + p->posAlignEncoder[i] = kProbInitValue; + + for (i = 0; i < kNumStates; i++) + { + unsigned j; + for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++) + { + p->isMatch[i][j] = kProbInitValue; + p->isRep0Long[i][j] = kProbInitValue; + } + p->isRep[i] = kProbInitValue; + p->isRepG0[i] = kProbInitValue; + p->isRepG1[i] = kProbInitValue; + p->isRepG2[i] = kProbInitValue; + } + + { + for (i = 0; i < kNumLenToPosStates; i++) + { + CLzmaProb *probs = p->posSlotEncoder[i]; + unsigned j; + for (j = 0; j < (1 << kNumPosSlotBits); j++) + probs[j] = kProbInitValue; + } + } + { + for (i = 0; i < kNumFullDistances; i++) + p->posEncoders[i] = kProbInitValue; + } + + { + UInt32 num = (UInt32)0x300 << (p->lp + p->lc); + UInt32 k; + CLzmaProb *probs = p->litProbs; + for (k = 0; k < num; k++) + probs[k] = kProbInitValue; + } + + + LenEnc_Init(&p->lenProbs); + LenEnc_Init(&p->repLenProbs); + + p->optEnd = 0; + p->optCur = 0; + + { + for (i = 0; i < kNumOpts; i++) + p->opt[i].price = kInfinityPrice; + } + + p->additionalOffset = 0; + + p->pbMask = (1 << p->pb) - 1; + p->lpMask = ((UInt32)0x100 << p->lp) - ((unsigned)0x100 >> p->lc); +} + + +void LzmaEnc_InitPrices(CLzmaEnc *p) +{ + if (!p->fastMode) + { + FillDistancesPrices(p); + FillAlignPrices(p); + } + + p->lenEnc.tableSize = + p->repLenEnc.tableSize = + p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN; + + p->repLenEncCounter = REP_LEN_COUNT; + + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); +} + +static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + unsigned i; + for (i = kEndPosModelIndex / 2; i < kDicLogSizeMax; i++) + if (p->dictSize <= ((UInt32)1 << i)) + break; + p->distTableSize = i * 2; + + p->finished = False; + p->result = SZ_OK; + RINOK(LzmaEnc_Alloc(p, keepWindowSize, alloc, allocBig)); + LzmaEnc_Init(p); + LzmaEnc_InitPrices(p); + p->nowPos64 = 0; + return SZ_OK; +} + +static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, + ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + p->matchFinderBase.stream = inStream; + p->needInit = 1; + p->rc.outStream = outStream; + return LzmaEnc_AllocAndInit(p, 0, alloc, allocBig); +} + +SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, + ISeqInStream *inStream, UInt32 keepWindowSize, + ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + p->matchFinderBase.stream = inStream; + p->needInit = 1; + return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); +} + +static void LzmaEnc_SetInputBuf(CLzmaEnc *p, const Byte *src, SizeT srcLen) +{ + p->matchFinderBase.directInput = 1; + p->matchFinderBase.bufferBase = (Byte *)src; + p->matchFinderBase.directInputRem = srcLen; +} + +SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + LzmaEnc_SetInputBuf(p, src, srcLen); + p->needInit = 1; + + LzmaEnc_SetDataSize(pp, srcLen); + return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); +} + +void LzmaEnc_Finish(CLzmaEncHandle pp) +{ + #ifndef _7ZIP_ST + CLzmaEnc *p = (CLzmaEnc *)pp; + if (p->mtMode) + MatchFinderMt_ReleaseStream(&p->matchFinderMt); + #else + UNUSED_VAR(pp); + #endif +} + + +typedef struct +{ + ISeqOutStream vt; + Byte *data; + SizeT rem; + BoolInt overflow; +} CLzmaEnc_SeqOutStreamBuf; + +static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size) +{ + CLzmaEnc_SeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CLzmaEnc_SeqOutStreamBuf, vt); + if (p->rem < size) + { + size = p->rem; + p->overflow = True; + } + memcpy(p->data, data, size); + p->rem -= size; + p->data += size; + return size; +} + + +UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp) +{ + const CLzmaEnc *p = (CLzmaEnc *)pp; + return p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); +} + + +const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp) +{ + const CLzmaEnc *p = (CLzmaEnc *)pp; + return p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; +} + + +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, + Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + UInt64 nowPos64; + SRes res; + CLzmaEnc_SeqOutStreamBuf outStream; + + outStream.vt.Write = SeqOutStreamBuf_Write; + outStream.data = dest; + outStream.rem = *destLen; + outStream.overflow = False; + + p->writeEndMark = False; + p->finished = False; + p->result = SZ_OK; + + if (reInit) + LzmaEnc_Init(p); + LzmaEnc_InitPrices(p); + + nowPos64 = p->nowPos64; + RangeEnc_Init(&p->rc); + p->rc.outStream = &outStream.vt; + + if (desiredPackSize == 0) + return SZ_ERROR_OUTPUT_EOF; + + res = LzmaEnc_CodeOneBlock(p, desiredPackSize, *unpackSize); + + *unpackSize = (UInt32)(p->nowPos64 - nowPos64); + *destLen -= outStream.rem; + if (outStream.overflow) + return SZ_ERROR_OUTPUT_EOF; + + return res; +} + + +static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) +{ + SRes res = SZ_OK; + + #ifndef _7ZIP_ST + Byte allocaDummy[0x300]; + allocaDummy[0] = 0; + allocaDummy[1] = allocaDummy[0]; + #endif + + for (;;) + { + res = LzmaEnc_CodeOneBlock(p, 0, 0); + if (res != SZ_OK || p->finished) + break; + if (progress) + { + res = ICompressProgress_Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc)); + if (res != SZ_OK) + { + res = SZ_ERROR_PROGRESS; + break; + } + } + } + + LzmaEnc_Finish(p); + + /* + if (res == SZ_OK && !Inline_MatchFinder_IsFinishedOK(&p->matchFinderBase)) + res = SZ_ERROR_FAIL; + } + */ + + return res; +} + + +SRes LzmaEnc_Encode(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress, + ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + RINOK(LzmaEnc_Prepare(pp, outStream, inStream, alloc, allocBig)); + return LzmaEnc_Encode2((CLzmaEnc *)pp, progress); +} + + +SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, Byte *props, SizeT *size) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + unsigned i; + UInt32 dictSize = p->dictSize; + if (*size < LZMA_PROPS_SIZE) + return SZ_ERROR_PARAM; + *size = LZMA_PROPS_SIZE; + props[0] = (Byte)((p->pb * 5 + p->lp) * 9 + p->lc); + + if (dictSize >= ((UInt32)1 << 22)) + { + UInt32 kDictMask = ((UInt32)1 << 20) - 1; + if (dictSize < (UInt32)0xFFFFFFFF - kDictMask) + dictSize = (dictSize + kDictMask) & ~kDictMask; + } + else for (i = 11; i <= 30; i++) + { + if (dictSize <= ((UInt32)2 << i)) { dictSize = (2 << i); break; } + if (dictSize <= ((UInt32)3 << i)) { dictSize = (3 << i); break; } + } + + for (i = 0; i < 4; i++) + props[1 + i] = (Byte)(dictSize >> (8 * i)); + return SZ_OK; +} + + +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle pp) +{ + return ((CLzmaEnc *)pp)->writeEndMark; +} + + +SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, + int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + SRes res; + CLzmaEnc *p = (CLzmaEnc *)pp; + + CLzmaEnc_SeqOutStreamBuf outStream; + + outStream.vt.Write = SeqOutStreamBuf_Write; + outStream.data = dest; + outStream.rem = *destLen; + outStream.overflow = False; + + p->writeEndMark = writeEndMark; + p->rc.outStream = &outStream.vt; + + res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig); + + if (res == SZ_OK) + { + res = LzmaEnc_Encode2(p, progress); + if (res == SZ_OK && p->nowPos64 != srcLen) + res = SZ_ERROR_FAIL; + } + + *destLen -= outStream.rem; + if (outStream.overflow) + return SZ_ERROR_OUTPUT_EOF; + return res; +} + + +SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, + const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CLzmaEnc *p = (CLzmaEnc *)LzmaEnc_Create(alloc); + SRes res; + if (!p) + return SZ_ERROR_MEM; + + res = LzmaEnc_SetProps(p, props); + if (res == SZ_OK) + { + res = LzmaEnc_WriteProperties(p, propsEncoded, propsSize); + if (res == SZ_OK) + res = LzmaEnc_MemEncode(p, dest, destLen, src, srcLen, + writeEndMark, progress, alloc, allocBig); + } + + LzmaEnc_Destroy(p, alloc, allocBig); + return res; +} diff --git a/code/nel/3rdparty/seven_zip/LzmaEnc.h b/code/nel/3rdparty/seven_zip/LzmaEnc.h new file mode 100644 index 000000000..9194ee576 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaEnc.h @@ -0,0 +1,76 @@ +/* LzmaEnc.h -- LZMA Encoder +2017-07-27 : Igor Pavlov : Public domain */ + +#ifndef __LZMA_ENC_H +#define __LZMA_ENC_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define LZMA_PROPS_SIZE 5 + +typedef struct _CLzmaEncProps +{ + int level; /* 0 <= level <= 9 */ + UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version + (1 << 12) <= dictSize <= (3 << 29) for 64-bit version + default = (1 << 24) */ + int lc; /* 0 <= lc <= 8, default = 3 */ + int lp; /* 0 <= lp <= 4, default = 0 */ + int pb; /* 0 <= pb <= 4, default = 2 */ + int algo; /* 0 - fast, 1 - normal, default = 1 */ + int fb; /* 5 <= fb <= 273, default = 32 */ + int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */ + int numHashBytes; /* 2, 3 or 4, default = 4 */ + UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ + unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */ + int numThreads; /* 1 or 2, default = 2 */ + + UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1. + Encoder uses this value to reduce dictionary size */ +} CLzmaEncProps; + +void LzmaEncProps_Init(CLzmaEncProps *p); +void LzmaEncProps_Normalize(CLzmaEncProps *p); +UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2); + + +/* ---------- CLzmaEncHandle Interface ---------- */ + +/* LzmaEnc* functions can return the following exit codes: +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater in props + SZ_ERROR_WRITE - ISeqOutStream write callback error + SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output + SZ_ERROR_PROGRESS - some break from progress callback + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) +*/ + +typedef void * CLzmaEncHandle; + +CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc); +void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig); + +SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props); +void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize); +SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size); +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p); + +SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, + int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); + + +/* ---------- One Call Interface ---------- */ + +SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, + const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/LzmaLib.c b/code/nel/3rdparty/seven_zip/LzmaLib.c new file mode 100644 index 000000000..706e9e58c --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaLib.c @@ -0,0 +1,40 @@ +/* LzmaLib.c -- LZMA library wrapper +2015-06-13 : Igor Pavlov : Public domain */ + +#include "Alloc.h" +#include "LzmaDec.h" +#include "LzmaEnc.h" +#include "LzmaLib.h" + +MY_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t srcLen, + unsigned char *outProps, size_t *outPropsSize, + int level, /* 0 <= level <= 9, default = 5 */ + unsigned dictSize, /* use (1 << N) or (3 << N). 4 KB < dictSize <= 128 MB */ + int lc, /* 0 <= lc <= 8, default = 3 */ + int lp, /* 0 <= lp <= 4, default = 0 */ + int pb, /* 0 <= pb <= 4, default = 2 */ + int fb, /* 5 <= fb <= 273, default = 32 */ + int numThreads /* 1 or 2, default = 2 */ +) +{ + CLzmaEncProps props; + LzmaEncProps_Init(&props); + props.level = level; + props.dictSize = dictSize; + props.lc = lc; + props.lp = lp; + props.pb = pb; + props.fb = fb; + props.numThreads = numThreads; + + return LzmaEncode(dest, destLen, src, srcLen, &props, outProps, outPropsSize, 0, + NULL, &g_Alloc, &g_Alloc); +} + + +MY_STDAPI LzmaUncompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t *srcLen, + const unsigned char *props, size_t propsSize) +{ + ELzmaStatus status; + return LzmaDecode(dest, destLen, src, srcLen, props, (unsigned)propsSize, LZMA_FINISH_ANY, &status, &g_Alloc); +} diff --git a/code/nel/3rdparty/seven_zip/LzmaLib.h b/code/nel/3rdparty/seven_zip/LzmaLib.h new file mode 100644 index 000000000..88fa87d35 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaLib.h @@ -0,0 +1,131 @@ +/* LzmaLib.h -- LZMA library interface +2013-01-18 : Igor Pavlov : Public domain */ + +#ifndef __LZMA_LIB_H +#define __LZMA_LIB_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define MY_STDAPI int MY_STD_CALL + +#define LZMA_PROPS_SIZE 5 + +/* +RAM requirements for LZMA: + for compression: (dictSize * 11.5 + 6 MB) + state_size + for decompression: dictSize + state_size + state_size = (4 + (1.5 << (lc + lp))) KB + by default (lc=3, lp=0), state_size = 16 KB. + +LZMA properties (5 bytes) format + Offset Size Description + 0 1 lc, lp and pb in encoded form. + 1 4 dictSize (little endian). +*/ + +/* +LzmaCompress +------------ + +outPropsSize - + In: the pointer to the size of outProps buffer; *outPropsSize = LZMA_PROPS_SIZE = 5. + Out: the pointer to the size of written properties in outProps buffer; *outPropsSize = LZMA_PROPS_SIZE = 5. + + LZMA Encoder will use defult values for any parameter, if it is + -1 for any from: level, loc, lp, pb, fb, numThreads + 0 for dictSize + +level - compression level: 0 <= level <= 9; + + level dictSize algo fb + 0: 16 KB 0 32 + 1: 64 KB 0 32 + 2: 256 KB 0 32 + 3: 1 MB 0 32 + 4: 4 MB 0 32 + 5: 16 MB 1 32 + 6: 32 MB 1 32 + 7+: 64 MB 1 64 + + The default value for "level" is 5. + + algo = 0 means fast method + algo = 1 means normal method + +dictSize - The dictionary size in bytes. The maximum value is + 128 MB = (1 << 27) bytes for 32-bit version + 1 GB = (1 << 30) bytes for 64-bit version + The default value is 16 MB = (1 << 24) bytes. + It's recommended to use the dictionary that is larger than 4 KB and + that can be calculated as (1 << N) or (3 << N) sizes. + +lc - The number of literal context bits (high bits of previous literal). + It can be in the range from 0 to 8. The default value is 3. + Sometimes lc=4 gives the gain for big files. + +lp - The number of literal pos bits (low bits of current position for literals). + It can be in the range from 0 to 4. The default value is 0. + The lp switch is intended for periodical data when the period is equal to 2^lp. + For example, for 32-bit (4 bytes) periodical data you can use lp=2. Often it's + better to set lc=0, if you change lp switch. + +pb - The number of pos bits (low bits of current position). + It can be in the range from 0 to 4. The default value is 2. + The pb switch is intended for periodical data when the period is equal 2^pb. + +fb - Word size (the number of fast bytes). + It can be in the range from 5 to 273. The default value is 32. + Usually, a big number gives a little bit better compression ratio and + slower compression process. + +numThreads - The number of thereads. 1 or 2. The default value is 2. + Fast mode (algo = 0) can use only 1 thread. + +Out: + destLen - processed output size +Returns: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater + SZ_ERROR_OUTPUT_EOF - output buffer overflow + SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) +*/ + +MY_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t srcLen, + unsigned char *outProps, size_t *outPropsSize, /* *outPropsSize must be = 5 */ + int level, /* 0 <= level <= 9, default = 5 */ + unsigned dictSize, /* default = (1 << 24) */ + int lc, /* 0 <= lc <= 8, default = 3 */ + int lp, /* 0 <= lp <= 4, default = 0 */ + int pb, /* 0 <= pb <= 4, default = 2 */ + int fb, /* 5 <= fb <= 273, default = 32 */ + int numThreads /* 1 or 2, default = 2 */ + ); + +/* +LzmaUncompress +-------------- +In: + dest - output data + destLen - output data size + src - input data + srcLen - input data size +Out: + destLen - processed output size + srcLen - processed input size +Returns: + SZ_OK - OK + SZ_ERROR_DATA - Data error + SZ_ERROR_MEM - Memory allocation arror + SZ_ERROR_UNSUPPORTED - Unsupported properties + SZ_ERROR_INPUT_EOF - it needs more bytes in input buffer (src) +*/ + +MY_STDAPI LzmaUncompress(unsigned char *dest, size_t *destLen, const unsigned char *src, SizeT *srcLen, + const unsigned char *props, size_t propsSize); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/LzmaUtil.c b/code/nel/3rdparty/seven_zip/LzmaUtil.c new file mode 100644 index 000000000..6bdb2ad6c --- /dev/null +++ b/code/nel/3rdparty/seven_zip/LzmaUtil.c @@ -0,0 +1,258 @@ +/* LzmaUtil.c -- Test application for LZMA compression +2018-07-04 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include +#include +#include + +#include "CpuArch.h" + +#include "Alloc.h" +#include "7zFile.h" +#include "7zVersion.h" +#include "LzmaDec.h" +#include "LzmaEnc.h" + +static const char * const kCantReadMessage = "Can not read input file"; +static const char * const kCantWriteMessage = "Can not write output file"; +static const char * const kCantAllocateMessage = "Can not allocate memory"; +static const char * const kDataErrorMessage = "Data error"; + +static void PrintHelp(char *buffer) +{ + strcat(buffer, + "\nLZMA-C " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n" + "Usage: lzma inputFile outputFile\n" + " e: encode file\n" + " d: decode file\n"); +} + +static int PrintError(char *buffer, const char *message) +{ + strcat(buffer, "\nError: "); + strcat(buffer, message); + strcat(buffer, "\n"); + return 1; +} + +static int PrintErrorNumber(char *buffer, SRes val) +{ + sprintf(buffer + strlen(buffer), "\nError code: %x\n", (unsigned)val); + return 1; +} + +static int PrintUserError(char *buffer) +{ + return PrintError(buffer, "Incorrect command"); +} + + +#define IN_BUF_SIZE (1 << 16) +#define OUT_BUF_SIZE (1 << 16) + + +static SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inStream, + UInt64 unpackSize) +{ + int thereIsSize = (unpackSize != (UInt64)(Int64)-1); + Byte inBuf[IN_BUF_SIZE]; + Byte outBuf[OUT_BUF_SIZE]; + size_t inPos = 0, inSize = 0, outPos = 0; + LzmaDec_Init(state); + for (;;) + { + if (inPos == inSize) + { + inSize = IN_BUF_SIZE; + RINOK(inStream->Read(inStream, inBuf, &inSize)); + inPos = 0; + } + { + SRes res; + SizeT inProcessed = inSize - inPos; + SizeT outProcessed = OUT_BUF_SIZE - outPos; + ELzmaFinishMode finishMode = LZMA_FINISH_ANY; + ELzmaStatus status; + if (thereIsSize && outProcessed > unpackSize) + { + outProcessed = (SizeT)unpackSize; + finishMode = LZMA_FINISH_END; + } + + res = LzmaDec_DecodeToBuf(state, outBuf + outPos, &outProcessed, + inBuf + inPos, &inProcessed, finishMode, &status); + inPos += inProcessed; + outPos += outProcessed; + unpackSize -= outProcessed; + + if (outStream) + if (outStream->Write(outStream, outBuf, outPos) != outPos) + return SZ_ERROR_WRITE; + + outPos = 0; + + if (res != SZ_OK || (thereIsSize && unpackSize == 0)) + return res; + + if (inProcessed == 0 && outProcessed == 0) + { + if (thereIsSize || status != LZMA_STATUS_FINISHED_WITH_MARK) + return SZ_ERROR_DATA; + return res; + } + } + } +} + + +static SRes Decode(ISeqOutStream *outStream, ISeqInStream *inStream) +{ + UInt64 unpackSize; + int i; + SRes res = 0; + + CLzmaDec state; + + /* header: 5 bytes of LZMA properties and 8 bytes of uncompressed size */ + unsigned char header[LZMA_PROPS_SIZE + 8]; + + /* Read and parse header */ + + RINOK(SeqInStream_Read(inStream, header, sizeof(header))); + + unpackSize = 0; + for (i = 0; i < 8; i++) + unpackSize += (UInt64)header[LZMA_PROPS_SIZE + i] << (i * 8); + + LzmaDec_Construct(&state); + RINOK(LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &g_Alloc)); + res = Decode2(&state, outStream, inStream, unpackSize); + LzmaDec_Free(&state, &g_Alloc); + return res; +} + +static SRes Encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize, char *rs) +{ + CLzmaEncHandle enc; + SRes res; + CLzmaEncProps props; + + UNUSED_VAR(rs); + + enc = LzmaEnc_Create(&g_Alloc); + if (enc == 0) + return SZ_ERROR_MEM; + + LzmaEncProps_Init(&props); + res = LzmaEnc_SetProps(enc, &props); + + if (res == SZ_OK) + { + Byte header[LZMA_PROPS_SIZE + 8]; + size_t headerSize = LZMA_PROPS_SIZE; + int i; + + res = LzmaEnc_WriteProperties(enc, header, &headerSize); + for (i = 0; i < 8; i++) + header[headerSize++] = (Byte)(fileSize >> (8 * i)); + if (outStream->Write(outStream, header, headerSize) != headerSize) + res = SZ_ERROR_WRITE; + else + { + if (res == SZ_OK) + res = LzmaEnc_Encode(enc, outStream, inStream, NULL, &g_Alloc, &g_Alloc); + } + } + LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc); + return res; +} + + +static int main2(int numArgs, const char *args[], char *rs) +{ + CFileSeqInStream inStream; + CFileOutStream outStream; + char c; + int res; + int encodeMode; + BoolInt useOutFile = False; + + FileSeqInStream_CreateVTable(&inStream); + File_Construct(&inStream.file); + + FileOutStream_CreateVTable(&outStream); + File_Construct(&outStream.file); + + if (numArgs == 1) + { + PrintHelp(rs); + return 0; + } + + if (numArgs < 3 || numArgs > 4 || strlen(args[1]) != 1) + return PrintUserError(rs); + + c = args[1][0]; + encodeMode = (c == 'e' || c == 'E'); + if (!encodeMode && c != 'd' && c != 'D') + return PrintUserError(rs); + + { + size_t t4 = sizeof(UInt32); + size_t t8 = sizeof(UInt64); + if (t4 != 4 || t8 != 8) + return PrintError(rs, "Incorrect UInt32 or UInt64"); + } + + if (InFile_Open(&inStream.file, args[2]) != 0) + return PrintError(rs, "Can not open input file"); + + if (numArgs > 3) + { + useOutFile = True; + if (OutFile_Open(&outStream.file, args[3]) != 0) + return PrintError(rs, "Can not open output file"); + } + else if (encodeMode) + PrintUserError(rs); + + if (encodeMode) + { + UInt64 fileSize; + File_GetLength(&inStream.file, &fileSize); + res = Encode(&outStream.vt, &inStream.vt, fileSize, rs); + } + else + { + res = Decode(&outStream.vt, useOutFile ? &inStream.vt : NULL); + } + + if (useOutFile) + File_Close(&outStream.file); + File_Close(&inStream.file); + + if (res != SZ_OK) + { + if (res == SZ_ERROR_MEM) + return PrintError(rs, kCantAllocateMessage); + else if (res == SZ_ERROR_DATA) + return PrintError(rs, kDataErrorMessage); + else if (res == SZ_ERROR_WRITE) + return PrintError(rs, kCantWriteMessage); + else if (res == SZ_ERROR_READ) + return PrintError(rs, kCantReadMessage); + return PrintErrorNumber(rs, res); + } + return 0; +} + + +int MY_CDECL main(int numArgs, const char *args[]) +{ + char rs[800] = { 0 }; + int res = main2(numArgs, args, rs); + fputs(rs, stdout); + return res; +} diff --git a/code/nel/3rdparty/seven_zip/Ppmd.h b/code/nel/3rdparty/seven_zip/Ppmd.h new file mode 100644 index 000000000..a5c1e3ef2 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Ppmd.h @@ -0,0 +1,85 @@ +/* Ppmd.h -- PPMD codec common code +2017-04-03 : Igor Pavlov : Public domain +This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ + +#ifndef __PPMD_H +#define __PPMD_H + +#include "CpuArch.h" + +EXTERN_C_BEGIN + +#ifdef MY_CPU_32BIT + #define PPMD_32BIT +#endif + +#define PPMD_INT_BITS 7 +#define PPMD_PERIOD_BITS 7 +#define PPMD_BIN_SCALE (1 << (PPMD_INT_BITS + PPMD_PERIOD_BITS)) + +#define PPMD_GET_MEAN_SPEC(summ, shift, round) (((summ) + (1 << ((shift) - (round)))) >> (shift)) +#define PPMD_GET_MEAN(summ) PPMD_GET_MEAN_SPEC((summ), PPMD_PERIOD_BITS, 2) +#define PPMD_UPDATE_PROB_0(prob) ((prob) + (1 << PPMD_INT_BITS) - PPMD_GET_MEAN(prob)) +#define PPMD_UPDATE_PROB_1(prob) ((prob) - PPMD_GET_MEAN(prob)) + +#define PPMD_N1 4 +#define PPMD_N2 4 +#define PPMD_N3 4 +#define PPMD_N4 ((128 + 3 - 1 * PPMD_N1 - 2 * PPMD_N2 - 3 * PPMD_N3) / 4) +#define PPMD_NUM_INDEXES (PPMD_N1 + PPMD_N2 + PPMD_N3 + PPMD_N4) + +#pragma pack(push, 1) +/* Most compilers works OK here even without #pragma pack(push, 1), but some GCC compilers need it. */ + +/* SEE-contexts for PPM-contexts with masked symbols */ +typedef struct +{ + UInt16 Summ; /* Freq */ + Byte Shift; /* Speed of Freq change; low Shift is for fast change */ + Byte Count; /* Count to next change of Shift */ +} CPpmd_See; + +#define Ppmd_See_Update(p) if ((p)->Shift < PPMD_PERIOD_BITS && --(p)->Count == 0) \ + { (p)->Summ <<= 1; (p)->Count = (Byte)(3 << (p)->Shift++); } + +typedef struct +{ + Byte Symbol; + Byte Freq; + UInt16 SuccessorLow; + UInt16 SuccessorHigh; +} CPpmd_State; + +#pragma pack(pop) + +typedef + #ifdef PPMD_32BIT + CPpmd_State * + #else + UInt32 + #endif + CPpmd_State_Ref; + +typedef + #ifdef PPMD_32BIT + void * + #else + UInt32 + #endif + CPpmd_Void_Ref; + +typedef + #ifdef PPMD_32BIT + Byte * + #else + UInt32 + #endif + CPpmd_Byte_Ref; + +#define PPMD_SetAllBitsIn256Bytes(p) \ + { size_t z; for (z = 0; z < 256 / sizeof(p[0]); z += 8) { \ + p[z+7] = p[z+6] = p[z+5] = p[z+4] = p[z+3] = p[z+2] = p[z+1] = p[z+0] = ~(size_t)0; }} + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/Ppmd7.c b/code/nel/3rdparty/seven_zip/Ppmd7.c new file mode 100644 index 000000000..470aadccf --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Ppmd7.c @@ -0,0 +1,712 @@ +/* Ppmd7.c -- PPMdH codec +2018-07-04 : Igor Pavlov : Public domain +This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ + +#include "Precomp.h" + +#include + +#include "Ppmd7.h" + +const Byte PPMD7_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 }; +static const UInt16 kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051}; + +#define MAX_FREQ 124 +#define UNIT_SIZE 12 + +#define U2B(nu) ((UInt32)(nu) * UNIT_SIZE) +#define U2I(nu) (p->Units2Indx[(size_t)(nu) - 1]) +#define I2U(indx) (p->Indx2Units[indx]) + +#ifdef PPMD_32BIT + #define REF(ptr) (ptr) +#else + #define REF(ptr) ((UInt32)((Byte *)(ptr) - (p)->Base)) +#endif + +#define STATS_REF(ptr) ((CPpmd_State_Ref)REF(ptr)) + +#define CTX(ref) ((CPpmd7_Context *)Ppmd7_GetContext(p, ref)) +#define STATS(ctx) Ppmd7_GetStats(p, ctx) +#define ONE_STATE(ctx) Ppmd7Context_OneState(ctx) +#define SUFFIX(ctx) CTX((ctx)->Suffix) + +typedef CPpmd7_Context * CTX_PTR; + +struct CPpmd7_Node_; + +typedef + #ifdef PPMD_32BIT + struct CPpmd7_Node_ * + #else + UInt32 + #endif + CPpmd7_Node_Ref; + +typedef struct CPpmd7_Node_ +{ + UInt16 Stamp; /* must be at offset 0 as CPpmd7_Context::NumStats. Stamp=0 means free */ + UInt16 NU; + CPpmd7_Node_Ref Next; /* must be at offset >= 4 */ + CPpmd7_Node_Ref Prev; +} CPpmd7_Node; + +#ifdef PPMD_32BIT + #define NODE(ptr) (ptr) +#else + #define NODE(offs) ((CPpmd7_Node *)(p->Base + (offs))) +#endif + +void Ppmd7_Construct(CPpmd7 *p) +{ + unsigned i, k, m; + + p->Base = 0; + + for (i = 0, k = 0; i < PPMD_NUM_INDEXES; i++) + { + unsigned step = (i >= 12 ? 4 : (i >> 2) + 1); + do { p->Units2Indx[k++] = (Byte)i; } while (--step); + p->Indx2Units[i] = (Byte)k; + } + + p->NS2BSIndx[0] = (0 << 1); + p->NS2BSIndx[1] = (1 << 1); + memset(p->NS2BSIndx + 2, (2 << 1), 9); + memset(p->NS2BSIndx + 11, (3 << 1), 256 - 11); + + for (i = 0; i < 3; i++) + p->NS2Indx[i] = (Byte)i; + for (m = i, k = 1; i < 256; i++) + { + p->NS2Indx[i] = (Byte)m; + if (--k == 0) + k = (++m) - 2; + } + + memset(p->HB2Flag, 0, 0x40); + memset(p->HB2Flag + 0x40, 8, 0x100 - 0x40); +} + +void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->Base); + p->Size = 0; + p->Base = 0; +} + +BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc) +{ + if (!p->Base || p->Size != size) + { + size_t size2; + Ppmd7_Free(p, alloc); + size2 = 0 + #ifndef PPMD_32BIT + + UNIT_SIZE + #endif + ; + p->AlignOffset = + #ifdef PPMD_32BIT + (4 - size) & 3; + #else + 4 - (size & 3); + #endif + if ((p->Base = (Byte *)ISzAlloc_Alloc(alloc, p->AlignOffset + size + size2)) == 0) + return False; + p->Size = size; + } + return True; +} + +static void InsertNode(CPpmd7 *p, void *node, unsigned indx) +{ + *((CPpmd_Void_Ref *)node) = p->FreeList[indx]; + p->FreeList[indx] = REF(node); +} + +static void *RemoveNode(CPpmd7 *p, unsigned indx) +{ + CPpmd_Void_Ref *node = (CPpmd_Void_Ref *)Ppmd7_GetPtr(p, p->FreeList[indx]); + p->FreeList[indx] = *node; + return node; +} + +static void SplitBlock(CPpmd7 *p, void *ptr, unsigned oldIndx, unsigned newIndx) +{ + unsigned i, nu = I2U(oldIndx) - I2U(newIndx); + ptr = (Byte *)ptr + U2B(I2U(newIndx)); + if (I2U(i = U2I(nu)) != nu) + { + unsigned k = I2U(--i); + InsertNode(p, ((Byte *)ptr) + U2B(k), nu - k - 1); + } + InsertNode(p, ptr, i); +} + +static void GlueFreeBlocks(CPpmd7 *p) +{ + #ifdef PPMD_32BIT + CPpmd7_Node headItem; + CPpmd7_Node_Ref head = &headItem; + #else + CPpmd7_Node_Ref head = p->AlignOffset + p->Size; + #endif + + CPpmd7_Node_Ref n = head; + unsigned i; + + p->GlueCount = 255; + + /* create doubly-linked list of free blocks */ + for (i = 0; i < PPMD_NUM_INDEXES; i++) + { + UInt16 nu = I2U(i); + CPpmd7_Node_Ref next = (CPpmd7_Node_Ref)p->FreeList[i]; + p->FreeList[i] = 0; + while (next != 0) + { + CPpmd7_Node *node = NODE(next); + node->Next = n; + n = NODE(n)->Prev = next; + next = *(const CPpmd7_Node_Ref *)node; + node->Stamp = 0; + node->NU = (UInt16)nu; + } + } + NODE(head)->Stamp = 1; + NODE(head)->Next = n; + NODE(n)->Prev = head; + if (p->LoUnit != p->HiUnit) + ((CPpmd7_Node *)p->LoUnit)->Stamp = 1; + + /* Glue free blocks */ + while (n != head) + { + CPpmd7_Node *node = NODE(n); + UInt32 nu = (UInt32)node->NU; + for (;;) + { + CPpmd7_Node *node2 = NODE(n) + nu; + nu += node2->NU; + if (node2->Stamp != 0 || nu >= 0x10000) + break; + NODE(node2->Prev)->Next = node2->Next; + NODE(node2->Next)->Prev = node2->Prev; + node->NU = (UInt16)nu; + } + n = node->Next; + } + + /* Fill lists of free blocks */ + for (n = NODE(head)->Next; n != head;) + { + CPpmd7_Node *node = NODE(n); + unsigned nu; + CPpmd7_Node_Ref next = node->Next; + for (nu = node->NU; nu > 128; nu -= 128, node += 128) + InsertNode(p, node, PPMD_NUM_INDEXES - 1); + if (I2U(i = U2I(nu)) != nu) + { + unsigned k = I2U(--i); + InsertNode(p, node + k, nu - k - 1); + } + InsertNode(p, node, i); + n = next; + } +} + +static void *AllocUnitsRare(CPpmd7 *p, unsigned indx) +{ + unsigned i; + void *retVal; + if (p->GlueCount == 0) + { + GlueFreeBlocks(p); + if (p->FreeList[indx] != 0) + return RemoveNode(p, indx); + } + i = indx; + do + { + if (++i == PPMD_NUM_INDEXES) + { + UInt32 numBytes = U2B(I2U(indx)); + p->GlueCount--; + return ((UInt32)(p->UnitsStart - p->Text) > numBytes) ? (p->UnitsStart -= numBytes) : (NULL); + } + } + while (p->FreeList[i] == 0); + retVal = RemoveNode(p, i); + SplitBlock(p, retVal, i, indx); + return retVal; +} + +static void *AllocUnits(CPpmd7 *p, unsigned indx) +{ + UInt32 numBytes; + if (p->FreeList[indx] != 0) + return RemoveNode(p, indx); + numBytes = U2B(I2U(indx)); + if (numBytes <= (UInt32)(p->HiUnit - p->LoUnit)) + { + void *retVal = p->LoUnit; + p->LoUnit += numBytes; + return retVal; + } + return AllocUnitsRare(p, indx); +} + +#define MyMem12Cpy(dest, src, num) \ + { UInt32 *d = (UInt32 *)dest; const UInt32 *s = (const UInt32 *)src; UInt32 n = num; \ + do { d[0] = s[0]; d[1] = s[1]; d[2] = s[2]; s += 3; d += 3; } while (--n); } + +static void *ShrinkUnits(CPpmd7 *p, void *oldPtr, unsigned oldNU, unsigned newNU) +{ + unsigned i0 = U2I(oldNU); + unsigned i1 = U2I(newNU); + if (i0 == i1) + return oldPtr; + if (p->FreeList[i1] != 0) + { + void *ptr = RemoveNode(p, i1); + MyMem12Cpy(ptr, oldPtr, newNU); + InsertNode(p, oldPtr, i0); + return ptr; + } + SplitBlock(p, oldPtr, i0, i1); + return oldPtr; +} + +#define SUCCESSOR(p) ((CPpmd_Void_Ref)((p)->SuccessorLow | ((UInt32)(p)->SuccessorHigh << 16))) + +static void SetSuccessor(CPpmd_State *p, CPpmd_Void_Ref v) +{ + (p)->SuccessorLow = (UInt16)((UInt32)(v) & 0xFFFF); + (p)->SuccessorHigh = (UInt16)(((UInt32)(v) >> 16) & 0xFFFF); +} + +static void RestartModel(CPpmd7 *p) +{ + unsigned i, k, m; + + memset(p->FreeList, 0, sizeof(p->FreeList)); + p->Text = p->Base + p->AlignOffset; + p->HiUnit = p->Text + p->Size; + p->LoUnit = p->UnitsStart = p->HiUnit - p->Size / 8 / UNIT_SIZE * 7 * UNIT_SIZE; + p->GlueCount = 0; + + p->OrderFall = p->MaxOrder; + p->RunLength = p->InitRL = -(Int32)((p->MaxOrder < 12) ? p->MaxOrder : 12) - 1; + p->PrevSuccess = 0; + + p->MinContext = p->MaxContext = (CTX_PTR)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */ + p->MinContext->Suffix = 0; + p->MinContext->NumStats = 256; + p->MinContext->SummFreq = 256 + 1; + p->FoundState = (CPpmd_State *)p->LoUnit; /* AllocUnits(p, PPMD_NUM_INDEXES - 1); */ + p->LoUnit += U2B(256 / 2); + p->MinContext->Stats = REF(p->FoundState); + for (i = 0; i < 256; i++) + { + CPpmd_State *s = &p->FoundState[i]; + s->Symbol = (Byte)i; + s->Freq = 1; + SetSuccessor(s, 0); + } + + for (i = 0; i < 128; i++) + for (k = 0; k < 8; k++) + { + UInt16 *dest = p->BinSumm[i] + k; + UInt16 val = (UInt16)(PPMD_BIN_SCALE - kInitBinEsc[k] / (i + 2)); + for (m = 0; m < 64; m += 8) + dest[m] = val; + } + + for (i = 0; i < 25; i++) + for (k = 0; k < 16; k++) + { + CPpmd_See *s = &p->See[i][k]; + s->Summ = (UInt16)((5 * i + 10) << (s->Shift = PPMD_PERIOD_BITS - 4)); + s->Count = 4; + } +} + +void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder) +{ + p->MaxOrder = maxOrder; + RestartModel(p); + p->DummySee.Shift = PPMD_PERIOD_BITS; + p->DummySee.Summ = 0; /* unused */ + p->DummySee.Count = 64; /* unused */ +} + +static CTX_PTR CreateSuccessors(CPpmd7 *p, BoolInt skip) +{ + CPpmd_State upState; + CTX_PTR c = p->MinContext; + CPpmd_Byte_Ref upBranch = (CPpmd_Byte_Ref)SUCCESSOR(p->FoundState); + CPpmd_State *ps[PPMD7_MAX_ORDER]; + unsigned numPs = 0; + + if (!skip) + ps[numPs++] = p->FoundState; + + while (c->Suffix) + { + CPpmd_Void_Ref successor; + CPpmd_State *s; + c = SUFFIX(c); + if (c->NumStats != 1) + { + for (s = STATS(c); s->Symbol != p->FoundState->Symbol; s++); + } + else + s = ONE_STATE(c); + successor = SUCCESSOR(s); + if (successor != upBranch) + { + c = CTX(successor); + if (numPs == 0) + return c; + break; + } + ps[numPs++] = s; + } + + upState.Symbol = *(const Byte *)Ppmd7_GetPtr(p, upBranch); + SetSuccessor(&upState, upBranch + 1); + + if (c->NumStats == 1) + upState.Freq = ONE_STATE(c)->Freq; + else + { + UInt32 cf, s0; + CPpmd_State *s; + for (s = STATS(c); s->Symbol != upState.Symbol; s++); + cf = s->Freq - 1; + s0 = c->SummFreq - c->NumStats - cf; + upState.Freq = (Byte)(1 + ((2 * cf <= s0) ? (5 * cf > s0) : ((2 * cf + 3 * s0 - 1) / (2 * s0)))); + } + + do + { + /* Create Child */ + CTX_PTR c1; /* = AllocContext(p); */ + if (p->HiUnit != p->LoUnit) + c1 = (CTX_PTR)(p->HiUnit -= UNIT_SIZE); + else if (p->FreeList[0] != 0) + c1 = (CTX_PTR)RemoveNode(p, 0); + else + { + c1 = (CTX_PTR)AllocUnitsRare(p, 0); + if (!c1) + return NULL; + } + c1->NumStats = 1; + *ONE_STATE(c1) = upState; + c1->Suffix = REF(c); + SetSuccessor(ps[--numPs], REF(c1)); + c = c1; + } + while (numPs != 0); + + return c; +} + +static void SwapStates(CPpmd_State *t1, CPpmd_State *t2) +{ + CPpmd_State tmp = *t1; + *t1 = *t2; + *t2 = tmp; +} + +static void UpdateModel(CPpmd7 *p) +{ + CPpmd_Void_Ref successor, fSuccessor = SUCCESSOR(p->FoundState); + CTX_PTR c; + unsigned s0, ns; + + if (p->FoundState->Freq < MAX_FREQ / 4 && p->MinContext->Suffix != 0) + { + c = SUFFIX(p->MinContext); + + if (c->NumStats == 1) + { + CPpmd_State *s = ONE_STATE(c); + if (s->Freq < 32) + s->Freq++; + } + else + { + CPpmd_State *s = STATS(c); + if (s->Symbol != p->FoundState->Symbol) + { + do { s++; } while (s->Symbol != p->FoundState->Symbol); + if (s[0].Freq >= s[-1].Freq) + { + SwapStates(&s[0], &s[-1]); + s--; + } + } + if (s->Freq < MAX_FREQ - 9) + { + s->Freq += 2; + c->SummFreq += 2; + } + } + } + + if (p->OrderFall == 0) + { + p->MinContext = p->MaxContext = CreateSuccessors(p, True); + if (p->MinContext == 0) + { + RestartModel(p); + return; + } + SetSuccessor(p->FoundState, REF(p->MinContext)); + return; + } + + *p->Text++ = p->FoundState->Symbol; + successor = REF(p->Text); + if (p->Text >= p->UnitsStart) + { + RestartModel(p); + return; + } + + if (fSuccessor) + { + if (fSuccessor <= successor) + { + CTX_PTR cs = CreateSuccessors(p, False); + if (cs == NULL) + { + RestartModel(p); + return; + } + fSuccessor = REF(cs); + } + if (--p->OrderFall == 0) + { + successor = fSuccessor; + p->Text -= (p->MaxContext != p->MinContext); + } + } + else + { + SetSuccessor(p->FoundState, successor); + fSuccessor = REF(p->MinContext); + } + + s0 = p->MinContext->SummFreq - (ns = p->MinContext->NumStats) - (p->FoundState->Freq - 1); + + for (c = p->MaxContext; c != p->MinContext; c = SUFFIX(c)) + { + unsigned ns1; + UInt32 cf, sf; + if ((ns1 = c->NumStats) != 1) + { + if ((ns1 & 1) == 0) + { + /* Expand for one UNIT */ + unsigned oldNU = ns1 >> 1; + unsigned i = U2I(oldNU); + if (i != U2I((size_t)oldNU + 1)) + { + void *ptr = AllocUnits(p, i + 1); + void *oldPtr; + if (!ptr) + { + RestartModel(p); + return; + } + oldPtr = STATS(c); + MyMem12Cpy(ptr, oldPtr, oldNU); + InsertNode(p, oldPtr, i); + c->Stats = STATS_REF(ptr); + } + } + c->SummFreq = (UInt16)(c->SummFreq + (2 * ns1 < ns) + 2 * ((4 * ns1 <= ns) & (c->SummFreq <= 8 * ns1))); + } + else + { + CPpmd_State *s = (CPpmd_State*)AllocUnits(p, 0); + if (!s) + { + RestartModel(p); + return; + } + *s = *ONE_STATE(c); + c->Stats = REF(s); + if (s->Freq < MAX_FREQ / 4 - 1) + s->Freq <<= 1; + else + s->Freq = MAX_FREQ - 4; + c->SummFreq = (UInt16)(s->Freq + p->InitEsc + (ns > 3)); + } + cf = 2 * (UInt32)p->FoundState->Freq * (c->SummFreq + 6); + sf = (UInt32)s0 + c->SummFreq; + if (cf < 6 * sf) + { + cf = 1 + (cf > sf) + (cf >= 4 * sf); + c->SummFreq += 3; + } + else + { + cf = 4 + (cf >= 9 * sf) + (cf >= 12 * sf) + (cf >= 15 * sf); + c->SummFreq = (UInt16)(c->SummFreq + cf); + } + { + CPpmd_State *s = STATS(c) + ns1; + SetSuccessor(s, successor); + s->Symbol = p->FoundState->Symbol; + s->Freq = (Byte)cf; + c->NumStats = (UInt16)(ns1 + 1); + } + } + p->MaxContext = p->MinContext = CTX(fSuccessor); +} + +static void Rescale(CPpmd7 *p) +{ + unsigned i, adder, sumFreq, escFreq; + CPpmd_State *stats = STATS(p->MinContext); + CPpmd_State *s = p->FoundState; + { + CPpmd_State tmp = *s; + for (; s != stats; s--) + s[0] = s[-1]; + *s = tmp; + } + escFreq = p->MinContext->SummFreq - s->Freq; + s->Freq += 4; + adder = (p->OrderFall != 0); + s->Freq = (Byte)((s->Freq + adder) >> 1); + sumFreq = s->Freq; + + i = p->MinContext->NumStats - 1; + do + { + escFreq -= (++s)->Freq; + s->Freq = (Byte)((s->Freq + adder) >> 1); + sumFreq += s->Freq; + if (s[0].Freq > s[-1].Freq) + { + CPpmd_State *s1 = s; + CPpmd_State tmp = *s1; + do + s1[0] = s1[-1]; + while (--s1 != stats && tmp.Freq > s1[-1].Freq); + *s1 = tmp; + } + } + while (--i); + + if (s->Freq == 0) + { + unsigned numStats = p->MinContext->NumStats; + unsigned n0, n1; + do { i++; } while ((--s)->Freq == 0); + escFreq += i; + p->MinContext->NumStats = (UInt16)(p->MinContext->NumStats - i); + if (p->MinContext->NumStats == 1) + { + CPpmd_State tmp = *stats; + do + { + tmp.Freq = (Byte)(tmp.Freq - (tmp.Freq >> 1)); + escFreq >>= 1; + } + while (escFreq > 1); + InsertNode(p, stats, U2I(((numStats + 1) >> 1))); + *(p->FoundState = ONE_STATE(p->MinContext)) = tmp; + return; + } + n0 = (numStats + 1) >> 1; + n1 = (p->MinContext->NumStats + 1) >> 1; + if (n0 != n1) + p->MinContext->Stats = STATS_REF(ShrinkUnits(p, stats, n0, n1)); + } + p->MinContext->SummFreq = (UInt16)(sumFreq + escFreq - (escFreq >> 1)); + p->FoundState = STATS(p->MinContext); +} + +CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *escFreq) +{ + CPpmd_See *see; + unsigned nonMasked = p->MinContext->NumStats - numMasked; + if (p->MinContext->NumStats != 256) + { + see = p->See[(unsigned)p->NS2Indx[(size_t)nonMasked - 1]] + + (nonMasked < (unsigned)SUFFIX(p->MinContext)->NumStats - p->MinContext->NumStats) + + 2 * (unsigned)(p->MinContext->SummFreq < 11 * p->MinContext->NumStats) + + 4 * (unsigned)(numMasked > nonMasked) + + p->HiBitsFlag; + { + unsigned r = (see->Summ >> see->Shift); + see->Summ = (UInt16)(see->Summ - r); + *escFreq = r + (r == 0); + } + } + else + { + see = &p->DummySee; + *escFreq = 1; + } + return see; +} + +static void NextContext(CPpmd7 *p) +{ + CTX_PTR c = CTX(SUCCESSOR(p->FoundState)); + if (p->OrderFall == 0 && (Byte *)c > p->Text) + p->MinContext = p->MaxContext = c; + else + UpdateModel(p); +} + +void Ppmd7_Update1(CPpmd7 *p) +{ + CPpmd_State *s = p->FoundState; + s->Freq += 4; + p->MinContext->SummFreq += 4; + if (s[0].Freq > s[-1].Freq) + { + SwapStates(&s[0], &s[-1]); + p->FoundState = --s; + if (s->Freq > MAX_FREQ) + Rescale(p); + } + NextContext(p); +} + +void Ppmd7_Update1_0(CPpmd7 *p) +{ + p->PrevSuccess = (2 * p->FoundState->Freq > p->MinContext->SummFreq); + p->RunLength += p->PrevSuccess; + p->MinContext->SummFreq += 4; + if ((p->FoundState->Freq += 4) > MAX_FREQ) + Rescale(p); + NextContext(p); +} + +void Ppmd7_UpdateBin(CPpmd7 *p) +{ + p->FoundState->Freq = (Byte)(p->FoundState->Freq + (p->FoundState->Freq < 128 ? 1: 0)); + p->PrevSuccess = 1; + p->RunLength++; + NextContext(p); +} + +void Ppmd7_Update2(CPpmd7 *p) +{ + p->MinContext->SummFreq += 4; + if ((p->FoundState->Freq += 4) > MAX_FREQ) + Rescale(p); + p->RunLength = p->InitRL; + UpdateModel(p); +} diff --git a/code/nel/3rdparty/seven_zip/Ppmd7.h b/code/nel/3rdparty/seven_zip/Ppmd7.h new file mode 100644 index 000000000..610539a04 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Ppmd7.h @@ -0,0 +1,142 @@ +/* Ppmd7.h -- PPMdH compression codec +2018-07-04 : Igor Pavlov : Public domain +This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ + +/* This code supports virtual RangeDecoder and includes the implementation +of RangeCoder from 7z, instead of RangeCoder from original PPMd var.H. +If you need the compatibility with original PPMd var.H, you can use external RangeDecoder */ + +#ifndef __PPMD7_H +#define __PPMD7_H + +#include "Ppmd.h" + +EXTERN_C_BEGIN + +#define PPMD7_MIN_ORDER 2 +#define PPMD7_MAX_ORDER 64 + +#define PPMD7_MIN_MEM_SIZE (1 << 11) +#define PPMD7_MAX_MEM_SIZE (0xFFFFFFFF - 12 * 3) + +struct CPpmd7_Context_; + +typedef + #ifdef PPMD_32BIT + struct CPpmd7_Context_ * + #else + UInt32 + #endif + CPpmd7_Context_Ref; + +typedef struct CPpmd7_Context_ +{ + UInt16 NumStats; + UInt16 SummFreq; + CPpmd_State_Ref Stats; + CPpmd7_Context_Ref Suffix; +} CPpmd7_Context; + +#define Ppmd7Context_OneState(p) ((CPpmd_State *)&(p)->SummFreq) + +typedef struct +{ + CPpmd7_Context *MinContext, *MaxContext; + CPpmd_State *FoundState; + unsigned OrderFall, InitEsc, PrevSuccess, MaxOrder, HiBitsFlag; + Int32 RunLength, InitRL; /* must be 32-bit at least */ + + UInt32 Size; + UInt32 GlueCount; + Byte *Base, *LoUnit, *HiUnit, *Text, *UnitsStart; + UInt32 AlignOffset; + + Byte Indx2Units[PPMD_NUM_INDEXES]; + Byte Units2Indx[128]; + CPpmd_Void_Ref FreeList[PPMD_NUM_INDEXES]; + Byte NS2Indx[256], NS2BSIndx[256], HB2Flag[256]; + CPpmd_See DummySee, See[25][16]; + UInt16 BinSumm[128][64]; +} CPpmd7; + +void Ppmd7_Construct(CPpmd7 *p); +BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc); +void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc); +void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder); +#define Ppmd7_WasAllocated(p) ((p)->Base != NULL) + + +/* ---------- Internal Functions ---------- */ + +extern const Byte PPMD7_kExpEscape[16]; + +#ifdef PPMD_32BIT + #define Ppmd7_GetPtr(p, ptr) (ptr) + #define Ppmd7_GetContext(p, ptr) (ptr) + #define Ppmd7_GetStats(p, ctx) ((ctx)->Stats) +#else + #define Ppmd7_GetPtr(p, offs) ((void *)((p)->Base + (offs))) + #define Ppmd7_GetContext(p, offs) ((CPpmd7_Context *)Ppmd7_GetPtr((p), (offs))) + #define Ppmd7_GetStats(p, ctx) ((CPpmd_State *)Ppmd7_GetPtr((p), ((ctx)->Stats))) +#endif + +void Ppmd7_Update1(CPpmd7 *p); +void Ppmd7_Update1_0(CPpmd7 *p); +void Ppmd7_Update2(CPpmd7 *p); +void Ppmd7_UpdateBin(CPpmd7 *p); + +#define Ppmd7_GetBinSumm(p) \ + &p->BinSumm[(size_t)(unsigned)Ppmd7Context_OneState(p->MinContext)->Freq - 1][p->PrevSuccess + \ + p->NS2BSIndx[(size_t)Ppmd7_GetContext(p, p->MinContext->Suffix)->NumStats - 1] + \ + (p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]) + \ + 2 * p->HB2Flag[(unsigned)Ppmd7Context_OneState(p->MinContext)->Symbol] + \ + ((p->RunLength >> 26) & 0x20)] + +CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *scale); + + +/* ---------- Decode ---------- */ + +typedef struct IPpmd7_RangeDec IPpmd7_RangeDec; + +struct IPpmd7_RangeDec +{ + UInt32 (*GetThreshold)(const IPpmd7_RangeDec *p, UInt32 total); + void (*Decode)(const IPpmd7_RangeDec *p, UInt32 start, UInt32 size); + UInt32 (*DecodeBit)(const IPpmd7_RangeDec *p, UInt32 size0); +}; + +typedef struct +{ + IPpmd7_RangeDec vt; + UInt32 Range; + UInt32 Code; + IByteIn *Stream; +} CPpmd7z_RangeDec; + +void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p); +BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p); +#define Ppmd7z_RangeDec_IsFinishedOK(p) ((p)->Code == 0) + +int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc); + + +/* ---------- Encode ---------- */ + +typedef struct +{ + UInt64 Low; + UInt32 Range; + Byte Cache; + UInt64 CacheSize; + IByteOut *Stream; +} CPpmd7z_RangeEnc; + +void Ppmd7z_RangeEnc_Init(CPpmd7z_RangeEnc *p); +void Ppmd7z_RangeEnc_FlushData(CPpmd7z_RangeEnc *p); + +void Ppmd7_EncodeSymbol(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/Ppmd7Dec.c b/code/nel/3rdparty/seven_zip/Ppmd7Dec.c new file mode 100644 index 000000000..311e9f9dd --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Ppmd7Dec.c @@ -0,0 +1,191 @@ +/* Ppmd7Dec.c -- PPMdH Decoder +2018-07-04 : Igor Pavlov : Public domain +This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ + +#include "Precomp.h" + +#include "Ppmd7.h" + +#define kTopValue (1 << 24) + +BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p) +{ + unsigned i; + p->Code = 0; + p->Range = 0xFFFFFFFF; + if (IByteIn_Read(p->Stream) != 0) + return False; + for (i = 0; i < 4; i++) + p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); + return (p->Code < 0xFFFFFFFF); +} + +#define GET_Ppmd7z_RangeDec CPpmd7z_RangeDec *p = CONTAINER_FROM_VTBL(pp, CPpmd7z_RangeDec, vt); + +static UInt32 Range_GetThreshold(const IPpmd7_RangeDec *pp, UInt32 total) +{ + GET_Ppmd7z_RangeDec + return p->Code / (p->Range /= total); +} + +static void Range_Normalize(CPpmd7z_RangeDec *p) +{ + if (p->Range < kTopValue) + { + p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); + p->Range <<= 8; + if (p->Range < kTopValue) + { + p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); + p->Range <<= 8; + } + } +} + +static void Range_Decode(const IPpmd7_RangeDec *pp, UInt32 start, UInt32 size) +{ + GET_Ppmd7z_RangeDec + p->Code -= start * p->Range; + p->Range *= size; + Range_Normalize(p); +} + +static UInt32 Range_DecodeBit(const IPpmd7_RangeDec *pp, UInt32 size0) +{ + GET_Ppmd7z_RangeDec + UInt32 newBound = (p->Range >> 14) * size0; + UInt32 symbol; + if (p->Code < newBound) + { + symbol = 0; + p->Range = newBound; + } + else + { + symbol = 1; + p->Code -= newBound; + p->Range -= newBound; + } + Range_Normalize(p); + return symbol; +} + +void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p) +{ + p->vt.GetThreshold = Range_GetThreshold; + p->vt.Decode = Range_Decode; + p->vt.DecodeBit = Range_DecodeBit; +} + + +#define MASK(sym) ((signed char *)charMask)[sym] + +int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc) +{ + size_t charMask[256 / sizeof(size_t)]; + if (p->MinContext->NumStats != 1) + { + CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext); + unsigned i; + UInt32 count, hiCnt; + if ((count = rc->GetThreshold(rc, p->MinContext->SummFreq)) < (hiCnt = s->Freq)) + { + Byte symbol; + rc->Decode(rc, 0, s->Freq); + p->FoundState = s; + symbol = s->Symbol; + Ppmd7_Update1_0(p); + return symbol; + } + p->PrevSuccess = 0; + i = p->MinContext->NumStats - 1; + do + { + if ((hiCnt += (++s)->Freq) > count) + { + Byte symbol; + rc->Decode(rc, hiCnt - s->Freq, s->Freq); + p->FoundState = s; + symbol = s->Symbol; + Ppmd7_Update1(p); + return symbol; + } + } + while (--i); + if (count >= p->MinContext->SummFreq) + return -2; + p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]; + rc->Decode(rc, hiCnt, p->MinContext->SummFreq - hiCnt); + PPMD_SetAllBitsIn256Bytes(charMask); + MASK(s->Symbol) = 0; + i = p->MinContext->NumStats - 1; + do { MASK((--s)->Symbol) = 0; } while (--i); + } + else + { + UInt16 *prob = Ppmd7_GetBinSumm(p); + if (rc->DecodeBit(rc, *prob) == 0) + { + Byte symbol; + *prob = (UInt16)PPMD_UPDATE_PROB_0(*prob); + symbol = (p->FoundState = Ppmd7Context_OneState(p->MinContext))->Symbol; + Ppmd7_UpdateBin(p); + return symbol; + } + *prob = (UInt16)PPMD_UPDATE_PROB_1(*prob); + p->InitEsc = PPMD7_kExpEscape[*prob >> 10]; + PPMD_SetAllBitsIn256Bytes(charMask); + MASK(Ppmd7Context_OneState(p->MinContext)->Symbol) = 0; + p->PrevSuccess = 0; + } + for (;;) + { + CPpmd_State *ps[256], *s; + UInt32 freqSum, count, hiCnt; + CPpmd_See *see; + unsigned i, num, numMasked = p->MinContext->NumStats; + do + { + p->OrderFall++; + if (!p->MinContext->Suffix) + return -1; + p->MinContext = Ppmd7_GetContext(p, p->MinContext->Suffix); + } + while (p->MinContext->NumStats == numMasked); + hiCnt = 0; + s = Ppmd7_GetStats(p, p->MinContext); + i = 0; + num = p->MinContext->NumStats - numMasked; + do + { + int k = (int)(MASK(s->Symbol)); + hiCnt += (s->Freq & k); + ps[i] = s++; + i -= k; + } + while (i != num); + + see = Ppmd7_MakeEscFreq(p, numMasked, &freqSum); + freqSum += hiCnt; + count = rc->GetThreshold(rc, freqSum); + + if (count < hiCnt) + { + Byte symbol; + CPpmd_State **pps = ps; + for (hiCnt = 0; (hiCnt += (*pps)->Freq) <= count; pps++); + s = *pps; + rc->Decode(rc, hiCnt - s->Freq, s->Freq); + Ppmd_See_Update(see); + p->FoundState = s; + symbol = s->Symbol; + Ppmd7_Update2(p); + return symbol; + } + if (count >= freqSum) + return -2; + rc->Decode(rc, hiCnt, freqSum - hiCnt); + see->Summ = (UInt16)(see->Summ + freqSum); + do { MASK(ps[--i]->Symbol) = 0; } while (i != 0); + } +} diff --git a/code/nel/3rdparty/seven_zip/Ppmd7Enc.c b/code/nel/3rdparty/seven_zip/Ppmd7Enc.c new file mode 100644 index 000000000..286b8712c --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Ppmd7Enc.c @@ -0,0 +1,187 @@ +/* Ppmd7Enc.c -- PPMdH Encoder +2017-04-03 : Igor Pavlov : Public domain +This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ + +#include "Precomp.h" + +#include "Ppmd7.h" + +#define kTopValue (1 << 24) + +void Ppmd7z_RangeEnc_Init(CPpmd7z_RangeEnc *p) +{ + p->Low = 0; + p->Range = 0xFFFFFFFF; + p->Cache = 0; + p->CacheSize = 1; +} + +static void RangeEnc_ShiftLow(CPpmd7z_RangeEnc *p) +{ + if ((UInt32)p->Low < (UInt32)0xFF000000 || (unsigned)(p->Low >> 32) != 0) + { + Byte temp = p->Cache; + do + { + IByteOut_Write(p->Stream, (Byte)(temp + (Byte)(p->Low >> 32))); + temp = 0xFF; + } + while (--p->CacheSize != 0); + p->Cache = (Byte)((UInt32)p->Low >> 24); + } + p->CacheSize++; + p->Low = (UInt32)p->Low << 8; +} + +static void RangeEnc_Encode(CPpmd7z_RangeEnc *p, UInt32 start, UInt32 size, UInt32 total) +{ + p->Low += start * (p->Range /= total); + p->Range *= size; + while (p->Range < kTopValue) + { + p->Range <<= 8; + RangeEnc_ShiftLow(p); + } +} + +static void RangeEnc_EncodeBit_0(CPpmd7z_RangeEnc *p, UInt32 size0) +{ + p->Range = (p->Range >> 14) * size0; + while (p->Range < kTopValue) + { + p->Range <<= 8; + RangeEnc_ShiftLow(p); + } +} + +static void RangeEnc_EncodeBit_1(CPpmd7z_RangeEnc *p, UInt32 size0) +{ + UInt32 newBound = (p->Range >> 14) * size0; + p->Low += newBound; + p->Range -= newBound; + while (p->Range < kTopValue) + { + p->Range <<= 8; + RangeEnc_ShiftLow(p); + } +} + +void Ppmd7z_RangeEnc_FlushData(CPpmd7z_RangeEnc *p) +{ + unsigned i; + for (i = 0; i < 5; i++) + RangeEnc_ShiftLow(p); +} + + +#define MASK(sym) ((signed char *)charMask)[sym] + +void Ppmd7_EncodeSymbol(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol) +{ + size_t charMask[256 / sizeof(size_t)]; + if (p->MinContext->NumStats != 1) + { + CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext); + UInt32 sum; + unsigned i; + if (s->Symbol == symbol) + { + RangeEnc_Encode(rc, 0, s->Freq, p->MinContext->SummFreq); + p->FoundState = s; + Ppmd7_Update1_0(p); + return; + } + p->PrevSuccess = 0; + sum = s->Freq; + i = p->MinContext->NumStats - 1; + do + { + if ((++s)->Symbol == symbol) + { + RangeEnc_Encode(rc, sum, s->Freq, p->MinContext->SummFreq); + p->FoundState = s; + Ppmd7_Update1(p); + return; + } + sum += s->Freq; + } + while (--i); + + p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]; + PPMD_SetAllBitsIn256Bytes(charMask); + MASK(s->Symbol) = 0; + i = p->MinContext->NumStats - 1; + do { MASK((--s)->Symbol) = 0; } while (--i); + RangeEnc_Encode(rc, sum, p->MinContext->SummFreq - sum, p->MinContext->SummFreq); + } + else + { + UInt16 *prob = Ppmd7_GetBinSumm(p); + CPpmd_State *s = Ppmd7Context_OneState(p->MinContext); + if (s->Symbol == symbol) + { + RangeEnc_EncodeBit_0(rc, *prob); + *prob = (UInt16)PPMD_UPDATE_PROB_0(*prob); + p->FoundState = s; + Ppmd7_UpdateBin(p); + return; + } + else + { + RangeEnc_EncodeBit_1(rc, *prob); + *prob = (UInt16)PPMD_UPDATE_PROB_1(*prob); + p->InitEsc = PPMD7_kExpEscape[*prob >> 10]; + PPMD_SetAllBitsIn256Bytes(charMask); + MASK(s->Symbol) = 0; + p->PrevSuccess = 0; + } + } + for (;;) + { + UInt32 escFreq; + CPpmd_See *see; + CPpmd_State *s; + UInt32 sum; + unsigned i, numMasked = p->MinContext->NumStats; + do + { + p->OrderFall++; + if (!p->MinContext->Suffix) + return; /* EndMarker (symbol = -1) */ + p->MinContext = Ppmd7_GetContext(p, p->MinContext->Suffix); + } + while (p->MinContext->NumStats == numMasked); + + see = Ppmd7_MakeEscFreq(p, numMasked, &escFreq); + s = Ppmd7_GetStats(p, p->MinContext); + sum = 0; + i = p->MinContext->NumStats; + do + { + int cur = s->Symbol; + if (cur == symbol) + { + UInt32 low = sum; + CPpmd_State *s1 = s; + do + { + sum += (s->Freq & (int)(MASK(s->Symbol))); + s++; + } + while (--i); + RangeEnc_Encode(rc, low, s1->Freq, sum + escFreq); + Ppmd_See_Update(see); + p->FoundState = s1; + Ppmd7_Update2(p); + return; + } + sum += (s->Freq & (int)(MASK(cur))); + MASK(cur) = 0; + s++; + } + while (--i); + + RangeEnc_Encode(rc, sum, escFreq, sum + escFreq); + see->Summ = (UInt16)(see->Summ + sum + escFreq); + } +} diff --git a/code/nel/3rdparty/seven_zip/Precomp.h b/code/nel/3rdparty/seven_zip/Precomp.h new file mode 100644 index 000000000..e8ff8b40e --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Precomp.h @@ -0,0 +1,10 @@ +/* Precomp.h -- StdAfx +2013-11-12 : Igor Pavlov : Public domain */ + +#ifndef __7Z_PRECOMP_H +#define __7Z_PRECOMP_H + +#include "Compiler.h" +/* #include "7zTypes.h" */ + +#endif diff --git a/code/nel/3rdparty/seven_zip/RotateDefs.h b/code/nel/3rdparty/seven_zip/RotateDefs.h new file mode 100644 index 000000000..8f01d1a6c --- /dev/null +++ b/code/nel/3rdparty/seven_zip/RotateDefs.h @@ -0,0 +1,30 @@ +/* RotateDefs.h -- Rotate functions +2015-03-25 : Igor Pavlov : Public domain */ + +#ifndef __ROTATE_DEFS_H +#define __ROTATE_DEFS_H + +#ifdef _MSC_VER + +#include + +/* don't use _rotl with MINGW. It can insert slow call to function. */ + +/* #if (_MSC_VER >= 1200) */ +#pragma intrinsic(_rotl) +#pragma intrinsic(_rotr) +/* #endif */ + +#define rotlFixed(x, n) _rotl((x), (n)) +#define rotrFixed(x, n) _rotr((x), (n)) + +#else + +/* new compilers can translate these macros to fast commands. */ + +#define rotlFixed(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) +#define rotrFixed(x, n) (((x) >> (n)) | ((x) << (32 - (n)))) + +#endif + +#endif diff --git a/code/nel/3rdparty/seven_zip/Sha256.c b/code/nel/3rdparty/seven_zip/Sha256.c new file mode 100644 index 000000000..04b688c6b --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Sha256.c @@ -0,0 +1,248 @@ +/* Crypto/Sha256.c -- SHA-256 Hash +2017-04-03 : Igor Pavlov : Public domain +This code is based on public domain code from Wei Dai's Crypto++ library. */ + +#include "Precomp.h" + +#include + +#include "CpuArch.h" +#include "RotateDefs.h" +#include "Sha256.h" + +/* define it for speed optimization */ +#ifndef _SFX +#define _SHA256_UNROLL +#define _SHA256_UNROLL2 +#endif + +/* #define _SHA256_UNROLL2 */ + +void Sha256_Init(CSha256 *p) +{ + p->state[0] = 0x6a09e667; + p->state[1] = 0xbb67ae85; + p->state[2] = 0x3c6ef372; + p->state[3] = 0xa54ff53a; + p->state[4] = 0x510e527f; + p->state[5] = 0x9b05688c; + p->state[6] = 0x1f83d9ab; + p->state[7] = 0x5be0cd19; + p->count = 0; +} + +#define S0(x) (rotrFixed(x, 2) ^ rotrFixed(x,13) ^ rotrFixed(x, 22)) +#define S1(x) (rotrFixed(x, 6) ^ rotrFixed(x,11) ^ rotrFixed(x, 25)) +#define s0(x) (rotrFixed(x, 7) ^ rotrFixed(x,18) ^ (x >> 3)) +#define s1(x) (rotrFixed(x,17) ^ rotrFixed(x,19) ^ (x >> 10)) + +#define blk0(i) (W[i]) +#define blk2(i) (W[i] += s1(W[((i)-2)&15]) + W[((i)-7)&15] + s0(W[((i)-15)&15])) + +#define Ch(x,y,z) (z^(x&(y^z))) +#define Maj(x,y,z) ((x&y)|(z&(x|y))) + +#ifdef _SHA256_UNROLL2 + +#define R(a,b,c,d,e,f,g,h, i) \ + h += S1(e) + Ch(e,f,g) + K[(i)+(size_t)(j)] + (j ? blk2(i) : blk0(i)); \ + d += h; \ + h += S0(a) + Maj(a, b, c) + +#define RX_8(i) \ + R(a,b,c,d,e,f,g,h, i); \ + R(h,a,b,c,d,e,f,g, i+1); \ + R(g,h,a,b,c,d,e,f, i+2); \ + R(f,g,h,a,b,c,d,e, i+3); \ + R(e,f,g,h,a,b,c,d, i+4); \ + R(d,e,f,g,h,a,b,c, i+5); \ + R(c,d,e,f,g,h,a,b, i+6); \ + R(b,c,d,e,f,g,h,a, i+7) + +#define RX_16 RX_8(0); RX_8(8); + +#else + +#define a(i) T[(0-(i))&7] +#define b(i) T[(1-(i))&7] +#define c(i) T[(2-(i))&7] +#define d(i) T[(3-(i))&7] +#define e(i) T[(4-(i))&7] +#define f(i) T[(5-(i))&7] +#define g(i) T[(6-(i))&7] +#define h(i) T[(7-(i))&7] + +#define R(i) \ + h(i) += S1(e(i)) + Ch(e(i),f(i),g(i)) + K[(i)+(size_t)(j)] + (j ? blk2(i) : blk0(i)); \ + d(i) += h(i); \ + h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) \ + +#ifdef _SHA256_UNROLL + +#define RX_8(i) R(i+0); R(i+1); R(i+2); R(i+3); R(i+4); R(i+5); R(i+6); R(i+7); +#define RX_16 RX_8(0); RX_8(8); + +#else + +#define RX_16 unsigned i; for (i = 0; i < 16; i++) { R(i); } + +#endif + +#endif + +static const UInt32 K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +}; + +static void Sha256_WriteByteBlock(CSha256 *p) +{ + UInt32 W[16]; + unsigned j; + UInt32 *state; + + #ifdef _SHA256_UNROLL2 + UInt32 a,b,c,d,e,f,g,h; + #else + UInt32 T[8]; + #endif + + for (j = 0; j < 16; j += 4) + { + const Byte *ccc = p->buffer + j * 4; + W[j ] = GetBe32(ccc); + W[j + 1] = GetBe32(ccc + 4); + W[j + 2] = GetBe32(ccc + 8); + W[j + 3] = GetBe32(ccc + 12); + } + + state = p->state; + + #ifdef _SHA256_UNROLL2 + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + f = state[5]; + g = state[6]; + h = state[7]; + #else + for (j = 0; j < 8; j++) + T[j] = state[j]; + #endif + + for (j = 0; j < 64; j += 16) + { + RX_16 + } + + #ifdef _SHA256_UNROLL2 + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + state[5] += f; + state[6] += g; + state[7] += h; + #else + for (j = 0; j < 8; j++) + state[j] += T[j]; + #endif + + /* Wipe variables */ + /* memset(W, 0, sizeof(W)); */ + /* memset(T, 0, sizeof(T)); */ +} + +#undef S0 +#undef S1 +#undef s0 +#undef s1 + +void Sha256_Update(CSha256 *p, const Byte *data, size_t size) +{ + if (size == 0) + return; + + { + unsigned pos = (unsigned)p->count & 0x3F; + unsigned num; + + p->count += size; + + num = 64 - pos; + if (num > size) + { + memcpy(p->buffer + pos, data, size); + return; + } + + size -= num; + memcpy(p->buffer + pos, data, num); + data += num; + } + + for (;;) + { + Sha256_WriteByteBlock(p); + if (size < 64) + break; + size -= 64; + memcpy(p->buffer, data, 64); + data += 64; + } + + if (size != 0) + memcpy(p->buffer, data, size); +} + +void Sha256_Final(CSha256 *p, Byte *digest) +{ + unsigned pos = (unsigned)p->count & 0x3F; + unsigned i; + + p->buffer[pos++] = 0x80; + + while (pos != (64 - 8)) + { + pos &= 0x3F; + if (pos == 0) + Sha256_WriteByteBlock(p); + p->buffer[pos++] = 0; + } + + { + UInt64 numBits = (p->count << 3); + SetBe32(p->buffer + 64 - 8, (UInt32)(numBits >> 32)); + SetBe32(p->buffer + 64 - 4, (UInt32)(numBits)); + } + + Sha256_WriteByteBlock(p); + + for (i = 0; i < 8; i += 2) + { + UInt32 v0 = p->state[i]; + UInt32 v1 = p->state[i + 1]; + SetBe32(digest , v0); + SetBe32(digest + 4, v1); + digest += 8; + } + + Sha256_Init(p); +} diff --git a/code/nel/3rdparty/seven_zip/Sha256.h b/code/nel/3rdparty/seven_zip/Sha256.h new file mode 100644 index 000000000..3f455dbc0 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Sha256.h @@ -0,0 +1,26 @@ +/* Sha256.h -- SHA-256 Hash +2013-01-18 : Igor Pavlov : Public domain */ + +#ifndef __CRYPTO_SHA256_H +#define __CRYPTO_SHA256_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define SHA256_DIGEST_SIZE 32 + +typedef struct +{ + UInt32 state[8]; + UInt64 count; + Byte buffer[64]; +} CSha256; + +void Sha256_Init(CSha256 *p); +void Sha256_Update(CSha256 *p, const Byte *data, size_t size); +void Sha256_Final(CSha256 *p, Byte *digest); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/Sort.c b/code/nel/3rdparty/seven_zip/Sort.c new file mode 100644 index 000000000..e1097e380 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Sort.c @@ -0,0 +1,141 @@ +/* Sort.c -- Sort functions +2014-04-05 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "Sort.h" + +#define HeapSortDown(p, k, size, temp) \ + { for (;;) { \ + size_t s = (k << 1); \ + if (s > size) break; \ + if (s < size && p[s + 1] > p[s]) s++; \ + if (temp >= p[s]) break; \ + p[k] = p[s]; k = s; \ + } p[k] = temp; } + +void HeapSort(UInt32 *p, size_t size) +{ + if (size <= 1) + return; + p--; + { + size_t i = size / 2; + do + { + UInt32 temp = p[i]; + size_t k = i; + HeapSortDown(p, k, size, temp) + } + while (--i != 0); + } + /* + do + { + size_t k = 1; + UInt32 temp = p[size]; + p[size--] = p[1]; + HeapSortDown(p, k, size, temp) + } + while (size > 1); + */ + while (size > 3) + { + UInt32 temp = p[size]; + size_t k = (p[3] > p[2]) ? 3 : 2; + p[size--] = p[1]; + p[1] = p[k]; + HeapSortDown(p, k, size, temp) + } + { + UInt32 temp = p[size]; + p[size] = p[1]; + if (size > 2 && p[2] < temp) + { + p[1] = p[2]; + p[2] = temp; + } + else + p[1] = temp; + } +} + +void HeapSort64(UInt64 *p, size_t size) +{ + if (size <= 1) + return; + p--; + { + size_t i = size / 2; + do + { + UInt64 temp = p[i]; + size_t k = i; + HeapSortDown(p, k, size, temp) + } + while (--i != 0); + } + /* + do + { + size_t k = 1; + UInt64 temp = p[size]; + p[size--] = p[1]; + HeapSortDown(p, k, size, temp) + } + while (size > 1); + */ + while (size > 3) + { + UInt64 temp = p[size]; + size_t k = (p[3] > p[2]) ? 3 : 2; + p[size--] = p[1]; + p[1] = p[k]; + HeapSortDown(p, k, size, temp) + } + { + UInt64 temp = p[size]; + p[size] = p[1]; + if (size > 2 && p[2] < temp) + { + p[1] = p[2]; + p[2] = temp; + } + else + p[1] = temp; + } +} + +/* +#define HeapSortRefDown(p, vals, n, size, temp) \ + { size_t k = n; UInt32 val = vals[temp]; for (;;) { \ + size_t s = (k << 1); \ + if (s > size) break; \ + if (s < size && vals[p[s + 1]] > vals[p[s]]) s++; \ + if (val >= vals[p[s]]) break; \ + p[k] = p[s]; k = s; \ + } p[k] = temp; } + +void HeapSortRef(UInt32 *p, UInt32 *vals, size_t size) +{ + if (size <= 1) + return; + p--; + { + size_t i = size / 2; + do + { + UInt32 temp = p[i]; + HeapSortRefDown(p, vals, i, size, temp); + } + while (--i != 0); + } + do + { + UInt32 temp = p[size]; + p[size--] = p[1]; + HeapSortRefDown(p, vals, 1, size, temp); + } + while (size > 1); +} +*/ diff --git a/code/nel/3rdparty/seven_zip/Sort.h b/code/nel/3rdparty/seven_zip/Sort.h new file mode 100644 index 000000000..2e2963a23 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Sort.h @@ -0,0 +1,18 @@ +/* Sort.h -- Sort functions +2014-04-05 : Igor Pavlov : Public domain */ + +#ifndef __7Z_SORT_H +#define __7Z_SORT_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +void HeapSort(UInt32 *p, size_t size); +void HeapSort64(UInt64 *p, size_t size); + +/* void HeapSortRef(UInt32 *p, UInt32 *vals, size_t size); */ + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/Xz.c b/code/nel/3rdparty/seven_zip/Xz.c new file mode 100644 index 000000000..d9f83df16 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Xz.c @@ -0,0 +1,90 @@ +/* Xz.c - Xz +2017-05-12 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "7zCrc.h" +#include "CpuArch.h" +#include "Xz.h" +#include "XzCrc64.h" + +const Byte XZ_SIG[XZ_SIG_SIZE] = { 0xFD, '7', 'z', 'X', 'Z', 0 }; +/* const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; */ + +unsigned Xz_WriteVarInt(Byte *buf, UInt64 v) +{ + unsigned i = 0; + do + { + buf[i++] = (Byte)((v & 0x7F) | 0x80); + v >>= 7; + } + while (v != 0); + buf[(size_t)i - 1] &= 0x7F; + return i; +} + +void Xz_Construct(CXzStream *p) +{ + p->numBlocks = 0; + p->blocks = NULL; + p->flags = 0; +} + +void Xz_Free(CXzStream *p, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, p->blocks); + p->numBlocks = 0; + p->blocks = NULL; +} + +unsigned XzFlags_GetCheckSize(CXzStreamFlags f) +{ + unsigned t = XzFlags_GetCheckType(f); + return (t == 0) ? 0 : (4 << ((t - 1) / 3)); +} + +void XzCheck_Init(CXzCheck *p, unsigned mode) +{ + p->mode = mode; + switch (mode) + { + case XZ_CHECK_CRC32: p->crc = CRC_INIT_VAL; break; + case XZ_CHECK_CRC64: p->crc64 = CRC64_INIT_VAL; break; + case XZ_CHECK_SHA256: Sha256_Init(&p->sha); break; + } +} + +void XzCheck_Update(CXzCheck *p, const void *data, size_t size) +{ + switch (p->mode) + { + case XZ_CHECK_CRC32: p->crc = CrcUpdate(p->crc, data, size); break; + case XZ_CHECK_CRC64: p->crc64 = Crc64Update(p->crc64, data, size); break; + case XZ_CHECK_SHA256: Sha256_Update(&p->sha, (const Byte *)data, size); break; + } +} + +int XzCheck_Final(CXzCheck *p, Byte *digest) +{ + switch (p->mode) + { + case XZ_CHECK_CRC32: + SetUi32(digest, CRC_GET_DIGEST(p->crc)); + break; + case XZ_CHECK_CRC64: + { + int i; + UInt64 v = CRC64_GET_DIGEST(p->crc64); + for (i = 0; i < 8; i++, v >>= 8) + digest[i] = (Byte)(v & 0xFF); + break; + } + case XZ_CHECK_SHA256: + Sha256_Final(&p->sha, digest); + break; + default: + return 0; + } + return 1; +} diff --git a/code/nel/3rdparty/seven_zip/Xz.h b/code/nel/3rdparty/seven_zip/Xz.h new file mode 100644 index 000000000..544ee18f2 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/Xz.h @@ -0,0 +1,460 @@ +/* Xz.h - Xz interface +2018-07-04 : Igor Pavlov : Public domain */ + +#ifndef __XZ_H +#define __XZ_H + +#include "Sha256.h" + +EXTERN_C_BEGIN + +#define XZ_ID_Subblock 1 +#define XZ_ID_Delta 3 +#define XZ_ID_X86 4 +#define XZ_ID_PPC 5 +#define XZ_ID_IA64 6 +#define XZ_ID_ARM 7 +#define XZ_ID_ARMT 8 +#define XZ_ID_SPARC 9 +#define XZ_ID_LZMA2 0x21 + +unsigned Xz_ReadVarInt(const Byte *p, size_t maxSize, UInt64 *value); +unsigned Xz_WriteVarInt(Byte *buf, UInt64 v); + +/* ---------- xz block ---------- */ + +#define XZ_BLOCK_HEADER_SIZE_MAX 1024 + +#define XZ_NUM_FILTERS_MAX 4 +#define XZ_BF_NUM_FILTERS_MASK 3 +#define XZ_BF_PACK_SIZE (1 << 6) +#define XZ_BF_UNPACK_SIZE (1 << 7) + +#define XZ_FILTER_PROPS_SIZE_MAX 20 + +typedef struct +{ + UInt64 id; + UInt32 propsSize; + Byte props[XZ_FILTER_PROPS_SIZE_MAX]; +} CXzFilter; + +typedef struct +{ + UInt64 packSize; + UInt64 unpackSize; + Byte flags; + CXzFilter filters[XZ_NUM_FILTERS_MAX]; +} CXzBlock; + +#define XzBlock_GetNumFilters(p) (((p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1) +#define XzBlock_HasPackSize(p) (((p)->flags & XZ_BF_PACK_SIZE) != 0) +#define XzBlock_HasUnpackSize(p) (((p)->flags & XZ_BF_UNPACK_SIZE) != 0) +#define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0) + +SRes XzBlock_Parse(CXzBlock *p, const Byte *header); +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes); + +/* ---------- xz stream ---------- */ + +#define XZ_SIG_SIZE 6 +#define XZ_FOOTER_SIG_SIZE 2 + +extern const Byte XZ_SIG[XZ_SIG_SIZE]; + +/* +extern const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE]; +*/ + +#define XZ_FOOTER_SIG_0 'Y' +#define XZ_FOOTER_SIG_1 'Z' + +#define XZ_STREAM_FLAGS_SIZE 2 +#define XZ_STREAM_CRC_SIZE 4 + +#define XZ_STREAM_HEADER_SIZE (XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE + XZ_STREAM_CRC_SIZE) +#define XZ_STREAM_FOOTER_SIZE (XZ_FOOTER_SIG_SIZE + XZ_STREAM_FLAGS_SIZE + XZ_STREAM_CRC_SIZE + 4) + +#define XZ_CHECK_MASK 0xF +#define XZ_CHECK_NO 0 +#define XZ_CHECK_CRC32 1 +#define XZ_CHECK_CRC64 4 +#define XZ_CHECK_SHA256 10 + +typedef struct +{ + unsigned mode; + UInt32 crc; + UInt64 crc64; + CSha256 sha; +} CXzCheck; + +void XzCheck_Init(CXzCheck *p, unsigned mode); +void XzCheck_Update(CXzCheck *p, const void *data, size_t size); +int XzCheck_Final(CXzCheck *p, Byte *digest); + +typedef UInt16 CXzStreamFlags; + +#define XzFlags_IsSupported(f) ((f) <= XZ_CHECK_MASK) +#define XzFlags_GetCheckType(f) ((f) & XZ_CHECK_MASK) +#define XzFlags_HasDataCrc32(f) (Xz_GetCheckType(f) == XZ_CHECK_CRC32) +unsigned XzFlags_GetCheckSize(CXzStreamFlags f); + +SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf); +SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream); + +typedef struct +{ + UInt64 unpackSize; + UInt64 totalSize; +} CXzBlockSizes; + +typedef struct +{ + CXzStreamFlags flags; + size_t numBlocks; + CXzBlockSizes *blocks; + UInt64 startOffset; +} CXzStream; + +void Xz_Construct(CXzStream *p); +void Xz_Free(CXzStream *p, ISzAllocPtr alloc); + +#define XZ_SIZE_OVERFLOW ((UInt64)(Int64)-1) + +UInt64 Xz_GetUnpackSize(const CXzStream *p); +UInt64 Xz_GetPackSize(const CXzStream *p); + +typedef struct +{ + size_t num; + size_t numAllocated; + CXzStream *streams; +} CXzs; + +void Xzs_Construct(CXzs *p); +void Xzs_Free(CXzs *p, ISzAllocPtr alloc); +SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc); + +UInt64 Xzs_GetNumBlocks(const CXzs *p); +UInt64 Xzs_GetUnpackSize(const CXzs *p); + + +// ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + +typedef enum +{ + CODER_STATUS_NOT_SPECIFIED, /* use main error code instead */ + CODER_STATUS_FINISHED_WITH_MARK, /* stream was finished with end mark. */ + CODER_STATUS_NOT_FINISHED, /* stream was not finished */ + CODER_STATUS_NEEDS_MORE_INPUT /* you must provide more input bytes */ +} ECoderStatus; + + +// ECoderFinishMode values are identical to ELzmaFinishMode + +typedef enum +{ + CODER_FINISH_ANY, /* finish at any point */ + CODER_FINISH_END /* block must be finished at the end */ +} ECoderFinishMode; + + +typedef struct _IStateCoder +{ + void *p; + void (*Free)(void *p, ISzAllocPtr alloc); + SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAllocPtr alloc); + void (*Init)(void *p); + SRes (*Code2)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status); + SizeT (*Filter)(void *p, Byte *data, SizeT size); +} IStateCoder; + + + +#define MIXCODER_NUM_FILTERS_MAX 4 + +typedef struct +{ + ISzAllocPtr alloc; + Byte *buf; + unsigned numCoders; + + Byte *outBuf; + size_t outBufSize; + size_t outWritten; // is equal to lzmaDecoder.dicPos (in outBuf mode) + BoolInt wasFinished; + SRes res; + ECoderStatus status; + // BoolInt SingleBufMode; + + int finished[MIXCODER_NUM_FILTERS_MAX - 1]; + size_t pos[MIXCODER_NUM_FILTERS_MAX - 1]; + size_t size[MIXCODER_NUM_FILTERS_MAX - 1]; + UInt64 ids[MIXCODER_NUM_FILTERS_MAX]; + SRes results[MIXCODER_NUM_FILTERS_MAX]; + IStateCoder coders[MIXCODER_NUM_FILTERS_MAX]; +} CMixCoder; + + +typedef enum +{ + XZ_STATE_STREAM_HEADER, + XZ_STATE_STREAM_INDEX, + XZ_STATE_STREAM_INDEX_CRC, + XZ_STATE_STREAM_FOOTER, + XZ_STATE_STREAM_PADDING, + XZ_STATE_BLOCK_HEADER, + XZ_STATE_BLOCK, + XZ_STATE_BLOCK_FOOTER +} EXzState; + + +typedef struct +{ + EXzState state; + UInt32 pos; + unsigned alignPos; + unsigned indexPreSize; + + CXzStreamFlags streamFlags; + + UInt32 blockHeaderSize; + UInt64 packSize; + UInt64 unpackSize; + + UInt64 numBlocks; // number of finished blocks in current stream + UInt64 indexSize; + UInt64 indexPos; + UInt64 padSize; + + UInt64 numStartedStreams; + UInt64 numFinishedStreams; + UInt64 numTotalBlocks; + + UInt32 crc; + CMixCoder decoder; + CXzBlock block; + CXzCheck check; + CSha256 sha; + + BoolInt parseMode; + BoolInt headerParsedOk; + BoolInt decodeToStreamSignature; + unsigned decodeOnlyOneBlock; + + Byte *outBuf; + size_t outBufSize; + size_t outDataWritten; // the size of data in (outBuf) that were fully unpacked + + Byte shaDigest[SHA256_DIGEST_SIZE]; + Byte buf[XZ_BLOCK_HEADER_SIZE_MAX]; +} CXzUnpacker; + +/* alloc : aligned for cache line allocation is better */ +void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc); +void XzUnpacker_Init(CXzUnpacker *p); +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize); +void XzUnpacker_Free(CXzUnpacker *p); + +/* + XzUnpacker + The sequence for decoding functions: + { + XzUnpacker_Construct() + [Decoding_Calls] + XzUnpacker_Free() + } + + [Decoding_Calls] + + There are 3 types of interfaces for [Decoding_Calls] calls: + + Interface-1 : Partial output buffers: + { + XzUnpacker_Init() + for() + XzUnpacker_Code(); + } + + Interface-2 : Direct output buffer: + Use it, if you know exact size of decoded data, and you need + whole xz unpacked data in one output buffer. + xz unpacker doesn't allocate additional buffer for lzma2 dictionary in that mode. + { + XzUnpacker_Init() + XzUnpacker_SetOutBufMode(); // to set output buffer and size + for() + XzUnpacker_Code(); // (dest = NULL) in XzUnpacker_Code() + } + + Interface-3 : Direct output buffer : One call full decoding + It unpacks whole input buffer to output buffer in one call. + It uses Interface-2 internally. + { + XzUnpacker_CodeFull() + } +*/ + +/* +finishMode: + It has meaning only if the decoding reaches output limit (*destLen). + CODER_FINISH_ANY - use smallest number of input bytes + CODER_FINISH_END - read EndOfStream marker after decoding + +Returns: + SZ_OK + status: + CODER_STATUS_NOT_FINISHED, + CODER_STATUS_NEEDS_MORE_INPUT - maybe there are more xz streams, + call XzUnpacker_IsStreamWasFinished to check that current stream was finished + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_DATA - Data error + SZ_ERROR_UNSUPPORTED - Unsupported method or method properties + SZ_ERROR_CRC - CRC error + // SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src). + + SZ_ERROR_NO_ARCHIVE - the error with xz Stream Header with one of the following reasons: + - xz Stream Signature failure + - CRC32 of xz Stream Header is failed + - The size of Stream padding is not multiple of four bytes. + It's possible to get that error, if xz stream was finished and the stream + contains some another data. In that case you can call XzUnpacker_GetExtraSize() + function to get real size of xz stream. +*/ + + +SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status); + +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status); + +BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); + +/* +XzUnpacker_GetExtraSize() returns then number of uncofirmed bytes, + if it's in (XZ_STATE_STREAM_HEADER) state or in (XZ_STATE_STREAM_PADDING) state. +These bytes can be some bytes after xz archive, or +it can be start of new xz stream. + +Call XzUnpacker_GetExtraSize() after XzUnpacker_Code() function to detect real size of +xz stream in two cases, if XzUnpacker_Code() returns: + res == SZ_OK && status == CODER_STATUS_NEEDS_MORE_INPUT + res == SZ_ERROR_NO_ARCHIVE +*/ + +UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p); + + +/* + for random block decoding: + XzUnpacker_Init(); + set CXzUnpacker::streamFlags + XzUnpacker_PrepareToRandomBlockDecoding() + loop + { + XzUnpacker_Code() + XzUnpacker_IsBlockFinished() + } +*/ + +void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p); +BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p); + +#define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags)) + + + +/* ---------- Multi Threading Decoding ---------- */ + + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + BoolInt ignoreErrors; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t memUseMax; + #endif +} CXzDecMtProps; + +void XzDecMtProps_Init(CXzDecMtProps *p); + + +typedef void * CXzDecMtHandle; + +/* + alloc : XzDecMt uses CAlignOffsetAlloc for addresses allocated by (alloc). + allocMid : for big allocations, aligned allocation is better +*/ + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void XzDecMt_Destroy(CXzDecMtHandle p); + + +typedef struct +{ + Byte UnpackSize_Defined; + Byte NumStreams_Defined; + Byte NumBlocks_Defined; + + Byte DataAfterEnd; + Byte DecodingTruncated; // Decoding was Truncated, we need only partial output data + + UInt64 InSize; // pack size processed + UInt64 OutSize; + + UInt64 NumStreams; + UInt64 NumBlocks; + + SRes DecodeRes; + SRes ReadRes; + SRes ProgressRes; + SRes CombinedRes; + SRes CombinedRes_Type; + +} CXzStatInfo; + +void XzStatInfo_Clear(CXzStatInfo *p); + +/* +XzDecMt_Decode() +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_NO_ARCHIVE - is not xz archive + SZ_ERROR_ARCHIVE - Headers error + SZ_ERROR_DATA - Data Error + SZ_ERROR_CRC - CRC Error + SZ_ERROR_INPUT_EOF - it needs more input data + SZ_ERROR_WRITE - ISeqOutStream error + (SZ_ERROR_READ) - ISeqInStream errors + (SZ_ERROR_PROGRESS) - ICompressProgress errors + // SZ_ERROR_THREAD - error in multi-threading functions + MY_SRes_HRESULT_FROM_WRes(WRes_error) - error in multi-threading function +*/ + +SRes XzDecMt_Decode(CXzDecMtHandle p, + const CXzDecMtProps *props, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - xz stream(s) must be finished + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, // 0 means that ST (Single-Thread) version was used + ICompressProgress *progress); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/XzCrc64.c b/code/nel/3rdparty/seven_zip/XzCrc64.c new file mode 100644 index 000000000..b6d02cbeb --- /dev/null +++ b/code/nel/3rdparty/seven_zip/XzCrc64.c @@ -0,0 +1,86 @@ +/* XzCrc64.c -- CRC64 calculation +2017-05-23 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "XzCrc64.h" +#include "CpuArch.h" + +#define kCrc64Poly UINT64_CONST(0xC96C5795D7870F42) + +#ifdef MY_CPU_LE + #define CRC64_NUM_TABLES 4 +#else + #define CRC64_NUM_TABLES 5 + #define CRC_UINT64_SWAP(v) \ + ((v >> 56) \ + | ((v >> 40) & ((UInt64)0xFF << 8)) \ + | ((v >> 24) & ((UInt64)0xFF << 16)) \ + | ((v >> 8) & ((UInt64)0xFF << 24)) \ + | ((v << 8) & ((UInt64)0xFF << 32)) \ + | ((v << 24) & ((UInt64)0xFF << 40)) \ + | ((v << 40) & ((UInt64)0xFF << 48)) \ + | ((v << 56))) + + UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table); +#endif + +#ifndef MY_CPU_BE + UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table); +#endif + +typedef UInt64 (MY_FAST_CALL *CRC64_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table); + +static CRC64_FUNC g_Crc64Update; +UInt64 g_Crc64Table[256 * CRC64_NUM_TABLES]; + +UInt64 MY_FAST_CALL Crc64Update(UInt64 v, const void *data, size_t size) +{ + return g_Crc64Update(v, data, size, g_Crc64Table); +} + +UInt64 MY_FAST_CALL Crc64Calc(const void *data, size_t size) +{ + return g_Crc64Update(CRC64_INIT_VAL, data, size, g_Crc64Table) ^ CRC64_INIT_VAL; +} + +void MY_FAST_CALL Crc64GenerateTable() +{ + UInt32 i; + for (i = 0; i < 256; i++) + { + UInt64 r = i; + unsigned j; + for (j = 0; j < 8; j++) + r = (r >> 1) ^ (kCrc64Poly & ((UInt64)0 - (r & 1))); + g_Crc64Table[i] = r; + } + for (i = 256; i < 256 * CRC64_NUM_TABLES; i++) + { + UInt64 r = g_Crc64Table[(size_t)i - 256]; + g_Crc64Table[i] = g_Crc64Table[r & 0xFF] ^ (r >> 8); + } + + #ifdef MY_CPU_LE + + g_Crc64Update = XzCrc64UpdateT4; + + #else + { + #ifndef MY_CPU_BE + UInt32 k = 1; + if (*(const Byte *)&k == 1) + g_Crc64Update = XzCrc64UpdateT4; + else + #endif + { + for (i = 256 * CRC64_NUM_TABLES - 1; i >= 256; i--) + { + UInt64 x = g_Crc64Table[(size_t)i - 256]; + g_Crc64Table[i] = CRC_UINT64_SWAP(x); + } + g_Crc64Update = XzCrc64UpdateT1_BeT4; + } + } + #endif +} diff --git a/code/nel/3rdparty/seven_zip/XzCrc64.h b/code/nel/3rdparty/seven_zip/XzCrc64.h new file mode 100644 index 000000000..08dbc330c --- /dev/null +++ b/code/nel/3rdparty/seven_zip/XzCrc64.h @@ -0,0 +1,26 @@ +/* XzCrc64.h -- CRC64 calculation +2013-01-18 : Igor Pavlov : Public domain */ + +#ifndef __XZ_CRC64_H +#define __XZ_CRC64_H + +#include + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +extern UInt64 g_Crc64Table[]; + +void MY_FAST_CALL Crc64GenerateTable(void); + +#define CRC64_INIT_VAL UINT64_CONST(0xFFFFFFFFFFFFFFFF) +#define CRC64_GET_DIGEST(crc) ((crc) ^ CRC64_INIT_VAL) +#define CRC64_UPDATE_BYTE(crc, b) (g_Crc64Table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) + +UInt64 MY_FAST_CALL Crc64Update(UInt64 crc, const void *data, size_t size); +UInt64 MY_FAST_CALL Crc64Calc(const void *data, size_t size); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/XzCrc64Opt.c b/code/nel/3rdparty/seven_zip/XzCrc64Opt.c new file mode 100644 index 000000000..b2852de4d --- /dev/null +++ b/code/nel/3rdparty/seven_zip/XzCrc64Opt.c @@ -0,0 +1,69 @@ +/* XzCrc64Opt.c -- CRC64 calculation +2017-06-30 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include "CpuArch.h" + +#ifndef MY_CPU_BE + +#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) + +UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table) +{ + const Byte *p = (const Byte *)data; + for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) + v = CRC64_UPDATE_BYTE_2(v, *p); + for (; size >= 4; size -= 4, p += 4) + { + UInt32 d = (UInt32)v ^ *(const UInt32 *)p; + v = (v >> 32) + ^ (table + 0x300)[((d ) & 0xFF)] + ^ (table + 0x200)[((d >> 8) & 0xFF)] + ^ (table + 0x100)[((d >> 16) & 0xFF)] + ^ (table + 0x000)[((d >> 24))]; + } + for (; size > 0; size--, p++) + v = CRC64_UPDATE_BYTE_2(v, *p); + return v; +} + +#endif + + +#ifndef MY_CPU_LE + +#define CRC_UINT64_SWAP(v) \ + ((v >> 56) \ + | ((v >> 40) & ((UInt64)0xFF << 8)) \ + | ((v >> 24) & ((UInt64)0xFF << 16)) \ + | ((v >> 8) & ((UInt64)0xFF << 24)) \ + | ((v << 8) & ((UInt64)0xFF << 32)) \ + | ((v << 24) & ((UInt64)0xFF << 40)) \ + | ((v << 40) & ((UInt64)0xFF << 48)) \ + | ((v << 56))) + +#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) + +UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table) +{ + const Byte *p = (const Byte *)data; + table += 0x100; + v = CRC_UINT64_SWAP(v); + for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) + v = CRC64_UPDATE_BYTE_2_BE(v, *p); + for (; size >= 4; size -= 4, p += 4) + { + UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)p; + v = (v << 32) + ^ (table + 0x000)[((d ) & 0xFF)] + ^ (table + 0x100)[((d >> 8) & 0xFF)] + ^ (table + 0x200)[((d >> 16) & 0xFF)] + ^ (table + 0x300)[((d >> 24))]; + } + for (; size > 0; size--, p++) + v = CRC64_UPDATE_BYTE_2_BE(v, *p); + return CRC_UINT64_SWAP(v); +} + +#endif diff --git a/code/nel/3rdparty/seven_zip/XzDec.c b/code/nel/3rdparty/seven_zip/XzDec.c new file mode 100644 index 000000000..395e83f67 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/XzDec.c @@ -0,0 +1,2766 @@ +/* XzDec.c -- Xz Decode +2019-02-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #include + +// #define XZ_DUMP + +/* #define XZ_DUMP */ + +#ifdef XZ_DUMP +#include +#endif + +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + +#include +#include + +#include "7zCrc.h" +#include "Alloc.h" +#include "Bra.h" +#include "CpuArch.h" +#include "Delta.h" +#include "Lzma2Dec.h" + +// #define USE_SUBBLOCK + +#ifdef USE_SUBBLOCK +#include "Bcj3Dec.c" +#include "SbDec.h" +#endif + +#include "Xz.h" + +#define XZ_CHECK_SIZE_MAX 64 + +#define CODER_BUF_SIZE ((size_t)1 << 17) + +unsigned Xz_ReadVarInt(const Byte *p, size_t maxSize, UInt64 *value) +{ + unsigned i, limit; + *value = 0; + limit = (maxSize > 9) ? 9 : (unsigned)maxSize; + + for (i = 0; i < limit;) + { + Byte b = p[i]; + *value |= (UInt64)(b & 0x7F) << (7 * i++); + if ((b & 0x80) == 0) + return (b == 0 && i != 1) ? 0 : i; + } + return 0; +} + +/* ---------- BraState ---------- */ + +#define BRA_BUF_SIZE (1 << 14) + +typedef struct +{ + size_t bufPos; + size_t bufConv; + size_t bufTotal; + + int encodeMode; + + UInt32 methodId; + UInt32 delta; + UInt32 ip; + UInt32 x86State; + Byte deltaState[DELTA_STATE_SIZE]; + + Byte buf[BRA_BUF_SIZE]; +} CBraState; + +static void BraState_Free(void *pp, ISzAllocPtr alloc) +{ + ISzAlloc_Free(alloc, pp); +} + +static SRes BraState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) +{ + CBraState *p = ((CBraState *)pp); + UNUSED_VAR(alloc); + p->ip = 0; + if (p->methodId == XZ_ID_Delta) + { + if (propSize != 1) + return SZ_ERROR_UNSUPPORTED; + p->delta = (unsigned)props[0] + 1; + } + else + { + if (propSize == 4) + { + UInt32 v = GetUi32(props); + switch (p->methodId) + { + case XZ_ID_PPC: + case XZ_ID_ARM: + case XZ_ID_SPARC: + if ((v & 3) != 0) + return SZ_ERROR_UNSUPPORTED; + break; + case XZ_ID_ARMT: + if ((v & 1) != 0) + return SZ_ERROR_UNSUPPORTED; + break; + case XZ_ID_IA64: + if ((v & 0xF) != 0) + return SZ_ERROR_UNSUPPORTED; + break; + } + p->ip = v; + } + else if (propSize != 0) + return SZ_ERROR_UNSUPPORTED; + } + return SZ_OK; +} + +static void BraState_Init(void *pp) +{ + CBraState *p = ((CBraState *)pp); + p->bufPos = p->bufConv = p->bufTotal = 0; + x86_Convert_Init(p->x86State); + if (p->methodId == XZ_ID_Delta) + Delta_Init(p->deltaState); +} + + +#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: size = isa ## _Convert(data, size, p->ip, p->encodeMode); break; + +static SizeT BraState_Filter(void *pp, Byte *data, SizeT size) +{ + CBraState *p = ((CBraState *)pp); + switch (p->methodId) + { + case XZ_ID_Delta: + if (p->encodeMode) + Delta_Encode(p->deltaState, p->delta, data, size); + else + Delta_Decode(p->deltaState, p->delta, data, size); + break; + case XZ_ID_X86: + size = x86_Convert(data, size, p->ip, &p->x86State, p->encodeMode); + break; + CASE_BRA_CONV(PPC) + CASE_BRA_CONV(IA64) + CASE_BRA_CONV(ARM) + CASE_BRA_CONV(ARMT) + CASE_BRA_CONV(SPARC) + } + p->ip += (UInt32)size; + return size; +} + + +static SRes BraState_Code2(void *pp, + Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, int srcWasFinished, + ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) +{ + CBraState *p = ((CBraState *)pp); + SizeT destRem = *destLen; + SizeT srcRem = *srcLen; + UNUSED_VAR(finishMode); + + *destLen = 0; + *srcLen = 0; + // *wasFinished = False; + *status = CODER_STATUS_NOT_FINISHED; + + while (destRem > 0) + { + if (p->bufPos != p->bufConv) + { + size_t size = p->bufConv - p->bufPos; + if (size > destRem) + size = destRem; + memcpy(dest, p->buf + p->bufPos, size); + p->bufPos += size; + *destLen += size; + dest += size; + destRem -= size; + continue; + } + + p->bufTotal -= p->bufPos; + memmove(p->buf, p->buf + p->bufPos, p->bufTotal); + p->bufPos = 0; + p->bufConv = 0; + { + size_t size = BRA_BUF_SIZE - p->bufTotal; + if (size > srcRem) + size = srcRem; + memcpy(p->buf + p->bufTotal, src, size); + *srcLen += size; + src += size; + srcRem -= size; + p->bufTotal += size; + } + if (p->bufTotal == 0) + break; + + p->bufConv = BraState_Filter(pp, p->buf, p->bufTotal); + + if (p->bufConv == 0) + { + if (!srcWasFinished) + break; + p->bufConv = p->bufTotal; + } + } + + if (p->bufTotal == p->bufPos && srcRem == 0 && srcWasFinished) + { + *status = CODER_STATUS_FINISHED_WITH_MARK; + // *wasFinished = 1; + } + + return SZ_OK; +} + + +SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc) +{ + CBraState *decoder; + if (id < XZ_ID_Delta || id > XZ_ID_SPARC) + return SZ_ERROR_UNSUPPORTED; + decoder = (CBraState *)p->p; + if (!decoder) + { + decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = BraState_Free; + p->SetProps = BraState_SetProps; + p->Init = BraState_Init; + p->Code2 = BraState_Code2; + p->Filter = BraState_Filter; + } + decoder->methodId = (UInt32)id; + decoder->encodeMode = encodeMode; + return SZ_OK; +} + + + +/* ---------- SbState ---------- */ + +#ifdef USE_SUBBLOCK + +static void SbState_Free(void *pp, ISzAllocPtr alloc) +{ + CSbDec *p = (CSbDec *)pp; + SbDec_Free(p); + ISzAlloc_Free(alloc, pp); +} + +static SRes SbState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) +{ + UNUSED_VAR(pp); + UNUSED_VAR(props); + UNUSED_VAR(alloc); + return (propSize == 0) ? SZ_OK : SZ_ERROR_UNSUPPORTED; +} + +static void SbState_Init(void *pp) +{ + SbDec_Init((CSbDec *)pp); +} + +static SRes SbState_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) +{ + CSbDec *p = (CSbDec *)pp; + SRes res; + UNUSED_VAR(srcWasFinished); + p->dest = dest; + p->destLen = *destLen; + p->src = src; + p->srcLen = *srcLen; + p->finish = finishMode; /* change it */ + res = SbDec_Decode((CSbDec *)pp); + *destLen -= p->destLen; + *srcLen -= p->srcLen; + // *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + *status = (*destLen == 0 && *srcLen == 0) ? + CODER_STATUS_FINISHED_WITH_MARK : + CODER_STATUS_NOT_FINISHED; + return res; +} + +static SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) +{ + CSbDec *decoder = (CSbDec *)p->p; + if (!decoder) + { + decoder = (CSbDec *)ISzAlloc_Alloc(alloc, sizeof(CSbDec)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = SbState_Free; + p->SetProps = SbState_SetProps; + p->Init = SbState_Init; + p->Code2 = SbState_Code2; + p->Filter = NULL; + } + SbDec_Construct(decoder); + SbDec_SetAlloc(decoder, alloc); + return SZ_OK; +} + +#endif + + + +/* ---------- Lzma2 ---------- */ + +typedef struct +{ + CLzma2Dec decoder; + BoolInt outBufMode; +} CLzma2Dec_Spec; + + +static void Lzma2State_Free(void *pp, ISzAllocPtr alloc) +{ + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + Lzma2Dec_FreeProbs(&p->decoder, alloc); + else + Lzma2Dec_Free(&p->decoder, alloc); + ISzAlloc_Free(alloc, pp); +} + +static SRes Lzma2State_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) +{ + if (propSize != 1) + return SZ_ERROR_UNSUPPORTED; + { + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + return Lzma2Dec_AllocateProbs(&p->decoder, props[0], alloc); + else + return Lzma2Dec_Allocate(&p->decoder, props[0], alloc); + } +} + +static void Lzma2State_Init(void *pp) +{ + Lzma2Dec_Init(&((CLzma2Dec_Spec *)pp)->decoder); +} + + +/* + if (outBufMode), then (dest) is not used. Use NULL. + Data is unpacked to (spec->decoder.decoder.dic) output buffer. +*/ + +static SRes Lzma2State_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status) +{ + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)pp; + ELzmaStatus status2; + /* ELzmaFinishMode fm = (finishMode == LZMA_FINISH_ANY) ? LZMA_FINISH_ANY : LZMA_FINISH_END; */ + SRes res; + UNUSED_VAR(srcWasFinished); + if (spec->outBufMode) + { + SizeT dicPos = spec->decoder.decoder.dicPos; + SizeT dicLimit = dicPos + *destLen; + res = Lzma2Dec_DecodeToDic(&spec->decoder, dicLimit, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + *destLen = spec->decoder.decoder.dicPos - dicPos; + } + else + res = Lzma2Dec_DecodeToBuf(&spec->decoder, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + // *wasFinished = (status2 == LZMA_STATUS_FINISHED_WITH_MARK); + // ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + *status = (ECoderStatus)status2; + return res; +} + + +static SRes Lzma2State_SetFromMethod(IStateCoder *p, Byte *outBuf, size_t outBufSize, ISzAllocPtr alloc) +{ + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if (!spec) + { + spec = (CLzma2Dec_Spec *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Dec_Spec)); + if (!spec) + return SZ_ERROR_MEM; + p->p = spec; + p->Free = Lzma2State_Free; + p->SetProps = Lzma2State_SetProps; + p->Init = Lzma2State_Init; + p->Code2 = Lzma2State_Code2; + p->Filter = NULL; + Lzma2Dec_Construct(&spec->decoder); + } + spec->outBufMode = False; + if (outBuf) + { + spec->outBufMode = True; + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } + return SZ_OK; +} + + +static SRes Lzma2State_ResetOutBuf(IStateCoder *p, Byte *outBuf, size_t outBufSize) +{ + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if ((spec->outBufMode && !outBuf) || (!spec->outBufMode && outBuf)) + return SZ_ERROR_FAIL; + if (outBuf) + { + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } + return SZ_OK; +} + + + +static void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc) +{ + unsigned i; + p->alloc = alloc; + p->buf = NULL; + p->numCoders = 0; + + p->outBufSize = 0; + p->outBuf = NULL; + // p->SingleBufMode = False; + + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) + p->coders[i].p = NULL; +} + + +static void MixCoder_Free(CMixCoder *p) +{ + unsigned i; + p->numCoders = 0; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) + { + IStateCoder *sc = &p->coders[i]; + if (sc->p) + { + sc->Free(sc->p, p->alloc); + sc->p = NULL; + } + } + if (p->buf) + { + ISzAlloc_Free(p->alloc, p->buf); + p->buf = NULL; /* 9.31: the BUG was fixed */ + } +} + +static void MixCoder_Init(CMixCoder *p) +{ + unsigned i; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX - 1; i++) + { + p->size[i] = 0; + p->pos[i] = 0; + p->finished[i] = 0; + } + for (i = 0; i < p->numCoders; i++) + { + IStateCoder *coder = &p->coders[i]; + coder->Init(coder->p); + p->results[i] = SZ_OK; + } + p->outWritten = 0; + p->wasFinished = False; + p->res = SZ_OK; + p->status = CODER_STATUS_NOT_SPECIFIED; +} + + +static SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) +{ + IStateCoder *sc = &p->coders[coderIndex]; + p->ids[coderIndex] = methodId; + switch (methodId) + { + case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, outBuf, outBufSize, p->alloc); + #ifdef USE_SUBBLOCK + case XZ_ID_Subblock: return SbState_SetFromMethod(sc, p->alloc); + #endif + } + if (coderIndex == 0) + return SZ_ERROR_UNSUPPORTED; + return BraState_SetFromMethod(sc, methodId, 0, p->alloc); +} + + +static SRes MixCoder_ResetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) +{ + IStateCoder *sc = &p->coders[coderIndex]; + switch (methodId) + { + case XZ_ID_LZMA2: return Lzma2State_ResetOutBuf(sc, outBuf, outBufSize); + } + return SZ_ERROR_UNSUPPORTED; +} + + + +/* + if (destFinish) - then unpack data block is finished at (*destLen) position, + and we can return data that were not processed by filter + +output (status) can be : + CODER_STATUS_NOT_FINISHED + CODER_STATUS_FINISHED_WITH_MARK + CODER_STATUS_NEEDS_MORE_INPUT - not implemented still +*/ + +static SRes MixCoder_Code(CMixCoder *p, + Byte *dest, SizeT *destLen, int destFinish, + const Byte *src, SizeT *srcLen, int srcWasFinished, + ECoderFinishMode finishMode) +{ + SizeT destLenOrig = *destLen; + SizeT srcLenOrig = *srcLen; + + *destLen = 0; + *srcLen = 0; + + if (p->wasFinished) + return p->res; + + p->status = CODER_STATUS_NOT_FINISHED; + + // if (p->SingleBufMode) + if (p->outBuf) + { + SRes res; + SizeT destLen2, srcLen2; + int wasFinished; + + PRF_STR("------- MixCoder Single ----------"); + + srcLen2 = srcLenOrig; + destLen2 = destLenOrig; + + { + IStateCoder *coder = &p->coders[0]; + res = coder->Code2(coder->p, NULL, &destLen2, src, &srcLen2, srcWasFinished, finishMode, + // &wasFinished, + &p->status); + wasFinished = (p->status == CODER_STATUS_FINISHED_WITH_MARK); + } + + p->res = res; + + /* + if (wasFinished) + p->status = CODER_STATUS_FINISHED_WITH_MARK; + else + { + if (res == SZ_OK) + if (destLen2 != destLenOrig) + p->status = CODER_STATUS_NEEDS_MORE_INPUT; + } + */ + + + *srcLen = srcLen2; + src += srcLen2; + p->outWritten += destLen2; + + if (res != SZ_OK || srcWasFinished || wasFinished) + p->wasFinished = True; + + if (p->numCoders == 1) + *destLen = destLen2; + else if (p->wasFinished) + { + unsigned i; + size_t processed = p->outWritten; + + for (i = 1; i < p->numCoders; i++) + { + IStateCoder *coder = &p->coders[i]; + processed = coder->Filter(coder->p, p->outBuf, processed); + if (wasFinished || (destFinish && p->outWritten == destLenOrig)) + processed = p->outWritten; + PRF_STR_INT("filter", i); + } + *destLen = processed; + } + return res; + } + + PRF_STR("standard mix"); + + if (p->numCoders != 1) + { + if (!p->buf) + { + p->buf = (Byte *)ISzAlloc_Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); + if (!p->buf) + return SZ_ERROR_MEM; + } + + finishMode = CODER_FINISH_ANY; + } + + for (;;) + { + BoolInt processed = False; + BoolInt allFinished = True; + SRes resMain = SZ_OK; + unsigned i; + + p->status = CODER_STATUS_NOT_FINISHED; + /* + if (p->numCoders == 1 && *destLen == destLenOrig && finishMode == LZMA_FINISH_ANY) + break; + */ + + for (i = 0; i < p->numCoders; i++) + { + SRes res; + IStateCoder *coder = &p->coders[i]; + Byte *dest2; + SizeT destLen2, srcLen2; // destLen2_Orig; + const Byte *src2; + int srcFinished2; + int encodingWasFinished; + ECoderStatus status2; + + if (i == 0) + { + src2 = src; + srcLen2 = srcLenOrig - *srcLen; + srcFinished2 = srcWasFinished; + } + else + { + size_t k = i - 1; + src2 = p->buf + (CODER_BUF_SIZE * k) + p->pos[k]; + srcLen2 = p->size[k] - p->pos[k]; + srcFinished2 = p->finished[k]; + } + + if (i == p->numCoders - 1) + { + dest2 = dest; + destLen2 = destLenOrig - *destLen; + } + else + { + if (p->pos[i] != p->size[i]) + continue; + dest2 = p->buf + (CODER_BUF_SIZE * i); + destLen2 = CODER_BUF_SIZE; + } + + // destLen2_Orig = destLen2; + + if (p->results[i] != SZ_OK) + { + if (resMain == SZ_OK) + resMain = p->results[i]; + continue; + } + + res = coder->Code2(coder->p, + dest2, &destLen2, + src2, &srcLen2, srcFinished2, + finishMode, + // &encodingWasFinished, + &status2); + + if (res != SZ_OK) + { + p->results[i] = res; + if (resMain == SZ_OK) + resMain = res; + } + + encodingWasFinished = (status2 == CODER_STATUS_FINISHED_WITH_MARK); + + if (!encodingWasFinished) + { + allFinished = False; + if (p->numCoders == 1 && res == SZ_OK) + p->status = status2; + } + + if (i == 0) + { + *srcLen += srcLen2; + src += srcLen2; + } + else + p->pos[(size_t)i - 1] += srcLen2; + + if (i == p->numCoders - 1) + { + *destLen += destLen2; + dest += destLen2; + } + else + { + p->size[i] = destLen2; + p->pos[i] = 0; + p->finished[i] = encodingWasFinished; + } + + if (destLen2 != 0 || srcLen2 != 0) + processed = True; + } + + if (!processed) + { + if (allFinished) + p->status = CODER_STATUS_FINISHED_WITH_MARK; + return resMain; + } + } +} + + +SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf) +{ + *p = (CXzStreamFlags)GetBe16(buf + XZ_SIG_SIZE); + if (CrcCalc(buf + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE) != + GetUi32(buf + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE)) + return SZ_ERROR_NO_ARCHIVE; + return XzFlags_IsSupported(*p) ? SZ_OK : SZ_ERROR_UNSUPPORTED; +} + +static BoolInt Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) +{ + return indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) + && GetUi32(buf) == CrcCalc(buf + 4, 6) + && flags == GetBe16(buf + 8) + && buf[10] == XZ_FOOTER_SIG_0 + && buf[11] == XZ_FOOTER_SIG_1; +} + +#define READ_VARINT_AND_CHECK(buf, pos, size, res) \ + { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ + if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } + + +static BoolInt XzBlock_AreSupportedFilters(const CXzBlock *p) +{ + unsigned numFilters = XzBlock_GetNumFilters(p) - 1; + unsigned i; + { + const CXzFilter *f = &p->filters[numFilters]; + if (f->id != XZ_ID_LZMA2 || f->propsSize != 1 || f->props[0] > 40) + return False; + } + + for (i = 0; i < numFilters; i++) + { + const CXzFilter *f = &p->filters[i]; + if (f->id == XZ_ID_Delta) + { + if (f->propsSize != 1) + return False; + } + else if (f->id < XZ_ID_Delta + || f->id > XZ_ID_SPARC + || (f->propsSize != 0 && f->propsSize != 4)) + return False; + } + return True; +} + + +SRes XzBlock_Parse(CXzBlock *p, const Byte *header) +{ + unsigned pos; + unsigned numFilters, i; + unsigned headerSize = (unsigned)header[0] << 2; + + /* (headerSize != 0) : another code checks */ + + if (CrcCalc(header, headerSize) != GetUi32(header + headerSize)) + return SZ_ERROR_ARCHIVE; + + pos = 1; + p->flags = header[pos++]; + + p->packSize = (UInt64)(Int64)-1; + if (XzBlock_HasPackSize(p)) + { + READ_VARINT_AND_CHECK(header, pos, headerSize, &p->packSize); + if (p->packSize == 0 || p->packSize + headerSize >= (UInt64)1 << 63) + return SZ_ERROR_ARCHIVE; + } + + p->unpackSize = (UInt64)(Int64)-1; + if (XzBlock_HasUnpackSize(p)) + READ_VARINT_AND_CHECK(header, pos, headerSize, &p->unpackSize); + + numFilters = XzBlock_GetNumFilters(p); + for (i = 0; i < numFilters; i++) + { + CXzFilter *filter = p->filters + i; + UInt64 size; + READ_VARINT_AND_CHECK(header, pos, headerSize, &filter->id); + READ_VARINT_AND_CHECK(header, pos, headerSize, &size); + if (size > headerSize - pos || size > XZ_FILTER_PROPS_SIZE_MAX) + return SZ_ERROR_ARCHIVE; + filter->propsSize = (UInt32)size; + memcpy(filter->props, header + pos, (size_t)size); + pos += (unsigned)size; + + #ifdef XZ_DUMP + printf("\nf[%u] = %2X: ", i, (unsigned)filter->id); + { + unsigned i; + for (i = 0; i < size; i++) + printf(" %2X", filter->props[i]); + } + #endif + } + + if (XzBlock_HasUnsupportedFlags(p)) + return SZ_ERROR_UNSUPPORTED; + + while (pos < headerSize) + if (header[pos++] != 0) + return SZ_ERROR_ARCHIVE; + return SZ_OK; +} + + + + +static SRes XzDecMix_Init(CMixCoder *p, const CXzBlock *block, Byte *outBuf, size_t outBufSize) +{ + unsigned i; + BoolInt needReInit = True; + unsigned numFilters = XzBlock_GetNumFilters(block); + + if (numFilters == p->numCoders && ((p->outBuf && outBuf) || (!p->outBuf && !outBuf))) + { + needReInit = False; + for (i = 0; i < numFilters; i++) + if (p->ids[i] != block->filters[numFilters - 1 - i].id) + { + needReInit = True; + break; + } + } + + // p->SingleBufMode = (outBuf != NULL); + p->outBuf = outBuf; + p->outBufSize = outBufSize; + + // p->SingleBufMode = False; + // outBuf = NULL; + + if (needReInit) + { + MixCoder_Free(p); + for (i = 0; i < numFilters; i++) + { + RINOK(MixCoder_SetFromMethod(p, i, block->filters[numFilters - 1 - i].id, outBuf, outBufSize)); + } + p->numCoders = numFilters; + } + else + { + RINOK(MixCoder_ResetFromMethod(p, 0, block->filters[numFilters - 1].id, outBuf, outBufSize)); + } + + for (i = 0; i < numFilters; i++) + { + const CXzFilter *f = &block->filters[numFilters - 1 - i]; + IStateCoder *sc = &p->coders[i]; + RINOK(sc->SetProps(sc->p, f->props, f->propsSize, p->alloc)); + } + + MixCoder_Init(p); + return SZ_OK; +} + + + +void XzUnpacker_Init(CXzUnpacker *p) +{ + p->state = XZ_STATE_STREAM_HEADER; + p->pos = 0; + p->numStartedStreams = 0; + p->numFinishedStreams = 0; + p->numTotalBlocks = 0; + p->padSize = 0; + p->decodeOnlyOneBlock = 0; + + p->parseMode = False; + p->decodeToStreamSignature = False; + + // p->outBuf = NULL; + // p->outBufSize = 0; + p->outDataWritten = 0; +} + + +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize) +{ + p->outBuf = outBuf; + p->outBufSize = outBufSize; +} + + +void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc) +{ + MixCoder_Construct(&p->decoder, alloc); + p->outBuf = NULL; + p->outBufSize = 0; + XzUnpacker_Init(p); +} + + +void XzUnpacker_Free(CXzUnpacker *p) +{ + MixCoder_Free(&p->decoder); +} + + +void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p) +{ + p->indexSize = 0; + p->numBlocks = 0; + Sha256_Init(&p->sha); + p->state = XZ_STATE_BLOCK_HEADER; + p->pos = 0; + p->decodeOnlyOneBlock = 1; +} + + +static void XzUnpacker_UpdateIndex(CXzUnpacker *p, UInt64 packSize, UInt64 unpackSize) +{ + Byte temp[32]; + unsigned num = Xz_WriteVarInt(temp, packSize); + num += Xz_WriteVarInt(temp + num, unpackSize); + Sha256_Update(&p->sha, temp, num); + p->indexSize += num; + p->numBlocks++; +} + + + +SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status) +{ + SizeT destLenOrig = *destLen; + SizeT srcLenOrig = *srcLen; + *destLen = 0; + *srcLen = 0; + *status = CODER_STATUS_NOT_SPECIFIED; + + for (;;) + { + SizeT srcRem; + + if (p->state == XZ_STATE_BLOCK) + { + SizeT destLen2 = destLenOrig - *destLen; + SizeT srcLen2 = srcLenOrig - *srcLen; + SRes res; + + ECoderFinishMode finishMode2 = finishMode; + BoolInt srcFinished2 = srcFinished; + BoolInt destFinish = False; + + if (p->block.packSize != (UInt64)(Int64)-1) + { + UInt64 rem = p->block.packSize - p->packSize; + if (srcLen2 >= rem) + { + srcFinished2 = True; + srcLen2 = (SizeT)rem; + } + if (rem == 0 && p->block.unpackSize == p->unpackSize) + return SZ_ERROR_DATA; + } + + if (p->block.unpackSize != (UInt64)(Int64)-1) + { + UInt64 rem = p->block.unpackSize - p->unpackSize; + if (destLen2 >= rem) + { + destFinish = True; + finishMode2 = CODER_FINISH_END; + destLen2 = (SizeT)rem; + } + } + + /* + if (srcLen2 == 0 && destLen2 == 0) + { + *status = CODER_STATUS_NOT_FINISHED; + return SZ_OK; + } + */ + + { + res = MixCoder_Code(&p->decoder, + (p->outBuf ? NULL : dest), &destLen2, destFinish, + src, &srcLen2, srcFinished2, + finishMode2); + + *status = p->decoder.status; + XzCheck_Update(&p->check, (p->outBuf ? p->outBuf + p->outDataWritten : dest), destLen2); + if (!p->outBuf) + dest += destLen2; + p->outDataWritten += destLen2; + } + + (*srcLen) += srcLen2; + src += srcLen2; + p->packSize += srcLen2; + (*destLen) += destLen2; + p->unpackSize += destLen2; + + RINOK(res); + + if (*status != CODER_STATUS_FINISHED_WITH_MARK) + { + if (p->block.packSize == p->packSize + && *status == CODER_STATUS_NEEDS_MORE_INPUT) + { + PRF_STR("CODER_STATUS_NEEDS_MORE_INPUT"); + *status = CODER_STATUS_NOT_SPECIFIED; + return SZ_ERROR_DATA; + } + + return SZ_OK; + } + { + XzUnpacker_UpdateIndex(p, XzUnpacker_GetPackSizeForIndex(p), p->unpackSize); + p->state = XZ_STATE_BLOCK_FOOTER; + p->pos = 0; + p->alignPos = 0; + *status = CODER_STATUS_NOT_SPECIFIED; + + if ((p->block.packSize != (UInt64)(Int64)-1 && p->block.packSize != p->packSize) + || (p->block.unpackSize != (UInt64)(Int64)-1 && p->block.unpackSize != p->unpackSize)) + { + PRF_STR("ERROR: block.size mismatch"); + return SZ_ERROR_DATA; + } + } + // continue; + } + + srcRem = srcLenOrig - *srcLen; + + // XZ_STATE_BLOCK_FOOTER can transit to XZ_STATE_BLOCK_HEADER without input bytes + if (srcRem == 0 && p->state != XZ_STATE_BLOCK_FOOTER) + { + *status = CODER_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + + switch (p->state) + { + case XZ_STATE_STREAM_HEADER: + { + if (p->pos < XZ_STREAM_HEADER_SIZE) + { + if (p->pos < XZ_SIG_SIZE && *src != XZ_SIG[p->pos]) + return SZ_ERROR_NO_ARCHIVE; + if (p->decodeToStreamSignature) + return SZ_OK; + p->buf[p->pos++] = *src++; + (*srcLen)++; + } + else + { + RINOK(Xz_ParseHeader(&p->streamFlags, p->buf)); + p->numStartedStreams++; + p->indexSize = 0; + p->numBlocks = 0; + Sha256_Init(&p->sha); + p->state = XZ_STATE_BLOCK_HEADER; + p->pos = 0; + } + break; + } + + case XZ_STATE_BLOCK_HEADER: + { + if (p->pos == 0) + { + p->buf[p->pos++] = *src++; + (*srcLen)++; + if (p->buf[0] == 0) + { + if (p->decodeOnlyOneBlock) + return SZ_ERROR_DATA; + p->indexPreSize = 1 + Xz_WriteVarInt(p->buf + 1, p->numBlocks); + p->indexPos = p->indexPreSize; + p->indexSize += p->indexPreSize; + Sha256_Final(&p->sha, p->shaDigest); + Sha256_Init(&p->sha); + p->crc = CrcUpdate(CRC_INIT_VAL, p->buf, p->indexPreSize); + p->state = XZ_STATE_STREAM_INDEX; + break; + } + p->blockHeaderSize = ((UInt32)p->buf[0] << 2) + 4; + break; + } + + if (p->pos != p->blockHeaderSize) + { + UInt32 cur = p->blockHeaderSize - p->pos; + if (cur > srcRem) + cur = (UInt32)srcRem; + memcpy(p->buf + p->pos, src, cur); + p->pos += cur; + (*srcLen) += cur; + src += cur; + } + else + { + RINOK(XzBlock_Parse(&p->block, p->buf)); + if (!XzBlock_AreSupportedFilters(&p->block)) + return SZ_ERROR_UNSUPPORTED; + p->numTotalBlocks++; + p->state = XZ_STATE_BLOCK; + p->packSize = 0; + p->unpackSize = 0; + XzCheck_Init(&p->check, XzFlags_GetCheckType(p->streamFlags)); + if (p->parseMode) + { + p->headerParsedOk = True; + return SZ_OK; + } + RINOK(XzDecMix_Init(&p->decoder, &p->block, p->outBuf, p->outBufSize)); + } + break; + } + + case XZ_STATE_BLOCK_FOOTER: + { + if ((((unsigned)p->packSize + p->alignPos) & 3) != 0) + { + if (srcRem == 0) + { + *status = CODER_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + (*srcLen)++; + p->alignPos++; + if (*src++ != 0) + return SZ_ERROR_CRC; + } + else + { + UInt32 checkSize = XzFlags_GetCheckSize(p->streamFlags); + UInt32 cur = checkSize - p->pos; + if (cur != 0) + { + if (srcRem == 0) + { + *status = CODER_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + if (cur > srcRem) + cur = (UInt32)srcRem; + memcpy(p->buf + p->pos, src, cur); + p->pos += cur; + (*srcLen) += cur; + src += cur; + if (checkSize != p->pos) + break; + } + { + Byte digest[XZ_CHECK_SIZE_MAX]; + p->state = XZ_STATE_BLOCK_HEADER; + p->pos = 0; + if (XzCheck_Final(&p->check, digest) && memcmp(digest, p->buf, checkSize) != 0) + return SZ_ERROR_CRC; + if (p->decodeOnlyOneBlock) + { + *status = CODER_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } + } + } + break; + } + + case XZ_STATE_STREAM_INDEX: + { + if (p->pos < p->indexPreSize) + { + (*srcLen)++; + if (*src++ != p->buf[p->pos++]) + return SZ_ERROR_CRC; + } + else + { + if (p->indexPos < p->indexSize) + { + UInt64 cur = p->indexSize - p->indexPos; + if (srcRem > cur) + srcRem = (SizeT)cur; + p->crc = CrcUpdate(p->crc, src, srcRem); + Sha256_Update(&p->sha, src, srcRem); + (*srcLen) += srcRem; + src += srcRem; + p->indexPos += srcRem; + } + else if ((p->indexPos & 3) != 0) + { + Byte b = *src++; + p->crc = CRC_UPDATE_BYTE(p->crc, b); + (*srcLen)++; + p->indexPos++; + p->indexSize++; + if (b != 0) + return SZ_ERROR_CRC; + } + else + { + Byte digest[SHA256_DIGEST_SIZE]; + p->state = XZ_STATE_STREAM_INDEX_CRC; + p->indexSize += 4; + p->pos = 0; + Sha256_Final(&p->sha, digest); + if (memcmp(digest, p->shaDigest, SHA256_DIGEST_SIZE) != 0) + return SZ_ERROR_CRC; + } + } + break; + } + + case XZ_STATE_STREAM_INDEX_CRC: + { + if (p->pos < 4) + { + (*srcLen)++; + p->buf[p->pos++] = *src++; + } + else + { + p->state = XZ_STATE_STREAM_FOOTER; + p->pos = 0; + if (CRC_GET_DIGEST(p->crc) != GetUi32(p->buf)) + return SZ_ERROR_CRC; + } + break; + } + + case XZ_STATE_STREAM_FOOTER: + { + UInt32 cur = XZ_STREAM_FOOTER_SIZE - p->pos; + if (cur > srcRem) + cur = (UInt32)srcRem; + memcpy(p->buf + p->pos, src, cur); + p->pos += cur; + (*srcLen) += cur; + src += cur; + if (p->pos == XZ_STREAM_FOOTER_SIZE) + { + p->state = XZ_STATE_STREAM_PADDING; + p->numFinishedStreams++; + p->padSize = 0; + if (!Xz_CheckFooter(p->streamFlags, p->indexSize, p->buf)) + return SZ_ERROR_CRC; + } + break; + } + + case XZ_STATE_STREAM_PADDING: + { + if (*src != 0) + { + if (((UInt32)p->padSize & 3) != 0) + return SZ_ERROR_NO_ARCHIVE; + p->pos = 0; + p->state = XZ_STATE_STREAM_HEADER; + } + else + { + (*srcLen)++; + src++; + p->padSize++; + } + break; + } + + case XZ_STATE_BLOCK: break; /* to disable GCC warning */ + } + } + /* + if (p->state == XZ_STATE_FINISHED) + *status = CODER_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + */ +} + + +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status) +{ + XzUnpacker_Init(p); + XzUnpacker_SetOutBuf(p, dest, *destLen); + + return XzUnpacker_Code(p, + NULL, destLen, + src, srcLen, True, + finishMode, status); +} + + +BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p) +{ + return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0); +} + +BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p) +{ + return (p->state == XZ_STATE_STREAM_PADDING) && (((UInt32)p->padSize & 3) == 0); +} + +UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p) +{ + UInt64 num = 0; + if (p->state == XZ_STATE_STREAM_PADDING) + num = p->padSize; + else if (p->state == XZ_STATE_STREAM_HEADER) + num = p->padSize + p->pos; + return num; +} + + + + + + + + + + + + + + + + + + + + + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +void XzDecMtProps_Init(CXzDecMtProps *p) +{ + p->inBufSize_ST = 1 << 18; + p->outStep_ST = 1 << 20; + p->ignoreErrors = False; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->memUseMax = sizeof(size_t) << 28; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CXzDecMtThread ---------- */ + +typedef struct +{ + Byte *outBuf; + size_t outBufSize; + size_t outPreSize; + size_t inPreSize; + size_t inPreHeaderSize; + size_t blockPackSize_for_Index; // including block header and checksum. + size_t blockPackTotal; // including stream header, block header and checksum. + size_t inCodeSize; + size_t outCodeSize; + ECoderStatus status; + SRes codeRes; + BoolInt skipMode; + // BoolInt finishedWithMark; + EMtDecParseState parseState; + BoolInt parsing_Truncated; + BoolInt atBlockHeader; + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + BoolInt dec_created; + CXzUnpacker dec; + + Byte mtPad[1 << 7]; +} CXzDecMtThread; + +#endif + + +/* ---------- CXzDecMt ---------- */ + +typedef struct +{ + CAlignOffsetAlloc alignOffsetAlloc; + ISzAllocPtr allocMid; + + CXzDecMtProps props; + size_t unpackBlockMaxSize; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + // CXzStatInfo *stat; + + BoolInt finishMode; + BoolInt outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + UInt64 readProcessed; + BoolInt readWasFinished; + SRes readRes; + SRes writeRes; + + Byte *outBuf; + size_t outBufSize; + Byte *inBuf; + size_t inBufSize; + + CXzUnpacker dec; + + ECoderStatus status; + SRes codeRes; + + #ifndef _7ZIP_ST + BoolInt mainDecoderWasCalled; + // int statErrorDefined; + int finishedDecoderIndex; + + // global values that are used in Parse stage + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + // UInt64 numBadBlocks; + SRes mainErrorCode; + + BoolInt isBlockHeaderState_Parse; + BoolInt isBlockHeaderState_Write; + UInt64 outProcessed_Parse; + BoolInt parsing_Truncated; + + BoolInt mtc_WasConstructed; + CMtDec mtc; + CXzDecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CXzDecMt; + + + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CXzDecMt *p = (CXzDecMt *)ISzAlloc_Alloc(alloc, sizeof(CXzDecMt)); + if (!p) + return NULL; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.baseAlloc = alloc; + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + + p->allocMid = allocMid; + + p->outBuf = NULL; + p->outBufSize = 0; + p->inBuf = NULL; + p->inBufSize = 0; + + XzUnpacker_Construct(&p->dec, &p->alignOffsetAlloc.vt); + + p->unpackBlockMaxSize = 0; + + XzDecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + coder->dec_created = False; + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void XzDecMt_FreeOutBufs(CXzDecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + if (coder->outBuf) + { + ISzAlloc_Free(p->allocMid, coder->outBuf); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + p->unpackBlockMaxSize = 0; +} + +#endif + + + +static void XzDecMt_FreeSt(CXzDecMt *p) +{ + XzUnpacker_Free(&p->dec); + + if (p->outBuf) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBuf = NULL; + } + p->outBufSize = 0; + + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void XzDecMt_Destroy(CXzDecMtHandle pp) +{ + CXzDecMt *p = (CXzDecMt *)pp; + + XzDecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + XzUnpacker_Free(&t->dec); + t->dec_created = False; + } + } + } + XzDecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void XzDecMt_Callback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CXzDecMt *me = (CXzDecMt *)obj; + CXzDecMtThread *coder = &me->coders[coderIndex]; + size_t srcSize = cc->srcSize; + + cc->srcSize = 0; + cc->outPos = 0; + cc->state = MTDEC_PARSE_CONTINUE; + + cc->canCreateNewThread = True; + + if (cc->startCall) + { + coder->outPreSize = 0; + coder->inPreSize = 0; + coder->inPreHeaderSize = 0; + coder->parseState = MTDEC_PARSE_CONTINUE; + coder->parsing_Truncated = False; + coder->skipMode = False; + coder->codeRes = SZ_OK; + coder->status = CODER_STATUS_NOT_SPECIFIED; + coder->inCodeSize = 0; + coder->outCodeSize = 0; + + coder->numStreams = me->numStreams; + coder->numTotalBlocks = me->numTotalBlocks; + coder->numBlocks = me->numBlocks; + + if (!coder->dec_created) + { + XzUnpacker_Construct(&coder->dec, &me->alignOffsetAlloc.vt); + coder->dec_created = True; + } + + XzUnpacker_Init(&coder->dec); + + if (me->isBlockHeaderState_Parse) + { + coder->dec.streamFlags = me->streamFlags; + coder->atBlockHeader = True; + XzUnpacker_PrepareToRandomBlockDecoding(&coder->dec); + } + else + { + coder->atBlockHeader = False; + me->isBlockHeaderState_Parse = True; + } + + coder->dec.numStartedStreams = me->numStreams; + coder->dec.numTotalBlocks = me->numTotalBlocks; + coder->dec.numBlocks = me->numBlocks; + } + + while (!coder->skipMode) + { + ECoderStatus status; + SRes res; + size_t srcSize2 = srcSize; + size_t destSize = (size_t)0 - 1; + + coder->dec.parseMode = True; + coder->dec.headerParsedOk = False; + + PRF_STR_INT("Parse", srcSize2); + + res = XzUnpacker_Code(&coder->dec, + NULL, &destSize, + cc->src, &srcSize2, cc->srcFinished, + CODER_FINISH_END, &status); + + // PRF(printf(" res = %d, srcSize2 = %d", res, (unsigned)srcSize2)); + + coder->codeRes = res; + coder->status = status; + cc->srcSize += srcSize2; + srcSize -= srcSize2; + coder->inPreHeaderSize += srcSize2; + coder->inPreSize = coder->inPreHeaderSize; + + if (res != SZ_OK) + { + cc->state = + coder->parseState = MTDEC_PARSE_END; + /* + if (res == SZ_ERROR_MEM) + return res; + return SZ_OK; + */ + return; // res; + } + + if (coder->dec.headerParsedOk) + { + const CXzBlock *block = &coder->dec.block; + if (XzBlock_HasUnpackSize(block) + // && block->unpackSize <= me->props.outBlockMax + && XzBlock_HasPackSize(block)) + { + { + if (block->unpackSize * 2 * me->mtc.numStartedThreads > me->props.memUseMax) + { + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + } + { + UInt64 packSize = block->packSize; + UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); + UInt32 checkSize = XzFlags_GetCheckSize(coder->dec.streamFlags); + UInt64 blockPackSum = coder->inPreSize + packSizeAligned + checkSize; + // if (blockPackSum <= me->props.inBlockMax) + // unpackBlockMaxSize + { + coder->blockPackSize_for_Index = (size_t)(coder->dec.blockHeaderSize + packSize + checkSize); + coder->blockPackTotal = (size_t)blockPackSum; + coder->outPreSize = (size_t)block->unpackSize; + coder->streamFlags = coder->dec.streamFlags; + me->streamFlags = coder->dec.streamFlags; + coder->skipMode = True; + break; + } + } + } + } + else + // if (coder->inPreSize <= me->props.inBlockMax) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = + coder->parseState = MTDEC_PARSE_END; + return; // SZ_OK; + } + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + + // ---------- skipMode ---------- + { + UInt64 rem = coder->blockPackTotal - coder->inPreSize; + size_t cur = srcSize; + if (cur > rem) + cur = (size_t)rem; + cc->srcSize += cur; + coder->inPreSize += cur; + srcSize -= cur; + + if (coder->inPreSize == coder->blockPackTotal) + { + if (srcSize == 0) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = MTDEC_PARSE_END; + } + else if ((cc->src)[cc->srcSize] == 0) // we check control byte of next block + cc->state = MTDEC_PARSE_END; + else + { + cc->state = MTDEC_PARSE_NEW; + + { + size_t blockMax = me->unpackBlockMaxSize; + if (blockMax < coder->outPreSize) + blockMax = coder->outPreSize; + { + UInt64 required = (UInt64)blockMax * (me->mtc.numStartedThreads + 1) * 2; + if (me->props.memUseMax < required) + cc->canCreateNewThread = False; + } + } + + if (me->outSize_Defined) + { + // next block can be zero size + const UInt64 rem2 = me->outSize - me->outProcessed_Parse; + if (rem2 < coder->outPreSize) + { + coder->parsing_Truncated = True; + cc->state = MTDEC_PARSE_END; + } + me->outProcessed_Parse += coder->outPreSize; + } + } + } + else if (cc->srcFinished) + cc->state = MTDEC_PARSE_END; + else + return; // SZ_OK; + + coder->parseState = cc->state; + cc->outPos = coder->outPreSize; + + me->numStreams = coder->dec.numStartedStreams; + me->numTotalBlocks = coder->dec.numTotalBlocks; + me->numBlocks = coder->dec.numBlocks + 1; + return; // SZ_OK; + } +} + + +static SRes XzDecMt_Callback_PreCode(void *pp, unsigned coderIndex) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + Byte *dest; + + if (!coder->dec.headerParsedOk) + return SZ_OK; + + dest = coder->outBuf; + + if (!dest || coder->outBufSize < coder->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + { + size_t outPreSize = coder->outPreSize; + if (outPreSize == 0) + outPreSize = 1; + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, outPreSize); + } + if (!dest) + return SZ_ERROR_MEM; + coder->outBuf = dest; + coder->outBufSize = coder->outPreSize; + + if (coder->outBufSize > me->unpackBlockMaxSize) + me->unpackBlockMaxSize = coder->outBufSize; + } + + // return SZ_ERROR_MEM; + + XzUnpacker_SetOutBuf(&coder->dec, coder->outBuf, coder->outBufSize); + + { + SRes res = XzDecMix_Init(&coder->dec.decoder, &coder->dec.block, coder->outBuf, coder->outBufSize); + // res = SZ_ERROR_UNSUPPORTED; // to test + coder->codeRes = res; + if (res != SZ_OK) + { + // if (res == SZ_ERROR_MEM) return res; + if (me->props.ignoreErrors && res != SZ_ERROR_MEM) + return S_OK; + return res; + } + } + + return SZ_OK; +} + + +static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + *stop = True; + + if (coder->inCodeSize < coder->inPreHeaderSize) + { + UInt64 rem = coder->inPreHeaderSize - coder->inCodeSize; + size_t step = srcSize; + if (step > rem) + step = (size_t)rem; + src += step; + srcSize -= step; + coder->inCodeSize += step; + if (coder->inCodeSize < coder->inPreHeaderSize) + { + *stop = False; + return SZ_OK; + } + } + + if (!coder->dec.headerParsedOk) + return SZ_OK; + if (!coder->outBuf) + return SZ_OK; + + if (coder->codeRes == SZ_OK) + { + ECoderStatus status; + SRes res; + size_t srcProcessed = srcSize; + size_t outSizeCur = coder->outPreSize - coder->dec.outDataWritten; + + // PRF(printf("\nCallback_Code: Code %d %d\n", (unsigned)srcSize, (unsigned)outSizeCur)); + + res = XzUnpacker_Code(&coder->dec, + NULL, &outSizeCur, + src, &srcProcessed, srcFinished, + // coder->finishedWithMark ? CODER_FINISH_END : CODER_FINISH_ANY, + CODER_FINISH_END, + &status); + + // PRF(printf(" res = %d, srcSize2 = %d, outSizeCur = %d", res, (unsigned)srcProcessed, (unsigned)outSizeCur)); + + coder->codeRes = res; + coder->status = status; + coder->inCodeSize += srcProcessed; + coder->outCodeSize = coder->dec.outDataWritten; + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + + if (res == SZ_OK) + { + if (srcProcessed == srcSize) + *stop = False; + return SZ_OK; + } + } + + if (me->props.ignoreErrors && coder->codeRes != SZ_ERROR_MEM) + { + *inCodePos = coder->inPreSize; + *outCodePos = coder->outPreSize; + return S_OK; + } + return coder->codeRes; +} + + +#define XZDECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, + // int srcFinished, + BoolInt *needContinue, + BoolInt *canRecode) +{ + CXzDecMt *me = (CXzDecMt *)pp; + const CXzDecMtThread *coder = &me->coders[coderIndex]; + + // PRF(printf("\nWrite processed = %d srcSize = %d\n", (unsigned)me->mtc.inProcessed, (unsigned)srcSize)); + + *needContinue = False; + *canRecode = True; + + if (!needWriteToStream) + return SZ_OK; + + if (!coder->dec.headerParsedOk || !coder->outBuf) + { + if (me->finishedDecoderIndex < 0) + me->finishedDecoderIndex = coderIndex; + return SZ_OK; + } + + if (me->finishedDecoderIndex >= 0) + return SZ_OK; + + me->mtc.inProcessed += coder->inCodeSize; + + *canRecode = False; + + { + SRes res; + size_t size = coder->outCodeSize; + Byte *data = coder->outBuf; + + // we use in me->dec: sha, numBlocks, indexSize + + if (!me->isBlockHeaderState_Write) + { + XzUnpacker_PrepareToRandomBlockDecoding(&me->dec); + me->dec.decodeOnlyOneBlock = False; + me->dec.numStartedStreams = coder->dec.numStartedStreams; + me->dec.streamFlags = coder->streamFlags; + + me->isBlockHeaderState_Write = True; + } + + me->dec.numTotalBlocks = coder->dec.numTotalBlocks; + XzUnpacker_UpdateIndex(&me->dec, coder->blockPackSize_for_Index, coder->outPreSize); + + if (coder->outPreSize != size) + { + if (me->props.ignoreErrors) + { + memset(data + size, 0, coder->outPreSize - size); + size = coder->outPreSize; + } + // me->numBadBlocks++; + if (me->mainErrorCode == SZ_OK) + { + if ((int)coder->status == LZMA_STATUS_NEEDS_MORE_INPUT) + me->mainErrorCode = SZ_ERROR_INPUT_EOF; + else + me->mainErrorCode = SZ_ERROR_DATA; + } + } + + if (me->writeRes != SZ_OK) + return me->writeRes; + + res = SZ_OK; + { + if (me->outSize_Defined) + { + const UInt64 rem = me->outSize - me->outProcessed; + if (size > rem) + size = (SizeT)rem; + } + + for (;;) + { + size_t cur = size; + size_t written; + if (cur > XZDECMT_STREAM_WRITE_STEP) + cur = XZDECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + // PRF(printf("\nWritten ask = %d written = %d\n", (unsigned)cur, (unsigned)written)); + + me->outProcessed += written; + if (written != cur) + { + me->writeRes = SZ_ERROR_WRITE; + res = me->writeRes; + break; + } + data += cur; + size -= cur; + // PRF_STR_INT("Written size =", size); + if (size == 0) + break; + res = MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0); + if (res != SZ_OK) + break; + } + } + + if (coder->codeRes != SZ_OK) + if (!me->props.ignoreErrors) + { + me->finishedDecoderIndex = coderIndex; + return res; + } + + RINOK(res); + + if (coder->inPreSize != coder->inCodeSize + || coder->blockPackTotal != coder->inCodeSize) + { + me->finishedDecoderIndex = coderIndex; + return SZ_OK; + } + + if (coder->parseState != MTDEC_PARSE_END) + { + *needContinue = True; + return SZ_OK; + } + } + + // (coder->state == MTDEC_PARSE_END) means that there are no other working threads + // so we can use mtc variables without lock + + PRF_STR_INT("Write MTDEC_PARSE_END", me->mtc.inProcessed); + + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + { + CXzUnpacker *dec = &me->dec; + + PRF_STR_INT("PostSingle", srcSize); + + { + size_t srcProcessed = srcSize; + ECoderStatus status; + size_t outSizeCur = 0; + SRes res; + + // dec->decodeOnlyOneBlock = False; + dec->decodeToStreamSignature = True; + + me->mainDecoderWasCalled = True; + + if (coder->parsing_Truncated) + { + me->parsing_Truncated = True; + return SZ_OK; + } + + res = XzUnpacker_Code(dec, + NULL, &outSizeCur, + src, &srcProcessed, + me->mtc.readWasFinished, // srcFinished + CODER_FINISH_END, // CODER_FINISH_ANY, + &status); + + me->status = status; + me->codeRes = res; + + me->mtc.inProcessed += srcProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return S_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + { + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + memcpy(crossBuf, src + srcProcessed, srcSize - srcProcessed); + } + me->mtc.crossStart = 0; + me->mtc.crossEnd = srcSize - srcProcessed; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + { + return E_FAIL; + } + + if (me->mtc.readWasFinished) + { + return SZ_OK; + } + } + + { + size_t inPos; + size_t inLim; + const Byte *inData; + UInt64 inProgressPrev = me->mtc.inProcessed; + + // XzDecMt_Prepare_InBuf_ST(p); + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + + inPos = 0; + inLim = 0; + // outProcessed = 0; + + inData = crossBuf; + + for (;;) + { + SizeT inProcessed; + SizeT outProcessed; + ECoderStatus status; + SRes res; + + if (inPos == inLim) + { + if (!me->mtc.readWasFinished) + { + inPos = 0; + inLim = me->mtc.inBufSize; + me->mtc.readRes = ISeqInStream_Read(me->inStream, (void *)inData, &inLim); + me->mtc.readProcessed += inLim; + if (inLim == 0 || me->mtc.readRes != SZ_OK) + me->mtc.readWasFinished = True; + } + } + + inProcessed = inLim - inPos; + outProcessed = 0; + + res = XzUnpacker_Code(dec, + NULL, &outProcessed, + inData + inPos, &inProcessed, + (inProcessed == 0), // srcFinished + CODER_FINISH_END, &status); + + me->codeRes = res; + me->status = status; + inPos += inProcessed; + me->mtc.inProcessed += inProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return S_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->mtc.crossStart = inPos; + me->mtc.crossEnd = inLim; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + return E_FAIL; + + if (me->mtc.progress) + { + UInt64 inDelta = me->mtc.inProcessed - inProgressPrev; + if (inDelta >= (1 << 22)) + { + RINOK(MtProgress_Progress_ST(&me->mtc.mtProgress)); + inProgressPrev = me->mtc.inProcessed; + } + } + if (me->mtc.readWasFinished) + return SZ_OK; + } + } + } +} + + +#endif + + + +void XzStatInfo_Clear(CXzStatInfo *p) +{ + p->InSize = 0; + p->OutSize = 0; + + p->NumStreams = 0; + p->NumBlocks = 0; + + p->UnpackSize_Defined = False; + + p->NumStreams_Defined = False; + p->NumBlocks_Defined = False; + + // p->IsArc = False; + // p->UnexpectedEnd = False; + // p->Unsupported = False; + // p->HeadersError = False; + // p->DataError = False; + // p->CrcError = False; + + p->DataAfterEnd = False; + p->DecodingTruncated = False; + + p->DecodeRes = SZ_OK; + p->ReadRes = SZ_OK; + p->ProgressRes = SZ_OK; + + p->CombinedRes = SZ_OK; + p->CombinedRes_Type = SZ_OK; +} + + + + +static SRes XzDecMt_Decode_ST(CXzDecMt *p + #ifndef _7ZIP_ST + , BoolInt tMode + #endif + , CXzStatInfo *stat) +{ + size_t outPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CXzUnpacker *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + XzDecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + if (!p->outBuf || p->outBufSize != p->props.outStep_ST) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBufSize = 0; + p->outBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.outStep_ST); + if (!p->outBuf) + return SZ_ERROR_MEM; + p->outBufSize = p->props.outStep_ST; + } + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + dec = &p->dec; + dec->decodeToStreamSignature = False; + // dec->decodeOnlyOneBlock = False; + + XzUnpacker_SetOutBuf(dec, NULL, 0); + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + outPos = 0; + + for (;;) + { + SizeT outSize; + BoolInt finished; + ECoderFinishMode finishMode; + SizeT inProcessed; + ECoderStatus status; + SRes res; + + SizeT outProcessed; + + + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + p->readProcessed += inLim; + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + outSize = p->props.outStep_ST - outPos; + + finishMode = CODER_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (outSize >= rem) + { + outSize = (SizeT)rem; + if (p->finishMode) + finishMode = CODER_FINISH_END; + } + } + + inProcessed = inLim - inPos; + outProcessed = outSize; + + res = XzUnpacker_Code(dec, p->outBuf + outPos, &outProcessed, + inData + inPos, &inProcessed, + (inPos == inLim), // srcFinished + finishMode, &status); + + p->codeRes = res; + p->status = status; + + inPos += inProcessed; + outPos += outProcessed; + p->inProcessed += inProcessed; + p->outProcessed += outProcessed; + + finished = ((inProcessed == 0 && outProcessed == 0) || res != SZ_OK); + + if (finished || outProcessed >= outSize) + if (outPos != 0) + { + size_t written = ISeqOutStream_Write(p->outStream, p->outBuf, outPos); + p->outProcessed += written; + if (written != outPos) + { + stat->CombinedRes_Type = SZ_ERROR_WRITE; + return SZ_ERROR_WRITE; + } + outPos = 0; + } + + if (p->progress && res == SZ_OK) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + res = ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed); + if (res != SZ_OK) + { + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + stat->ProgressRes = res; + return res; + } + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + + if (finished) + return res; + } +} + +static SRes XzStatInfo_SetStat(const CXzUnpacker *dec, + int finishMode, + UInt64 readProcessed, UInt64 inProcessed, + SRes res, ECoderStatus status, + BoolInt decodingTruncated, + CXzStatInfo *stat) +{ + UInt64 extraSize; + + stat->DecodingTruncated = (Byte)(decodingTruncated ? 1 : 0); + stat->InSize = inProcessed; + stat->NumStreams = dec->numStartedStreams; + stat->NumBlocks = dec->numTotalBlocks; + + stat->UnpackSize_Defined = True; + stat->NumStreams_Defined = True; + stat->NumBlocks_Defined = True; + + extraSize = XzUnpacker_GetExtraSize(dec); + + if (res == SZ_OK) + { + if (status == CODER_STATUS_NEEDS_MORE_INPUT) + { + // CODER_STATUS_NEEDS_MORE_INPUT is expected status for correct xz streams + extraSize = 0; + if (!XzUnpacker_IsStreamWasFinished(dec)) + res = SZ_ERROR_INPUT_EOF; + } + else if (!decodingTruncated || finishMode) // (status == CODER_STATUS_NOT_FINISHED) + res = SZ_ERROR_DATA; + } + else if (res == SZ_ERROR_NO_ARCHIVE) + { + /* + SZ_ERROR_NO_ARCHIVE is possible for 2 states: + XZ_STATE_STREAM_HEADER - if bad signature or bad CRC + XZ_STATE_STREAM_PADDING - if non-zero padding data + extraSize / inProcessed don't include "bad" byte + */ + if (inProcessed != extraSize) // if good streams before error + if (extraSize != 0 || readProcessed != inProcessed) + { + stat->DataAfterEnd = True; + // there is some good xz stream before. So we set SZ_OK + res = SZ_OK; + } + } + + stat->DecodeRes = res; + + stat->InSize -= extraSize; + return res; +} + + +SRes XzDecMt_Decode(CXzDecMtHandle pp, + const CXzDecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, + ICompressProgress *progress) +{ + CXzDecMt *p = (CXzDecMt *)pp; + #ifndef _7ZIP_ST + BoolInt tMode; + #endif + + XzStatInfo_Clear(stat); + + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + // p->stat = stat; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + + p->finishMode = finishMode; + + // p->outSize = 457; p->outSize_Defined = True; p->finishMode = False; // for test + + p->writeRes = SZ_OK; + p->outProcessed = 0; + p->inProcessed = 0; + p->readProcessed = 0; + p->readWasFinished = False; + + p->codeRes = 0; + p->status = CODER_STATUS_NOT_SPECIFIED; + + XzUnpacker_Init(&p->dec); + + *isMT = False; + + /* + p->outBuf = NULL; + p->outBufSize = 0; + if (!outStream) + { + p->outBuf = outBuf; + p->outBufSize = *outBufSize; + *outBufSize = 0; + } + */ + + + #ifndef _7ZIP_ST + + p->isBlockHeaderState_Parse = False; + p->isBlockHeaderState_Write = False; + // p->numBadBlocks = 0; + p->mainErrorCode = SZ_OK; + p->mainDecoderWasCalled = False; + + tMode = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + // we just free ST buffers here + // but we still keep state variables, that was set in XzUnpacker_Init() + XzDecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + p->parsing_Truncated = False; + + p->numStreams = 0; + p->numTotalBlocks = 0; + p->numBlocks = 0; + p->finishedDecoderIndex = -1; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.inBufSize = p->props.inBufSize_MT; + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = XzDecMt_Callback_Parse; + vt.PreCode = XzDecMt_Callback_PreCode; + vt.Code = XzDecMt_Callback_Code; + vt.Write = XzDecMt_Callback_Write; + + { + BoolInt needContinue; + + SRes res = MtDec_Code(&p->mtc); + + stat->InSize = p->mtc.inProcessed; + + p->inProcessed = p->mtc.inProcessed; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->readProcessed = p->mtc.readProcessed; + + tMode = True; + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + { + res = p->mtc.mtProgress.res; + stat->ProgressRes = res; + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + } + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + SRes codeRes; + BoolInt truncated = False; + ECoderStatus status; + CXzUnpacker *dec; + + stat->OutSize = p->outProcessed; + + if (p->finishedDecoderIndex >= 0) + { + CXzDecMtThread *coder = &p->coders[(unsigned)p->finishedDecoderIndex]; + codeRes = coder->codeRes; + dec = &coder->dec; + status = coder->status; + } + else if (p->mainDecoderWasCalled) + { + codeRes = p->codeRes; + dec = &p->dec; + status = p->status; + truncated = p->parsing_Truncated; + } + else + return E_FAIL; + + XzStatInfo_SetStat(dec, p->finishMode, + p->mtc.readProcessed, p->mtc.inProcessed, + codeRes, status, + truncated, + stat); + + if (res == SZ_OK) + { + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else if (p->mtc.readRes != SZ_OK && p->mtc.inProcessed == p->mtc.readProcessed) + { + res = p->mtc.readRes; + stat->ReadRes = res; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + else if (p->mainErrorCode != SZ_OK) + { + res = p->mainErrorCode; + } + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = XzDecMt_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + , stat + ); + + XzStatInfo_SetStat(&p->dec, + p->finishMode, + p->readProcessed, p->inProcessed, + p->codeRes, p->status, + False, // truncated + stat); + + if (res == SZ_OK) + { + /* + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else + */ + if (p->readRes != SZ_OK && p->inProcessed == p->readProcessed) + { + res = p->readRes; + stat->ReadRes = res; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + #ifndef _7ZIP_ST + else if (p->mainErrorCode != SZ_OK) + res = p->mainErrorCode; + #endif + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } +} diff --git a/code/nel/3rdparty/seven_zip/XzEnc.c b/code/nel/3rdparty/seven_zip/XzEnc.c new file mode 100644 index 000000000..d0a8b4489 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/XzEnc.c @@ -0,0 +1,1329 @@ +/* XzEnc.c -- Xz Encode +2019-02-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include +#include + +#include "7zCrc.h" +#include "Bra.h" +#include "CpuArch.h" + +#ifdef USE_SUBBLOCK +#include "Bcj3Enc.c" +#include "SbFind.c" +#include "SbEnc.c" +#endif + +#include "XzEnc.h" + +// #define _7ZIP_ST + +#ifndef _7ZIP_ST +#include "MtCoder.h" +#else +#define MTCODER__THREADS_MAX 1 +#define MTCODER__BLOCKS_MAX 1 +#endif + +#define XZ_GET_PAD_SIZE(dataSize) ((4 - ((unsigned)(dataSize) & 3)) & 3) + +/* max pack size for LZMA2 block + check-64bytrs: */ +#define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + 64) + +#define XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(unpackSize) (XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize)) + + +#define XzBlock_ClearFlags(p) (p)->flags = 0; +#define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1); +#define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE; +#define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE; + + +static SRes WriteBytes(ISeqOutStream *s, const void *buf, size_t size) +{ + return (ISeqOutStream_Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; +} + +static SRes WriteBytesUpdateCrc(ISeqOutStream *s, const void *buf, size_t size, UInt32 *crc) +{ + *crc = CrcUpdate(*crc, buf, size); + return WriteBytes(s, buf, size); +} + + +static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s) +{ + UInt32 crc; + Byte header[XZ_STREAM_HEADER_SIZE]; + memcpy(header, XZ_SIG, XZ_SIG_SIZE); + header[XZ_SIG_SIZE] = (Byte)(f >> 8); + header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF); + crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE); + SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc); + return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE); +} + + +static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s) +{ + Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; + + unsigned pos = 1; + unsigned numFilters, i; + header[pos++] = p->flags; + + if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize); + if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize); + numFilters = XzBlock_GetNumFilters(p); + + for (i = 0; i < numFilters; i++) + { + const CXzFilter *f = &p->filters[i]; + pos += Xz_WriteVarInt(header + pos, f->id); + pos += Xz_WriteVarInt(header + pos, f->propsSize); + memcpy(header + pos, f->props, f->propsSize); + pos += f->propsSize; + } + + while ((pos & 3) != 0) + header[pos++] = 0; + + header[0] = (Byte)(pos >> 2); + SetUi32(header + pos, CrcCalc(header, pos)); + return WriteBytes(s, header, pos + 4); +} + + + + +typedef struct +{ + size_t numBlocks; + size_t size; + size_t allocated; + Byte *blocks; +} CXzEncIndex; + + +static void XzEncIndex_Construct(CXzEncIndex *p) +{ + p->numBlocks = 0; + p->size = 0; + p->allocated = 0; + p->blocks = NULL; +} + +static void XzEncIndex_Init(CXzEncIndex *p) +{ + p->numBlocks = 0; + p->size = 0; +} + +static void XzEncIndex_Free(CXzEncIndex *p, ISzAllocPtr alloc) +{ + if (p->blocks) + { + ISzAlloc_Free(alloc, p->blocks); + p->blocks = NULL; + } + p->numBlocks = 0; + p->size = 0; + p->allocated = 0; +} + + +static SRes XzEncIndex_ReAlloc(CXzEncIndex *p, size_t newSize, ISzAllocPtr alloc) +{ + Byte *blocks = (Byte *)ISzAlloc_Alloc(alloc, newSize); + if (!blocks) + return SZ_ERROR_MEM; + if (p->size != 0) + memcpy(blocks, p->blocks, p->size); + if (p->blocks) + ISzAlloc_Free(alloc, p->blocks); + p->blocks = blocks; + p->allocated = newSize; + return SZ_OK; +} + + +static SRes XzEncIndex_PreAlloc(CXzEncIndex *p, UInt64 numBlocks, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) +{ + UInt64 pos; + { + Byte buf[32]; + unsigned pos2 = Xz_WriteVarInt(buf, totalSize); + pos2 += Xz_WriteVarInt(buf + pos2, unpackSize); + pos = numBlocks * pos2; + } + + if (pos <= p->allocated - p->size) + return SZ_OK; + { + UInt64 newSize64 = p->size + pos; + size_t newSize = (size_t)newSize64; + if (newSize != newSize64) + return SZ_ERROR_MEM; + return XzEncIndex_ReAlloc(p, newSize, alloc); + } +} + + +static SRes XzEncIndex_AddIndexRecord(CXzEncIndex *p, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) +{ + Byte buf[32]; + unsigned pos = Xz_WriteVarInt(buf, totalSize); + pos += Xz_WriteVarInt(buf + pos, unpackSize); + + if (pos > p->allocated - p->size) + { + size_t newSize = p->allocated * 2 + 16 * 2; + if (newSize < p->size + pos) + return SZ_ERROR_MEM; + RINOK(XzEncIndex_ReAlloc(p, newSize, alloc)); + } + memcpy(p->blocks + p->size, buf, pos); + p->size += pos; + p->numBlocks++; + return SZ_OK; +} + + +static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, ISeqOutStream *s) +{ + Byte buf[32]; + UInt64 globalPos; + UInt32 crc = CRC_INIT_VAL; + unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); + + globalPos = pos; + buf[0] = 0; + RINOK(WriteBytesUpdateCrc(s, buf, pos, &crc)); + RINOK(WriteBytesUpdateCrc(s, p->blocks, p->size, &crc)); + globalPos += p->size; + + pos = XZ_GET_PAD_SIZE(globalPos); + buf[1] = 0; + buf[2] = 0; + buf[3] = 0; + globalPos += pos; + + crc = CrcUpdate(crc, buf + 4 - pos, pos); + SetUi32(buf + 4, CRC_GET_DIGEST(crc)); + + SetUi32(buf + 8 + 4, (UInt32)(globalPos >> 2)); + buf[8 + 8] = (Byte)(flags >> 8); + buf[8 + 9] = (Byte)(flags & 0xFF); + SetUi32(buf + 8, CrcCalc(buf + 8 + 4, 6)); + buf[8 + 10] = XZ_FOOTER_SIG_0; + buf[8 + 11] = XZ_FOOTER_SIG_1; + + return WriteBytes(s, buf + 4 - pos, pos + 4 + 12); +} + + + +/* ---------- CSeqCheckInStream ---------- */ + +typedef struct +{ + ISeqInStream vt; + ISeqInStream *realStream; + const Byte *data; + UInt64 limit; + UInt64 processed; + int realStreamFinished; + CXzCheck check; +} CSeqCheckInStream; + +static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned checkMode) +{ + p->limit = (UInt64)(Int64)-1; + p->processed = 0; + p->realStreamFinished = 0; + XzCheck_Init(&p->check, checkMode); +} + +static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest) +{ + XzCheck_Final(&p->check, digest); +} + +static SRes SeqCheckInStream_Read(const ISeqInStream *pp, void *data, size_t *size) +{ + CSeqCheckInStream *p = CONTAINER_FROM_VTBL(pp, CSeqCheckInStream, vt); + size_t size2 = *size; + SRes res = SZ_OK; + + if (p->limit != (UInt64)(Int64)-1) + { + UInt64 rem = p->limit - p->processed; + if (size2 > rem) + size2 = (size_t)rem; + } + if (size2 != 0) + { + if (p->realStream) + { + res = ISeqInStream_Read(p->realStream, data, &size2); + p->realStreamFinished = (size2 == 0) ? 1 : 0; + } + else + memcpy(data, p->data + (size_t)p->processed, size2); + XzCheck_Update(&p->check, data, size2); + p->processed += size2; + } + *size = size2; + return res; +} + + +/* ---------- CSeqSizeOutStream ---------- */ + +typedef struct +{ + ISeqOutStream vt; + ISeqOutStream *realStream; + Byte *outBuf; + size_t outBufLimit; + UInt64 processed; +} CSeqSizeOutStream; + +static size_t SeqSizeOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) +{ + CSeqSizeOutStream *p = CONTAINER_FROM_VTBL(pp, CSeqSizeOutStream, vt); + if (p->realStream) + size = ISeqOutStream_Write(p->realStream, data, size); + else + { + if (size > p->outBufLimit - (size_t)p->processed) + return 0; + memcpy(p->outBuf + (size_t)p->processed, data, size); + } + p->processed += size; + return size; +} + + +/* ---------- CSeqInFilter ---------- */ + +#define FILTER_BUF_SIZE (1 << 20) + +typedef struct +{ + ISeqInStream p; + ISeqInStream *realStream; + IStateCoder StateCoder; + Byte *buf; + size_t curPos; + size_t endPos; + int srcWasFinished; +} CSeqInFilter; + + +SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc); + +static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPtr alloc) +{ + if (!p->buf) + { + p->buf = (Byte *)ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); + if (!p->buf) + return SZ_ERROR_MEM; + } + p->curPos = p->endPos = 0; + p->srcWasFinished = 0; + RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, alloc)); + RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, alloc)); + p->StateCoder.Init(p->StateCoder.p); + return SZ_OK; +} + + +static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) +{ + CSeqInFilter *p = CONTAINER_FROM_VTBL(pp, CSeqInFilter, p); + size_t sizeOriginal = *size; + if (sizeOriginal == 0) + return SZ_OK; + *size = 0; + + for (;;) + { + if (!p->srcWasFinished && p->curPos == p->endPos) + { + p->curPos = 0; + p->endPos = FILTER_BUF_SIZE; + RINOK(ISeqInStream_Read(p->realStream, p->buf, &p->endPos)); + if (p->endPos == 0) + p->srcWasFinished = 1; + } + { + SizeT srcLen = p->endPos - p->curPos; + ECoderStatus status; + SRes res; + *size = sizeOriginal; + res = p->StateCoder.Code2(p->StateCoder.p, + (Byte *)data, size, + p->buf + p->curPos, &srcLen, + p->srcWasFinished, CODER_FINISH_ANY, + &status); + p->curPos += srcLen; + if (*size != 0 || srcLen == 0 || res != SZ_OK) + return res; + } + } +} + +static void SeqInFilter_Construct(CSeqInFilter *p) +{ + p->buf = NULL; + p->StateCoder.p = NULL; + p->p.Read = SeqInFilter_Read; +} + +static void SeqInFilter_Free(CSeqInFilter *p, ISzAllocPtr alloc) +{ + if (p->StateCoder.p) + { + p->StateCoder.Free(p->StateCoder.p, alloc); + p->StateCoder.p = NULL; + } + if (p->buf) + { + ISzAlloc_Free(alloc, p->buf); + p->buf = NULL; + } +} + + +/* ---------- CSbEncInStream ---------- */ + +#ifdef USE_SUBBLOCK + +typedef struct +{ + ISeqInStream vt; + ISeqInStream *inStream; + CSbEnc enc; +} CSbEncInStream; + +static SRes SbEncInStream_Read(const ISeqInStream *pp, void *data, size_t *size) +{ + CSbEncInStream *p = CONTAINER_FROM_VTBL(pp, CSbEncInStream, vt); + size_t sizeOriginal = *size; + if (sizeOriginal == 0) + return SZ_OK; + + for (;;) + { + if (p->enc.needRead && !p->enc.readWasFinished) + { + size_t processed = p->enc.needReadSizeMax; + RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed)); + p->enc.readPos += processed; + if (processed == 0) + { + p->enc.readWasFinished = True; + p->enc.isFinalFinished = True; + } + p->enc.needRead = False; + } + + *size = sizeOriginal; + RINOK(SbEnc_Read(&p->enc, data, size)); + if (*size != 0 || !p->enc.needRead) + return SZ_OK; + } +} + +void SbEncInStream_Construct(CSbEncInStream *p, ISzAllocPtr alloc) +{ + SbEnc_Construct(&p->enc, alloc); + p->vt.Read = SbEncInStream_Read; +} + +SRes SbEncInStream_Init(CSbEncInStream *p) +{ + return SbEnc_Init(&p->enc); +} + +void SbEncInStream_Free(CSbEncInStream *p) +{ + SbEnc_Free(&p->enc); +} + +#endif + + + +/* ---------- CXzProps ---------- */ + + +void XzFilterProps_Init(CXzFilterProps *p) +{ + p->id = 0; + p->delta = 0; + p->ip = 0; + p->ipDefined = False; +} + +void XzProps_Init(CXzProps *p) +{ + p->checkId = XZ_CHECK_CRC32; + p->blockSize = XZ_PROPS__BLOCK_SIZE__AUTO; + p->numBlockThreads_Reduced = -1; + p->numBlockThreads_Max = -1; + p->numTotalThreads = -1; + p->reduceSize = (UInt64)(Int64)-1; + p->forceWriteSizesInHeader = 0; + // p->forceWriteSizesInHeader = 1; + + XzFilterProps_Init(&p->filterProps); + Lzma2EncProps_Init(&p->lzma2Props); +} + + +static void XzEncProps_Normalize_Fixed(CXzProps *p) +{ + UInt64 fileSize; + int t1, t1n, t2, t2r, t3; + { + CLzma2EncProps tp = p->lzma2Props; + if (tp.numTotalThreads <= 0) + tp.numTotalThreads = p->numTotalThreads; + Lzma2EncProps_Normalize(&tp); + t1n = tp.numTotalThreads; + } + + t1 = p->lzma2Props.numTotalThreads; + t2 = p->numBlockThreads_Max; + t3 = p->numTotalThreads; + + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; + + if (t3 <= 0) + { + if (t2 <= 0) + t2 = 1; + t3 = t1n * t2; + } + else if (t2 <= 0) + { + t2 = t3 / t1n; + if (t2 == 0) + { + t1 = 1; + t2 = t3; + } + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; + } + else if (t1 <= 0) + { + t1 = t3 / t2; + if (t1 == 0) + t1 = 1; + } + else + t3 = t1n * t2; + + p->lzma2Props.numTotalThreads = t1; + + t2r = t2; + + fileSize = p->reduceSize; + + if ((p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) + p->lzma2Props.lzmaProps.reduceSize = p->blockSize; + + Lzma2EncProps_Normalize(&p->lzma2Props); + + t1 = p->lzma2Props.numTotalThreads; + + { + if (t2 > 1 && fileSize != (UInt64)(Int64)-1) + { + UInt64 numBlocks = fileSize / p->blockSize; + if (numBlocks * p->blockSize != fileSize) + numBlocks++; + if (numBlocks < (unsigned)t2) + { + t2r = (unsigned)numBlocks; + if (t2r == 0) + t2r = 1; + t3 = t1 * t2r; + } + } + } + + p->numBlockThreads_Max = t2; + p->numBlockThreads_Reduced = t2r; + p->numTotalThreads = t3; +} + + +static void XzProps_Normalize(CXzProps *p) +{ + /* we normalize xzProps properties, but we normalize only some of CXzProps::lzma2Props properties. + Lzma2Enc_SetProps() will normalize lzma2Props later. */ + + if (p->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID) + { + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + p->numBlockThreads_Reduced = 1; + p->numBlockThreads_Max = 1; + if (p->lzma2Props.numTotalThreads <= 0) + p->lzma2Props.numTotalThreads = p->numTotalThreads; + return; + } + else + { + CLzma2EncProps *lzma2 = &p->lzma2Props; + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + { + // xz-auto + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + { + // if (xz-auto && lzma2-solid) - we use solid for both + p->blockSize = XZ_PROPS__BLOCK_SIZE__SOLID; + p->numBlockThreads_Reduced = 1; + p->numBlockThreads_Max = 1; + if (p->lzma2Props.numTotalThreads <= 0) + p->lzma2Props.numTotalThreads = p->numTotalThreads; + } + else + { + // if (xz-auto && (lzma2-auto || lzma2-fixed_) + // we calculate block size for lzma2 and use that block size for xz, lzma2 uses single-chunk per block + CLzma2EncProps tp = p->lzma2Props; + if (tp.numTotalThreads <= 0) + tp.numTotalThreads = p->numTotalThreads; + + Lzma2EncProps_Normalize(&tp); + + p->blockSize = tp.blockSize; // fixed or solid + p->numBlockThreads_Reduced = tp.numBlockThreads_Reduced; + p->numBlockThreads_Max = tp.numBlockThreads_Max; + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + lzma2->blockSize = tp.blockSize; // fixed or solid, LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + if (lzma2->lzmaProps.reduceSize > tp.blockSize && tp.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + lzma2->lzmaProps.reduceSize = tp.blockSize; + lzma2->numBlockThreads_Reduced = 1; + lzma2->numBlockThreads_Max = 1; + return; + } + } + else + { + // xz-fixed + // we can use xz::reduceSize or xz::blockSize as base for lzmaProps::reduceSize + + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + { + UInt64 r = p->reduceSize; + if (r > p->blockSize || r == (UInt64)(Int64)-1) + r = p->blockSize; + lzma2->lzmaProps.reduceSize = r; + } + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + lzma2->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; + else if (lzma2->blockSize > p->blockSize && lzma2->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + lzma2->blockSize = p->blockSize; + + XzEncProps_Normalize_Fixed(p); + } + } +} + + +/* ---------- CLzma2WithFilters ---------- */ + +typedef struct +{ + CLzma2EncHandle lzma2; + CSeqInFilter filter; + + #ifdef USE_SUBBLOCK + CSbEncInStream sb; + #endif +} CLzma2WithFilters; + + +static void Lzma2WithFilters_Construct(CLzma2WithFilters *p) +{ + p->lzma2 = NULL; + SeqInFilter_Construct(&p->filter); + + #ifdef USE_SUBBLOCK + SbEncInStream_Construct(&p->sb, alloc); + #endif +} + + +static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p, ISzAllocPtr alloc, ISzAllocPtr bigAlloc) +{ + if (!p->lzma2) + { + p->lzma2 = Lzma2Enc_Create(alloc, bigAlloc); + if (!p->lzma2) + return SZ_ERROR_MEM; + } + return SZ_OK; +} + + +static void Lzma2WithFilters_Free(CLzma2WithFilters *p, ISzAllocPtr alloc) +{ + #ifdef USE_SUBBLOCK + SbEncInStream_Free(&p->sb); + #endif + + SeqInFilter_Free(&p->filter, alloc); + if (p->lzma2) + { + Lzma2Enc_Destroy(p->lzma2); + p->lzma2 = NULL; + } +} + + +typedef struct +{ + UInt64 unpackSize; + UInt64 totalSize; + size_t headerSize; +} CXzEncBlockInfo; + + +static SRes Xz_CompressBlock( + CLzma2WithFilters *lzmaf, + + ISeqOutStream *outStream, + Byte *outBufHeader, + Byte *outBufData, size_t outBufDataLimit, + + ISeqInStream *inStream, + // UInt64 expectedSize, + const Byte *inBuf, // used if (!inStream) + size_t inBufSize, // used if (!inStream), it's block size, props->blockSize is ignored + + const CXzProps *props, + ICompressProgress *progress, + int *inStreamFinished, /* only for inStream version */ + CXzEncBlockInfo *blockSizes, + ISzAllocPtr alloc, + ISzAllocPtr allocBig) +{ + CSeqCheckInStream checkInStream; + CSeqSizeOutStream seqSizeOutStream; + CXzBlock block; + unsigned filterIndex = 0; + CXzFilter *filter = NULL; + const CXzFilterProps *fp = &props->filterProps; + if (fp->id == 0) + fp = NULL; + + *inStreamFinished = False; + + RINOK(Lzma2WithFilters_Create(lzmaf, alloc, allocBig)); + + RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, &props->lzma2Props)); + + XzBlock_ClearFlags(&block); + XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0)); + + if (fp) + { + filter = &block.filters[filterIndex++]; + filter->id = fp->id; + filter->propsSize = 0; + + if (fp->id == XZ_ID_Delta) + { + filter->props[0] = (Byte)(fp->delta - 1); + filter->propsSize = 1; + } + else if (fp->ipDefined) + { + SetUi32(filter->props, fp->ip); + filter->propsSize = 4; + } + } + + { + CXzFilter *f = &block.filters[filterIndex++]; + f->id = XZ_ID_LZMA2; + f->propsSize = 1; + f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); + } + + seqSizeOutStream.vt.Write = SeqSizeOutStream_Write; + seqSizeOutStream.realStream = outStream; + seqSizeOutStream.outBuf = outBufData; + seqSizeOutStream.outBufLimit = outBufDataLimit; + seqSizeOutStream.processed = 0; + + /* + if (expectedSize != (UInt64)(Int64)-1) + { + block.unpackSize = expectedSize; + if (props->blockSize != (UInt64)(Int64)-1) + if (expectedSize > props->blockSize) + block.unpackSize = props->blockSize; + XzBlock_SetHasUnpackSize(&block); + } + */ + + if (outStream) + { + RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + } + + checkInStream.vt.Read = SeqCheckInStream_Read; + SeqCheckInStream_Init(&checkInStream, props->checkId); + + checkInStream.realStream = inStream; + checkInStream.data = inBuf; + checkInStream.limit = props->blockSize; + if (!inStream) + checkInStream.limit = inBufSize; + + if (fp) + { + #ifdef USE_SUBBLOCK + if (fp->id == XZ_ID_Subblock) + { + lzmaf->sb.inStream = &checkInStream.vt; + RINOK(SbEncInStream_Init(&lzmaf->sb)); + } + else + #endif + { + lzmaf->filter.realStream = &checkInStream.vt; + RINOK(SeqInFilter_Init(&lzmaf->filter, filter, alloc)); + } + } + + { + SRes res; + Byte *outBuf = NULL; + size_t outSize = 0; + BoolInt useStream = (fp || inStream); + // useStream = True; + + if (!useStream) + { + XzCheck_Update(&checkInStream.check, inBuf, inBufSize); + checkInStream.processed = inBufSize; + } + + if (!outStream) + { + outBuf = seqSizeOutStream.outBuf; // + (size_t)seqSizeOutStream.processed; + outSize = seqSizeOutStream.outBufLimit; // - (size_t)seqSizeOutStream.processed; + } + + res = Lzma2Enc_Encode2(lzmaf->lzma2, + outBuf ? NULL : &seqSizeOutStream.vt, + outBuf, + outBuf ? &outSize : NULL, + + useStream ? + (fp ? + ( + #ifdef USE_SUBBLOCK + (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.vt: + #endif + &lzmaf->filter.p) : + &checkInStream.vt) : NULL, + + useStream ? NULL : inBuf, + useStream ? 0 : inBufSize, + + progress); + + if (outBuf) + seqSizeOutStream.processed += outSize; + + RINOK(res); + blockSizes->unpackSize = checkInStream.processed; + } + { + Byte buf[4 + 64]; + unsigned padSize = XZ_GET_PAD_SIZE(seqSizeOutStream.processed); + UInt64 packSize = seqSizeOutStream.processed; + + buf[0] = 0; + buf[1] = 0; + buf[2] = 0; + buf[3] = 0; + + SeqCheckInStream_GetDigest(&checkInStream, buf + 4); + RINOK(WriteBytes(&seqSizeOutStream.vt, buf + (4 - padSize), padSize + XzFlags_GetCheckSize((CXzStreamFlags)props->checkId))); + + blockSizes->totalSize = seqSizeOutStream.processed - padSize; + + if (!outStream) + { + seqSizeOutStream.outBuf = outBufHeader; + seqSizeOutStream.outBufLimit = XZ_BLOCK_HEADER_SIZE_MAX; + seqSizeOutStream.processed = 0; + + block.unpackSize = blockSizes->unpackSize; + XzBlock_SetHasUnpackSize(&block); + + block.packSize = packSize; + XzBlock_SetHasPackSize(&block); + + RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + + blockSizes->headerSize = (size_t)seqSizeOutStream.processed; + blockSizes->totalSize += seqSizeOutStream.processed; + } + } + + if (inStream) + *inStreamFinished = checkInStream.realStreamFinished; + else + { + *inStreamFinished = False; + if (checkInStream.processed != inBufSize) + return SZ_ERROR_FAIL; + } + + return SZ_OK; +} + + + +typedef struct +{ + ICompressProgress vt; + ICompressProgress *progress; + UInt64 inOffset; + UInt64 outOffset; +} CCompressProgress_XzEncOffset; + + +static SRes CompressProgress_XzEncOffset_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) +{ + const CCompressProgress_XzEncOffset *p = CONTAINER_FROM_VTBL(pp, CCompressProgress_XzEncOffset, vt); + inSize += p->inOffset; + outSize += p->outOffset; + return ICompressProgress_Progress(p->progress, inSize, outSize); +} + + + + +typedef struct +{ + ISzAllocPtr alloc; + ISzAllocPtr allocBig; + + CXzProps xzProps; + UInt64 expectedDataSize; + + CXzEncIndex xzIndex; + + CLzma2WithFilters lzmaf_Items[MTCODER__THREADS_MAX]; + + size_t outBufSize; /* size of allocated outBufs[i] */ + Byte *outBufs[MTCODER__BLOCKS_MAX]; + + #ifndef _7ZIP_ST + unsigned checkType; + ISeqOutStream *outStream; + BoolInt mtCoder_WasConstructed; + CMtCoder mtCoder; + CXzEncBlockInfo EncBlocks[MTCODER__BLOCKS_MAX]; + #endif + +} CXzEnc; + + +static void XzEnc_Construct(CXzEnc *p) +{ + unsigned i; + + XzEncIndex_Construct(&p->xzIndex); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + Lzma2WithFilters_Construct(&p->lzmaf_Items[i]); + + #ifndef _7ZIP_ST + p->mtCoder_WasConstructed = False; + { + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->outBufs[i] = NULL; + p->outBufSize = 0; + } + #endif +} + + +static void XzEnc_FreeOutBufs(CXzEnc *p) +{ + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + if (p->outBufs[i]) + { + ISzAlloc_Free(p->alloc, p->outBufs[i]); + p->outBufs[i] = NULL; + } + p->outBufSize = 0; +} + + +static void XzEnc_Free(CXzEnc *p, ISzAllocPtr alloc) +{ + unsigned i; + + XzEncIndex_Free(&p->xzIndex, alloc); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + Lzma2WithFilters_Free(&p->lzmaf_Items[i], alloc); + + #ifndef _7ZIP_ST + if (p->mtCoder_WasConstructed) + { + MtCoder_Destruct(&p->mtCoder); + p->mtCoder_WasConstructed = False; + } + XzEnc_FreeOutBufs(p); + #endif +} + + +CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) +{ + CXzEnc *p = (CXzEnc *)ISzAlloc_Alloc(alloc, sizeof(CXzEnc)); + if (!p) + return NULL; + XzEnc_Construct(p); + XzProps_Init(&p->xzProps); + XzProps_Normalize(&p->xzProps); + p->expectedDataSize = (UInt64)(Int64)-1; + p->alloc = alloc; + p->allocBig = allocBig; + return p; +} + + +void XzEnc_Destroy(CXzEncHandle pp) +{ + CXzEnc *p = (CXzEnc *)pp; + XzEnc_Free(p, p->alloc); + ISzAlloc_Free(p->alloc, p); +} + + +SRes XzEnc_SetProps(CXzEncHandle pp, const CXzProps *props) +{ + CXzEnc *p = (CXzEnc *)pp; + p->xzProps = *props; + XzProps_Normalize(&p->xzProps); + return SZ_OK; +} + + +void XzEnc_SetDataSize(CXzEncHandle pp, UInt64 expectedDataSiize) +{ + CXzEnc *p = (CXzEnc *)pp; + p->expectedDataSize = expectedDataSiize; +} + + + + +#ifndef _7ZIP_ST + +static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, + const Byte *src, size_t srcSize, int finished) +{ + CXzEnc *me = (CXzEnc *)pp; + SRes res; + CMtProgressThunk progressThunk; + + Byte *dest = me->outBufs[outBufIndex]; + + UNUSED_VAR(finished) + + { + CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; + bInfo->totalSize = 0; + bInfo->unpackSize = 0; + bInfo->headerSize = 0; + } + + if (!dest) + { + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); + if (!dest) + return SZ_ERROR_MEM; + me->outBufs[outBufIndex] = dest; + } + + MtProgressThunk_CreateVTable(&progressThunk); + progressThunk.mtProgress = &me->mtCoder.mtProgress; + MtProgressThunk_Init(&progressThunk); + + { + CXzEncBlockInfo blockSizes; + int inStreamFinished; + + res = Xz_CompressBlock( + &me->lzmaf_Items[coderIndex], + + NULL, + dest, + dest + XZ_BLOCK_HEADER_SIZE_MAX, me->outBufSize - XZ_BLOCK_HEADER_SIZE_MAX, + + NULL, + // srcSize, // expectedSize + src, srcSize, + + &me->xzProps, + &progressThunk.vt, + &inStreamFinished, + &blockSizes, + me->alloc, + me->allocBig); + + if (res == SZ_OK) + me->EncBlocks[outBufIndex] = blockSizes; + + return res; + } +} + + +static SRes XzEnc_MtCallback_Write(void *pp, unsigned outBufIndex) +{ + CXzEnc *me = (CXzEnc *)pp; + + const CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; + const Byte *data = me->outBufs[outBufIndex]; + + RINOK(WriteBytes(me->outStream, data, bInfo->headerSize)); + + { + UInt64 totalPackFull = bInfo->totalSize + XZ_GET_PAD_SIZE(bInfo->totalSize); + RINOK(WriteBytes(me->outStream, data + XZ_BLOCK_HEADER_SIZE_MAX, (size_t)totalPackFull - bInfo->headerSize)); + } + + return XzEncIndex_AddIndexRecord(&me->xzIndex, bInfo->unpackSize, bInfo->totalSize, me->alloc); +} + +#endif + + + +SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) +{ + CXzEnc *p = (CXzEnc *)pp; + + const CXzProps *props = &p->xzProps; + + XzEncIndex_Init(&p->xzIndex); + { + UInt64 numBlocks = 1; + UInt64 blockSize = props->blockSize; + + if (blockSize != XZ_PROPS__BLOCK_SIZE__SOLID + && props->reduceSize != (UInt64)(Int64)-1) + { + numBlocks = props->reduceSize / blockSize; + if (numBlocks * blockSize != props->reduceSize) + numBlocks++; + } + else + blockSize = (UInt64)1 << 62; + + RINOK(XzEncIndex_PreAlloc(&p->xzIndex, numBlocks, blockSize, XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(blockSize), p->alloc)); + } + + RINOK(Xz_WriteHeader((CXzStreamFlags)props->checkId, outStream)); + + + #ifndef _7ZIP_ST + if (props->numBlockThreads_Reduced > 1) + { + IMtCoderCallback2 vt; + + if (!p->mtCoder_WasConstructed) + { + p->mtCoder_WasConstructed = True; + MtCoder_Construct(&p->mtCoder); + } + + vt.Code = XzEnc_MtCallback_Code; + vt.Write = XzEnc_MtCallback_Write; + + p->checkType = props->checkId; + p->xzProps = *props; + + p->outStream = outStream; + + p->mtCoder.allocBig = p->allocBig; + p->mtCoder.progress = progress; + p->mtCoder.inStream = inStream; + p->mtCoder.inData = NULL; + p->mtCoder.inDataSize = 0; + p->mtCoder.mtCallback = &vt; + p->mtCoder.mtCallbackObject = p; + + if ( props->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID + || props->blockSize == XZ_PROPS__BLOCK_SIZE__AUTO) + return SZ_ERROR_FAIL; + + p->mtCoder.blockSize = (size_t)props->blockSize; + if (p->mtCoder.blockSize != props->blockSize) + return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ + + { + size_t destBlockSize = XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(p->mtCoder.blockSize); + if (destBlockSize < p->mtCoder.blockSize) + return SZ_ERROR_PARAM; + if (p->outBufSize != destBlockSize) + XzEnc_FreeOutBufs(p); + p->outBufSize = destBlockSize; + } + + p->mtCoder.numThreadsMax = props->numBlockThreads_Max; + p->mtCoder.expectedDataSize = p->expectedDataSize; + + RINOK(MtCoder_Code(&p->mtCoder)); + } + else + #endif + { + int writeStartSizes; + CCompressProgress_XzEncOffset progress2; + Byte *bufData = NULL; + size_t bufSize = 0; + + progress2.vt.Progress = CompressProgress_XzEncOffset_Progress; + progress2.inOffset = 0; + progress2.outOffset = 0; + progress2.progress = progress; + + writeStartSizes = 0; + + if (props->blockSize != XZ_PROPS__BLOCK_SIZE__SOLID) + { + writeStartSizes = (props->forceWriteSizesInHeader > 0); + + if (writeStartSizes) + { + size_t t2; + size_t t = (size_t)props->blockSize; + if (t != props->blockSize) + return SZ_ERROR_PARAM; + t = XZ_GET_MAX_BLOCK_PACK_SIZE(t); + if (t < props->blockSize) + return SZ_ERROR_PARAM; + t2 = XZ_BLOCK_HEADER_SIZE_MAX + t; + if (!p->outBufs[0] || t2 != p->outBufSize) + { + XzEnc_FreeOutBufs(p); + p->outBufs[0] = (Byte *)ISzAlloc_Alloc(p->alloc, t2); + if (!p->outBufs[0]) + return SZ_ERROR_MEM; + p->outBufSize = t2; + } + bufData = p->outBufs[0] + XZ_BLOCK_HEADER_SIZE_MAX; + bufSize = t; + } + } + + for (;;) + { + CXzEncBlockInfo blockSizes; + int inStreamFinished; + + /* + UInt64 rem = (UInt64)(Int64)-1; + if (props->reduceSize != (UInt64)(Int64)-1 + && props->reduceSize >= progress2.inOffset) + rem = props->reduceSize - progress2.inOffset; + */ + + blockSizes.headerSize = 0; // for GCC + + RINOK(Xz_CompressBlock( + &p->lzmaf_Items[0], + + writeStartSizes ? NULL : outStream, + writeStartSizes ? p->outBufs[0] : NULL, + bufData, bufSize, + + inStream, + // rem, + NULL, 0, + + props, + progress ? &progress2.vt : NULL, + &inStreamFinished, + &blockSizes, + p->alloc, + p->allocBig)); + + { + UInt64 totalPackFull = blockSizes.totalSize + XZ_GET_PAD_SIZE(blockSizes.totalSize); + + if (writeStartSizes) + { + RINOK(WriteBytes(outStream, p->outBufs[0], blockSizes.headerSize)); + RINOK(WriteBytes(outStream, bufData, (size_t)totalPackFull - blockSizes.headerSize)); + } + + RINOK(XzEncIndex_AddIndexRecord(&p->xzIndex, blockSizes.unpackSize, blockSizes.totalSize, p->alloc)); + + progress2.inOffset += blockSizes.unpackSize; + progress2.outOffset += totalPackFull; + } + + if (inStreamFinished) + break; + } + } + + return XzEncIndex_WriteFooter(&p->xzIndex, (CXzStreamFlags)props->checkId, outStream); +} + + +#include "Alloc.h" + +SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, + const CXzProps *props, ICompressProgress *progress) +{ + SRes res; + CXzEncHandle xz = XzEnc_Create(&g_Alloc, &g_BigAlloc); + if (!xz) + return SZ_ERROR_MEM; + res = XzEnc_SetProps(xz, props); + if (res == SZ_OK) + res = XzEnc_Encode(xz, outStream, inStream, progress); + XzEnc_Destroy(xz); + return res; +} + + +SRes Xz_EncodeEmpty(ISeqOutStream *outStream) +{ + SRes res; + CXzEncIndex xzIndex; + XzEncIndex_Construct(&xzIndex); + res = Xz_WriteHeader((CXzStreamFlags)0, outStream); + if (res == SZ_OK) + res = XzEncIndex_WriteFooter(&xzIndex, (CXzStreamFlags)0, outStream); + XzEncIndex_Free(&xzIndex, NULL); // g_Alloc + return res; +} diff --git a/code/nel/3rdparty/seven_zip/XzEnc.h b/code/nel/3rdparty/seven_zip/XzEnc.h new file mode 100644 index 000000000..0c29e7e1e --- /dev/null +++ b/code/nel/3rdparty/seven_zip/XzEnc.h @@ -0,0 +1,60 @@ +/* XzEnc.h -- Xz Encode +2017-06-27 : Igor Pavlov : Public domain */ + +#ifndef __XZ_ENC_H +#define __XZ_ENC_H + +#include "Lzma2Enc.h" + +#include "Xz.h" + +EXTERN_C_BEGIN + + +#define XZ_PROPS__BLOCK_SIZE__AUTO LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO +#define XZ_PROPS__BLOCK_SIZE__SOLID LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + + +typedef struct +{ + UInt32 id; + UInt32 delta; + UInt32 ip; + int ipDefined; +} CXzFilterProps; + +void XzFilterProps_Init(CXzFilterProps *p); + + +typedef struct +{ + CLzma2EncProps lzma2Props; + CXzFilterProps filterProps; + unsigned checkId; + UInt64 blockSize; + int numBlockThreads_Reduced; + int numBlockThreads_Max; + int numTotalThreads; + int forceWriteSizesInHeader; + UInt64 reduceSize; +} CXzProps; + +void XzProps_Init(CXzProps *p); + + +typedef void * CXzEncHandle; + +CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); +void XzEnc_Destroy(CXzEncHandle p); +SRes XzEnc_SetProps(CXzEncHandle p, const CXzProps *props); +void XzEnc_SetDataSize(CXzEncHandle p, UInt64 expectedDataSiize); +SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress); + +SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, + const CXzProps *props, ICompressProgress *progress); + +SRes Xz_EncodeEmpty(ISeqOutStream *outStream); + +EXTERN_C_END + +#endif diff --git a/code/nel/3rdparty/seven_zip/XzIn.c b/code/nel/3rdparty/seven_zip/XzIn.c new file mode 100644 index 000000000..ff48e2dd4 --- /dev/null +++ b/code/nel/3rdparty/seven_zip/XzIn.c @@ -0,0 +1,319 @@ +/* XzIn.c - Xz input +2018-07-04 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#include + +#include "7zCrc.h" +#include "CpuArch.h" +#include "Xz.h" + +/* +#define XZ_FOOTER_SIG_CHECK(p) (memcmp((p), XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0) +*/ +#define XZ_FOOTER_SIG_CHECK(p) ((p)[0] == XZ_FOOTER_SIG_0 && (p)[1] == XZ_FOOTER_SIG_1) + + +SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream) +{ + Byte sig[XZ_STREAM_HEADER_SIZE]; + RINOK(SeqInStream_Read2(inStream, sig, XZ_STREAM_HEADER_SIZE, SZ_ERROR_NO_ARCHIVE)); + if (memcmp(sig, XZ_SIG, XZ_SIG_SIZE) != 0) + return SZ_ERROR_NO_ARCHIVE; + return Xz_ParseHeader(p, sig); +} + +#define READ_VARINT_AND_CHECK(buf, pos, size, res) \ + { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ + if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } + +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes) +{ + Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; + unsigned headerSize; + *headerSizeRes = 0; + RINOK(SeqInStream_ReadByte(inStream, &header[0])); + headerSize = (unsigned)header[0]; + if (headerSize == 0) + { + *headerSizeRes = 1; + *isIndex = True; + return SZ_OK; + } + + *isIndex = False; + headerSize = (headerSize << 2) + 4; + *headerSizeRes = headerSize; + RINOK(SeqInStream_Read(inStream, header + 1, headerSize - 1)); + return XzBlock_Parse(p, header); +} + +#define ADD_SIZE_CHECK(size, val) \ + { UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; } + +UInt64 Xz_GetUnpackSize(const CXzStream *p) +{ + UInt64 size = 0; + size_t i; + for (i = 0; i < p->numBlocks; i++) + ADD_SIZE_CHECK(size, p->blocks[i].unpackSize); + return size; +} + +UInt64 Xz_GetPackSize(const CXzStream *p) +{ + UInt64 size = 0; + size_t i; + for (i = 0; i < p->numBlocks; i++) + ADD_SIZE_CHECK(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3); + return size; +} + +/* +SRes XzBlock_ReadFooter(CXzBlock *p, CXzStreamFlags f, ISeqInStream *inStream) +{ + return SeqInStream_Read(inStream, p->check, XzFlags_GetCheckSize(f)); +} +*/ + +static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPtr alloc) +{ + size_t numBlocks, pos = 1; + UInt32 crc; + + if (size < 5 || buf[0] != 0) + return SZ_ERROR_ARCHIVE; + + size -= 4; + crc = CrcCalc(buf, size); + if (crc != GetUi32(buf + size)) + return SZ_ERROR_ARCHIVE; + + { + UInt64 numBlocks64; + READ_VARINT_AND_CHECK(buf, pos, size, &numBlocks64); + numBlocks = (size_t)numBlocks64; + if (numBlocks != numBlocks64 || numBlocks * 2 > size) + return SZ_ERROR_ARCHIVE; + } + + Xz_Free(p, alloc); + if (numBlocks != 0) + { + size_t i; + p->numBlocks = numBlocks; + p->blocks = (CXzBlockSizes *)ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); + if (!p->blocks) + return SZ_ERROR_MEM; + for (i = 0; i < numBlocks; i++) + { + CXzBlockSizes *block = &p->blocks[i]; + READ_VARINT_AND_CHECK(buf, pos, size, &block->totalSize); + READ_VARINT_AND_CHECK(buf, pos, size, &block->unpackSize); + if (block->totalSize == 0) + return SZ_ERROR_ARCHIVE; + } + } + while ((pos & 3) != 0) + if (buf[pos++] != 0) + return SZ_ERROR_ARCHIVE; + return (pos == size) ? SZ_OK : SZ_ERROR_ARCHIVE; +} + +static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, ISzAllocPtr alloc) +{ + SRes res; + size_t size; + Byte *buf; + if (indexSize > ((UInt32)1 << 31)) + return SZ_ERROR_UNSUPPORTED; + size = (size_t)indexSize; + if (size != indexSize) + return SZ_ERROR_UNSUPPORTED; + buf = (Byte *)ISzAlloc_Alloc(alloc, size); + if (!buf) + return SZ_ERROR_MEM; + res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED); + if (res == SZ_OK) + res = Xz_ReadIndex2(p, buf, size, alloc); + ISzAlloc_Free(alloc, buf); + return res; +} + +static SRes LookInStream_SeekRead_ForArc(ILookInStream *stream, UInt64 offset, void *buf, size_t size) +{ + RINOK(LookInStream_SeekTo(stream, offset)); + return LookInStream_Read(stream, buf, size); + /* return LookInStream_Read2(stream, buf, size, SZ_ERROR_NO_ARCHIVE); */ +} + +static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOffset, ISzAllocPtr alloc) +{ + UInt64 indexSize; + Byte buf[XZ_STREAM_FOOTER_SIZE]; + UInt64 pos = *startOffset; + + if ((pos & 3) != 0 || pos < XZ_STREAM_FOOTER_SIZE) + return SZ_ERROR_NO_ARCHIVE; + + pos -= XZ_STREAM_FOOTER_SIZE; + RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); + + if (!XZ_FOOTER_SIG_CHECK(buf + 10)) + { + UInt32 total = 0; + pos += XZ_STREAM_FOOTER_SIZE; + + for (;;) + { + size_t i; + #define TEMP_BUF_SIZE (1 << 10) + Byte temp[TEMP_BUF_SIZE]; + + i = (pos > TEMP_BUF_SIZE) ? TEMP_BUF_SIZE : (size_t)pos; + pos -= i; + RINOK(LookInStream_SeekRead_ForArc(stream, pos, temp, i)); + total += (UInt32)i; + for (; i != 0; i--) + if (temp[i - 1] != 0) + break; + if (i != 0) + { + if ((i & 3) != 0) + return SZ_ERROR_NO_ARCHIVE; + pos += i; + break; + } + if (pos < XZ_STREAM_FOOTER_SIZE || total > (1 << 16)) + return SZ_ERROR_NO_ARCHIVE; + } + + if (pos < XZ_STREAM_FOOTER_SIZE) + return SZ_ERROR_NO_ARCHIVE; + pos -= XZ_STREAM_FOOTER_SIZE; + RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); + if (!XZ_FOOTER_SIG_CHECK(buf + 10)) + return SZ_ERROR_NO_ARCHIVE; + } + + p->flags = (CXzStreamFlags)GetBe16(buf + 8); + + if (!XzFlags_IsSupported(p->flags)) + return SZ_ERROR_UNSUPPORTED; + + if (GetUi32(buf) != CrcCalc(buf + 4, 6)) + return SZ_ERROR_ARCHIVE; + + indexSize = ((UInt64)GetUi32(buf + 4) + 1) << 2; + + if (pos < indexSize) + return SZ_ERROR_ARCHIVE; + + pos -= indexSize; + RINOK(LookInStream_SeekTo(stream, pos)); + RINOK(Xz_ReadIndex(p, stream, indexSize, alloc)); + + { + UInt64 totalSize = Xz_GetPackSize(p); + if (totalSize == XZ_SIZE_OVERFLOW + || totalSize >= ((UInt64)1 << 63) + || pos < totalSize + XZ_STREAM_HEADER_SIZE) + return SZ_ERROR_ARCHIVE; + pos -= (totalSize + XZ_STREAM_HEADER_SIZE); + RINOK(LookInStream_SeekTo(stream, pos)); + *startOffset = pos; + } + { + CXzStreamFlags headerFlags; + CSecToRead secToRead; + SecToRead_CreateVTable(&secToRead); + secToRead.realStream = stream; + + RINOK(Xz_ReadHeader(&headerFlags, &secToRead.vt)); + return (p->flags == headerFlags) ? SZ_OK : SZ_ERROR_ARCHIVE; + } +} + + +/* ---------- Xz Streams ---------- */ + +void Xzs_Construct(CXzs *p) +{ + p->num = p->numAllocated = 0; + p->streams = 0; +} + +void Xzs_Free(CXzs *p, ISzAllocPtr alloc) +{ + size_t i; + for (i = 0; i < p->num; i++) + Xz_Free(&p->streams[i], alloc); + ISzAlloc_Free(alloc, p->streams); + p->num = p->numAllocated = 0; + p->streams = 0; +} + +UInt64 Xzs_GetNumBlocks(const CXzs *p) +{ + UInt64 num = 0; + size_t i; + for (i = 0; i < p->num; i++) + num += p->streams[i].numBlocks; + return num; +} + +UInt64 Xzs_GetUnpackSize(const CXzs *p) +{ + UInt64 size = 0; + size_t i; + for (i = 0; i < p->num; i++) + ADD_SIZE_CHECK(size, Xz_GetUnpackSize(&p->streams[i])); + return size; +} + +/* +UInt64 Xzs_GetPackSize(const CXzs *p) +{ + UInt64 size = 0; + size_t i; + for (i = 0; i < p->num; i++) + ADD_SIZE_CHECK(size, Xz_GetTotalSize(&p->streams[i])); + return size; +} +*/ + +SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc) +{ + Int64 endOffset = 0; + RINOK(ILookInStream_Seek(stream, &endOffset, SZ_SEEK_END)); + *startOffset = endOffset; + for (;;) + { + CXzStream st; + SRes res; + Xz_Construct(&st); + res = Xz_ReadBackward(&st, stream, startOffset, alloc); + st.startOffset = *startOffset; + RINOK(res); + if (p->num == p->numAllocated) + { + size_t newNum = p->num + p->num / 4 + 1; + Byte *data = (Byte *)ISzAlloc_Alloc(alloc, newNum * sizeof(CXzStream)); + if (!data) + return SZ_ERROR_MEM; + p->numAllocated = newNum; + if (p->num != 0) + memcpy(data, p->streams, p->num * sizeof(CXzStream)); + ISzAlloc_Free(alloc, p->streams); + p->streams = (CXzStream *)data; + } + p->streams[p->num++] = st; + if (*startOffset == 0) + break; + RINOK(LookInStream_SeekTo(stream, *startOffset)); + if (progress && ICompressProgress_Progress(progress, endOffset - *startOffset, (UInt64)(Int64)-1) != SZ_OK) + return SZ_ERROR_PROGRESS; + } + return SZ_OK; +} diff --git a/code/ryzom/tools/patch_gen/patch_gen_common.cpp b/code/ryzom/tools/patch_gen/patch_gen_common.cpp index 18d1e76ec..4ef82a350 100644 --- a/code/ryzom/tools/patch_gen/patch_gen_common.cpp +++ b/code/ryzom/tools/patch_gen/patch_gen_common.cpp @@ -26,8 +26,8 @@ #include "nel/misc/file.h" #include "nel/misc/command.h" #include "nel/misc/sstring.h" +#include "nel/misc/seven_zip.h" #include "game_share/singleton_registry.h" -#include "seven_zip.h" using namespace std; using namespace NLMISC; From b32b1becec0f63e67e57eecb16076b249db33fff Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 27 Nov 2019 17:38:29 +0800 Subject: [PATCH 26/27] Decouple downloader from streamed package manager --- .../include/nel/misc/http_package_provider.h | 50 ++++++ .../nel/misc/i_streamed_package_provider.h | 44 ++++++ code/nel/include/nel/misc/streamed_package.h | 2 + .../nel/misc/streamed_package_manager.h | 15 +- code/nel/include/nel/misc/types_nl.h | 6 + code/nel/src/misc/http_package_provider.cpp | 145 ++++++++++++++++++ .../src/misc/i_streamed_package_provider.cpp | 37 +++++ .../nel/src/misc/streamed_package_manager.cpp | 102 +----------- code/ryzom/client/src/global.cpp | 3 + code/ryzom/client/src/global.h | 8 + code/ryzom/client/src/init.cpp | 10 +- code/ryzom/client/src/release.cpp | 2 + 12 files changed, 319 insertions(+), 105 deletions(-) create mode 100644 code/nel/include/nel/misc/http_package_provider.h create mode 100644 code/nel/include/nel/misc/i_streamed_package_provider.h create mode 100644 code/nel/src/misc/http_package_provider.cpp create mode 100644 code/nel/src/misc/i_streamed_package_provider.cpp diff --git a/code/nel/include/nel/misc/http_package_provider.h b/code/nel/include/nel/misc/http_package_provider.h new file mode 100644 index 000000000..b4b82d70a --- /dev/null +++ b/code/nel/include/nel/misc/http_package_provider.h @@ -0,0 +1,50 @@ +// NeL - MMORPG Framework +// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_HTTP_PACKAGE_PROVIDER_H +#define NLMISC_HTTP_PACKAGE_PROVIDER_H + +#include +#include + +namespace NLMISC { + +class CHttpPackageProvider : public IStreamedPackageProvider +{ +public: + CHttpPackageProvider(); + virtual ~CHttpPackageProvider(); + + /// Download a file. This call is blocking + /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. + /// hash: [in] + virtual bool getFile(std::string &filePath, const CHashKey &hash, const std::string &name) NL_OVERRIDE; + +public: + /// Set storage path (ex. stream/) + std::string Path; + + /// Loads a package into the package manager (ex. http://cdn.ryzom.dev/open/stream/) + typedef std::vector THosts; + THosts Hosts; + +}; /* class CHttpPackageProvider */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */ + +/* end of file */ diff --git a/code/nel/include/nel/misc/i_streamed_package_provider.h b/code/nel/include/nel/misc/i_streamed_package_provider.h new file mode 100644 index 000000000..6e74d866d --- /dev/null +++ b/code/nel/include/nel/misc/i_streamed_package_provider.h @@ -0,0 +1,44 @@ +// NeL - MMORPG Framework +// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_PROVIDER_H +#define NLMISC_STREAMED_PACKAGE_PROVIDER_H + +#include +#include +#include + +namespace NLMISC { + +class IStreamedPackageProvider +{ +public: + IStreamedPackageProvider(); + virtual ~IStreamedPackageProvider(); + + /// Download a file. This call is blocking + /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. + /// hash: [in] + /// name: [in] name for debugging + virtual bool getFile(std::string &filePath, const CHashKey &hash, const std::string &name = ""); + +}; /* class IStreamedPackageProvider */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_PROVIDER_H */ + +/* end of file */ diff --git a/code/nel/include/nel/misc/streamed_package.h b/code/nel/include/nel/misc/streamed_package.h index c777d8a02..9040598d6 100644 --- a/code/nel/include/nel/misc/streamed_package.h +++ b/code/nel/include/nel/misc/streamed_package.h @@ -42,6 +42,8 @@ public: void serial(NLMISC::IStream &f) throw(NLMISC::EStream); + /// result: [out] ex. /00/00/000000000.. + /// hash: [in] static void makePath(std::string &result, const CHashKey &hash); public: diff --git a/code/nel/include/nel/misc/streamed_package_manager.h b/code/nel/include/nel/misc/streamed_package_manager.h index e23344a70..85d13db59 100644 --- a/code/nel/include/nel/misc/streamed_package_manager.h +++ b/code/nel/include/nel/misc/streamed_package_manager.h @@ -20,6 +20,7 @@ #include #include #include +#include namespace NLMISC { @@ -40,21 +41,19 @@ public: /// Unload all packages void unloadAll(); - /// Get an existing file or download if necessary + /// Get an existing file or download if necessary. This call is blocking /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. - /// fileName: ex. fy_hof_underwear.dds + /// fileName: [in] ex. fy_hof_underwear.dds bool getFile(std::string &filePath, const std::string &fileName); /// Get file size + /// fileSize: [out] + /// fileName: [in] ex. fy_hof_underwear.dds bool getFileSize(uint32 &fileSize, const std::string &fileName); public: - /// Set storage path (ex. stream/) - std::string Path; - - /// Loads a package into the package manager (ex. http://patch.live.polyverse.org/stream/) - typedef std::vector THosts; - THosts Hosts; + /// Streamed package provider. This downloads the files + IStreamedPackageProvider *Provider; private: typedef std::map TPackages; diff --git a/code/nel/include/nel/misc/types_nl.h b/code/nel/include/nel/misc/types_nl.h index 353f50490..f42f410e6 100644 --- a/code/nel/include/nel/misc/types_nl.h +++ b/code/nel/include/nel/misc/types_nl.h @@ -479,6 +479,7 @@ extern void operator delete[](void *p) throw(); # define CHashMultiMap NL_ISO_STDTR1_NAMESPACE::unordered_multimap # define CUniquePtr ::std::auto_ptr # define CUniquePtrMove +# define NL_OVERRIDE #elif defined(NL_COMP_VC) && (NL_COMP_VC_VERSION >= 70 && NL_COMP_VC_VERSION <= 90) // VC7 through 9 # include # include @@ -487,6 +488,7 @@ extern void operator delete[](void *p) throw(); # define CHashMultiMap stdext::hash_multimap # define CUniquePtr ::std::auto_ptr # define CUniquePtrMove +# define NL_OVERRIDE #elif defined(NL_COMP_GCC) // GCC4 # include # include @@ -526,6 +528,10 @@ template<> struct hash */ typedef uint16 ucchar; +#ifndef NL_OVERRIDE +#define NL_OVERRIDE override +#endif + #if defined(NL_OS_WINDOWS) && (defined(UNICODE) || defined(_UNICODE)) #define nltmain wmain #define nltWinMain wWinMain diff --git a/code/nel/src/misc/http_package_provider.cpp b/code/nel/src/misc/http_package_provider.cpp new file mode 100644 index 000000000..035fa2a5d --- /dev/null +++ b/code/nel/src/misc/http_package_provider.cpp @@ -0,0 +1,145 @@ +// NeL - MMORPG Framework +// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// 3rd Party includes +#include + +// Project includes +#include +#include +#include +#include +#include +#include + +namespace NLMISC +{ + +CHttpPackageProvider::CHttpPackageProvider() +{ + // init +} + +CHttpPackageProvider::~CHttpPackageProvider() +{ + // release +} + +bool CHttpPackageProvider::getFile(std::string &filePath, const CHashKey &hash, const std::string &name) +{ + CStreamedPackage::makePath(filePath, hash); + std::string downloadUrlFile = filePath + ".lzma"; + filePath = Path + filePath; + std::string downloadPath = filePath + ".download." + toString(rand() * rand()); + + std::string storageDirectory = CFile::getPath(downloadPath); + CFile::createDirectoryTree(storageDirectory); + /*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory)) + { + nldebug("Unable to create directory '%s'", storageDirectory.c_str()); + return false; + }*/ + + // download + for (;;) + { + if (CFile::fileExists(filePath)) + return true; + + std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile; + nldebug("Download streamed package '%s' from '%s'", name.c_str(), downloadUrl.c_str()); + + FILE *fp = fopen(downloadPath.c_str(), "wb"); + if (fp == NULL) + { + nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), name.c_str()); + return false; + } + + CURL *curl; + CURLcode res; + curl = curl_easy_init(); + if (curl) + { + curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_FILE, fp); + res = curl_easy_perform(curl); + long r; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r); + curl_easy_cleanup(curl); + + bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/; + fclose(fp); + + if (diskFull) + { + CFile::deleteFile(downloadPath); + throw EDiskFullError(downloadPath); + } + + if (res != CURLE_OK || r < 200 || r >= 300) + { + CFile::deleteFile(downloadPath); + nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str()); + nlSleep(1000); + continue; + } + } + else + { + nldebug("Curl initialize failed"); + fclose(fp); + CFile::deleteFile(downloadPath); + return false; + } + + // ok! + break; + } + + // extract into file + std::string unpackPath = filePath + ".extract." + toString(rand() * rand()); + + CHashKey outHash; + if (!unpackLZMA(downloadPath, unpackPath, outHash)) + { + return false; + } + + if (!(outHash == hash)) + { + std::string wantHashS = hash.toString(); + std::string outHashS = outHash.toString(); + nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str()); + return false; + } + + if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str())) + { + nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str()); + // in case downloaded from another thread + return CFile::fileExists(filePath); + } + + return true; +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/src/misc/i_streamed_package_provider.cpp b/code/nel/src/misc/i_streamed_package_provider.cpp new file mode 100644 index 000000000..12c873760 --- /dev/null +++ b/code/nel/src/misc/i_streamed_package_provider.cpp @@ -0,0 +1,37 @@ +// NeL - MMORPG Framework +// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// Project includes +#include + +namespace NLMISC +{ + +IStreamedPackageProvider::IStreamedPackageProvider() +{ + // init +} + +IStreamedPackageProvider::~IStreamedPackageProvider() +{ + // release +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 483df6a52..ac605b049 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -16,14 +16,10 @@ #include "stdmisc.h" -// 3rd Party includes -#include - // Project includes #include #include #include -#include #include namespace NLMISC @@ -96,105 +92,21 @@ bool CStreamedPackageManager::getFile(std::string &filePath, const std::string & { // nldebug("Get file path for streamed file '%s'", fileName.c_str()); - TEntries::iterator it = m_Entries.find(fileName); - if (it == m_Entries.end()) - return false; - const CStreamedPackage::CEntry *entry = it->second; - - CStreamedPackage::makePath(filePath, entry->Hash); - std::string downloadUrlFile = filePath + ".lzma"; - filePath = Path + filePath; - std::string downloadPath = filePath + ".download." + toString(rand() * rand()); - - std::string storageDirectory = CFile::getPath(downloadPath); - CFile::createDirectoryTree(storageDirectory); - /*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory)) + IStreamedPackageProvider *provider = Provider; + if (!provider) { - nldebug("Unable to create directory '%s'", storageDirectory.c_str()); + nlerrornoex("No streamed package provider was set"); return false; - }*/ - - // download - for (;;) - { - if (CFile::fileExists(filePath)) - return true; - - std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile; - nldebug("Download streamed package '%s' from '%s'", fileName.c_str(), downloadUrl.c_str()); - - FILE *fp = fopen(downloadPath.c_str(), "wb"); - if (fp == NULL) - { - nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), fileName.c_str()); - return false; - } - - CURL *curl; - CURLcode res; - curl = curl_easy_init(); - if (curl) - { - curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str()); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_FILE, fp); - res = curl_easy_perform(curl); - long r; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r); - curl_easy_cleanup(curl); - - bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/; - fclose(fp); - - if (diskFull) - { - CFile::deleteFile(downloadPath); - throw EDiskFullError(downloadPath); - } - - if (res != CURLE_OK || r < 200 || r >= 300) - { - CFile::deleteFile(downloadPath); - nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str()); - nlSleep(1000); - continue; - } - } - else - { - nldebug("Curl initialize failed"); - fclose(fp); - CFile::deleteFile(downloadPath); - return false; - } - - // ok! - break; } - // extract into file - std::string unpackPath = filePath + ".extract." + toString(rand() * rand()); - - CHashKey outHash; - if (!unpackLZMA(downloadPath, unpackPath, outHash)) - { - return false; - } - - if (!(outHash == entry->Hash)) + TEntries::iterator it = m_Entries.find(fileName); + if (it == m_Entries.end()) { - std::string wantHashS = entry->Hash.toString(); - std::string outHashS = outHash.toString(); - nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str()); return false; } + const CStreamedPackage::CEntry *entry = it->second; - if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str())) - { - nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str()); - // in case downloaded from another thread - return CFile::fileExists(filePath); - } + return provider->getFile(filePath, entry->Hash, fileName); return true; } diff --git a/code/ryzom/client/src/global.cpp b/code/ryzom/client/src/global.cpp index 4b8e1aad7..41d41160c 100644 --- a/code/ryzom/client/src/global.cpp +++ b/code/ryzom/client/src/global.cpp @@ -24,6 +24,9 @@ using namespace NLMISC; // *************************************************************************** +// Data +NLMISC::CHttpPackageProvider *HttpPackageProvider = NULL; + // Main System NL3D::UDriver *Driver = NULL; // The main 3D Driver NL3D::IStereoDisplay *StereoDisplay = NULL; // Stereo display diff --git a/code/ryzom/client/src/global.h b/code/ryzom/client/src/global.h index 78cebcec9..75ba2381d 100644 --- a/code/ryzom/client/src/global.h +++ b/code/ryzom/client/src/global.h @@ -28,6 +28,11 @@ // *************************************************************************** +namespace NLMISC +{ + class CHttpPackageProvider; +} + namespace NL3D { class UDriver; @@ -78,6 +83,9 @@ const float ExtraZoneLoadingVision = 100.f; // *************************************************************************** +// Data +extern NLMISC::CHttpPackageProvider *HttpPackageProvider; // Http provider from on-the-fly downloaded game data + // Main System extern NL3D::UDriver *Driver; // The main 3D Driver extern NL3D::IStereoDisplay *StereoDisplay; // Stereo display diff --git a/code/ryzom/client/src/init.cpp b/code/ryzom/client/src/init.cpp index 1f9c10b34..85eb50b51 100644 --- a/code/ryzom/client/src/init.cpp +++ b/code/ryzom/client/src/init.cpp @@ -35,6 +35,7 @@ #include "nel/misc/block_memory.h" #include "nel/misc/system_utils.h" #include "nel/misc/streamed_package_manager.h" +#include "nel/misc/http_package_provider.h" #include "nel/misc/cmd_args.h" // 3D Interface. #include "nel/3d/bloom_effect.h" @@ -740,9 +741,14 @@ static void addPaths(IProgressCallback &progress, const std::vector void initStreamedPackageManager(NLMISC::IProgressCallback &progress) { CStreamedPackageManager &spm = CStreamedPackageManager::getInstance(); - spm.Path = ClientCfg.StreamedPackagePath; + nlassert(!spm.Provider); // If this asserts, init was called twice without release + nlassert(!HttpPackageProvider); // Idem + CHttpPackageProvider *hpp = new CHttpPackageProvider(); + hpp->Path = ClientCfg.StreamedPackagePath; for (uint i = 0; i < ClientCfg.StreamedPackageHosts.size(); i++) - spm.Hosts.push_back(ClientCfg.StreamedPackageHosts[i]); + hpp->Hosts.push_back(ClientCfg.StreamedPackageHosts[i]); + spm.Provider = hpp; + HttpPackageProvider = hpp; } void addSearchPaths(IProgressCallback &progress) diff --git a/code/ryzom/client/src/release.cpp b/code/ryzom/client/src/release.cpp index be34d9ba8..6686fea5d 100644 --- a/code/ryzom/client/src/release.cpp +++ b/code/ryzom/client/src/release.cpp @@ -668,6 +668,8 @@ void release() NLMISC::CBigFile::getInstance().removeAll(); NLMISC::CBigFile::releaseInstance(); NLMISC::CStreamedPackageManager::releaseInstance(); + delete HttpPackageProvider; + HttpPackageProvider = NULL; NL3D::CFastHLSModifier::releaseInstance(); CLandscapePolyDrawer::releaseInstance(); NL3D::CParticleSystemShape::releaseInstance(); From 09e31cf75a2e883a2bcc1fb2db84fb9a531aacf3 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Wed, 27 Nov 2019 17:48:54 +0800 Subject: [PATCH 27/27] Fix --- code/nel/include/nel/misc/i_streamed_package_provider.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nel/include/nel/misc/i_streamed_package_provider.h b/code/nel/include/nel/misc/i_streamed_package_provider.h index 6e74d866d..320e3cdfd 100644 --- a/code/nel/include/nel/misc/i_streamed_package_provider.h +++ b/code/nel/include/nel/misc/i_streamed_package_provider.h @@ -33,7 +33,7 @@ public: /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. /// hash: [in] /// name: [in] name for debugging - virtual bool getFile(std::string &filePath, const CHashKey &hash, const std::string &name = ""); + virtual bool getFile(std::string &filePath, const CHashKey &hash, const std::string &name = "") = 0; }; /* class IStreamedPackageProvider */