diff --git a/code/nel/include/nel/misc/streamed_package.h b/code/nel/include/nel/misc/streamed_package.h new file mode 100644 index 000000000..b27e6658f --- /dev/null +++ b/code/nel/include/nel/misc/streamed_package.h @@ -0,0 +1,56 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_H +#define NLMISC_STREAMED_PACKAGE_H + +#include + +namespace NLMISC { + +class CStreamedPackage +{ +public: + struct CEntry + { + std::string Name; + CHashKey Hash; + uint32 Size; + uint32 LastModified; + + void serial(NLMISC::IStream &f) throw(NLMISC::EStream); + + }; + +public: + CStreamedPackage(); + ~CStreamedPackage(); + + void serial(NLMISC::IStream &f) throw(NLMISC::EStream); + + static void makePath(std::string &result, const CHashKey &hash); + +public: + typedef std::vector TEntries; + TEntries Entries; + +}; /* class CStreamedPackage */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */ + +/* end of file */ diff --git a/code/nel/include/nel/misc/streamed_package_manager.h b/code/nel/include/nel/misc/streamed_package_manager.h new file mode 100644 index 000000000..0bec38584 --- /dev/null +++ b/code/nel/include/nel/misc/streamed_package_manager.h @@ -0,0 +1,34 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_H +#define NLMISC_STREAMED_PACKAGE_H + +namespace NLMISC { + +class CStreamedPackageManager +{ +public: + CStreamedPackageManager(); + ~CStreamedPackageManager(); + +}; /* class CStreamedPackageManager */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */ + +/* end of file */ diff --git a/code/nel/src/misc/streamed_package.cpp b/code/nel/src/misc/streamed_package.cpp new file mode 100644 index 000000000..fed7d1dc3 --- /dev/null +++ b/code/nel/src/misc/streamed_package.cpp @@ -0,0 +1,66 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// Project includes +#include +#include + +namespace NLMISC { + +CStreamedPackage::CStreamedPackage() +{ + // init +} + +CStreamedPackage::~CStreamedPackage() +{ + // release +} + +void CStreamedPackage::serial(NLMISC::IStream &f) throw(NLMISC::EStream) +{ + f.serialCheck(NELID("SNPK")); + + uint version = 1; + f.serialVersion(version); + + f.serialCont(Entries); +} + +void CStreamedPackage::CEntry::serial(NLMISC::IStream &f) throw(NLMISC::EStream) +{ + uint version = 1; + f.serialVersion(version); + + f.serial(Name); + f.serial(Hash); + f.serial(Size); + f.serial(LastModified); +} + +void CStreamedPackage::makePath(std::string &result, const CHashKey &hash) +{ + std::string lowerHash = NLMISC::toLower(hash.toString()); + result = std::string("/") + lowerHash.substr(0, 2) + + "/" + lowerHash.substr(2, 2) + + "/" + lowerHash.substr(4); +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp new file mode 100644 index 000000000..2d04cb95a --- /dev/null +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -0,0 +1,36 @@ +// NeL - MMORPG Framework +// Copyright (C) 2014 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// Project includes +#include + +namespace NLMISC { + +CStreamedPackageManager::CStreamedPackageManager() +{ + // init +} + +CStreamedPackageManager::~CStreamedPackageManager() +{ + // release +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/tools/misc/CMakeLists.txt b/code/nel/tools/misc/CMakeLists.txt index 5386cbbc6..76399f6a3 100644 --- a/code/nel/tools/misc/CMakeLists.txt +++ b/code/nel/tools/misc/CMakeLists.txt @@ -1,4 +1,4 @@ -SUBDIRS(bnp_make disp_sheet_id extract_filename lock make_sheet_id xml_packer) +SUBDIRS(bnp_make snp_make disp_sheet_id extract_filename lock make_sheet_id xml_packer) IF(WITH_QT) ADD_SUBDIRECTORY(words_dic_qt) diff --git a/code/nel/tools/misc/snp_make/CMakeLists.txt b/code/nel/tools/misc/snp_make/CMakeLists.txt new file mode 100644 index 000000000..a1b5f388d --- /dev/null +++ b/code/nel/tools/misc/snp_make/CMakeLists.txt @@ -0,0 +1,9 @@ +FILE(GLOB SRC *.cpp *.h) + +ADD_EXECUTABLE(snp_make ${SRC}) + +TARGET_LINK_LIBRARIES(snp_make nelmisc) +NL_DEFAULT_PROPS(snp_make "NeL, Tools, Misc: snp_make") +NL_ADD_RUNTIME_FLAGS(snp_make) + +INSTALL(TARGETS snp_make RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT toolsmisc) diff --git a/code/nel/tools/misc/snp_make/main.cpp b/code/nel/tools/misc/snp_make/main.cpp new file mode 100644 index 000000000..61fc11810 --- /dev/null +++ b/code/nel/tools/misc/snp_make/main.cpp @@ -0,0 +1,355 @@ +// NeL - MMORPG Framework +// Copyright (C) 2010 Winch Gate Property Limited +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "nel/misc/types_nl.h" + +#include +#include + +#ifdef NL_OS_WINDOWS +# include +# include +#endif + +#include +#include + +#include "nel/misc/debug.h" +#include "nel/misc/file.h" +#include "nel/misc/path.h" +#include "nel/misc/algo.h" +#include "nel/misc/common.h" +#include "nel/misc/streamed_package.h" + +using namespace std; +using namespace NLMISC; + +// --------------------------------------------------------------------------- + +class CWildCard +{ +public: + string Expression; + bool Not; +}; +std::vector WildCards; + +std::string SourceDirectory; +std::string PackageFileName; +std::string StreamDirectory; + +CStreamedPackage Package; + +// --------------------------------------------------------------------------- + +bool keepFile (const char *fileName) +{ + uint i; + bool ifPresent = false; + bool ifTrue = false; + string file = toLower(CFile::getFilename (fileName)); + for (i=0; i [option] ... [option]\n"); + printf (" option : \n"); + printf (" -if wildcard : add the file if it matches the wilcard (at least one 'if' conditions must be met for a file to be adding)\n"); + printf (" -ifnot wildcard : add the file if it doesn't match the wilcard (all the 'ifnot' conditions must be met for a file to be adding)\n"); + printf (" Pack the directory to a snp file\n"); + printf (" snp_make /l \n"); + printf (" List the files contained in the snp file\n"); +} + +// --------------------------------------------------------------------------- + +void generateLZMA(const std::string sourceFile, const std::string &outputFile) +{ + std::string cmd="lzma e "; + cmd+=" "+sourceFile+" "+outputFile; + nlinfo("executing system command: %s",cmd.c_str()); +#ifdef NL_OS_WINDOWS + _spawnlp(_P_WAIT, "lzma.exe","lzma.exe", "e", sourceFile.c_str(), outputFile.c_str(), NULL); +#else // NL_OS_WINDOWS + sint error = system (cmd.c_str()); + if (error) + nlwarning("'%s' failed with error code %d", cmd.c_str(), error); +#endif // NL_OS_WINDOWS +} + +// --------------------------------------------------------------------------- + +uint readOptions (int nNbArg, char **ppArgs) +{ + uint i; + uint optionCount = 0; + for (i=0; i<(uint)nNbArg; i++) + { + // If ? + if ((strcmp (ppArgs[i], "-if") == 0) && ((i+1)<(uint)nNbArg)) + { + CWildCard card; + card.Expression = toLower(string(ppArgs[i+1])); + card.Not = false; + WildCards.push_back (card); + optionCount += 2; + } + // If not ? + if ((strcmp (ppArgs[i], "-ifnot") == 0) && ((i+1)<(uint)nNbArg)) + { + CWildCard card; + card.Expression = toLower(string(ppArgs[i+1])); + card.Not = true; + WildCards.push_back (card); + optionCount += 2; + } + } + return optionCount; +} + +// --------------------------------------------------------------------------- +int main (int nNbArg, char **ppArgs) +{ + NLMISC::CApplicationContext myApplicationContext; + + if (nNbArg < 3) + { + usage(); + return -1; + } + + if ((strcmp(ppArgs[1], "/p") == 0) || (strcmp(ppArgs[1], "/P") == 0) || + (strcmp(ppArgs[1], "-p") == 0) || (strcmp(ppArgs[1], "-P") == 0)) + { + if (nNbArg < 5) + { + usage(); + return -1; + } + + SourceDirectory = ppArgs[2]; + PackageFileName = ppArgs[3]; + StreamDirectory = ppArgs[4]; + readOptions(nNbArg, ppArgs); + + nldebug("Make streamed package: '%s'", PackageFileName.c_str()); + + if (CFile::fileExists(PackageFileName)) + { + nldebug("Update existing package"); + try + { + CIFile fi; + fi.open(PackageFileName); + fi.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + } + else + { + nldebug("New package"); + } + + std::vector pathContent; // contains full pathnames + std::vector nameContent; // only filename + CPath::getPathContent(SourceDirectory, true, false, true, pathContent); + nameContent.reserve(pathContent.size()); + for (std::vector::size_type i = 0; i < pathContent.size(); ++i) + { + const std::string &file = pathContent[i]; + if (keepFile(file.c_str())) + { + std::string fileName = NLMISC::toLower(CFile::getFilename(file)); + // nldebug("File: '%s' ('%s')", file.c_str(), fileName.c_str()); + nameContent.push_back(fileName); + nlassert(nameContent.size() == (i + 1)); + } + else + { + // Not included in this package + pathContent.erase(pathContent.begin() + i); + --i; + } + } + + std::vector packageIndex; // index of file in package + packageIndex.resize(pathContent.size(), -1); + + for (CStreamedPackage::TEntries::size_type i = 0; i < Package.Entries.size(); ++i) + { + const CStreamedPackage::CEntry &entry = Package.Entries[i]; + + sint foundIndex = -1; // find index in found file list + for (std::vector::size_type j = 0; j < pathContent.size(); ++j) + { + if (nameContent[j] == entry.Name) + { + foundIndex = j; + break; + } + } + + if (foundIndex < 0) + { + nlinfo("File no longer exists: '%s'", entry.Name.c_str()); + Package.Entries.erase(Package.Entries.begin() + i); + --i; + } + else + { + // File still exists, map it + packageIndex[foundIndex] = i; + } + } + + for (std::vector::size_type i = 0; i < pathContent.size(); ++i) + { + sint pidx = packageIndex[i]; + const std::string &name = nameContent[i]; + const std::string &path = pathContent[i]; + + if (pidx < 0) + { + nlinfo("File added: '%s'", name.c_str()); + pidx = Package.Entries.size(); + Package.Entries.push_back(CStreamedPackage::CEntry()); + Package.Entries[pidx].Name = name; + Package.Entries[pidx].LastModified = 0; + Package.Entries[pidx].Size = 0; + } + else + { + nlinfo("File check for changes: '%s'", name.c_str()); + } + + CStreamedPackage::CEntry &entry = Package.Entries[pidx]; + + std::string targetLzmaOld; // in case lzma wasn't made make sure it exists a second run + CStreamedPackage::makePath(targetLzmaOld, entry.Hash); + targetLzmaOld = StreamDirectory + targetLzmaOld + ".lzma"; + + uint32 lastModified = CFile::getFileModificationDate(path); + uint32 fileSize = CFile::getFileSize(path); + if (lastModified > entry.LastModified || fileSize != entry.Size || !CFile::fileExists(targetLzmaOld)) + { + entry.LastModified = lastModified; + + nlinfo("Calculate file hash"); + CHashKey hash = getSHA1(path, true); + /*nldebug("%s", hash.toString().c_str()); + std::string hashPath; + CStreamedPackage::makePath(hashPath, hash); + nldebug("%s", hashPath.c_str());*/ + + if (hash == entry.Hash && fileSize == entry.Size) + { + // File has not changed + } + else + { + nlinfo("File changed"); + entry.Hash = hash; + entry.Size = fileSize; + } + + std::string targetLzma; // in case lzma wasn't made make sure it exists a second run + CStreamedPackage::makePath(targetLzma, entry.Hash); + targetLzma = StreamDirectory + targetLzma + ".lzma"; + + if (!CFile::fileExists(targetLzma)) + { + // make the compressed file + nlinfo("%s -> %s", path.c_str(), targetLzma.c_str()); + CFile::createDirectoryTree(CFile::getPath(targetLzma)); + generateLZMA(path, targetLzma); + } + } + } + + try + { + nldebug("Store package '%s'", PackageFileName.c_str()); + COFile fo; + fo.open(PackageFileName); + fo.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + + return 0; + } + + if ((strcmp(ppArgs[1], "/l") == 0) || (strcmp(ppArgs[1], "/L") == 0) || + (strcmp(ppArgs[1], "-l") == 0) || (strcmp(ppArgs[1], "-L") == 0)) + { + PackageFileName = ppArgs[2]; + if (!CFile::fileExists(PackageFileName)) + { + nlwarning("ERROR (snp_make) : package doesn't exist: '%s'", PackageFileName.c_str()); + return -1; + } + + try + { + CIFile fi; + fi.open(PackageFileName); + fi.serial(Package); + } + catch (Exception &e) + { + nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what()); + return -1; + } + + for (CStreamedPackage::TEntries::const_iterator it(Package.Entries.begin()), end(Package.Entries.end()); it != end; ++it) + { + const CStreamedPackage::CEntry &entry = (*it); + + printf("List files in '%s'", PackageFileName.c_str()); + printf("%s { Hash: '%s', Size: '%u', LastModified: '%u' }", entry.Name.c_str(), entry.Hash.toString().c_str(), entry.Size, entry.LastModified); + } + + return 0; + } + + usage (); + return -1; +} diff --git a/code/ryzom/tools/patch_gen/patch_gen_common.cpp b/code/ryzom/tools/patch_gen/patch_gen_common.cpp index 3d28a2439..127be5053 100644 --- a/code/ryzom/tools/patch_gen/patch_gen_common.cpp +++ b/code/ryzom/tools/patch_gen/patch_gen_common.cpp @@ -442,7 +442,8 @@ void CPackageDescription::buildDefaultFileList() std::vector fileList; NLMISC::CPath::getPathContent(_BnpDirectory,false,false,true,fileList); for (uint32 i=0;i