Add tool to build streamed package, ref #179

--HG--
branch : feature-streamed-package

code/nel/include/nel/misc/streamed_package.h

@@ -0,0 +1,56 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2014  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef NLMISC_STREAMED_PACKAGE_H
+#define NLMISC_STREAMED_PACKAGE_H
+
+#include <nel/misc/sha1.h>
+
+namespace NLMISC {
+
+class CStreamedPackage
+{
+public:
+	struct CEntry
+	{
+		std::string Name;
+		CHashKey Hash;
+		uint32 Size;
+		uint32 LastModified;
+
+		void serial(NLMISC::IStream &f) throw(NLMISC::EStream);
+
+	};
+
+public:
+	CStreamedPackage();
+	~CStreamedPackage();
+
+	void serial(NLMISC::IStream &f) throw(NLMISC::EStream);
+
+	static void makePath(std::string &result, const CHashKey &hash);
+
+public:
+	typedef std::vector<CEntry> TEntries;
+	TEntries Entries;
+	
+}; /* class CStreamedPackage */
+
+} /* namespace NLMISC */
+
+#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */
+
+/* end of file */

code/nel/include/nel/misc/streamed_package_manager.h

@@ -0,0 +1,34 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2014  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef NLMISC_STREAMED_PACKAGE_H
+#define NLMISC_STREAMED_PACKAGE_H
+
+namespace NLMISC {
+
+class CStreamedPackageManager
+{
+public:
+	CStreamedPackageManager();
+	~CStreamedPackageManager();
+	
+}; /* class CStreamedPackageManager */
+
+} /* namespace NLMISC */
+
+#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */
+
+/* end of file */

code/nel/src/misc/streamed_package.cpp

@@ -0,0 +1,66 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2014  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "stdmisc.h"
+
+// Project includes
+#include <nel/misc/streamed_package.h>
+#include <nel/misc/stream.h>
+
+namespace NLMISC {
+
+CStreamedPackage::CStreamedPackage()
+{
+	// init
+}
+
+CStreamedPackage::~CStreamedPackage()
+{
+	// release
+}
+
+void CStreamedPackage::serial(NLMISC::IStream &f) throw(NLMISC::EStream)
+{
+	f.serialCheck(NELID("SNPK"));
+
+	uint version = 1;
+	f.serialVersion(version);
+
+	f.serialCont(Entries);
+}
+
+void CStreamedPackage::CEntry::serial(NLMISC::IStream &f) throw(NLMISC::EStream)
+{
+	uint version = 1;
+	f.serialVersion(version);
+
+	f.serial(Name);
+	f.serial(Hash);
+	f.serial(Size);
+	f.serial(LastModified);
+}
+
+void CStreamedPackage::makePath(std::string &result, const CHashKey &hash)
+{
+	std::string lowerHash = NLMISC::toLower(hash.toString());
+	result = std::string("/") + lowerHash.substr(0, 2)
+		+ "/" + lowerHash.substr(2, 2)
+		+ "/" + lowerHash.substr(4);
+}
+
+} /* namespace NLMISC */
+
+/* end of file */

code/nel/src/misc/streamed_package_manager.cpp

@@ -0,0 +1,36 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2014  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "stdmisc.h"
+
+// Project includes
+#include <nel/misc/streamed_package_manager.h>
+
+namespace NLMISC {
+
+CStreamedPackageManager::CStreamedPackageManager()
+{
+	// init
+}
+
+CStreamedPackageManager::~CStreamedPackageManager()
+{
+	// release
+}
+
+} /* namespace NLMISC */
+
+/* end of file */

code/nel/tools/misc/CMakeLists.txt

@@ -1,4 +1,4 @@
-SUBDIRS(bnp_make disp_sheet_id extract_filename lock make_sheet_id xml_packer)
+SUBDIRS(bnp_make snp_make disp_sheet_id extract_filename lock make_sheet_id xml_packer)
 
 IF(WITH_QT)
   ADD_SUBDIRECTORY(words_dic_qt)

code/nel/tools/misc/snp_make/CMakeLists.txt

@@ -0,0 +1,9 @@
+FILE(GLOB SRC *.cpp *.h)
+
+ADD_EXECUTABLE(snp_make ${SRC})
+
+TARGET_LINK_LIBRARIES(snp_make nelmisc)
+NL_DEFAULT_PROPS(snp_make "NeL, Tools, Misc: snp_make")
+NL_ADD_RUNTIME_FLAGS(snp_make)
+
+INSTALL(TARGETS snp_make RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT toolsmisc)

code/nel/tools/misc/snp_make/main.cpp

@@ -0,0 +1,355 @@
+// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+// Copyright (C) 2010  Winch Gate Property Limited
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "nel/misc/types_nl.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef NL_OS_WINDOWS
+#	include <io.h>
+#	include <direct.h>
+#endif
+
+#include <vector>
+#include <string>
+
+#include "nel/misc/debug.h"
+#include "nel/misc/file.h"
+#include "nel/misc/path.h"
+#include "nel/misc/algo.h"
+#include "nel/misc/common.h"
+#include "nel/misc/streamed_package.h"
+
+using namespace std;
+using namespace NLMISC;
+
+// ---------------------------------------------------------------------------
+
+class CWildCard
+{
+public:
+	string		Expression;
+	bool		Not;
+};
+std::vector<CWildCard>	WildCards;
+
+std::string SourceDirectory;
+std::string PackageFileName;
+std::string StreamDirectory;
+
+CStreamedPackage Package;
+
+// ---------------------------------------------------------------------------
+
+bool keepFile (const char *fileName)
+{
+	uint i;
+	bool ifPresent = false;
+	bool ifTrue = false;
+	string file = toLower(CFile::getFilename (fileName));
+	for (i=0; i<WildCards.size(); i++)
+	{
+		if (WildCards[i].Not)
+		{
+			// One ifnot condition met and the file is not added
+			if (testWildCard(file.c_str(), WildCards[i].Expression.c_str()))
+				return false;
+		}
+		else
+		{
+			ifPresent = true;
+			ifTrue |= testWildCard(file.c_str(), WildCards[i].Expression.c_str());
+		}
+	}
+
+	return !ifPresent || ifTrue;
+}
+
+// ---------------------------------------------------------------------------
+void usage()
+{
+	printf ("USAGE : \n");
+	printf ("   snp_make /p <directory_name> <package_file> <stream_directory> [option] ... [option]\n");
+	printf ("   option : \n");
+	printf ("      -if wildcard : add the file if it matches the wilcard (at least one 'if' conditions must be met for a file to be adding)\n");
+	printf ("      -ifnot wildcard : add the file if it doesn't match the wilcard (all the 'ifnot' conditions must be met for a file to be adding)\n");
+	printf (" Pack the directory to a snp file\n");
+	printf ("   snp_make /l <package_file>\n");
+	printf (" List the files contained in the snp file\n");
+}
+
+// ---------------------------------------------------------------------------
+
+void generateLZMA(const std::string sourceFile, const std::string &outputFile)
+{
+	std::string cmd="lzma e ";
+	cmd+=" "+sourceFile+" "+outputFile;
+	nlinfo("executing system command: %s",cmd.c_str());
+#ifdef NL_OS_WINDOWS
+	_spawnlp(_P_WAIT, "lzma.exe","lzma.exe", "e", sourceFile.c_str(), outputFile.c_str(), NULL);
+#else // NL_OS_WINDOWS
+	sint error = system (cmd.c_str());
+	if (error)
+		nlwarning("'%s' failed with error code %d", cmd.c_str(), error);
+#endif // NL_OS_WINDOWS
+}
+
+// ---------------------------------------------------------------------------
+
+uint readOptions (int nNbArg, char **ppArgs)
+{
+	uint i;
+	uint optionCount = 0;
+	for (i=0; i<(uint)nNbArg; i++)
+	{
+		// If ?
+		if ((strcmp (ppArgs[i], "-if") == 0) && ((i+1)<(uint)nNbArg))
+		{
+			CWildCard card;
+			card.Expression = toLower(string(ppArgs[i+1]));
+			card.Not = false;
+			WildCards.push_back (card);
+			optionCount += 2;
+		}
+		// If not ?
+		if ((strcmp (ppArgs[i], "-ifnot") == 0) && ((i+1)<(uint)nNbArg))
+		{
+			CWildCard card;
+			card.Expression = toLower(string(ppArgs[i+1]));
+			card.Not = true;
+			WildCards.push_back (card);
+			optionCount += 2;
+		}
+	}
+	return optionCount;
+}
+
+// ---------------------------------------------------------------------------
+int main (int nNbArg, char **ppArgs)
+{
+	NLMISC::CApplicationContext myApplicationContext;
+
+	if (nNbArg < 3)
+	{
+		usage();
+		return -1;
+	}
+
+	if ((strcmp(ppArgs[1], "/p") == 0) || (strcmp(ppArgs[1], "/P") == 0) ||
+		(strcmp(ppArgs[1], "-p") == 0) || (strcmp(ppArgs[1], "-P") == 0))
+	{
+		if (nNbArg < 5)
+		{
+			usage();
+			return -1;
+		}
+
+		SourceDirectory = ppArgs[2];
+		PackageFileName = ppArgs[3];
+		StreamDirectory = ppArgs[4];
+		readOptions(nNbArg, ppArgs);
+		
+		nldebug("Make streamed package: '%s'", PackageFileName.c_str());
+
+		if (CFile::fileExists(PackageFileName))
+		{
+			nldebug("Update existing package");
+			try
+			{
+				CIFile fi;
+				fi.open(PackageFileName);
+				fi.serial(Package);
+			}
+			catch (Exception &e)
+			{
+				nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what());
+				return -1;
+			}
+		}
+		else
+		{
+			nldebug("New package");
+		}
+
+		std::vector<std::string> pathContent; // contains full pathnames
+		std::vector<std::string> nameContent; // only filename
+		CPath::getPathContent(SourceDirectory, true, false, true, pathContent);
+		nameContent.reserve(pathContent.size());
+		for (std::vector<std::string>::size_type i = 0; i < pathContent.size(); ++i)
+		{
+			const std::string &file = pathContent[i];
+			if (keepFile(file.c_str()))
+			{
+				std::string fileName = NLMISC::toLower(CFile::getFilename(file));
+				// nldebug("File: '%s' ('%s')", file.c_str(), fileName.c_str());
+				nameContent.push_back(fileName);
+				nlassert(nameContent.size() == (i + 1));
+			}
+			else
+			{
+				// Not included in this package
+				pathContent.erase(pathContent.begin() + i);
+				--i;
+			}
+		}
+
+		std::vector<sint> packageIndex; // index of file in package
+		packageIndex.resize(pathContent.size(), -1);
+
+		for (CStreamedPackage::TEntries::size_type i = 0; i < Package.Entries.size(); ++i)
+		{
+			const CStreamedPackage::CEntry &entry = Package.Entries[i];
+			
+			sint foundIndex = -1; // find index in found file list
+			for (std::vector<std::string>::size_type j = 0; j < pathContent.size(); ++j)
+			{
+				if (nameContent[j] == entry.Name)
+				{
+					foundIndex = j;
+					break;
+				}
+			}
+
+			if (foundIndex < 0)
+			{
+				nlinfo("File no longer exists: '%s'", entry.Name.c_str());
+				Package.Entries.erase(Package.Entries.begin() + i);
+				--i;
+			}
+			else
+			{
+				// File still exists, map it
+				packageIndex[foundIndex] = i;
+			}
+		}
+
+		for (std::vector<std::string>::size_type i = 0; i < pathContent.size(); ++i)
+		{
+			sint pidx = packageIndex[i];
+			const std::string &name = nameContent[i];
+			const std::string &path = pathContent[i];
+
+			if (pidx < 0)
+			{
+				nlinfo("File added: '%s'", name.c_str());
+				pidx = Package.Entries.size();
+				Package.Entries.push_back(CStreamedPackage::CEntry());
+				Package.Entries[pidx].Name = name;
+				Package.Entries[pidx].LastModified = 0;
+				Package.Entries[pidx].Size = 0;
+			}
+			else
+			{
+				nlinfo("File check for changes: '%s'", name.c_str());
+			}
+
+			CStreamedPackage::CEntry &entry = Package.Entries[pidx];
+
+			std::string targetLzmaOld; // in case lzma wasn't made make sure it exists a second run
+			CStreamedPackage::makePath(targetLzmaOld, entry.Hash);
+			targetLzmaOld = StreamDirectory + targetLzmaOld + ".lzma";
+
+			uint32 lastModified = CFile::getFileModificationDate(path);
+			uint32 fileSize = CFile::getFileSize(path);
+			if (lastModified > entry.LastModified || fileSize != entry.Size || !CFile::fileExists(targetLzmaOld))
+			{
+				entry.LastModified = lastModified;
+
+				nlinfo("Calculate file hash");
+				CHashKey hash = getSHA1(path, true);
+				/*nldebug("%s", hash.toString().c_str());
+				std::string hashPath;
+				CStreamedPackage::makePath(hashPath, hash);
+				nldebug("%s", hashPath.c_str());*/
+
+				if (hash == entry.Hash && fileSize == entry.Size)
+				{
+					// File has not changed
+				}
+				else
+				{
+					nlinfo("File changed");
+					entry.Hash = hash;
+					entry.Size = fileSize;
+				}
+
+				std::string targetLzma; // in case lzma wasn't made make sure it exists a second run
+				CStreamedPackage::makePath(targetLzma, entry.Hash);
+				targetLzma = StreamDirectory + targetLzma + ".lzma";
+
+				if (!CFile::fileExists(targetLzma))
+				{
+					// make the compressed file
+					nlinfo("%s -> %s", path.c_str(), targetLzma.c_str());
+					CFile::createDirectoryTree(CFile::getPath(targetLzma));
+					generateLZMA(path, targetLzma);
+				}
+			}
+		}
+
+		try
+		{
+			nldebug("Store package '%s'", PackageFileName.c_str());
+			COFile fo;
+			fo.open(PackageFileName);
+			fo.serial(Package);
+		}
+		catch (Exception &e)
+		{
+			nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what());
+			return -1;
+		}
+
+		return 0;
+	}	
+
+	if ((strcmp(ppArgs[1], "/l") == 0) || (strcmp(ppArgs[1], "/L") == 0) ||
+		(strcmp(ppArgs[1], "-l") == 0) || (strcmp(ppArgs[1], "-L") == 0))
+	{
+		PackageFileName = ppArgs[2];
+		if (!CFile::fileExists(PackageFileName))
+		{
+			nlwarning("ERROR (snp_make) : package doesn't exist: '%s'", PackageFileName.c_str());
+			return -1;
+		}
+
+		try
+		{
+			CIFile fi;
+			fi.open(PackageFileName);
+			fi.serial(Package);
+		}
+		catch (Exception &e)
+		{
+			nlwarning("ERROR (snp_make) : serial exception: '%s'", e.what());
+			return -1;
+		}
+
+		for (CStreamedPackage::TEntries::const_iterator it(Package.Entries.begin()), end(Package.Entries.end()); it != end; ++it)
+		{
+			const CStreamedPackage::CEntry &entry = (*it);
+
+			printf("List files in '%s'", PackageFileName.c_str());
+			printf("%s { Hash: '%s', Size: '%u', LastModified: '%u' }", entry.Name.c_str(), entry.Hash.toString().c_str(), entry.Size, entry.LastModified);
+		}
+		
+		return 0;
+	}
+
+	usage ();
+	return -1;
+}

code/ryzom/tools/patch_gen/patch_gen_common.cpp

@@ -442,7 +442,8 @@ void CPackageDescription::buildDefaultFileList()
 	std::vector<std::string> fileList;
 	NLMISC::CPath::getPathContent(_BnpDirectory,false,false,true,fileList);
 	for (uint32 i=0;i<fileList.size();++i)
-		if (NLMISC::toLower(NLMISC::CFile::getExtension(fileList[i]))=="bnp")
+		if (NLMISC::toLower(NLMISC::CFile::getExtension(fileList[i]))=="bnp"
+			|| NLMISC::toLower(NLMISC::CFile::getExtension(fileList[i]))=="snp")
 			_Categories.addFile("main",NLMISC::toLower(NLMISC::CFile::getFilename(fileList[i])));
 
 	_Categories.addFile("unpacked","root.bnp");