Add streamed package manager

--HG--
branch : feature-streamed-package

code/nel/include/nel/misc/app_context.h

@@ -229,6 +229,20 @@ namespace NLMISC
 		} \
 	private:
 
+#define NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(className) \
+	NLMISC_SAFE_SINGLETON_DECL(className); \
+	\
+	void className::releaseInstance() \
+	{ \
+		if (_Instance) \
+		{ \
+			NLMISC::INelContext::getInstance().releaseSingletonPointer(#className, _Instance); \
+			delete _Instance; \
+			_Instance = NULL; \
+		} \
+	} \
+
+
 	/** The same as above, but generate a getInstance method that
 	 *	return a pointer instead of a reference
 	 */
@@ -296,5 +310,6 @@ void initNelLibrary(CLibrary &lib);
 
 } // namespace NLMISC
 
+#include <nel/misc/debug.h>
 
 #endif //APP_CONTEXT_H

code/nel/include/nel/misc/path.h

@@ -74,6 +74,9 @@ public:
 
 	/** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included  (the extention (Nel Big File) is used to know that it's a big file) */
 	void			addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
+	
+	/** Sale but for .snp (Streamed NeL Package) */
+	void			addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
 
 	/** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included   */
 	void			addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
@@ -371,6 +374,9 @@ public:
 	/** Same as AddSearchPath but with a big file "c:/test.nbf" all files name contained in the big file will be included  (the extention (Nel Big File) is used to know that it's a big file) */
 	static void			addSearchBigFile (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
 
+	/** Same but Streamed Package */
+	static void			addSearchStreamedPackage (const std::string &filename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
+
 	/** Same as AddSearchPath but with a xml pack file "c:/test.xml_pack" all files name contained in the xml pack will be included   */
 	static void			addSearchXmlpackFile (const std::string &sXmlpackFilename, bool recurse, bool alternative, class NLMISC::IProgressCallback *progressCallBack = NULL);
 

code/nel/include/nel/misc/streamed_package.h

@@ -17,6 +17,7 @@
 #ifndef NLMISC_STREAMED_PACKAGE_H
 #define NLMISC_STREAMED_PACKAGE_H
 
+#include <nel/misc/types_nl.h>
 #include <nel/misc/sha1.h>
 
 namespace NLMISC {

code/nel/include/nel/misc/streamed_package_manager.h

@@ -14,21 +14,49 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-#ifndef NLMISC_STREAMED_PACKAGE_H
-#define NLMISC_STREAMED_PACKAGE_H
+#ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H
+#define NLMISC_STREAMED_PACKAGE_MANAGER_H
+
+#include <nel/misc/types_nl.h>
+#include <nel/misc/app_context.h>
+#include <nel/misc/streamed_package.h>
 
 namespace NLMISC {
 
 class CStreamedPackageManager
 {
+	NLMISC_SAFE_RELEASABLE_SINGLETON_DECL(CStreamedPackageManager);
+
 public:
 	CStreamedPackageManager();
 	~CStreamedPackageManager();
+
+	/// Loads a package into the package manager
+	/// package: ex. /games/nel/data/characters_maps_hr.snp
+	bool loadPackage(const std::string &package);
+
+	/// Get a file list
+	void list(std::vector<std::string> &fileNames, const std::string &package);
+
+	/// Unload all packages
+	void unloadAll();
+
+	/// Get an existing file or download if necessary
+	/// filePath: [out] ex. /games/nel/stream/00/00/000000000..
+	/// fileName: ex. fy_hof_underwear.dds
+	bool getFile(std::string &filePath, const std::string &fileName);
+
+	/// Get file size
+	bool getFileSize(uint32 &fileSize, const std::string &fileName);
+
+private:
+	std::map<std::string, CStreamedPackage> m_Packages;
+	std::map<std::string, const CStreamedPackage::CEntry *> m_Entries;
 	
 }; /* class CStreamedPackageManager */
 
 } /* namespace NLMISC */
 
-#endif /* #ifndef NLMISC_STREAMED_PACKAGE_H */
+#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */
 
 /* end of file */

code/nel/src/misc/file.cpp

@@ -23,6 +23,7 @@
 #include "nel/misc/command.h"
 #include "nel/misc/sstring.h"
 #include "nel/misc/xml_pack.h"
+#include "nel/misc/streamed_package_manager.h"
 
 #ifndef NL_OS_WINDOWS
 #include <errno.h>
@@ -203,6 +204,39 @@ bool		CIFile::open(const std::string &path, bool text)
 				_F = CXMLPack::getInstance().getFile (path, _FileSize, _BigFileOffset, dummy, _AlwaysOpened);
 			}
 		}
+		else if (pos > 3 && path[pos-3] == 's' && path[pos-2] == 'n' && path[pos-1] == 'p')
+		{
+			nldebug("Opening a streamed package file");
+
+			_IsInXMLPackFile = false;
+			_IsInBigFile = false;
+			_BigFileOffset = 0;
+			_AlwaysOpened = false;
+			std::string filePath;
+			if (CStreamedPackageManager::getInstance().getFile (filePath, path))
+			{
+				_F = fopen (filePath.c_str(), mode);
+				if (_F != NULL)
+				{
+					_FileSize=CFile::getFileSize(_F);
+					if (_FileSize == 0)
+					{
+						nlwarning("FILE: Size of file '%s' is 0", path.c_str());
+						fclose(_F);
+						_F = NULL;
+					}
+				}
+				else
+				{
+					nlwarning("Failed to open file '%s', error %u : %s", path.c_str(), errno, strerror(errno));
+					_FileSize = 0;
+				}
+			}
+			else
+			{
+				nlerror("File '%s' not in streamed package", path.c_str());
+			}
+		}
 		else
 		{
 			// bnp file

code/nel/src/misc/path.cpp

@@ -23,6 +23,7 @@
 #include "nel/misc/progress_callback.h"
 #include "nel/misc/file.h"
 #include "nel/misc/xml_pack.h"
+#include "nel/misc/streamed_package_manager.h"
 
 #ifdef NL_OS_WINDOWS
 #	ifndef NL_COMP_MINGW
@@ -292,6 +293,7 @@ void CFileContainer::clearMap ()
 	nlassert(!_MemoryCompressed);
 	_Files.clear ();
 	CBigFile::getInstance().removeAll ();
+	CStreamedPackageManager::getInstance().unloadAll ();
 	NL_DISPLAY_PATH("PATH: CPath::clearMap(): map directory cleared");
 }
 
@@ -1161,16 +1163,26 @@ void CFileContainer::addSearchFile (const string &file, bool remap, const string
 		return;
 	}
 
+	std::string fileExtension = CFile::getExtension(newFile);
+
 	// check if it s a big file
-	if (CFile::getExtension(newFile) == "bnp")
+	if (fileExtension == "bnp")
 	{
 		NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is a big file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str());
 		addSearchBigFile(file, false, false, progressCallBack);
 		return;
 	}
 
+	// check if it s a streamed package
+	if (fileExtension == "snp")
+	{
+		NL_DISPLAY_PATH ("PATH: CPath::addSearchStreamedPackage(%s, %d, '%s'): '%s' is a streamed package, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str());
+		addSearchStreamedPackage(file, false, false, progressCallBack);
+		return;
+	}
+
 	// check if it s an xml pack file
-	if (CFile::getExtension(newFile) == "xml_pack")
+	if (fileExtension == "xml_pack")
 	{
 		NL_DISPLAY_PATH ("PATH: CPath::addSearchFile(%s, %d, '%s'): '%s' is an xml pack file, add it", file.c_str(), remap, virtual_ext.c_str(), newFile.c_str());
 		addSearchXmlpackFile(file, false, false, progressCallBack);
@@ -1391,6 +1403,52 @@ void CFileContainer::addSearchBigFile (const string &sBigFilename, bool recurse,
 	fclose (Handle);
 }
 
+void CPath::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack)
+{
+	getInstance()->_FileContainer.addSearchBigFile(filename, recurse, alternative, progressCallBack);
+}
+
+void CFileContainer::addSearchStreamedPackage (const string &filename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack)
+{
+	// Check if filename is not empty
+	if (filename.empty())
+	{
+		nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): can't add empty file, skip it", filename.c_str(), recurse, alternative);
+		return;
+	}
+	// Check if the file exists
+	if (!CFile::isExists(filename))
+	{
+		nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not found, skip it", filename.c_str(), recurse, alternative, filename.c_str());
+		return;
+	}
+	// Check if it s a file
+	if (CFile::isDirectory(filename))
+	{
+		nlwarning ("PATH: CPath::addSearchStreamedPackage(%s, %d, %d): '%s' is not a file, skip it", filename.c_str(), recurse, alternative, filename.c_str());
+		return;
+	}
+
+	// Add the file itself
+	std::string packname = NLMISC::toLower(CFile::getFilename(filename));
+	insertFileInMap(packname, filename, false, CFile::getExtension(filename));
+
+	// Add the package to the package manager
+	if (CStreamedPackageManager::getInstance().loadPackage(filename))
+	{
+		std::vector<std::string> fileNames;
+		CStreamedPackageManager::getInstance().list(fileNames, packname);
+
+		for (std::vector<std::string>::iterator it(fileNames.begin()), end(fileNames.end()); it != end; ++it)
+		{
+			// Add the file to the lookup
+			std::string filePackageName = packname + "@" + (*it);
+			nldebug("Insert '%s'", filePackageName.c_str());
+			insertFileInMap((*it), filePackageName, false, CFile::getExtension(*it));
+		}
+	}
+}
+
 // WARNING : recurse is not used
 void CPath::addSearchXmlpackFile (const string &sXmlpackFilename, bool recurse, bool alternative, NLMISC::IProgressCallback *progressCallBack)
 {
@@ -1983,6 +2041,7 @@ string CFile::findNewFile (const string &filename)
 // \warning doesn't work with big file
 uint32	CFile::getFileSize (const std::string &filename)
 {
+	std::string::size_type pos;
 	if (filename.find("@@") != string::npos)
 	{
 		uint32 fs = 0, bfo;
@@ -1990,12 +2049,21 @@ uint32	CFile::getFileSize (const std::string &filename)
 		CXMLPack::getInstance().getFile (filename, fs, bfo, c, d);
 		return fs;
 	}
-	else if (filename.find('@') != string::npos)
+	else if ((pos = filename.find('@')) != string::npos)
 	{
-		uint32 fs = 0, bfo;
-		bool c, d;
-		CBigFile::getInstance().getFile (filename, fs, bfo, c, d);
-		return fs;
+		if (pos > 3 && filename[pos-3] == 's' && filename[pos-2] == 'n' && filename[pos-1] == 'p')
+		{
+			uint32 fs = 0;
+			CStreamedPackageManager::getInstance().getFileSize (fs, filename);
+			return fs;
+		}
+		else
+		{
+			uint32 fs = 0, bfo;
+			bool c, d;
+			CBigFile::getInstance().getFile (filename, fs, bfo, c, d);
+			return fs;
+		}
 	}
 	else
 	{

code/nel/src/misc/streamed_package_manager.cpp

@@ -18,9 +18,13 @@
 
 // Project includes
 #include <nel/misc/streamed_package_manager.h>
+#include <nel/misc/file.h>
+#include <nel/misc/path.h>
 
 namespace NLMISC {
 
+NLMISC_SAFE_SINGLETON_IMPL(CStreamedPackageManager);
+
 CStreamedPackageManager::CStreamedPackageManager()
 {
 	// init
@@ -31,6 +35,70 @@ CStreamedPackageManager::~CStreamedPackageManager()
 	// release
 }
 
+bool CStreamedPackageManager::loadPackage(const std::string &package)
+{
+	nldebug("Load package '%s'", package.c_str());
+
+	std::string packname = NLMISC::toLower(CFile::getFilename(package));
+	m_Packages[packname] = CStreamedPackage();
+	std::map<std::string, CStreamedPackage>::iterator it = m_Packages.find(packname);
+	CStreamedPackage &p = it->second;
+
+	try
+	{
+		CIFile fi;
+		fi.open(package);
+		fi.serial(p);
+	}
+	catch (Exception &e)
+	{
+		nlerror("Package serial exception: '%s'", e.what());
+		m_Packages.erase(it);
+		return false;
+	}
+
+	for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it)
+	{
+		const CStreamedPackage::CEntry &entry = (*it);
+		m_Entries[entry.Name] = &entry;
+	}
+
+	return true;
+}
+
+void CStreamedPackageManager::list(std::vector<std::string> &fileNames, const std::string &package)
+{
+	nldebug("List package '%s'", package.c_str());
+
+	std::map<std::string, CStreamedPackage>::iterator it = m_Packages.find(package);
+	CStreamedPackage &p = it->second;
+
+	for (CStreamedPackage::TEntries::const_iterator it(p.Entries.begin()), end(p.Entries.end()); it != end; ++it)
+	{
+		const CStreamedPackage::CEntry &entry = (*it);
+		fileNames.push_back(entry.Name);
+	}
+}
+
+void CStreamedPackageManager::unloadAll()
+{
+	m_Packages.clear();
+	m_Entries.clear();
+}
+
+bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName)
+{
+	// .. :)
+	nldebug("Get file path for streamed file '%s'", fileName.c_str());
+	return false;
+}
+
+bool CStreamedPackageManager::getFileSize(uint32 &fileSize, const std::string &fileName)
+{
+	nldebug("Get file size for streamed file '%s'", fileName.c_str());
+	return false;
+}
+
 } /* namespace NLMISC */
 
 /* end of file */