Implement streamed data download, ref #179

--HG--
branch : feature-streamed-package

code/nel/include/nel/misc/path.h

@@ -279,8 +279,8 @@ private:
 	struct CMCFileEntry
 	{
 		char *Name;				// Normal case (the search is done by using nlstricmp)
-		uint32	idPath	 : 16;	// Path (not with file at the end) - look in the SSMpath (65536 different path allowed)
-		uint32	idExt	 : 15;	// real extension of the file if remapped - look in the SSMext (32768 different extension allowed)
+		uint32	idPath	 : 20;	// Path (not with file at the end) - look in the SSMpath (1048576 different path allowed)
+		uint32	idExt	 : 11;	// real extension of the file if remapped - look in the SSMext (2048 different extension allowed)
 		uint32	Remapped : 1;	// true if the file is remapped
 	};
 

code/nel/include/nel/misc/streamed_package_manager.h

@@ -31,7 +31,6 @@ public:
 	CStreamedPackageManager();
 	~CStreamedPackageManager();
 
-	/// Loads a package into the package manager
 	/// package: ex. /games/nel/data/characters_maps_hr.snp
 	bool loadPackage(const std::string &package);
 
@@ -49,9 +48,20 @@ public:
 	/// Get file size
 	bool getFileSize(uint32 &fileSize, const std::string &fileName);
 
+public:
+	/// Set storage path (ex. stream/)
+	std::string Path;
+
+	/// Loads a package into the package manager (ex. http://patch.live.polyverse.org/stream/)
+	typedef std::vector<std::string> THosts;
+	THosts Hosts;
+
 private:
-	std::map<std::string, CStreamedPackage> m_Packages;
-	std::map<std::string, const CStreamedPackage::CEntry *> m_Entries;
+	typedef std::map<std::string, CStreamedPackage> TPackages;
+	TPackages m_Packages;
+
+	typedef std::map<std::string, const CStreamedPackage::CEntry *> TEntries;
+	TEntries m_Entries;
 	
 }; /* class CStreamedPackageManager */
 

code/nel/src/misc/file.cpp

@@ -234,6 +234,7 @@ bool		CIFile::open(const std::string &path, bool text)
 			}
 			else
 			{
+				// TEMPORARY ERROR
 				nlerror("File '%s' not in streamed package", path.c_str());
 			}
 		}

code/nel/src/misc/path.cpp

@@ -342,7 +342,7 @@ void CFileContainer::remapExtension (const string &ext1, const string &ext2, boo
 		nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): can't remap empty extension", ext1lwr.c_str(), ext2lwr.c_str(), substitute);
 	}
 
-	if (ext1lwr == "bnp" || ext2lwr == "bnp")
+	if (ext1lwr == "bnp" || ext2lwr == "bnp" || ext1lwr == "snp" || ext2lwr == "snp")
 	{
 		nlwarning ("PATH: CPath::remapExtension(%s, %s, %d): you can't remap a big file", ext1lwr.c_str(), ext2lwr.c_str(), substitute);
 	}
@@ -1445,6 +1445,18 @@ void CFileContainer::addSearchStreamedPackage (const string &filename, bool recu
 			std::string filePackageName = packname + "@" + (*it);
 			nldebug("Insert '%s'", filePackageName.c_str());
 			insertFileInMap((*it), filePackageName, false, CFile::getExtension(*it));
+
+			// Remapped extensions
+			std::string ext = CFile::getExtension(*it);
+			for (uint j = 0; j < _Extensions.size (); j++)
+			{
+				if (_Extensions[j].first == ext)
+				{
+					// Need to remap
+					insertFileInMap(CFile::getFilenameWithoutExtension(*it) + "." + _Extensions[j].second,
+						filePackageName, true, _Extensions[j].first);
+				}
+			}
 		}
 	}
 }

code/nel/src/misc/streamed_package_manager.cpp

@@ -16,6 +16,13 @@
 
 #include "stdmisc.h"
 
+// 3rd Party includes
+#include <curl/curl.h>
+#include <seven_zip/7zCrc.h>
+#include <seven_zip/7zIn.h>
+#include <seven_zip/7zExtract.h>
+#include <seven_zip/LzmaDecode.h>
+
 // Project includes
 #include <nel/misc/streamed_package_manager.h>
 #include <nel/misc/file.h>
@@ -88,15 +95,174 @@ void CStreamedPackageManager::unloadAll()
 
 bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &fileName)
 {
-	// .. :)
 	nldebug("Get file path for streamed file '%s'", fileName.c_str());
-	return false;
+
+	TEntries::iterator it = m_Entries.find(fileName);
+	if (it == m_Entries.end())
+		return false;
+	const CStreamedPackage::CEntry *entry = it->second;
+
+	CStreamedPackage::makePath(filePath, entry->Hash);
+	std::string downloadUrlFile = filePath + ".lzma";
+	filePath = Path + filePath;
+	std::string downloadPath = filePath + ".download." + toString(rand());
+
+	std::string storageDirectory = CFile::getPath(downloadPath);
+	CFile::createDirectoryTree(storageDirectory);
+	/*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory))
+	{
+		nldebug("Unable to create directory '%s'", storageDirectory.c_str());
+		return false;
+	}*/
+
+	// download
+	for (; ; )
+	{
+		if (CFile::fileExists(filePath))
+			return true;
+
+		std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile;
+		nldebug("Download streamed package '%s' from '%s'", fileName.c_str(), downloadUrl.c_str());
+
+		FILE *fp = fopen(downloadPath.c_str(), "wb");
+		if (fp == NULL)
+		{
+			nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), fileName.c_str());
+			return false;
+		}
+
+		CURL *curl;
+		CURLcode res;
+		curl = curl_easy_init();
+		if (curl)
+		{
+			curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str());
+			curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+			curl_easy_setopt(curl, CURLOPT_FILE, fp);
+			res = curl_easy_perform(curl);
+			long r;
+			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r);
+			curl_easy_cleanup(curl);
+
+			bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/;
+			fclose(fp);
+
+			if (diskFull)
+			{
+				CFile::deleteFile(downloadPath);
+				throw EDiskFullError(downloadPath);
+			}
+
+			if (res != CURLE_OK || r < 200 || r >= 300)
+			{
+				CFile::deleteFile(downloadPath);
+				nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str());
+				nlSleep(1000);
+				continue;
+			}
+		}
+		else
+		{
+			nldebug("Curl initialize failed");
+			fclose(fp);
+			CFile::deleteFile(downloadPath);
+			return false;
+		}
+
+		// ok!
+		break;
+	}
+
+	// read into memory :(
+	std::string unpackPath = filePath + ".extract." + toString(rand());
+
+	std::vector<uint8> inBuffer;
+	{
+		CIFile inStream(downloadPath);
+		uint32 inSize = inStream.getFileSize();
+		inBuffer.resize(inSize);
+		inStream.serialBuffer(&inBuffer[0], inSize);
+	}
+	CFile::deleteFile(downloadPath);
+
+	if (inBuffer.size() < LZMA_PROPERTIES_SIZE + 8)
+	{
+		nlwarning("Invalid file size %u, too small file '%s'", inBuffer.size(), downloadPath.c_str());
+		return false;
+	}
+
+	// extract
+	{
+		CLzmaDecoderState state;
+		uint8 *pos = &inBuffer[0];
+		int ret = LzmaDecodeProperties(&state.Properties, (unsigned char *)pos, LZMA_PROPERTIES_SIZE);
+		if (ret != 0)
+		{
+			nlwarning("Failed to decode lzma properies in file '%s'", downloadPath.c_str());
+			return false;
+		}
+
+		// FROM login_patch.cpp
+		// alloc the probs, making sure they are deleted in function exit
+		size_t nbProb = LzmaGetNumProbs(&state.Properties);
+		std::vector<CProb> probs;
+		probs.resize(nbProb);
+		state.Probs = &probs[0];
+
+		pos += LZMA_PROPERTIES_SIZE;
+
+		// read the output file size
+		size_t fileSize = 0;
+		for (int i = 0; i < 8; i++)
+		{
+			//Byte b;
+			if (pos >= &inBuffer[0] + inBuffer.size())
+			{
+				nlerror("pos >= inBuffer.get() + inSize");
+				return false;
+			}
+			fileSize |= ((UInt64)*pos++) << (8 * i);
+		}
+
+		nlassert(fileSize == entry->Size);
+
+		SizeT outProcessed = 0;
+		SizeT inProcessed = 0;
+		// allocate the output buffer :(
+		std::vector<uint8> outBuffer;
+		outBuffer.resize(fileSize);
+		// decompress the file in memory
+		ret = LzmaDecode(&state, (unsigned char *)pos, (SizeT)(inBuffer.size() - (pos - &inBuffer[0])), &inProcessed, (unsigned char*)&outBuffer[0], (SizeT)fileSize, &outProcessed);
+		if (ret != 0 || outProcessed != fileSize)
+		{
+			nlwarning("Failed to decode lzma file '%s'", downloadPath.c_str());
+			return false;
+		}
+
+		{
+			COFile outStream(unpackPath);
+			outStream.serialBuffer(&outBuffer[0], (uint)fileSize);
+		}
+
+		if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str()))
+		{
+			nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str());
+			// in case downloaded from another thread
+			return CFile::fileExists(filePath);
+		}
+
+		return true;
+	}
 }
 
 bool CStreamedPackageManager::getFileSize(uint32 &fileSize, const std::string &fileName)
 {
-	nldebug("Get file size for streamed file '%s'", fileName.c_str());
-	return false;
+	// nldebug("Get file size for streamed file '%s'", fileName.c_str());
+	TEntries::iterator it = m_Entries.find(fileName);
+	if (it == m_Entries.end())
+		return false;
+	fileSize = it->second->Size;
+	return true;
 }
 
 } /* namespace NLMISC */

code/nel/tools/build_gamedata/d1_client_patch.py

@@ -97,7 +97,7 @@ else:
 							if category["UnpackTo"] != "":
 								cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./" + category["UnpackTo"] + "/\"/>\n")
 							else:
-								cfg.write("\t\t\t<_UnpackTo type=\"SINT32\" value=\"./\"/>\n")
+								cfg.write("\t\t\t<_UnpackTo type=\"STRING\" value=\"./\"/>\n")
 						cfg.write("\t\t\t<_IsOptional type=\"SINT32\" value=\"" + str(category["IsOptional"]) + "\"/>\n")
 						cfg.write("\t\t\t<_IsIncremental type=\"SINT32\" value=\"" + str(category["IsIncremental"]) + "\"/>\n")
 						for package in category["Packages"]:
@@ -148,7 +148,7 @@ else:
 					os.chdir(cwDir)
 				else:
 					printLog(log, "BNP " + targetBnp)
-					subprocess.call([ BnpMake, "/p", sourcePath, targetPath, targetBnp ] + package[1])
+					subprocess.call([ BnpMake, "/p", sourcePath, targetPath ] + package[1])
 			else:
 				printLog(log, "SKIP " + targetBnp)
 	printLog(log, "")

code/ryzom/client/src/client_cfg.cpp

@@ -464,6 +464,8 @@ CClientConfig::CClientConfig()
 	ColorShout			= CRGBA(150,0,0,255);		// Default Shout color.
 	ColorTalk			= CRGBA(255,255,255,255);	// Default Talk color.
 
+	StreamedPackagePath = "stream";
+
 //	PreDataPath.push_back("data/gamedev/language/");	// Default Path for the language data
 
 //	DataPath.push_back("data/");					// Default Path for the Data.
@@ -1247,18 +1249,25 @@ void CClientConfig::setValues()
 
 	//////////
 	// MISC //
+
 	// Pre Data Path.
 	READ_STRINGVECTOR_FV(PreDataPath);
 
 	// Data Path.
 	READ_STRINGVECTOR_FV(DataPath);
 
-	// List of files that trigger R2ED reload when touched
-	READ_STRINGVECTOR_FV(R2EDReloadFiles);
-
 	// Data Path no recurse.
 	READ_STRINGVECTOR_FV(DataPathNoRecurse);
 
+	// Streamed package path
+	READ_STRING_FV(StreamedPackagePath);
+
+	// Streamed package hosts
+	READ_STRINGVECTOR_FV(StreamedPackageHosts);
+
+	// List of files that trigger R2ED reload when touched
+	READ_STRINGVECTOR_FV(R2EDReloadFiles);
+
 	// Update packed sheet Path
 	READ_STRINGVECTOR_FV(UpdatePackedSheetPath);
 

code/ryzom/client/src/client_cfg.h

@@ -360,6 +360,10 @@ struct CClientConfig
 	std::vector<string>			DataPath;
 	/// Data Path no recurse.
 	std::vector<string>			DataPathNoRecurse;
+	/// Streamed package path
+	std::string					StreamedPackagePath;
+	/// Streamed package hosts
+	std::vector<string>			StreamedPackageHosts; // TODO: From 'domain' SQL table
 	/// Update packed sheet Path.
 	std::vector<string>			UpdatePackedSheetPath;
 	/// True if we want the packed sheet to be updated if needed

code/ryzom/client/src/init.cpp

@@ -34,6 +34,7 @@
 #include "nel/misc/system_info.h"
 #include "nel/misc/block_memory.h"
 #include "nel/misc/system_utils.h"
+#include "nel/misc/streamed_package_manager.h"
 // 3D Interface.
 #include "nel/3d/bloom_effect.h"
 #include "nel/3d/u_driver.h"
@@ -664,6 +665,14 @@ void initStereoDisplayDevice()
 	IStereoDisplay::releaseUnusedLibraries();
 }
 
+void initStreamedPackageManager(NLMISC::IProgressCallback &progress)
+{
+	CStreamedPackageManager &spm = CStreamedPackageManager::getInstance();
+	spm.Path = replaceApplicationDirToken(ClientCfg.StreamedPackagePath);
+	for (uint i = 0; i < ClientCfg.StreamedPackageHosts.size(); i++)
+		spm.Hosts.push_back(ClientCfg.StreamedPackageHosts[i]);
+}
+
 void addSearchPaths(IProgressCallback &progress)
 {
 	// Add search path of UI addon. Allow only a subset of files.
@@ -848,6 +857,7 @@ void prelogInit()
 		CPath::remapExtension ("png", "tga", true);
 		FPU_CHECKER_ONCE
 
+		initStreamedPackageManager(ProgressBar);
 		addPreDataPaths(ProgressBar);
 
 		FPU_CHECKER_ONCE

code/ryzom/client/src/init.h

@@ -33,6 +33,7 @@ void prelogInit();
 // Initialize the application after login step
 void postlogInit();
 
+void initStreamedPackageManager(NLMISC::IProgressCallback &progress);
 void addSearchPaths(NLMISC::IProgressCallback &progress);
 void addPreDataPaths(NLMISC::IProgressCallback &progress);
 

code/ryzom/client/src/streamable_ig.cpp

@@ -89,8 +89,10 @@ CStreamableIG::~CStreamableIG()
 	H_AUTO_USE(RZ_StremableIG)
 	if (!_Linked)
 	{
+		#ifdef NL_DEBUG
 		if(!ClientCfg.Light)
 			nlwarning("Loading async %p", this);
+		#endif
 		#ifdef NL_DEBUG
 				//nlinfo("Loading async : %s", Name.c_str());
 		#endif