Decouple downloader from streamed package manager

code/nel/include/nel/misc/http_package_provider.h

@@ -0,0 +1,50 @@
+// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
+// Copyright (C) 2019  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef NLMISC_HTTP_PACKAGE_PROVIDER_H
+#define NLMISC_HTTP_PACKAGE_PROVIDER_H
+
+#include <nel/misc/types_nl.h>
+#include <nel/misc/i_streamed_package_provider.h>
+
+namespace NLMISC {
+
+class CHttpPackageProvider : public IStreamedPackageProvider
+{
+public:
+	CHttpPackageProvider();
+	virtual ~CHttpPackageProvider();
+
+	/// Download a file. This call is blocking
+	/// filePath: [out] ex. /games/nel/stream/00/00/000000000..
+	/// hash: [in]
+	virtual bool getFile(std::string &filePath, const CHashKey &hash, const std::string &name) NL_OVERRIDE;
+	
+public:
+	/// Set storage path (ex. stream/)
+	std::string Path;
+
+	/// Loads a package into the package manager (ex. http://cdn.ryzom.dev/open/stream/)
+	typedef std::vector<std::string> THosts;
+	THosts Hosts;
+	
+}; /* class CHttpPackageProvider */
+
+} /* namespace NLMISC */
+
+#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */
+
+/* end of file */

code/nel/include/nel/misc/i_streamed_package_provider.h

@@ -0,0 +1,44 @@
+// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
+// Copyright (C) 2019  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef NLMISC_STREAMED_PACKAGE_PROVIDER_H
+#define NLMISC_STREAMED_PACKAGE_PROVIDER_H
+
+#include <nel/misc/types_nl.h>
+#include <nel/misc/sha1.h>
+#include <string>
+
+namespace NLMISC {
+
+class IStreamedPackageProvider
+{
+public:
+	IStreamedPackageProvider();
+	virtual ~IStreamedPackageProvider();
+
+	/// Download a file. This call is blocking
+	/// filePath: [out] ex. /games/nel/stream/00/00/000000000..
+	/// hash: [in]
+	/// name: [in] name for debugging
+	virtual bool getFile(std::string &filePath, const CHashKey &hash, const std::string &name = "");
+	
+}; /* class IStreamedPackageProvider */
+
+} /* namespace NLMISC */
+
+#endif /* #ifndef NLMISC_STREAMED_PACKAGE_PROVIDER_H */
+
+/* end of file */

code/nel/include/nel/misc/streamed_package.h

@@ -42,6 +42,8 @@ public:
 
 	void serial(NLMISC::IStream &f) throw(NLMISC::EStream);
 
+	/// result: [out] ex. /00/00/000000000..
+	/// hash: [in]
 	static void makePath(std::string &result, const CHashKey &hash);
 
 public:

code/nel/include/nel/misc/streamed_package_manager.h

@@ -20,6 +20,7 @@
 #include <nel/misc/types_nl.h>
 #include <nel/misc/app_context.h>
 #include <nel/misc/streamed_package.h>
+#include <nel/misc/i_streamed_package_provider.h>
 
 namespace NLMISC {
 
@@ -40,21 +41,19 @@ public:
 	/// Unload all packages
 	void unloadAll();
 
-	/// Get an existing file or download if necessary
+	/// Get an existing file or download if necessary. This call is blocking
 	/// filePath: [out] ex. /games/nel/stream/00/00/000000000..
-	/// fileName: ex. fy_hof_underwear.dds
+	/// fileName: [in] ex. fy_hof_underwear.dds
 	bool getFile(std::string &filePath, const std::string &fileName);
 
 	/// Get file size
+	/// fileSize: [out]
+	/// fileName: [in] ex. fy_hof_underwear.dds
 	bool getFileSize(uint32 &fileSize, const std::string &fileName);
 
 public:
-	/// Set storage path (ex. stream/)
+	/// Streamed package provider. This downloads the files
-	std::string Path;
+	IStreamedPackageProvider *Provider;
-
-	/// Loads a package into the package manager (ex. http://patch.live.polyverse.org/stream/)
-	typedef std::vector<std::string> THosts;
-	THosts Hosts;
 
 private:
 	typedef std::map<std::string, CStreamedPackage> TPackages;

code/nel/include/nel/misc/types_nl.h

@@ -479,6 +479,7 @@ extern void operator delete[](void *p) throw();
 #	define CHashMultiMap NL_ISO_STDTR1_NAMESPACE::unordered_multimap
 #	define CUniquePtr ::std::auto_ptr
 #	define CUniquePtrMove
+#	define NL_OVERRIDE
 #elif defined(NL_COMP_VC) && (NL_COMP_VC_VERSION >= 70 && NL_COMP_VC_VERSION <= 90) // VC7 through 9
 #	include <hash_map>
 #	include <hash_set>
@@ -487,6 +488,7 @@ extern void operator delete[](void *p) throw();
 #	define CHashMultiMap stdext::hash_multimap
 #	define CUniquePtr ::std::auto_ptr
 #	define CUniquePtrMove
+#	define NL_OVERRIDE
 #elif defined(NL_COMP_GCC) // GCC4
 #	include <ext/hash_map>
 #	include <ext/hash_set>
@@ -526,6 +528,10 @@ template<> struct hash<uint64>
  */
 typedef	uint16	ucchar;
 
+#ifndef NL_OVERRIDE
+#define NL_OVERRIDE override
+#endif
+
 #if defined(NL_OS_WINDOWS) && (defined(UNICODE) || defined(_UNICODE))
 #define nltmain wmain
 #define nltWinMain wWinMain

code/nel/src/misc/http_package_provider.cpp

@@ -0,0 +1,145 @@
+// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
+// Copyright (C) 2019  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "stdmisc.h"
+
+// 3rd Party includes
+#include <curl/curl.h>
+
+// Project includes
+#include <nel/misc/http_package_provider.h>
+#include <nel/misc/streamed_package.h>
+#include <nel/misc/file.h>
+#include <nel/misc/path.h>
+#include <nel/misc/seven_zip.h>
+#include <nel/misc/sha1.h>
+
+namespace NLMISC
+{
+
+CHttpPackageProvider::CHttpPackageProvider()
+{
+	// init
+}
+
+CHttpPackageProvider::~CHttpPackageProvider()
+{
+	// release
+}
+
+bool CHttpPackageProvider::getFile(std::string &filePath, const CHashKey &hash, const std::string &name)
+{
+	CStreamedPackage::makePath(filePath, hash);
+	std::string downloadUrlFile = filePath + ".lzma";
+	filePath = Path + filePath;
+	std::string downloadPath = filePath + ".download." + toString(rand() * rand());
+
+	std::string storageDirectory = CFile::getPath(downloadPath);
+	CFile::createDirectoryTree(storageDirectory);
+	/*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory))
+	{
+		nldebug("Unable to create directory '%s'", storageDirectory.c_str());
+		return false;
+	}*/
+
+	// download
+	for (;;)
+	{
+		if (CFile::fileExists(filePath))
+			return true;
+
+		std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile;
+		nldebug("Download streamed package '%s' from '%s'", name.c_str(), downloadUrl.c_str());
+
+		FILE *fp = fopen(downloadPath.c_str(), "wb");
+		if (fp == NULL)
+		{
+			nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), name.c_str());
+			return false;
+		}
+
+		CURL *curl;
+		CURLcode res;
+		curl = curl_easy_init();
+		if (curl)
+		{
+			curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str());
+			curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+			curl_easy_setopt(curl, CURLOPT_FILE, fp);
+			res = curl_easy_perform(curl);
+			long r;
+			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r);
+			curl_easy_cleanup(curl);
+
+			bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/;
+			fclose(fp);
+
+			if (diskFull)
+			{
+				CFile::deleteFile(downloadPath);
+				throw EDiskFullError(downloadPath);
+			}
+
+			if (res != CURLE_OK || r < 200 || r >= 300)
+			{
+				CFile::deleteFile(downloadPath);
+				nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str());
+				nlSleep(1000);
+				continue;
+			}
+		}
+		else
+		{
+			nldebug("Curl initialize failed");
+			fclose(fp);
+			CFile::deleteFile(downloadPath);
+			return false;
+		}
+
+		// ok!
+		break;
+	}
+
+	// extract into file
+	std::string unpackPath = filePath + ".extract." + toString(rand() * rand());
+
+	CHashKey outHash;
+	if (!unpackLZMA(downloadPath, unpackPath, outHash))
+	{
+		return false;
+	}
+
+	if (!(outHash == hash))
+	{
+		std::string wantHashS = hash.toString();
+		std::string outHashS = outHash.toString();
+		nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str());
+		return false;
+	}
+
+	if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str()))
+	{
+		nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str());
+		// in case downloaded from another thread
+		return CFile::fileExists(filePath);
+	}
+
+	return true;
+}
+
+} /* namespace NLMISC */
+
+/* end of file */

code/nel/src/misc/i_streamed_package_provider.cpp

@@ -0,0 +1,37 @@
+// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
+// Copyright (C) 2019  Jan BOON (jan.boon@kaetemi.be)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "stdmisc.h"
+
+// Project includes
+#include <nel/misc/i_streamed_package_provider.h>
+
+namespace NLMISC
+{
+
+IStreamedPackageProvider::IStreamedPackageProvider()
+{
+	// init
+}
+
+IStreamedPackageProvider::~IStreamedPackageProvider()
+{
+	// release
+}
+
+} /* namespace NLMISC */
+
+/* end of file */

code/nel/src/misc/streamed_package_manager.cpp

@@ -16,14 +16,10 @@
 
 #include "stdmisc.h"
 
-// 3rd Party includes
-#include <curl/curl.h>
-
 // Project includes
 #include <nel/misc/streamed_package_manager.h>
 #include <nel/misc/file.h>
 #include <nel/misc/path.h>
-#include <nel/misc/seven_zip.h>
 #include <nel/misc/sha1.h>
 
 namespace NLMISC
@@ -96,105 +92,21 @@ bool CStreamedPackageManager::getFile(std::string &filePath, const std::string &
 {
 	// nldebug("Get file path for streamed file '%s'", fileName.c_str());
 
-	TEntries::iterator it = m_Entries.find(fileName);
+	IStreamedPackageProvider *provider = Provider;
-	if (it == m_Entries.end())
+	if (!provider)
-		return false;
-	const CStreamedPackage::CEntry *entry = it->second;
-
-	CStreamedPackage::makePath(filePath, entry->Hash);
-	std::string downloadUrlFile = filePath + ".lzma";
-	filePath = Path + filePath;
-	std::string downloadPath = filePath + ".download." + toString(rand() * rand());
-
-	std::string storageDirectory = CFile::getPath(downloadPath);
-	CFile::createDirectoryTree(storageDirectory);
-	/*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory))
 	{
-		nldebug("Unable to create directory '%s'", storageDirectory.c_str());
+		nlerrornoex("No streamed package provider was set");
 		return false;
-	}*/
-
-	// download
-	for (;;)
-	{
-		if (CFile::fileExists(filePath))
-			return true;
-
-		std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile;
-		nldebug("Download streamed package '%s' from '%s'", fileName.c_str(), downloadUrl.c_str());
-
-		FILE *fp = fopen(downloadPath.c_str(), "wb");
-		if (fp == NULL)
-		{
-			nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), fileName.c_str());
-			return false;
-		}
-
-		CURL *curl;
-		CURLcode res;
-		curl = curl_easy_init();
-		if (curl)
-		{
-			curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str());
-			curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
-			curl_easy_setopt(curl, CURLOPT_FILE, fp);
-			res = curl_easy_perform(curl);
-			long r;
-			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r);
-			curl_easy_cleanup(curl);
-
-			bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/;
-			fclose(fp);
-
-			if (diskFull)
-			{
-				CFile::deleteFile(downloadPath);
-				throw EDiskFullError(downloadPath);
-			}
-
-			if (res != CURLE_OK || r < 200 || r >= 300)
-			{
-				CFile::deleteFile(downloadPath);
-				nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str());
-				nlSleep(1000);
-				continue;
-			}
-		}
-		else
-		{
-			nldebug("Curl initialize failed");
-			fclose(fp);
-			CFile::deleteFile(downloadPath);
-			return false;
-		}
-
-		// ok!
-		break;
 	}
 
-	// extract into file
+	TEntries::iterator it = m_Entries.find(fileName);
-	std::string unpackPath = filePath + ".extract." + toString(rand() * rand());
+	if (it == m_Entries.end())
-
-	CHashKey outHash;
-	if (!unpackLZMA(downloadPath, unpackPath, outHash))
-	{
-		return false;
-	}
-
-	if (!(outHash == entry->Hash))
 	{
-		std::string wantHashS = entry->Hash.toString();
-		std::string outHashS = outHash.toString();
-		nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str());
 		return false;
 	}
+	const CStreamedPackage::CEntry *entry = it->second;
 
-	if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str()))
+	return provider->getFile(filePath, entry->Hash, fileName);
-	{
-		nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str());
-		// in case downloaded from another thread
-		return CFile::fileExists(filePath);
-	}
 
 	return true;
 }

code/ryzom/client/src/global.cpp

@@ -24,6 +24,9 @@
 using namespace NLMISC;
 
 // ***************************************************************************
+// Data
+NLMISC::CHttpPackageProvider *HttpPackageProvider = NULL;
+
 // Main System
 NL3D::UDriver				*Driver = NULL;		// The main 3D Driver
 NL3D::IStereoDisplay		*StereoDisplay = NULL; // Stereo display

code/ryzom/client/src/global.h

@@ -28,6 +28,11 @@
 
 
 // ***************************************************************************
+namespace NLMISC
+{
+	class CHttpPackageProvider;
+}
+
 namespace NL3D
 {
 	class UDriver;
@@ -78,6 +83,9 @@ const float		ExtraZoneLoadingVision	= 100.f;
 
 
 // ***************************************************************************
+// Data
+extern NLMISC::CHttpPackageProvider *HttpPackageProvider; // Http provider from on-the-fly downloaded game data
+
 // Main System
 extern NL3D::UDriver				*Driver;		// The main 3D Driver
 extern NL3D::IStereoDisplay			*StereoDisplay; // Stereo display

code/ryzom/client/src/init.cpp

@@ -35,6 +35,7 @@
 #include "nel/misc/block_memory.h"
 #include "nel/misc/system_utils.h"
 #include "nel/misc/streamed_package_manager.h"
+#include "nel/misc/http_package_provider.h"
 #include "nel/misc/cmd_args.h"
 // 3D Interface.
 #include "nel/3d/bloom_effect.h"
@@ -740,9 +741,14 @@ static void addPaths(IProgressCallback &progress, const std::vector<std::string>
 void initStreamedPackageManager(NLMISC::IProgressCallback &progress)
 {
 	CStreamedPackageManager &spm = CStreamedPackageManager::getInstance();
-	spm.Path = ClientCfg.StreamedPackagePath;
+	nlassert(!spm.Provider); // If this asserts, init was called twice without release
+	nlassert(!HttpPackageProvider); // Idem
+	CHttpPackageProvider *hpp = new CHttpPackageProvider();
+	hpp->Path = ClientCfg.StreamedPackagePath;
 	for (uint i = 0; i < ClientCfg.StreamedPackageHosts.size(); i++)
-		spm.Hosts.push_back(ClientCfg.StreamedPackageHosts[i]);
+		hpp->Hosts.push_back(ClientCfg.StreamedPackageHosts[i]);
+	spm.Provider = hpp;
+	HttpPackageProvider = hpp;
 }
 
 void addSearchPaths(IProgressCallback &progress)

code/ryzom/client/src/release.cpp

@@ -668,6 +668,8 @@ void release()
 	NLMISC::CBigFile::getInstance().removeAll();
 	NLMISC::CBigFile::releaseInstance();
 	NLMISC::CStreamedPackageManager::releaseInstance();
+	delete HttpPackageProvider;
+	HttpPackageProvider = NULL;
 	NL3D::CFastHLSModifier::releaseInstance();
 	CLandscapePolyDrawer::releaseInstance();
 	NL3D::CParticleSystemShape::releaseInstance();