diff --git a/code/nel/include/nel/misc/http_package_provider.h b/code/nel/include/nel/misc/http_package_provider.h new file mode 100644 index 000000000..b4b82d70a --- /dev/null +++ b/code/nel/include/nel/misc/http_package_provider.h @@ -0,0 +1,50 @@ +// NeL - MMORPG Framework +// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_HTTP_PACKAGE_PROVIDER_H +#define NLMISC_HTTP_PACKAGE_PROVIDER_H + +#include +#include + +namespace NLMISC { + +class CHttpPackageProvider : public IStreamedPackageProvider +{ +public: + CHttpPackageProvider(); + virtual ~CHttpPackageProvider(); + + /// Download a file. This call is blocking + /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. + /// hash: [in] + virtual bool getFile(std::string &filePath, const CHashKey &hash, const std::string &name) NL_OVERRIDE; + +public: + /// Set storage path (ex. stream/) + std::string Path; + + /// Loads a package into the package manager (ex. http://cdn.ryzom.dev/open/stream/) + typedef std::vector THosts; + THosts Hosts; + +}; /* class CHttpPackageProvider */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_MANAGER_H */ + +/* end of file */ diff --git a/code/nel/include/nel/misc/i_streamed_package_provider.h b/code/nel/include/nel/misc/i_streamed_package_provider.h new file mode 100644 index 000000000..6e74d866d --- /dev/null +++ b/code/nel/include/nel/misc/i_streamed_package_provider.h @@ -0,0 +1,44 @@ +// NeL - MMORPG Framework +// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#ifndef NLMISC_STREAMED_PACKAGE_PROVIDER_H +#define NLMISC_STREAMED_PACKAGE_PROVIDER_H + +#include +#include +#include + +namespace NLMISC { + +class IStreamedPackageProvider +{ +public: + IStreamedPackageProvider(); + virtual ~IStreamedPackageProvider(); + + /// Download a file. This call is blocking + /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. + /// hash: [in] + /// name: [in] name for debugging + virtual bool getFile(std::string &filePath, const CHashKey &hash, const std::string &name = ""); + +}; /* class IStreamedPackageProvider */ + +} /* namespace NLMISC */ + +#endif /* #ifndef NLMISC_STREAMED_PACKAGE_PROVIDER_H */ + +/* end of file */ diff --git a/code/nel/include/nel/misc/streamed_package.h b/code/nel/include/nel/misc/streamed_package.h index c777d8a02..9040598d6 100644 --- a/code/nel/include/nel/misc/streamed_package.h +++ b/code/nel/include/nel/misc/streamed_package.h @@ -42,6 +42,8 @@ public: void serial(NLMISC::IStream &f) throw(NLMISC::EStream); + /// result: [out] ex. /00/00/000000000.. + /// hash: [in] static void makePath(std::string &result, const CHashKey &hash); public: diff --git a/code/nel/include/nel/misc/streamed_package_manager.h b/code/nel/include/nel/misc/streamed_package_manager.h index e23344a70..85d13db59 100644 --- a/code/nel/include/nel/misc/streamed_package_manager.h +++ b/code/nel/include/nel/misc/streamed_package_manager.h @@ -20,6 +20,7 @@ #include #include #include +#include namespace NLMISC { @@ -40,21 +41,19 @@ public: /// Unload all packages void unloadAll(); - /// Get an existing file or download if necessary + /// Get an existing file or download if necessary. This call is blocking /// filePath: [out] ex. /games/nel/stream/00/00/000000000.. - /// fileName: ex. fy_hof_underwear.dds + /// fileName: [in] ex. fy_hof_underwear.dds bool getFile(std::string &filePath, const std::string &fileName); /// Get file size + /// fileSize: [out] + /// fileName: [in] ex. fy_hof_underwear.dds bool getFileSize(uint32 &fileSize, const std::string &fileName); public: - /// Set storage path (ex. stream/) - std::string Path; - - /// Loads a package into the package manager (ex. http://patch.live.polyverse.org/stream/) - typedef std::vector THosts; - THosts Hosts; + /// Streamed package provider. This downloads the files + IStreamedPackageProvider *Provider; private: typedef std::map TPackages; diff --git a/code/nel/include/nel/misc/types_nl.h b/code/nel/include/nel/misc/types_nl.h index 353f50490..f42f410e6 100644 --- a/code/nel/include/nel/misc/types_nl.h +++ b/code/nel/include/nel/misc/types_nl.h @@ -479,6 +479,7 @@ extern void operator delete[](void *p) throw(); # define CHashMultiMap NL_ISO_STDTR1_NAMESPACE::unordered_multimap # define CUniquePtr ::std::auto_ptr # define CUniquePtrMove +# define NL_OVERRIDE #elif defined(NL_COMP_VC) && (NL_COMP_VC_VERSION >= 70 && NL_COMP_VC_VERSION <= 90) // VC7 through 9 # include # include @@ -487,6 +488,7 @@ extern void operator delete[](void *p) throw(); # define CHashMultiMap stdext::hash_multimap # define CUniquePtr ::std::auto_ptr # define CUniquePtrMove +# define NL_OVERRIDE #elif defined(NL_COMP_GCC) // GCC4 # include # include @@ -526,6 +528,10 @@ template<> struct hash */ typedef uint16 ucchar; +#ifndef NL_OVERRIDE +#define NL_OVERRIDE override +#endif + #if defined(NL_OS_WINDOWS) && (defined(UNICODE) || defined(_UNICODE)) #define nltmain wmain #define nltWinMain wWinMain diff --git a/code/nel/src/misc/http_package_provider.cpp b/code/nel/src/misc/http_package_provider.cpp new file mode 100644 index 000000000..035fa2a5d --- /dev/null +++ b/code/nel/src/misc/http_package_provider.cpp @@ -0,0 +1,145 @@ +// NeL - MMORPG Framework +// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// 3rd Party includes +#include + +// Project includes +#include +#include +#include +#include +#include +#include + +namespace NLMISC +{ + +CHttpPackageProvider::CHttpPackageProvider() +{ + // init +} + +CHttpPackageProvider::~CHttpPackageProvider() +{ + // release +} + +bool CHttpPackageProvider::getFile(std::string &filePath, const CHashKey &hash, const std::string &name) +{ + CStreamedPackage::makePath(filePath, hash); + std::string downloadUrlFile = filePath + ".lzma"; + filePath = Path + filePath; + std::string downloadPath = filePath + ".download." + toString(rand() * rand()); + + std::string storageDirectory = CFile::getPath(downloadPath); + CFile::createDirectoryTree(storageDirectory); + /*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory)) + { + nldebug("Unable to create directory '%s'", storageDirectory.c_str()); + return false; + }*/ + + // download + for (;;) + { + if (CFile::fileExists(filePath)) + return true; + + std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile; + nldebug("Download streamed package '%s' from '%s'", name.c_str(), downloadUrl.c_str()); + + FILE *fp = fopen(downloadPath.c_str(), "wb"); + if (fp == NULL) + { + nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), name.c_str()); + return false; + } + + CURL *curl; + CURLcode res; + curl = curl_easy_init(); + if (curl) + { + curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_FILE, fp); + res = curl_easy_perform(curl); + long r; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r); + curl_easy_cleanup(curl); + + bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/; + fclose(fp); + + if (diskFull) + { + CFile::deleteFile(downloadPath); + throw EDiskFullError(downloadPath); + } + + if (res != CURLE_OK || r < 200 || r >= 300) + { + CFile::deleteFile(downloadPath); + nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str()); + nlSleep(1000); + continue; + } + } + else + { + nldebug("Curl initialize failed"); + fclose(fp); + CFile::deleteFile(downloadPath); + return false; + } + + // ok! + break; + } + + // extract into file + std::string unpackPath = filePath + ".extract." + toString(rand() * rand()); + + CHashKey outHash; + if (!unpackLZMA(downloadPath, unpackPath, outHash)) + { + return false; + } + + if (!(outHash == hash)) + { + std::string wantHashS = hash.toString(); + std::string outHashS = outHash.toString(); + nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str()); + return false; + } + + if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str())) + { + nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str()); + // in case downloaded from another thread + return CFile::fileExists(filePath); + } + + return true; +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/src/misc/i_streamed_package_provider.cpp b/code/nel/src/misc/i_streamed_package_provider.cpp new file mode 100644 index 000000000..12c873760 --- /dev/null +++ b/code/nel/src/misc/i_streamed_package_provider.cpp @@ -0,0 +1,37 @@ +// NeL - MMORPG Framework +// Copyright (C) 2019 Jan BOON (jan.boon@kaetemi.be) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include "stdmisc.h" + +// Project includes +#include + +namespace NLMISC +{ + +IStreamedPackageProvider::IStreamedPackageProvider() +{ + // init +} + +IStreamedPackageProvider::~IStreamedPackageProvider() +{ + // release +} + +} /* namespace NLMISC */ + +/* end of file */ diff --git a/code/nel/src/misc/streamed_package_manager.cpp b/code/nel/src/misc/streamed_package_manager.cpp index 483df6a52..ac605b049 100644 --- a/code/nel/src/misc/streamed_package_manager.cpp +++ b/code/nel/src/misc/streamed_package_manager.cpp @@ -16,14 +16,10 @@ #include "stdmisc.h" -// 3rd Party includes -#include - // Project includes #include #include #include -#include #include namespace NLMISC @@ -96,105 +92,21 @@ bool CStreamedPackageManager::getFile(std::string &filePath, const std::string & { // nldebug("Get file path for streamed file '%s'", fileName.c_str()); - TEntries::iterator it = m_Entries.find(fileName); - if (it == m_Entries.end()) - return false; - const CStreamedPackage::CEntry *entry = it->second; - - CStreamedPackage::makePath(filePath, entry->Hash); - std::string downloadUrlFile = filePath + ".lzma"; - filePath = Path + filePath; - std::string downloadPath = filePath + ".download." + toString(rand() * rand()); - - std::string storageDirectory = CFile::getPath(downloadPath); - CFile::createDirectoryTree(storageDirectory); - /*if (!CFile::isDirectory(storageDirectory) || !CFile::createDirectoryTree(storageDirectory)) + IStreamedPackageProvider *provider = Provider; + if (!provider) { - nldebug("Unable to create directory '%s'", storageDirectory.c_str()); + nlerrornoex("No streamed package provider was set"); return false; - }*/ - - // download - for (;;) - { - if (CFile::fileExists(filePath)) - return true; - - std::string downloadUrl = Hosts[rand() % Hosts.size()] + downloadUrlFile; - nldebug("Download streamed package '%s' from '%s'", fileName.c_str(), downloadUrl.c_str()); - - FILE *fp = fopen(downloadPath.c_str(), "wb"); - if (fp == NULL) - { - nldebug("Unable to create file '%s' for '%s'", downloadPath.c_str(), fileName.c_str()); - return false; - } - - CURL *curl; - CURLcode res; - curl = curl_easy_init(); - if (curl) - { - curl_easy_setopt(curl, CURLOPT_URL, downloadUrl.c_str()); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_FILE, fp); - res = curl_easy_perform(curl); - long r; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r); - curl_easy_cleanup(curl); - - bool diskFull = ferror(fp) && errno == 28 /*ENOSPC*/; - fclose(fp); - - if (diskFull) - { - CFile::deleteFile(downloadPath); - throw EDiskFullError(downloadPath); - } - - if (res != CURLE_OK || r < 200 || r >= 300) - { - CFile::deleteFile(downloadPath); - nldebug("Download failed '%s', retry in 1s", downloadUrl.c_str()); - nlSleep(1000); - continue; - } - } - else - { - nldebug("Curl initialize failed"); - fclose(fp); - CFile::deleteFile(downloadPath); - return false; - } - - // ok! - break; } - // extract into file - std::string unpackPath = filePath + ".extract." + toString(rand() * rand()); - - CHashKey outHash; - if (!unpackLZMA(downloadPath, unpackPath, outHash)) - { - return false; - } - - if (!(outHash == entry->Hash)) + TEntries::iterator it = m_Entries.find(fileName); + if (it == m_Entries.end()) { - std::string wantHashS = entry->Hash.toString(); - std::string outHashS = outHash.toString(); - nlwarning("Invalid SHA1 hash for file '%s', download has hash '%s'", wantHashS.c_str(), outHashS.c_str()); return false; } + const CStreamedPackage::CEntry *entry = it->second; - if (!CFile::moveFile(filePath.c_str(), unpackPath.c_str())) - { - nldebug("Failed moving '%s' to '%s'", unpackPath.c_str(), filePath.c_str()); - // in case downloaded from another thread - return CFile::fileExists(filePath); - } + return provider->getFile(filePath, entry->Hash, fileName); return true; } diff --git a/code/ryzom/client/src/global.cpp b/code/ryzom/client/src/global.cpp index 4b8e1aad7..41d41160c 100644 --- a/code/ryzom/client/src/global.cpp +++ b/code/ryzom/client/src/global.cpp @@ -24,6 +24,9 @@ using namespace NLMISC; // *************************************************************************** +// Data +NLMISC::CHttpPackageProvider *HttpPackageProvider = NULL; + // Main System NL3D::UDriver *Driver = NULL; // The main 3D Driver NL3D::IStereoDisplay *StereoDisplay = NULL; // Stereo display diff --git a/code/ryzom/client/src/global.h b/code/ryzom/client/src/global.h index 78cebcec9..75ba2381d 100644 --- a/code/ryzom/client/src/global.h +++ b/code/ryzom/client/src/global.h @@ -28,6 +28,11 @@ // *************************************************************************** +namespace NLMISC +{ + class CHttpPackageProvider; +} + namespace NL3D { class UDriver; @@ -78,6 +83,9 @@ const float ExtraZoneLoadingVision = 100.f; // *************************************************************************** +// Data +extern NLMISC::CHttpPackageProvider *HttpPackageProvider; // Http provider from on-the-fly downloaded game data + // Main System extern NL3D::UDriver *Driver; // The main 3D Driver extern NL3D::IStereoDisplay *StereoDisplay; // Stereo display diff --git a/code/ryzom/client/src/init.cpp b/code/ryzom/client/src/init.cpp index 1f9c10b34..85eb50b51 100644 --- a/code/ryzom/client/src/init.cpp +++ b/code/ryzom/client/src/init.cpp @@ -35,6 +35,7 @@ #include "nel/misc/block_memory.h" #include "nel/misc/system_utils.h" #include "nel/misc/streamed_package_manager.h" +#include "nel/misc/http_package_provider.h" #include "nel/misc/cmd_args.h" // 3D Interface. #include "nel/3d/bloom_effect.h" @@ -740,9 +741,14 @@ static void addPaths(IProgressCallback &progress, const std::vector void initStreamedPackageManager(NLMISC::IProgressCallback &progress) { CStreamedPackageManager &spm = CStreamedPackageManager::getInstance(); - spm.Path = ClientCfg.StreamedPackagePath; + nlassert(!spm.Provider); // If this asserts, init was called twice without release + nlassert(!HttpPackageProvider); // Idem + CHttpPackageProvider *hpp = new CHttpPackageProvider(); + hpp->Path = ClientCfg.StreamedPackagePath; for (uint i = 0; i < ClientCfg.StreamedPackageHosts.size(); i++) - spm.Hosts.push_back(ClientCfg.StreamedPackageHosts[i]); + hpp->Hosts.push_back(ClientCfg.StreamedPackageHosts[i]); + spm.Provider = hpp; + HttpPackageProvider = hpp; } void addSearchPaths(IProgressCallback &progress) diff --git a/code/ryzom/client/src/release.cpp b/code/ryzom/client/src/release.cpp index be34d9ba8..6686fea5d 100644 --- a/code/ryzom/client/src/release.cpp +++ b/code/ryzom/client/src/release.cpp @@ -668,6 +668,8 @@ void release() NLMISC::CBigFile::getInstance().removeAll(); NLMISC::CBigFile::releaseInstance(); NLMISC::CStreamedPackageManager::releaseInstance(); + delete HttpPackageProvider; + HttpPackageProvider = NULL; NL3D::CFastHLSModifier::releaseInstance(); CLandscapePolyDrawer::releaseInstance(); NL3D::CParticleSystemShape::releaseInstance();