Merge with develop

--HG--
branch : compatibility-develop

code/nel/include/nel/gui/group_html.h

@@ -821,13 +821,13 @@ namespace NLGUI
 		{
 		public:
 			CDataDownload(const std::string &u, const std::string &d, TDataType t, CViewBase *i, const std::string &s, const std::string &m, const CStyleParams &style = CStyleParams())
-				: curl(NULL), fp(NULL), url(u), dest(d), type(t), luaScript(s), md5sum(m)
+				: data(NULL), fp(NULL), url(u), dest(d), type(t), luaScript(s), md5sum(m)
 			{
 				if (t == ImgType) imgs.push_back(CDataImageDownload(i, style));
 			}
 
 		public:
-			CURL *curl;
+			CCurlWWWData *data;
 			std::string url;
 			std::string dest;
 			std::string luaScript;

code/nel/include/nel/gui/http_cache.h

@@ -0,0 +1,77 @@
+// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
+// Copyright (C) 2010  Winch Gate Property Limited
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef CL_HTTP_CACHE_H
+#define CL_HTTP_CACHE_H
+
+#include "nel/misc/types_nl.h"
+
+namespace NLGUI
+{
+	struct CHttpCacheObject
+	{
+		CHttpCacheObject(uint32 expires = 0, const std::string& lastModified = "", const std::string& etag = "")
+			: Expires(expires)
+			, LastModified(lastModified)
+			, Etag(etag){};
+
+		uint32 Expires;
+		std::string LastModified;
+		std::string Etag;
+
+		void serial(NLMISC::IStream& f);
+	};
+
+	/**
+	 * Keeping track of downloaded files cache related headers
+	 * \author Meelis Mägi (nimetu)
+	 * \date 2017
+	 */
+	class CHttpCache
+	{
+		typedef std::map<std::string, CHttpCacheObject> THttpCacheMap;
+
+	public:
+		static CHttpCache* getInstance();
+		static void release();
+
+	public:
+		void setCacheIndex(const std::string& fname);
+		void init();
+
+		CHttpCacheObject lookup(const std::string& fname);
+		void store(const std::string& fname, const CHttpCacheObject& data);
+
+		void flushCache();
+
+		void serial(NLMISC::IStream& f);
+
+	private:
+		CHttpCache();
+		~CHttpCache();
+
+		void pruneCache();
+
+		static CHttpCache* instance;
+
+		THttpCacheMap _List;
+
+		std::string _IndexFilename;
+		bool _Initialized;
+		size_t _MaxObjects;
+	};
+}
+#endif

code/nel/src/3d/meshvp_wind_tree.cpp

@@ -226,7 +226,8 @@ void CMeshVPWindTree::initVertexPrograms()
 		{
 			// setup of the VPLight fragment
 			uint	numPls= i/4;
-			bool	normalize= (i&1)!=0;
+			// FIXME: normalize=true makes trees dance, workaround for issue #160
+			bool	normalize= false; //(i&1)!=0;
 			bool	specular= (i&2)!=0;
 
 			// combine

code/nel/src/gui/group_html.cpp

@@ -45,6 +45,7 @@
 #include "nel/3d/texture_file.h"
 #include "nel/misc/big_file.h"
 #include "nel/gui/url_parser.h"
+#include "nel/gui/http_cache.h"
 
 using namespace std;
 using namespace NLMISC;
@@ -87,6 +88,15 @@ namespace NLGUI
 					curl_slist_free_all(HeadersSent);
 			}
 
+			void sendHeaders(const std::vector<std::string> headers)
+			{
+				for(uint i = 0; i < headers.size(); ++i)
+				{
+					HeadersSent = curl_slist_append(HeadersSent, headers[i].c_str());
+				}
+				curl_easy_setopt(Request, CURLOPT_HTTPHEADER, HeadersSent);
+			}
+
 			void setRecvHeader(const std::string &header)
 			{
 				size_t pos = header.find(": ");
@@ -110,12 +120,42 @@ namespace NLGUI
 				return "";
 			}
 
+			const uint32 getExpires()
+			{
+				time_t ret = 0;
+				if (HeadersRecv.count("expires") > 0)
+					ret = curl_getdate(HeadersRecv["expires"].c_str(), NULL);
+
+				return ret > -1 ? ret : 0;
+			}
+
+			const std::string getLastModified()
+			{
+				if (HeadersRecv.count("last-modified") > 0)
+				{
+					return HeadersRecv["last-modified"];
+				}
+
+				return "";
+			}
+
+			const std::string getEtag()
+			{
+				if (HeadersRecv.count("etag") > 0)
+				{
+					return HeadersRecv["etag"];
+				}
+
+				return "";
+			}
+
 		public:
 			CURL *Request;
 
 			std::string Url;
 			std::string Content;
 
+		private:
 			// headers sent with curl request, must be released after transfer
 			curl_slist * HeadersSent;
 
@@ -274,19 +314,16 @@ namespace NLGUI
 			return false;
 		}
 
-		// TODO: replace with expire and etag headers
+		time_t currentTime;
-		if (CFile::fileExists(download.dest))
+		time(&currentTime);
+
+		CHttpCacheObject cache = CHttpCache::getInstance()->lookup(download.dest);
+		if (cache.Expires > currentTime)
 		{
-			time_t currentTime;
-			time(&currentTime);
-			uint32 mtime = CFile::getFileModificationDate(download.dest);
-			if (mtime + 3600 > currentTime)
-			{
 	#ifdef LOG_DL
-				nlwarning("Cache for (%s) is not expired (%s, age:%d)", download.url.c_str(), download.dest.c_str(), currentTime - mtime);
+			nlwarning("Cache for (%s) is not expired (%s, expires:%d)", download.url.c_str(), download.dest.c_str(), cache.Expires - currentTime);
 	#endif
-				return false;
+			return false;
-			}
 		}
 
 		string tmpdest = download.dest + ".tmp";
@@ -312,7 +349,7 @@ namespace NLGUI
 			return false;
 		}
 
-		download.curl = curl;
+		download.data = new CCurlWWWData(curl, download.url);
 		download.fp = fp;
 
 		curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
@@ -322,6 +359,20 @@ namespace NLGUI
 		curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
 		curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
 
+		std::vector<std::string> headers;
+		if (!cache.Etag.empty())
+			headers.push_back("If-None-Match: " + cache.Etag);
+
+		if (!cache.LastModified.empty())
+			headers.push_back("If-Modified-Since: " + cache.LastModified);
+
+		if (headers.size() > 0)
+			download.data->sendHeaders(headers);
+
+		// catch headers
+		curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);
+		curl_easy_setopt(curl, CURLOPT_WRITEHEADER, download.data);
+
 		std::string userAgent = options.appName + "/" + options.appVersion;
 		curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str());
 
@@ -369,7 +420,7 @@ namespace NLGUI
 
 			RunningCurls++;
 	#ifdef LOG_DL
-			nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size());
+			nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().data->Request, Curls.size());
 		}
 		else
 		{
@@ -443,7 +494,7 @@ namespace NLGUI
 				}
 				RunningCurls++;
 	#ifdef LOG_DL
-				nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size());
+				nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().data->Request, Curls.size());
 			}
 			else
 			{
@@ -580,26 +631,44 @@ namespace NLGUI
 
 					for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); it++)
 					{
-						if(msg->easy_handle == it->curl)
+						if(it->data && it->data->Request == msg->easy_handle)
 						{
 							CURLcode res = msg->data.result;
 							long r;
-							curl_easy_getinfo(it->curl, CURLINFO_RESPONSE_CODE, &r);
+							curl_easy_getinfo(it->data->Request, CURLINFO_RESPONSE_CODE, &r);
 							fclose(it->fp);
 
+							receiveCookies(it->data->Request, _DocumentDomain, _TrustedDomain);
 	#ifdef LOG_DL
-							nlwarning("(%s) transfer '%p' completed with status %d, http %d, url (len %d) '%s'", _Id.c_str(), it->curl, res, r, it->url.size(), it->url.c_str());
+							nlwarning("(%s) transfer '%p' completed with status %d, http %d, url (len %d) '%s'", _Id.c_str(), it->data->Request, res, r, it->url.size(), it->url.c_str());
 	#endif
-							curl_multi_remove_handle(MultiCurl, it->curl);
+							curl_multi_remove_handle(MultiCurl, it->data->Request);
-							curl_easy_cleanup(it->curl);
 
 							string tmpfile = it->dest + ".tmp";
 							if(res != CURLE_OK || r < 200 || r >= 300 || (!it->md5sum.empty() && (it->md5sum != getMD5(tmpfile).toString())))
 							{
 								NLMISC::CFile::deleteFile(tmpfile.c_str());
+
+								// 304 Not Modified
+								if (res == CURLE_OK && r == 304)
+								{
+									CHttpCacheObject obj;
+									obj.Expires = it->data->getExpires();
+									obj.Etag = it->data->getEtag();
+									obj.LastModified = it->data->getLastModified();
+
+									CHttpCache::getInstance()->store(it->dest, obj);
+								}
 							}
 							else
 							{
+								CHttpCacheObject obj;
+								obj.Expires = it->data->getExpires();
+								obj.Etag = it->data->getEtag();
+								obj.LastModified = it->data->getLastModified();
+
+								CHttpCache::getInstance()->store(it->dest, obj);
+
 								string finalUrl;
 								if (it->type == ImgType)
 								{
@@ -642,6 +711,9 @@ namespace NLGUI
 								}
 							}
 
+							// release CCurlWWWData
+							delete it->data;
+
 							Curls.erase(it);
 							break;
 						}
@@ -656,7 +728,7 @@ namespace NLGUI
 		{
 			for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); it++)
 			{
-				if (it->curl == NULL) {
+				if (it->data == NULL) {
 	#ifdef LOG_DL
 					nlwarning("(%s) starting new download '%s'", _Id.c_str(), it->url.c_str());
 	#endif
@@ -4575,7 +4647,7 @@ namespace NLGUI
 		// remove download that are still queued
 		for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); )
 		{
-			if (it->curl == NULL) {
+			if (it->data == NULL) {
 	#ifdef LOG_DL
 		nlwarning("Remove waiting curl download (%s)", it->url.c_str());
 	#endif
@@ -5172,11 +5244,7 @@ namespace NLGUI
 		std::vector<std::string> headers;
 		headers.push_back("Accept-Language: "+options.languageCode);
 		headers.push_back("Accept-Charset: utf-8");
-		for(uint i=0; i< headers.size(); ++i)
+		_CurlWWW->sendHeaders(headers);
-		{
-			_CurlWWW->HeadersSent = curl_slist_append(_CurlWWW->HeadersSent, headers[i].c_str());
-		}
-		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, _CurlWWW->HeadersSent);
 
 		// catch headers for redirect
 		curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);

code/nel/src/gui/http_cache.cpp

@@ -0,0 +1,200 @@
+// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
+// Copyright (C) 2010  Winch Gate Property Limited
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "stdpch.h"
+#include "nel/gui/http_cache.h"
+
+using namespace std;
+using namespace NLMISC;
+
+#ifdef DEBUG_NEW
+#define new DEBUG_NEW
+#endif
+
+namespace NLGUI
+{
+	CHttpCache* CHttpCache::instance = NULL;
+
+	CHttpCache* CHttpCache::getInstance()
+	{
+		if (!instance)
+		{
+			instance = new CHttpCache();
+		}
+
+		return instance;
+	}
+
+	void CHttpCache::release()
+	{
+		delete instance;
+		instance = NULL;
+	}
+
+	CHttpCache::CHttpCache()
+		: _Initialized(false)
+		, _MaxObjects(100)
+	{ };
+
+	CHttpCache::~CHttpCache()
+	{
+		flushCache();
+	}
+
+	void CHttpCache::setCacheIndex(const std::string& fname)
+	{
+		_IndexFilename = fname;
+		_Initialized = false;
+	}
+
+	CHttpCacheObject CHttpCache::lookup(const std::string& fname)
+	{
+		if (!_Initialized)
+			init();
+
+		if (_List.count(fname) > 0)
+			return _List[fname];
+
+		return CHttpCacheObject();
+	}
+
+	void CHttpCache::store(const std::string& fname, const CHttpCacheObject& data)
+	{
+		if (!_Initialized)
+			init();
+
+		_List[fname] = data;
+	}
+
+	void CHttpCache::init()
+	{
+		if (_Initialized)
+			return;
+
+		_Initialized = true;
+
+		if (_IndexFilename.empty() || !CFile::fileExists(_IndexFilename))
+			return;
+
+		CIFile in;
+		if (!in.open(_IndexFilename)) {
+			nlwarning("Unable to open %s for reading", _IndexFilename.c_str());
+			return;
+		}
+
+		serial(in);
+	}
+
+	void CHttpCacheObject::serial(NLMISC::IStream& f)
+	{
+		f.serialVersion(1);
+		f.serial(Expires);
+		f.serial(LastModified);
+		f.serial(Etag);
+	}
+
+	void CHttpCache::serial(NLMISC::IStream& f)
+	{
+		// saved state is ignored when version checks fail
+		try {
+			f.serialVersion(1);
+
+			// CacheIdx
+			f.serialCheck(NELID("hcaC"));
+			f.serialCheck(NELID("xdIe"));
+
+			if (f.isReading())
+			{
+				uint32 numFiles;
+				f.serial(numFiles);
+
+				_List.clear();
+				for (uint k = 0; k < numFiles; ++k)
+				{
+					std::string fname;
+					f.serial(fname);
+
+					CHttpCacheObject obj;
+					obj.serial(f);
+
+					_List[fname] = obj;
+				}
+			}
+			else
+			{
+				uint32 numFiles = _List.size();
+				f.serial(numFiles);
+
+				for (THttpCacheMap::iterator it = _List.begin(); it != _List.end(); ++it)
+				{
+					std::string fname(it->first);
+					f.serial(fname);
+
+					(*it).second.serial(f);
+				}
+			}
+		} catch (...) {
+			_List.clear();
+			nlwarning("Invalid cache index format (%s)", _IndexFilename.c_str());
+			return;
+		}
+	}
+
+	void CHttpCache::pruneCache()
+	{
+		if (_List.size() < _MaxObjects)
+			return;
+
+		size_t mustDrop = _List.size() - _MaxObjects;
+
+		time_t currentTime;
+		time(&currentTime);
+
+		// if we over object limit, then start removing expired objects
+		// this does not guarantee that max limit is reached
+		for (THttpCacheMap::iterator it = _List.begin(); it != _List.end();) {
+			if (it->second.Expires <= currentTime) {
+				it = _List.erase(it);
+
+				--mustDrop;
+				if (mustDrop == 0)
+					break;
+			}
+			else
+			{
+				++it;
+			}
+		}
+	}
+
+	void CHttpCache::flushCache()
+	{
+		if (_IndexFilename.empty())
+			return;
+
+		pruneCache();
+
+		COFile out;
+		if (!out.open(_IndexFilename))
+		{
+			nlwarning("Unable to open %s for writing", _IndexFilename.c_str());
+			return;
+		}
+
+		serial(out);
+		out.close();
+	}
+}

code/nel/src/gui/stdpch.h

@@ -26,6 +26,7 @@
 
 #include <string>
 #include <limits>
+#include <ctime>
 
 #include "nel/misc/types_nl.h"
 #include "nel/misc/algo.h"

code/ryzom/client/src/init.cpp

@@ -68,6 +68,7 @@
 
 #include "interface_v3/sbrick_manager.h"
 #include "nel/gui/widget_manager.h"
+#include "nel/gui/http_cache.h"
 //
 #include "gabarit.h"
 #include "hair_set.h"
@@ -1365,6 +1366,10 @@ void prelogInit()
 			//nlinfo ("PROFILE: %d seconds for Add search paths Data", (uint32)(ryzomGetLocalTime ()-initPaths)/1000);
 		}
 
+		// Initialize HTTP cache
+		CHttpCache::getInstance()->setCacheIndex("cache/cache.index");
+		CHttpCache::getInstance()->init();
+
 		// Register the reflected classes
 		registerInterfaceElements();
 

code/ryzom/client/src/release.cpp

@@ -94,7 +94,7 @@
 #include "bg_downloader_access.h"
 #include "nel/gui/lua_manager.h"
 #include "item_group_manager.h"
-
+#include "nel/gui/http_cache.h"
 
 ///////////
 // USING //
@@ -687,6 +687,7 @@ void release()
 	CWidgetManager::release();
 	CViewRenderer::release();
 	CIXml::releaseLibXml();
+	CHttpCache::release();
 
 #if FINAL_VERSION
 	// openURL ("http://ryzom.com/exit/");