From 20c216ecda7a08e73e6005b201b7e56078df3763 Mon Sep 17 00:00:00 2001 From: Nimetu Date: Sun, 16 Apr 2017 22:01:44 +0300 Subject: [PATCH] Added: Basic queue for curl downloads --HG-- branch : develop --- code/nel/include/nel/gui/group_html.h | 5 +- code/nel/src/gui/group_html.cpp | 189 ++++++++++++++++---------- 2 files changed, 120 insertions(+), 74 deletions(-) diff --git a/code/nel/include/nel/gui/group_html.h b/code/nel/include/nel/gui/group_html.h index 479fc7384..2dc5e80bc 100644 --- a/code/nel/include/nel/gui/group_html.h +++ b/code/nel/include/nel/gui/group_html.h @@ -820,7 +820,8 @@ namespace NLGUI struct CDataDownload { public: - CDataDownload(CURL *c, const std::string &u, const std::string &d, FILE *f, TDataType t, CViewBase *i, const std::string &s, const std::string &m, const CStyleParams &style = CStyleParams()) : curl(c), url(u), dest(d), luaScript(s), md5sum(m), type(t), fp(f) + CDataDownload(const std::string &u, const std::string &d, TDataType t, CViewBase *i, const std::string &s, const std::string &m, const CStyleParams &style = CStyleParams()) + : curl(NULL), fp(NULL), url(u), dest(d), type(t), luaScript(s), md5sum(m) { if (t == ImgType) imgs.push_back(CDataImageDownload(i, style)); } @@ -840,6 +841,8 @@ namespace NLGUI CURLM *MultiCurl; int RunningCurls; + bool startCurlDownload(CDataDownload &download); + void initImageDownload(); void checkImageDownload(); void addImageDownload(const std::string &url, CViewBase *img, const CStyleParams &style = CStyleParams()); diff --git a/code/nel/src/gui/group_html.cpp b/code/nel/src/gui/group_html.cpp index c5c4bb532..418704262 100644 --- a/code/nel/src/gui/group_html.cpp +++ b/code/nel/src/gui/group_html.cpp @@ -265,6 +265,61 @@ namespace NLGUI return dest; } + // Add url to MultiCurl queue and return cURL handle + bool CGroupHTML::startCurlDownload(CDataDownload &download) + { + if (!MultiCurl) + { + nlwarning("Invalid MultiCurl handle, unable to download '%s'", download.url.c_str()); + return false; + } + + string tmpdest = download.dest + ".tmp"; + + // erase the tmp file if exists + if (CFile::fileExists(tmpdest)) + CFile::deleteFile(tmpdest); + + FILE *fp = nlfopen (tmpdest, "wb"); + if (fp == NULL) + { + nlwarning("Can't open file '%s' for writing: code=%d '%s'", tmpdest.c_str (), errno, strerror(errno)); + return false; + } + + CURL *curl = curl_easy_init(); + if (!curl) + { + fclose(fp); + CFile::deleteFile(tmpdest); + + nlwarning("Creating cURL handle failed, unable to download '%s'", download.url.c_str()); + return false; + } + + download.curl = curl; + download.fp = fp; + + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true); + curl_easy_setopt(curl, CURLOPT_URL, download.url.c_str()); + + // limit curl to HTTP and HTTPS protocols only + curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + + std::string userAgent = options.appName + "/" + options.appVersion; + curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str()); + + sendCookies(curl, _DocumentDomain, _TrustedDomain); + + curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite); + + curl_multi_add_handle(MultiCurl, curl); + + return true; + } + // Add a image download request in the multi_curl void CGroupHTML::addImageDownload(const string &url, CViewBase *img, const CStyleParams &style) { @@ -285,55 +340,29 @@ namespace NLGUI // use requested url for local name string dest = localImageName(url); - string tmpdest = localImageName(url)+".tmp"; #ifdef LOG_DL nlwarning("add to download '%s' dest '%s' img %p", finalUrl.c_str(), dest.c_str(), img); #endif - // erase the tmp file if exists - if (NLMISC::CFile::fileExists(tmpdest)) - NLMISC::CFile::deleteFile(tmpdest); - if (!NLMISC::CFile::fileExists(dest)) { - if (!MultiCurl) - { - nlwarning("Invalid MultiCurl handle, unable to download '%s'", finalUrl.c_str()); - return; - } + Curls.push_back(CDataDownload(finalUrl, dest, ImgType, img, "", "", style)); + if (Curls.size() < options.curlMaxConnections) { + if (!startCurlDownload(Curls.back())) + { + Curls.pop_back(); + return; + } - CURL *curl = curl_easy_init(); - if (!curl) - { - nlwarning("Creating cURL handle failed, unable to download '%s'", finalUrl.c_str()); - return; + RunningCurls++; + #ifdef LOG_DL + nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size()); } - - FILE *fp = nlfopen(tmpdest, "wb"); - if (fp == NULL) + else { - curl_easy_cleanup(curl); - - nlwarning("Can't open file '%s' for writing: code=%d '%s'", tmpdest.c_str (), errno, strerror(errno)); - return; - } - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true); - curl_easy_setopt(curl, CURLOPT_URL, finalUrl.c_str()); - - std::string userAgent = options.appName + "/" + options.appVersion; - curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str()); - - sendCookies(curl, _DocumentDomain, _TrustedDomain); - - curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite); - - curl_multi_add_handle(MultiCurl, curl); - Curls.push_back(CDataDownload(curl, finalUrl, dest, fp, ImgType, img, "", "", style)); - #ifdef LOG_DL - nlwarning("adding handle %x, %d curls", curl, Curls.size()); + nlwarning("(%s) download queued, %d curls", _Id.c_str(), Curls.size()); #endif - RunningCurls++; + } } else { @@ -378,14 +407,9 @@ namespace NLGUI } string dest = localBnpName(url); - string tmpdest = localBnpName(url)+".tmp"; #ifdef LOG_DL nlwarning("add to download '%s' dest '%s'", url.c_str(), dest.c_str()); #endif - - // erase the tmp file if exists - if (NLMISC::CFile::fileExists(tmpdest)) - NLMISC::CFile::deleteFile(tmpdest); // create/delete the local file if (NLMISC::CFile::fileExists(dest)) @@ -402,39 +426,23 @@ namespace NLGUI } if (action != "delete") { - if (!MultiCurl) - { - nlwarning("Invalid MultiCurl handle, unable to download '%s'", url.c_str()); - return false; - } - - CURL *curl = curl_easy_init(); - if (!curl) + Curls.push_back(CDataDownload(url, dest, BnpType, NULL, script, md5sum)); + if (Curls.size() < options.curlMaxConnections) { - nlwarning("Creating cURL handle failed, unable to download '%s'", url.c_str()); - return false; + if (!startCurlDownload(Curls.back())) + { + Curls.pop_back(); + return false; + } + RunningCurls++; + #ifdef LOG_DL + nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size()); } - - FILE *fp = nlfopen (tmpdest, "wb"); - if (fp == NULL) + else { - curl_easy_cleanup(curl); - nlwarning("Can't open file '%s' for writing: code=%d '%s'", tmpdest.c_str (), errno, strerror(errno)); - return false; - } - - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - - curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite); - - curl_multi_add_handle(MultiCurl, curl); - Curls.push_back(CDataDownload(curl, url, dest, fp, BnpType, NULL, script, md5sum)); - #ifdef LOG_DL - nlwarning("adding handle %x, %d curls", curl, Curls.size()); + nlwarning("(%s) download queued, %d curls", _Id.c_str(), Curls.size()); #endif - RunningCurls++; + } } else return true; @@ -631,7 +639,30 @@ namespace NLGUI } } } + RunningCurls = NewRunningCurls; + + if (RunningCurls < options.curlMaxConnections) + { + for (vector::iterator it=Curls.begin(); itcurl == NULL) { + #ifdef LOG_DL + nlwarning("(%s) starting new download '%s'", _Id.c_str(), it->url.c_str()); + #endif + if (!startCurlDownload(*it)) + { + Curls.erase(it); + break; + } + + RunningCurls++; + if (RunningCurls >= options.curlMaxConnections) + break; + } + } + } + #ifdef LOG_DL if (RunningCurls > 0 || !Curls.empty()) nlwarning("(%s) RunningCurls %d, _Curls %d", _Id.c_str(), RunningCurls, Curls.size()); @@ -4531,6 +4562,18 @@ namespace NLGUI Curls[i].imgs.clear(); } + // remove download that are still queued + for (vector::iterator it=Curls.begin(); itcurl == NULL) { + #ifdef LOG_DL + nlwarning("Remove waiting curl download (%s)", it->url.c_str()); + #endif + it = Curls.erase(it); + } else { + ++it; + } + } } // ***************************************************************************