From 74acdedeb24bbcf32faec2830dccf2146e8c3bea Mon Sep 17 00:00:00 2001 From: Thulinma Date: Sat, 10 Jun 2017 12:28:43 +0200 Subject: [PATCH] Added HTTP::Downloader class --- CMakeLists.txt | 2 + lib/downloader.cpp | 117 ++++++++++++++++++++++++++++++++++++++++++++ lib/downloader.h | 27 ++++++++++ lib/http_parser.cpp | 2 +- lib/http_parser.h | 2 +- 5 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 lib/downloader.cpp create mode 100644 lib/downloader.h diff --git a/CMakeLists.txt b/CMakeLists.txt index cbf7be2e..85850a08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,6 +109,7 @@ set(libHeaders ${SOURCE_DIR}/lib/flv_tag.h ${SOURCE_DIR}/lib/h264.h ${SOURCE_DIR}/lib/http_parser.h + ${SOURCE_DIR}/lib/downloader.h ${SOURCE_DIR}/lib/json.h ${SOURCE_DIR}/lib/langcodes.h ${SOURCE_DIR}/lib/mp4_adobe.h @@ -146,6 +147,7 @@ set(libSources ${SOURCE_DIR}/lib/flv_tag.cpp ${SOURCE_DIR}/lib/h264.cpp ${SOURCE_DIR}/lib/http_parser.cpp + ${SOURCE_DIR}/lib/downloader.cpp ${SOURCE_DIR}/lib/json.cpp ${SOURCE_DIR}/lib/langcodes.cpp ${SOURCE_DIR}/lib/mp4_adobe.cpp diff --git a/lib/downloader.cpp b/lib/downloader.cpp new file mode 100644 index 00000000..2a07f654 --- /dev/null +++ b/lib/downloader.cpp @@ -0,0 +1,117 @@ +#include "downloader.h" +#include "defines.h" +#include "timing.h" + +namespace HTTP{ + + /// Returns a reference to the internal HTTP::Parser body element + std::string &Downloader::data(){return H.body;} + + /// Returns the status text of the HTTP Request. + std::string &Downloader::getStatusText(){return H.method;} + + /// Returns the status code of the HTTP Request. + uint32_t Downloader::getStatusCode(){return atoi(H.url.c_str());} + + /// Returns true if the HTTP Request is OK + bool Downloader::isOk(){return (getStatusCode() == 200);} + + /// Returns the given header from the response, or empty string if it does not exist. + std::string Downloader::getHeader(const std::string &headerName){ + return H.GetHeader(headerName); + } + + /// Simply turns link into a HTTP::URL and calls get(const HTTP::URL&) + bool Downloader::get(const std::string &link){ + HTTP::URL uri(link); + return get(uri); + } + + /// Sets an extra (or overridden) header to be sent with outgoing requests. + void Downloader::setHeader(const std::string &name, const std::string &val){ + extraHeaders[name] = val; + } + + /// Clears all extra/override headers for outgoing requests. + void Downloader::clearHeaders(){extraHeaders.clear();} + + /// Downloads the given URL into 'H', returns true on success. + /// Makes at most 5 attempts, and will wait no longer than 5 seconds without receiving data. + bool Downloader::get(const HTTP::URL &link, uint8_t maxRecursiveDepth){ + if (!link.host.size()){return false;} + if (link.protocol != "http"){ + FAIL_MSG("Protocol not supported: %s", link.protocol.c_str()); + return false; + } + INFO_MSG("Retrieving %s", link.getUrl().c_str()); + unsigned int loop = 6; // max 5 attempts + + while (--loop){// loop while we are unsuccessful + H.Clean(); + // Reconnect if needed + if (!S || link.host != connectedHost || link.getPort() != connectedPort){ + S.close(); + connectedHost = link.host; + connectedPort = link.getPort(); + S = Socket::Connection(connectedHost, connectedPort, true); + } + H.url = "/" + link.path; + if (link.args.size()){H.url += "?" + link.args;} + if (link.port.size()){ + H.SetHeader("Host", link.host + ":" + link.port); + }else{ + H.SetHeader("Host", link.host); + } + H.SetHeader("User-Agent", "MistServer " PACKAGE_VERSION); + H.SetHeader("X-Version", PACKAGE_VERSION); + H.SetHeader("Accept", "*/*"); + if (extraHeaders.size()){ + for (std::map::iterator it = extraHeaders.begin(); + it != extraHeaders.end(); ++it){ + H.SetHeader(it->first, it->second); + } + } + H.SendRequest(S); + H.Clean(); + uint64_t reqTime = Util::bootSecs(); + while (S && Util::bootSecs() < reqTime + 5){ + // No data? Wait for a second or so. + if (!S.spool()){ + if (progressCallback != 0){ + if (!progressCallback()){ + WARN_MSG("Download aborted by callback"); + return false; + } + } + Util::sleep(250); + continue; + } + // Data! Check if we can parse it... + if (H.Read(S)){ + if (getStatusCode() >= 300 && getStatusCode() < 400){ + // follow redirect + std::string location = getHeader("Location"); + if (maxRecursiveDepth == 0){ + FAIL_MSG("Maximum redirect depth reached: %s", location.c_str()); + return false; + }else{ + FAIL_MSG("Following redirect to %s", location.c_str()); + return get(link.link(location), maxRecursiveDepth--); + } + } + return true; // Success! + } + // reset the 5 second timeout + reqTime = Util::bootSecs(); + } + if (S){ + FAIL_MSG("Timeout while retrieving %s", link.getUrl().c_str()); + return false; + } + Util::sleep(500); // wait a bit before retrying + } + FAIL_MSG("Could not retrieve %s", link.getUrl().c_str()); + return false; + } +} + diff --git a/lib/downloader.h b/lib/downloader.h new file mode 100644 index 00000000..0c40b707 --- /dev/null +++ b/lib/downloader.h @@ -0,0 +1,27 @@ +#include "http_parser.h" +#include "socket.h" + +namespace HTTP{ + class Downloader{ + public: + Downloader(){progressCallback = 0;} + std::string &data(); + bool get(const std::string &link); + bool get(const HTTP::URL &link, uint8_t maxRecursiveDepth = 6); + std::string getHeader(const std::string &headerName); + std::string &getStatusText(); + uint32_t getStatusCode(); + bool isOk(); + bool (*progressCallback)(); ///< Called every time the socket stalls, up to 4X per second. + void setHeader(const std::string &name, const std::string &val); + void clearHeaders(); + + private: + std::map extraHeaders; ///< Holds extra headers to sent with request + std::string connectedHost; ///< Currently connected host name + uint32_t connectedPort; ///< Currently connected port number + Parser H; ///< HTTP parser for downloader + Socket::Connection S; ///< TCP socket for downloader + }; +} + diff --git a/lib/http_parser.cpp b/lib/http_parser.cpp index 188bdf77..3b4bf7dc 100644 --- a/lib/http_parser.cpp +++ b/lib/http_parser.cpp @@ -158,7 +158,7 @@ std::string HTTP::URL::getBareUrl() const{ } ///Returns a URL object for the given link, resolved relative to the current URL object. -HTTP::URL HTTP::URL::link(const std::string &l){ +HTTP::URL HTTP::URL::link(const std::string &l) const{ //Full link if (l.find("://") < l.find('/') && l.find('/' != std::string::npos)){ DONTEVEN_MSG("Full link: %s", l.c_str()); diff --git a/lib/http_parser.h b/lib/http_parser.h index 692febd8..fdeb32d1 100644 --- a/lib/http_parser.h +++ b/lib/http_parser.h @@ -83,7 +83,7 @@ namespace HTTP { std::string path;///