Added HTTP::Downloader class

CMakeLists.txt

@@ -109,6 +109,7 @@ set(libHeaders
   ${SOURCE_DIR}/lib/flv_tag.h
   ${SOURCE_DIR}/lib/h264.h
   ${SOURCE_DIR}/lib/http_parser.h
+  ${SOURCE_DIR}/lib/downloader.h
   ${SOURCE_DIR}/lib/json.h
   ${SOURCE_DIR}/lib/langcodes.h
   ${SOURCE_DIR}/lib/mp4_adobe.h
@@ -146,6 +147,7 @@ set(libSources
   ${SOURCE_DIR}/lib/flv_tag.cpp
   ${SOURCE_DIR}/lib/h264.cpp
   ${SOURCE_DIR}/lib/http_parser.cpp
+  ${SOURCE_DIR}/lib/downloader.cpp
   ${SOURCE_DIR}/lib/json.cpp
   ${SOURCE_DIR}/lib/langcodes.cpp
   ${SOURCE_DIR}/lib/mp4_adobe.cpp

lib/downloader.cpp

@@ -0,0 +1,117 @@
+#include "downloader.h"
+#include "defines.h"
+#include "timing.h"
+
+namespace HTTP{
+
+  /// Returns a reference to the internal HTTP::Parser body element
+  std::string &Downloader::data(){return H.body;}
+
+  /// Returns the status text of the HTTP Request.
+  std::string &Downloader::getStatusText(){return H.method;}
+
+  /// Returns the status code of the HTTP Request.
+  uint32_t Downloader::getStatusCode(){return atoi(H.url.c_str());}
+
+  /// Returns true if the HTTP Request is OK
+  bool Downloader::isOk(){return (getStatusCode() == 200);}
+
+  /// Returns the given header from the response, or empty string if it does not exist.
+  std::string Downloader::getHeader(const std::string &headerName){
+    return H.GetHeader(headerName);
+  }
+
+  /// Simply turns link into a HTTP::URL and calls get(const HTTP::URL&)
+  bool Downloader::get(const std::string &link){
+    HTTP::URL uri(link);
+    return get(uri);
+  }
+
+  /// Sets an extra (or overridden) header to be sent with outgoing requests.
+  void Downloader::setHeader(const std::string &name, const std::string &val){
+    extraHeaders[name] = val;
+  }
+
+  /// Clears all extra/override headers for outgoing requests.
+  void Downloader::clearHeaders(){extraHeaders.clear();}
+
+  /// Downloads the given URL into 'H', returns true on success.
+  /// Makes at most 5 attempts, and will wait no longer than 5 seconds without receiving data.
+  bool Downloader::get(const HTTP::URL &link, uint8_t maxRecursiveDepth){
+    if (!link.host.size()){return false;}
+    if (link.protocol != "http"){
+      FAIL_MSG("Protocol not supported: %s", link.protocol.c_str());
+      return false;
+    }
+    INFO_MSG("Retrieving %s", link.getUrl().c_str());
+    unsigned int loop = 6; // max 5 attempts
+
+    while (--loop){// loop while we are unsuccessful
+      H.Clean();
+      // Reconnect if needed
+      if (!S || link.host != connectedHost || link.getPort() != connectedPort){
+        S.close();
+        connectedHost = link.host;
+        connectedPort = link.getPort();
+        S = Socket::Connection(connectedHost, connectedPort, true);
+      }
+      H.url = "/" + link.path;
+      if (link.args.size()){H.url += "?" + link.args;}
+      if (link.port.size()){
+        H.SetHeader("Host", link.host + ":" + link.port);
+      }else{
+        H.SetHeader("Host", link.host);
+      }
+      H.SetHeader("User-Agent", "MistServer " PACKAGE_VERSION);
+      H.SetHeader("X-Version", PACKAGE_VERSION);
+      H.SetHeader("Accept", "*/*");
+      if (extraHeaders.size()){
+        for (std::map<std::string, std::string>::iterator it = extraHeaders.begin();
+             it != extraHeaders.end(); ++it){
+          H.SetHeader(it->first, it->second);
+        }
+      }
+      H.SendRequest(S);
+      H.Clean();
+      uint64_t reqTime = Util::bootSecs();
+      while (S && Util::bootSecs() < reqTime + 5){
+        // No data? Wait for a second or so.
+        if (!S.spool()){
+          if (progressCallback != 0){
+            if (!progressCallback()){
+              WARN_MSG("Download aborted by callback");
+              return false;
+            }
+          }
+          Util::sleep(250);
+          continue;
+        }
+        // Data! Check if we can parse it...
+        if (H.Read(S)){
+          if (getStatusCode() >= 300 && getStatusCode() < 400){
+            // follow redirect
+            std::string location = getHeader("Location");
+            if (maxRecursiveDepth == 0){
+              FAIL_MSG("Maximum redirect depth reached: %s", location.c_str());
+              return false;
+            }else{
+              FAIL_MSG("Following redirect to %s", location.c_str());
+              return get(link.link(location), maxRecursiveDepth--);
+            }
+          }
+          return true; // Success!
+        }
+        // reset the 5 second timeout
+        reqTime = Util::bootSecs();
+      }
+      if (S){
+        FAIL_MSG("Timeout while retrieving %s", link.getUrl().c_str());
+        return false;
+      }
+      Util::sleep(500); // wait a bit before retrying
+    }
+    FAIL_MSG("Could not retrieve %s", link.getUrl().c_str());
+    return false;
+  }
+}
+

lib/downloader.h

@@ -0,0 +1,27 @@
+#include "http_parser.h"
+#include "socket.h"
+
+namespace HTTP{
+  class Downloader{
+  public:
+    Downloader(){progressCallback = 0;}
+    std::string &data();
+    bool get(const std::string &link);
+    bool get(const HTTP::URL &link, uint8_t maxRecursiveDepth = 6);
+    std::string getHeader(const std::string &headerName);
+    std::string &getStatusText();
+    uint32_t getStatusCode();
+    bool isOk();
+    bool (*progressCallback)(); ///< Called every time the socket stalls, up to 4X per second.
+    void setHeader(const std::string &name, const std::string &val);
+    void clearHeaders();
+
+  private:
+    std::map<std::string, std::string> extraHeaders; ///< Holds extra headers to sent with request
+    std::string connectedHost;                       ///< Currently connected host name
+    uint32_t connectedPort;                          ///< Currently connected port number
+    Parser H;                                        ///< HTTP parser for downloader
+    Socket::Connection S;                            ///< TCP socket for downloader
+  };
+}
+

lib/http_parser.cpp

@@ -158,7 +158,7 @@ std::string HTTP::URL::getBareUrl() const{
 }
 
 ///Returns a URL object for the given link, resolved relative to the current URL object.
-HTTP::URL HTTP::URL::link(const std::string &l){
+HTTP::URL HTTP::URL::link(const std::string &l) const{
   //Full link
   if (l.find("://") < l.find('/') && l.find('/' != std::string::npos)){
     DONTEVEN_MSG("Full link: %s", l.c_str());

lib/http_parser.h

@@ -83,7 +83,7 @@ namespace HTTP {
       std::string path;///<Path after the first slash (not inclusive) but before any question mark
       std::string args;///<Everything after the question mark in the path, if it was present
       std::string frag;///<Everything after the # in the path, if it was present
-      URL link(const std::string &l);
+      URL link(const std::string &l) const;
   };
 
 }//HTTP namespace