Fixes to Downloader and URIReader classes, fix S3 support not liking range requests anymore after 15 minutes

lib/downloader.cpp

@@ -188,6 +188,9 @@ namespace HTTP{
             if (!progressCallback()){
               WARN_MSG("Download aborted by callback");
               H.headerOnly = false;
+              H.Clean();
+              getSocket().close();
+              getSocket().Received().clear();
               return false;
             }
           }
@@ -231,7 +234,9 @@ namespace HTTP{
             if (!progressCallback()){
               WARN_MSG("Download aborted by callback");
               H.headerOnly = false;
+              H.Clean();
               getSocket().close();
+              getSocket().Received().clear();
               return false;
             }
           }
@@ -245,7 +250,6 @@ namespace HTTP{
 
       if (getSocket()){
         FAIL_MSG("Timeout while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount);
-        H.Clean();
         getSocket().close();
       }else{
         if (loop > 1){
@@ -253,8 +257,9 @@ namespace HTTP{
         }else{
           MEDIUM_MSG("Lost connection while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount);
         }
-        H.Clean();
       }
+      H.Clean();
+      getSocket().Received().clear();
       Util::sleep(100); // wait a bit before retrying
     }
     FAIL_MSG("Could not retrieve %s", link.getUrl().c_str());

lib/urireader.cpp

@@ -11,27 +11,6 @@
 
 namespace HTTP{
 
-  // When another protocol needs this, rename struct to HeaderOverride or similar
-  struct HTTPHeadThenGet {
-    bool continueOperation;
-    std::string date, headAuthorization, getAuthorization;
-
-    HTTPHeadThenGet() : continueOperation(false) {}
-
-    void prepareHeadHeaders(HTTP::Downloader& downloader) {
-      if(!continueOperation) return;
-      downloader.setHeader("Date", date);
-      downloader.setHeader("Authorization", headAuthorization);
-    }
-
-    void prepareGetHeaders(HTTP::Downloader& downloader) {
-      if(!continueOperation) return;
-      // .setHeader() overwrites existing header value
-      downloader.setHeader("Date", date);
-      downloader.setHeader("Authorization", getAuthorization);
-    }
-  };
-
 #ifndef NOSSL
   inline bool s3CalculateSignature(std::string& signature, const std::string method, const std::string date, const std::string& requestPath, const std::string& accessKey, const std::string& secret) {
     std::string toSign = method + "\n\n\n" + date + "\n" + requestPath;
@@ -53,36 +32,40 @@ namespace HTTP{
     signature = "AWS " + accessKey + ":" + base64encoded;
     return true;
   }
-  // Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
-  // Transform to: 
-  // url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
-  // header Date: ${Util::getDateString(()}
-  // header Authorization: AWS ${url.user}:${signature}
-  inline HTTPHeadThenGet s3TransformToHttp(HTTP::URL& url) {
-    HTTPHeadThenGet result;
-    result.date = Util::getDateString();
-    // remove "s3+" prefix
-    url.protocol = url.protocol.erase(0, 3); 
-    // Use user and pass to create signature and remove from HTTP request
-    std::string accessKey(url.user), secret(url.pass);
-    url.user = "";
-    url.pass = "";
-    std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&");
-    if(url.args.size()) requestPath += "?" + url.args;
-    // Calculate Authorization data
-    if(!s3CalculateSignature(result.headAuthorization, "HEAD", result.date, requestPath, accessKey, secret)) {
-      result.continueOperation = false;
-      return result;
-    }
-    if(!s3CalculateSignature(result.getAuthorization, "GET", result.date, requestPath, accessKey, secret)) {
-      result.continueOperation = false;
-      return result;
-    }
-    result.continueOperation = true;
-    return result;
-  }
 #endif // ifndef NOSSL
 
+
+  inline HTTP::URL injectHeaders(const HTTP::URL& url, const std::string & method, HTTP::Downloader & downer) {
+#ifndef NOSSL
+    // Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
+    // Transform to: 
+    // url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
+    // header Date: ${Util::getDateString(()}
+    // header Authorization: AWS ${url.user}:${signature}
+    if (url.protocol.size() > 3 && url.protocol.substr(0, 3) == "s3+"){
+      std::string date = Util::getDateString();
+      HTTP::URL newUrl = url;
+      // remove "s3+" prefix
+      newUrl.protocol = newUrl.protocol.erase(0, 3); 
+      // Use user and pass to create signature and remove from HTTP request
+      std::string accessKey(url.user), secret(url.pass);
+      newUrl.user = "";
+      newUrl.pass = "";
+      std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&");
+      if(url.args.size()) requestPath += "?" + url.args;
+      std::string authLine;
+      // Calculate Authorization data
+      if(method.size() && s3CalculateSignature(authLine, method, date, requestPath, accessKey, secret)) {
+        downer.setHeader("Date", date);
+        downer.setHeader("Authorization", authLine);
+      }
+      return newUrl;
+    }
+#endif // ifndef NOSSL
+    return url;
+  }
+
+
   void URIReader::init(){
     handle = -1;
     mapped = 0;
@@ -120,13 +103,11 @@ namespace HTTP{
   void URIReader::dataCallback(const char *ptr, size_t size){allData.append(ptr, size);}
 
   bool URIReader::open(const HTTP::URL &uri){
+    close();
     myURI = uri;
-    curPos = 0;
+    originalUrl = myURI;
-    allData.truncate(0);
-    bufPos = 0;
 
     if (!myURI.protocol.size() || myURI.protocol == "file"){
-      close();
       if (!myURI.path.size() || myURI.path == "-"){
         downer.getSocket().open(-1, fileno(stdin));
         stateType = HTTP::Stream;
@@ -173,8 +154,6 @@ namespace HTTP{
     }
 
     // prepare for s3 and http
-    HTTPHeadThenGet httpHeaderOverride;
-
 #ifndef NOSSL
     // In case of s3 URI we prepare HTTP request with AWS authorization and rely on HTTP logic below
     if (myURI.protocol == "s3+https" || myURI.protocol == "s3+http"){
@@ -195,10 +174,7 @@ namespace HTTP{
         }
         myURI.pass = envValue;
       }
-      // Transform s3 url to HTTP request:
+      myURI = injectHeaders(originalUrl, "", downer);
-      httpHeaderOverride = s3TransformToHttp(myURI);
-      bool errorInSignatureCalculation = !httpHeaderOverride.continueOperation;
-      if(errorInSignatureCalculation) return false;
       // Do not return, continue to HTTP case
     }
 #endif // ifndef NOSSL
@@ -209,7 +185,7 @@ namespace HTTP{
       downer.clearHeaders();
 
       // One set of headers specified for HEAD request
-      httpHeaderOverride.prepareHeadHeaders(downer);
+      injectHeaders(originalUrl, "HEAD", downer);
       // Send HEAD request to determine range request is supported, and get total length
       if (userAgentOverride.size()){downer.setHeader("User-Agent", userAgentOverride);}
       if (!downer.head(myURI) || !downer.isOk()){
@@ -218,6 +194,7 @@ namespace HTTP{
         // Close the socket, and clean up the buffer
         downer.getSocket().close();
         downer.getSocket().Received().clear();
+        allData.truncate(0);
         if (!downer.isOk()){return false;}
         supportRangeRequest = false;
         totalSize = std::string::npos;
@@ -229,7 +206,7 @@ namespace HTTP{
       }
 
       // Other set of headers specified for GET request
-      httpHeaderOverride.prepareGetHeaders(downer);
+      injectHeaders(originalUrl, "GET", downer);
       // streaming mode when size is unknown
       if (!supportRangeRequest){
         MEDIUM_MSG("URI get without range request: %s, totalsize: %zu", myURI.getUrl().c_str(), totalSize);
@@ -272,6 +249,7 @@ namespace HTTP{
     if (stateType == HTTP::HTTP && supportRangeRequest){
       downer.getSocket().close();
       downer.getSocket().Received().clear();
+      injectHeaders(originalUrl, "GET", downer);
       if (!downer.getRangeNonBlocking(myURI.getUrl(), pos, 0)){
         FAIL_MSG("Error making range request");
         return false;
@@ -371,6 +349,10 @@ namespace HTTP{
   }
 
   void URIReader::close(){
+    //Wipe internal state
+    curPos = 0;
+    allData.truncate(0);
+    bufPos = 0;
     // Close downloader socket if open
     downer.getSocket().close();
     downer.getSocket().Received().clear();

lib/urireader.h

@@ -77,6 +77,7 @@ namespace HTTP{
     size_t bufPos; ///< Current read position in buffer
     int handle;    ///< Open file handle, if file-based.
     char *mapped;  ///< Memory-map of open file handle, if file-based.
+    HTTP::URL originalUrl;
     bool supportRangeRequest;
     Util::ResizeablePointer rPtr;
     Util::ResizeablePointer allData;