From d9c7e0f3ca5a46752a75ac91a5c0408f4e9fe2cb Mon Sep 17 00:00:00 2001 From: Thulinma Date: Thu, 1 Sep 2022 10:05:40 +0200 Subject: [PATCH] Fixes to Downloader and URIReader classes, fix S3 support not liking range requests anymore after 15 minutes --- lib/downloader.cpp | 9 +++- lib/urireader.cpp | 104 +++++++++++++++++++-------------------------- lib/urireader.h | 1 + 3 files changed, 51 insertions(+), 63 deletions(-) diff --git a/lib/downloader.cpp b/lib/downloader.cpp index 1f430972..8a09a881 100644 --- a/lib/downloader.cpp +++ b/lib/downloader.cpp @@ -188,6 +188,9 @@ namespace HTTP{ if (!progressCallback()){ WARN_MSG("Download aborted by callback"); H.headerOnly = false; + H.Clean(); + getSocket().close(); + getSocket().Received().clear(); return false; } } @@ -231,7 +234,9 @@ namespace HTTP{ if (!progressCallback()){ WARN_MSG("Download aborted by callback"); H.headerOnly = false; + H.Clean(); getSocket().close(); + getSocket().Received().clear(); return false; } } @@ -245,7 +250,6 @@ namespace HTTP{ if (getSocket()){ FAIL_MSG("Timeout while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount); - H.Clean(); getSocket().close(); }else{ if (loop > 1){ @@ -253,8 +257,9 @@ namespace HTTP{ }else{ MEDIUM_MSG("Lost connection while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount); } - H.Clean(); } + H.Clean(); + getSocket().Received().clear(); Util::sleep(100); // wait a bit before retrying } FAIL_MSG("Could not retrieve %s", link.getUrl().c_str()); diff --git a/lib/urireader.cpp b/lib/urireader.cpp index d4dd0e31..f4bd9353 100644 --- a/lib/urireader.cpp +++ b/lib/urireader.cpp @@ -11,27 +11,6 @@ namespace HTTP{ - // When another protocol needs this, rename struct to HeaderOverride or similar - struct HTTPHeadThenGet { - bool continueOperation; - std::string date, headAuthorization, getAuthorization; - - HTTPHeadThenGet() : continueOperation(false) {} - - void prepareHeadHeaders(HTTP::Downloader& downloader) { - if(!continueOperation) return; - downloader.setHeader("Date", date); - downloader.setHeader("Authorization", headAuthorization); - } - - void prepareGetHeaders(HTTP::Downloader& downloader) { - if(!continueOperation) return; - // .setHeader() overwrites existing header value - downloader.setHeader("Date", date); - downloader.setHeader("Authorization", getAuthorization); - } - }; - #ifndef NOSSL inline bool s3CalculateSignature(std::string& signature, const std::string method, const std::string date, const std::string& requestPath, const std::string& accessKey, const std::string& secret) { std::string toSign = method + "\n\n\n" + date + "\n" + requestPath; @@ -53,36 +32,40 @@ namespace HTTP{ signature = "AWS " + accessKey + ":" + base64encoded; return true; } - // Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts - // Transform to: - // url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts - // header Date: ${Util::getDateString(()} - // header Authorization: AWS ${url.user}:${signature} - inline HTTPHeadThenGet s3TransformToHttp(HTTP::URL& url) { - HTTPHeadThenGet result; - result.date = Util::getDateString(); - // remove "s3+" prefix - url.protocol = url.protocol.erase(0, 3); - // Use user and pass to create signature and remove from HTTP request - std::string accessKey(url.user), secret(url.pass); - url.user = ""; - url.pass = ""; - std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&"); - if(url.args.size()) requestPath += "?" + url.args; - // Calculate Authorization data - if(!s3CalculateSignature(result.headAuthorization, "HEAD", result.date, requestPath, accessKey, secret)) { - result.continueOperation = false; - return result; - } - if(!s3CalculateSignature(result.getAuthorization, "GET", result.date, requestPath, accessKey, secret)) { - result.continueOperation = false; - return result; - } - result.continueOperation = true; - return result; - } #endif // ifndef NOSSL + + inline HTTP::URL injectHeaders(const HTTP::URL& url, const std::string & method, HTTP::Downloader & downer) { +#ifndef NOSSL + // Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts + // Transform to: + // url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts + // header Date: ${Util::getDateString(()} + // header Authorization: AWS ${url.user}:${signature} + if (url.protocol.size() > 3 && url.protocol.substr(0, 3) == "s3+"){ + std::string date = Util::getDateString(); + HTTP::URL newUrl = url; + // remove "s3+" prefix + newUrl.protocol = newUrl.protocol.erase(0, 3); + // Use user and pass to create signature and remove from HTTP request + std::string accessKey(url.user), secret(url.pass); + newUrl.user = ""; + newUrl.pass = ""; + std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&"); + if(url.args.size()) requestPath += "?" + url.args; + std::string authLine; + // Calculate Authorization data + if(method.size() && s3CalculateSignature(authLine, method, date, requestPath, accessKey, secret)) { + downer.setHeader("Date", date); + downer.setHeader("Authorization", authLine); + } + return newUrl; + } +#endif // ifndef NOSSL + return url; + } + + void URIReader::init(){ handle = -1; mapped = 0; @@ -120,13 +103,11 @@ namespace HTTP{ void URIReader::dataCallback(const char *ptr, size_t size){allData.append(ptr, size);} bool URIReader::open(const HTTP::URL &uri){ + close(); myURI = uri; - curPos = 0; - allData.truncate(0); - bufPos = 0; + originalUrl = myURI; if (!myURI.protocol.size() || myURI.protocol == "file"){ - close(); if (!myURI.path.size() || myURI.path == "-"){ downer.getSocket().open(-1, fileno(stdin)); stateType = HTTP::Stream; @@ -173,8 +154,6 @@ namespace HTTP{ } // prepare for s3 and http - HTTPHeadThenGet httpHeaderOverride; - #ifndef NOSSL // In case of s3 URI we prepare HTTP request with AWS authorization and rely on HTTP logic below if (myURI.protocol == "s3+https" || myURI.protocol == "s3+http"){ @@ -195,10 +174,7 @@ namespace HTTP{ } myURI.pass = envValue; } - // Transform s3 url to HTTP request: - httpHeaderOverride = s3TransformToHttp(myURI); - bool errorInSignatureCalculation = !httpHeaderOverride.continueOperation; - if(errorInSignatureCalculation) return false; + myURI = injectHeaders(originalUrl, "", downer); // Do not return, continue to HTTP case } #endif // ifndef NOSSL @@ -209,7 +185,7 @@ namespace HTTP{ downer.clearHeaders(); // One set of headers specified for HEAD request - httpHeaderOverride.prepareHeadHeaders(downer); + injectHeaders(originalUrl, "HEAD", downer); // Send HEAD request to determine range request is supported, and get total length if (userAgentOverride.size()){downer.setHeader("User-Agent", userAgentOverride);} if (!downer.head(myURI) || !downer.isOk()){ @@ -218,6 +194,7 @@ namespace HTTP{ // Close the socket, and clean up the buffer downer.getSocket().close(); downer.getSocket().Received().clear(); + allData.truncate(0); if (!downer.isOk()){return false;} supportRangeRequest = false; totalSize = std::string::npos; @@ -229,7 +206,7 @@ namespace HTTP{ } // Other set of headers specified for GET request - httpHeaderOverride.prepareGetHeaders(downer); + injectHeaders(originalUrl, "GET", downer); // streaming mode when size is unknown if (!supportRangeRequest){ MEDIUM_MSG("URI get without range request: %s, totalsize: %zu", myURI.getUrl().c_str(), totalSize); @@ -272,6 +249,7 @@ namespace HTTP{ if (stateType == HTTP::HTTP && supportRangeRequest){ downer.getSocket().close(); downer.getSocket().Received().clear(); + injectHeaders(originalUrl, "GET", downer); if (!downer.getRangeNonBlocking(myURI.getUrl(), pos, 0)){ FAIL_MSG("Error making range request"); return false; @@ -371,6 +349,10 @@ namespace HTTP{ } void URIReader::close(){ + //Wipe internal state + curPos = 0; + allData.truncate(0); + bufPos = 0; // Close downloader socket if open downer.getSocket().close(); downer.getSocket().Received().clear(); diff --git a/lib/urireader.h b/lib/urireader.h index 11b34aec..72fd90bb 100644 --- a/lib/urireader.h +++ b/lib/urireader.h @@ -77,6 +77,7 @@ namespace HTTP{ size_t bufPos; ///< Current read position in buffer int handle; ///< Open file handle, if file-based. char *mapped; ///< Memory-map of open file handle, if file-based. + HTTP::URL originalUrl; bool supportRangeRequest; Util::ResizeablePointer rPtr; Util::ResizeablePointer allData;