From 6c117b63cfe85a204b48dfddddb355f416b39364 Mon Sep 17 00:00:00 2001 From: Alex Kordic Date: Thu, 4 Aug 2022 14:49:57 +0200 Subject: [PATCH] Add s3 protocol to `URIReader` --- lib/timing.cpp | 12 +++++ lib/timing.h | 1 + lib/urireader.cpp | 116 ++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 126 insertions(+), 3 deletions(-) diff --git a/lib/timing.cpp b/lib/timing.cpp index 036f0b29..f339f4fe 100644 --- a/lib/timing.cpp +++ b/lib/timing.cpp @@ -114,3 +114,15 @@ std::string Util::getUTCString(uint64_t epoch){ ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec); return std::string(result); } + +std::string Util::getDateString(uint64_t epoch){ + char buffer[80]; + time_t rawtime = epoch; + if (!epoch) { + time(&rawtime); + } + struct tm * timeinfo; + timeinfo = localtime(&rawtime); + strftime(buffer, sizeof(buffer), "%a, %d %b %Y %H:%M:%S %z", timeinfo); + return std::string(buffer); +} diff --git a/lib/timing.h b/lib/timing.h index 107ccee5..b2a4d612 100644 --- a/lib/timing.h +++ b/lib/timing.h @@ -17,4 +17,5 @@ namespace Util{ uint64_t getNTP(); uint64_t epoch(); ///< Gets the amount of seconds since 01/01/1970. std::string getUTCString(uint64_t epoch = 0); + std::string getDateString(uint64_t epoch = 0); }// namespace Util diff --git a/lib/urireader.cpp b/lib/urireader.cpp index 80f204c8..ad85b052 100644 --- a/lib/urireader.cpp +++ b/lib/urireader.cpp @@ -3,11 +3,86 @@ #include "timing.h" #include "urireader.h" #include "util.h" +#include "encode.h" #include #include +#include +#include namespace HTTP{ + // When another protocol needs this, rename struct to HeaderOverride or similar + struct HTTPHeadThenGet { + bool continueOperation; + std::string date, headAuthorization, getAuthorization; + + HTTPHeadThenGet() : continueOperation(false) {} + + void prepareHeadHeaders(HTTP::Downloader& downloader) { + if(!continueOperation) return; + downloader.setHeader("Date", date); + downloader.setHeader("Authorization", headAuthorization); + } + + void prepareGetHeaders(HTTP::Downloader& downloader) { + if(!continueOperation) return; + // .setHeader() overwrites existing header value + downloader.setHeader("Date", date); + downloader.setHeader("Authorization", getAuthorization); + } + }; + +#ifndef NOSSL + inline bool s3CalculateSignature(std::string& signature, const std::string method, const std::string date, const std::string& requestPath, const std::string& accessKey, const std::string& secret) { + std::string toSign = method + "\n\n\n" + date + "\n" + requestPath; + unsigned char signatureBytes[MBEDTLS_MD_MAX_SIZE]; + const int sha1Size = 20; + mbedtls_md_context_t md_ctx = {0}; + // TODO: When we use MBEDTLS_MD_SHA512 ? Consult documentation/code + const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type(MBEDTLS_MD_SHA1); + if (!md_info){ FAIL_MSG("error s3 MBEDTLS_MD_SHA1 unavailable"); return false; } + int status = mbedtls_md_setup(&md_ctx, md_info, 1); + if(status != 0) { FAIL_MSG("error s3 mbedtls_md_setup error %d", status); return false; } + status = mbedtls_md_hmac_starts(&md_ctx, (const unsigned char *)secret.c_str(), secret.size()); + if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_starts error %d", status); return false; } + status = mbedtls_md_hmac_update(&md_ctx, (const unsigned char *)toSign.c_str(), toSign.size()); + if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_update error %d", status); return false; } + status = mbedtls_md_hmac_finish(&md_ctx, signatureBytes); + if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_finish error %d", status); return false; } + std::string base64encoded = Encodings::Base64::encode(std::string((const char*)signatureBytes, sha1Size)); + signature = "AWS " + accessKey + ":" + base64encoded; + return true; + } + // Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts + // Transform to: + // url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts + // header Date: ${Util::getDateString(()} + // header Authorization: AWS ${url.user}:${signature} + inline HTTPHeadThenGet s3TransformToHttp(HTTP::URL& url) { + HTTPHeadThenGet result; + result.date = Util::getDateString(); + // remove "s3+" prefix + url.protocol = url.protocol.erase(0, 3); + // Use user and pass to create signature and remove from HTTP request + std::string accessKey(url.user), secret(url.pass); + url.user = ""; + url.pass = ""; + std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&"); + if(url.args.size()) requestPath += "?" + url.args; + // Calculate Authorization data + if(!s3CalculateSignature(result.headAuthorization, "HEAD", result.date, requestPath, accessKey, secret)) { + result.continueOperation = false; + return result; + } + if(!s3CalculateSignature(result.getAuthorization, "GET", result.date, requestPath, accessKey, secret)) { + result.continueOperation = false; + return result; + } + result.continueOperation = true; + return result; + } +#endif // ifndef NOSSL + void URIReader::init(){ handle = -1; mapped = 0; @@ -97,12 +172,45 @@ namespace HTTP{ } } + // prepare for s3 and http + HTTPHeadThenGet httpHeaderOverride; + +#ifndef NOSSL + // In case of s3 URI we prepare HTTP request with AWS authorization and rely on HTTP logic below + if (myURI.protocol == "s3+https" || myURI.protocol == "s3+http"){ + // Check fallback to global credentials in env vars + bool haveCredentials = myURI.user.size() && myURI.pass.size(); + if(!haveCredentials) { + // Use environment variables + char * envValue = std::getenv("S3_ACCESS_KEY_ID"); + if(envValue == NULL) { + FAIL_MSG("error s3 uri without credentials. Consider setting S3_ACCESS_KEY_ID env var"); + return false; + } + myURI.user = envValue; + envValue = std::getenv("S3_SECRET_ACCESS_KEY"); + if(envValue == NULL) { + FAIL_MSG("error s3 uri without credentials. Consider setting S3_SECRET_ACCESS_KEY env var"); + return false; + } + myURI.pass = envValue; + } + // Transform s3 url to HTTP request: + httpHeaderOverride = s3TransformToHttp(myURI); + bool errorInSignatureCalculation = !httpHeaderOverride.continueOperation; + if(errorInSignatureCalculation) return false; + // Do not return, continue to HTTP case + } +#endif // ifndef NOSSL + // HTTP, stream or regular download? if (myURI.protocol == "http" || myURI.protocol == "https"){ - stateType = HTTP::HTTP; - - // Send HEAD request to determine range request is supported, and get total length + stateType = HTTP; downer.clearHeaders(); + + // One set of headers specified for HEAD request + httpHeaderOverride.prepareHeadHeaders(downer); + // Send HEAD request to determine range request is supported, and get total length if (userAgentOverride.size()){downer.setHeader("User-Agent", userAgentOverride);} if (!downer.head(myURI) || !downer.isOk()){ FAIL_MSG("Error getting URI info for '%s': %" PRIu32 " %s", myURI.getUrl().c_str(), @@ -120,6 +228,8 @@ namespace HTTP{ myURI = downer.lastURL(); } + // Other set of headers specified for GET request + httpHeaderOverride.prepareGetHeaders(downer); // streaming mode when size is unknown if (!supportRangeRequest){ MEDIUM_MSG("URI get without range request: %s, totalsize: %zu", myURI.getUrl().c_str(), totalSize);