diff --git a/lib/timing.cpp b/lib/timing.cpp
index 036f0b29..f339f4fe 100644
--- a/lib/timing.cpp
+++ b/lib/timing.cpp
@@ -114,3 +114,15 @@ std::string Util::getUTCString(uint64_t epoch){
ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec);
return std::string(result);
}
+
+std::string Util::getDateString(uint64_t epoch){
+ char buffer[80];
+ time_t rawtime = epoch;
+ if (!epoch) {
+ time(&rawtime);
+ }
+ struct tm * timeinfo;
+ timeinfo = localtime(&rawtime);
+ strftime(buffer, sizeof(buffer), "%a, %d %b %Y %H:%M:%S %z", timeinfo);
+ return std::string(buffer);
+}
diff --git a/lib/timing.h b/lib/timing.h
index 107ccee5..b2a4d612 100644
--- a/lib/timing.h
+++ b/lib/timing.h
@@ -17,4 +17,5 @@ namespace Util{
uint64_t getNTP();
uint64_t epoch(); ///< Gets the amount of seconds since 01/01/1970.
std::string getUTCString(uint64_t epoch = 0);
+ std::string getDateString(uint64_t epoch = 0);
}// namespace Util
diff --git a/lib/urireader.cpp b/lib/urireader.cpp
index 80f204c8..ad85b052 100644
--- a/lib/urireader.cpp
+++ b/lib/urireader.cpp
@@ -3,11 +3,86 @@
#include "timing.h"
#include "urireader.h"
#include "util.h"
+#include "encode.h"
#include <sys/mman.h>
#include <sys/stat.h>
+#include <cstdlib>
+#include <ctime>
namespace HTTP{
+ // When another protocol needs this, rename struct to HeaderOverride or similar
+ struct HTTPHeadThenGet {
+ bool continueOperation;
+ std::string date, headAuthorization, getAuthorization;
+
+ HTTPHeadThenGet() : continueOperation(false) {}
+
+ void prepareHeadHeaders(HTTP::Downloader& downloader) {
+ if(!continueOperation) return;
+ downloader.setHeader("Date", date);
+ downloader.setHeader("Authorization", headAuthorization);
+ }
+
+ void prepareGetHeaders(HTTP::Downloader& downloader) {
+ if(!continueOperation) return;
+ // .setHeader() overwrites existing header value
+ downloader.setHeader("Date", date);
+ downloader.setHeader("Authorization", getAuthorization);
+ }
+ };
+
+#ifndef NOSSL
+ inline bool s3CalculateSignature(std::string& signature, const std::string method, const std::string date, const std::string& requestPath, const std::string& accessKey, const std::string& secret) {
+ std::string toSign = method + "\n\n\n" + date + "\n" + requestPath;
+ unsigned char signatureBytes[MBEDTLS_MD_MAX_SIZE];
+ const int sha1Size = 20;
+ mbedtls_md_context_t md_ctx = {0};
+ // TODO: When we use MBEDTLS_MD_SHA512 ? Consult documentation/code
+ const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type(MBEDTLS_MD_SHA1);
+ if (!md_info){ FAIL_MSG("error s3 MBEDTLS_MD_SHA1 unavailable"); return false; }
+ int status = mbedtls_md_setup(&md_ctx, md_info, 1);
+ if(status != 0) { FAIL_MSG("error s3 mbedtls_md_setup error %d", status); return false; }
+ status = mbedtls_md_hmac_starts(&md_ctx, (const unsigned char *)secret.c_str(), secret.size());
+ if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_starts error %d", status); return false; }
+ status = mbedtls_md_hmac_update(&md_ctx, (const unsigned char *)toSign.c_str(), toSign.size());
+ if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_update error %d", status); return false; }
+ status = mbedtls_md_hmac_finish(&md_ctx, signatureBytes);
+ if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_finish error %d", status); return false; }
+ std::string base64encoded = Encodings::Base64::encode(std::string((const char*)signatureBytes, sha1Size));
+ signature = "AWS " + accessKey + ":" + base64encoded;
+ return true;
+ }
+ // Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
+ // Transform to:
+ // url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
+ // header Date: ${Util::getDateString(()}
+ // header Authorization: AWS ${url.user}:${signature}
+ inline HTTPHeadThenGet s3TransformToHttp(HTTP::URL& url) {
+ HTTPHeadThenGet result;
+ result.date = Util::getDateString();
+ // remove "s3+" prefix
+ url.protocol = url.protocol.erase(0, 3);
+ // Use user and pass to create signature and remove from HTTP request
+ std::string accessKey(url.user), secret(url.pass);
+ url.user = "";
+ url.pass = "";
+ std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&");
+ if(url.args.size()) requestPath += "?" + url.args;
+ // Calculate Authorization data
+ if(!s3CalculateSignature(result.headAuthorization, "HEAD", result.date, requestPath, accessKey, secret)) {
+ result.continueOperation = false;
+ return result;
+ }
+ if(!s3CalculateSignature(result.getAuthorization, "GET", result.date, requestPath, accessKey, secret)) {
+ result.continueOperation = false;
+ return result;
+ }
+ result.continueOperation = true;
+ return result;
+ }
+#endif // ifndef NOSSL
+
void URIReader::init(){
handle = -1;
mapped = 0;
@@ -97,12 +172,45 @@ namespace HTTP{
}
}
+ // prepare for s3 and http
+ HTTPHeadThenGet httpHeaderOverride;
+
+#ifndef NOSSL
+ // In case of s3 URI we prepare HTTP request with AWS authorization and rely on HTTP logic below
+ if (myURI.protocol == "s3+https" || myURI.protocol == "s3+http"){
+ // Check fallback to global credentials in env vars
+ bool haveCredentials = myURI.user.size() && myURI.pass.size();
+ if(!haveCredentials) {
+ // Use environment variables
+ char * envValue = std::getenv("S3_ACCESS_KEY_ID");
+ if(envValue == NULL) {
+ FAIL_MSG("error s3 uri without credentials. Consider setting S3_ACCESS_KEY_ID env var");
+ return false;
+ }
+ myURI.user = envValue;
+ envValue = std::getenv("S3_SECRET_ACCESS_KEY");
+ if(envValue == NULL) {
+ FAIL_MSG("error s3 uri without credentials. Consider setting S3_SECRET_ACCESS_KEY env var");
+ return false;
+ }
+ myURI.pass = envValue;
+ }
+ // Transform s3 url to HTTP request:
+ httpHeaderOverride = s3TransformToHttp(myURI);
+ bool errorInSignatureCalculation = !httpHeaderOverride.continueOperation;
+ if(errorInSignatureCalculation) return false;
+ // Do not return, continue to HTTP case
+ }
+#endif // ifndef NOSSL
+
// HTTP, stream or regular download?
if (myURI.protocol == "http" || myURI.protocol == "https"){
- stateType = HTTP::HTTP;
-
- // Send HEAD request to determine range request is supported, and get total length
+ stateType = HTTP;
downer.clearHeaders();
+
+ // One set of headers specified for HEAD request
+ httpHeaderOverride.prepareHeadHeaders(downer);
+ // Send HEAD request to determine range request is supported, and get total length
if (userAgentOverride.size()){downer.setHeader("User-Agent", userAgentOverride);}
if (!downer.head(myURI) || !downer.isOk()){
FAIL_MSG("Error getting URI info for '%s': %" PRIu32 " %s", myURI.getUrl().c_str(),
@@ -120,6 +228,8 @@ namespace HTTP{
myURI = downer.lastURL();
}
+ // Other set of headers specified for GET request
+ httpHeaderOverride.prepareGetHeaders(downer);
// streaming mode when size is unknown
if (!supportRangeRequest){
MEDIUM_MSG("URI get without range request: %s, totalsize: %zu", myURI.getUrl().c_str(), totalSize);