Add s3 protocol to URIReader

lib/timing.cpp

@@ -114,3 +114,15 @@ std::string Util::getUTCString(uint64_t epoch){
            ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec);
   return std::string(result);
 }
+
+std::string Util::getDateString(uint64_t epoch){
+  char buffer[80];
+  time_t rawtime = epoch;
+  if (!epoch) {
+    time(&rawtime);
+  }
+  struct tm * timeinfo;
+  timeinfo = localtime(&rawtime);
+  strftime(buffer, sizeof(buffer), "%a, %d %b %Y %H:%M:%S %z", timeinfo);
+  return std::string(buffer);
+}

lib/timing.h

@@ -17,4 +17,5 @@ namespace Util{
   uint64_t getNTP();
   uint64_t epoch(); ///< Gets the amount of seconds since 01/01/1970.
   std::string getUTCString(uint64_t epoch = 0);
+  std::string getDateString(uint64_t epoch = 0);
 }// namespace Util

lib/urireader.cpp

@@ -3,11 +3,86 @@
 #include "timing.h"
 #include "urireader.h"
 #include "util.h"
+#include "encode.h"
 #include <sys/mman.h>
 #include <sys/stat.h>
+#include <cstdlib>
+#include <ctime>
 
 namespace HTTP{
 
+  // When another protocol needs this, rename struct to HeaderOverride or similar
+  struct HTTPHeadThenGet {
+    bool continueOperation;
+    std::string date, headAuthorization, getAuthorization;
+
+    HTTPHeadThenGet() : continueOperation(false) {}
+
+    void prepareHeadHeaders(HTTP::Downloader& downloader) {
+      if(!continueOperation) return;
+      downloader.setHeader("Date", date);
+      downloader.setHeader("Authorization", headAuthorization);
+    }
+
+    void prepareGetHeaders(HTTP::Downloader& downloader) {
+      if(!continueOperation) return;
+      // .setHeader() overwrites existing header value
+      downloader.setHeader("Date", date);
+      downloader.setHeader("Authorization", getAuthorization);
+    }
+  };
+
+#ifndef NOSSL
+  inline bool s3CalculateSignature(std::string& signature, const std::string method, const std::string date, const std::string& requestPath, const std::string& accessKey, const std::string& secret) {
+    std::string toSign = method + "\n\n\n" + date + "\n" + requestPath;
+    unsigned char signatureBytes[MBEDTLS_MD_MAX_SIZE];
+    const int sha1Size = 20;
+    mbedtls_md_context_t md_ctx = {0};
+    // TODO: When we use MBEDTLS_MD_SHA512 ? Consult documentation/code
+    const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type(MBEDTLS_MD_SHA1);
+    if (!md_info){ FAIL_MSG("error s3 MBEDTLS_MD_SHA1 unavailable"); return false; }
+    int status = mbedtls_md_setup(&md_ctx, md_info, 1);
+    if(status != 0) { FAIL_MSG("error s3 mbedtls_md_setup error %d", status); return false; }
+    status = mbedtls_md_hmac_starts(&md_ctx, (const unsigned char *)secret.c_str(), secret.size());
+    if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_starts error %d", status); return false; }
+    status = mbedtls_md_hmac_update(&md_ctx, (const unsigned char *)toSign.c_str(), toSign.size());
+    if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_update error %d", status); return false; }
+    status = mbedtls_md_hmac_finish(&md_ctx, signatureBytes);
+    if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_finish error %d", status); return false; }
+    std::string base64encoded = Encodings::Base64::encode(std::string((const char*)signatureBytes, sha1Size));
+    signature = "AWS " + accessKey + ":" + base64encoded;
+    return true;
+  }
+  // Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
+  // Transform to: 
+  // url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
+  // header Date: ${Util::getDateString(()}
+  // header Authorization: AWS ${url.user}:${signature}
+  inline HTTPHeadThenGet s3TransformToHttp(HTTP::URL& url) {
+    HTTPHeadThenGet result;
+    result.date = Util::getDateString();
+    // remove "s3+" prefix
+    url.protocol = url.protocol.erase(0, 3); 
+    // Use user and pass to create signature and remove from HTTP request
+    std::string accessKey(url.user), secret(url.pass);
+    url.user = "";
+    url.pass = "";
+    std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&");
+    if(url.args.size()) requestPath += "?" + url.args;
+    // Calculate Authorization data
+    if(!s3CalculateSignature(result.headAuthorization, "HEAD", result.date, requestPath, accessKey, secret)) {
+      result.continueOperation = false;
+      return result;
+    }
+    if(!s3CalculateSignature(result.getAuthorization, "GET", result.date, requestPath, accessKey, secret)) {
+      result.continueOperation = false;
+      return result;
+    }
+    result.continueOperation = true;
+    return result;
+  }
+#endif // ifndef NOSSL
+
   void URIReader::init(){
     handle = -1;
     mapped = 0;
@@ -97,12 +172,45 @@ namespace HTTP{
       }
     }
 
+    // prepare for s3 and http
+    HTTPHeadThenGet httpHeaderOverride;
+
+#ifndef NOSSL
+    // In case of s3 URI we prepare HTTP request with AWS authorization and rely on HTTP logic below
+    if (myURI.protocol == "s3+https" || myURI.protocol == "s3+http"){
+      // Check fallback to global credentials in env vars
+      bool haveCredentials = myURI.user.size() && myURI.pass.size();
+      if(!haveCredentials) {
+        // Use environment variables
+        char * envValue = std::getenv("S3_ACCESS_KEY_ID");
+        if(envValue == NULL) {
+          FAIL_MSG("error s3 uri without credentials. Consider setting S3_ACCESS_KEY_ID env var");
+          return false;
+        }
+        myURI.user = envValue;
+        envValue = std::getenv("S3_SECRET_ACCESS_KEY");
+        if(envValue == NULL) {
+          FAIL_MSG("error s3 uri without credentials. Consider setting S3_SECRET_ACCESS_KEY env var");
+          return false;
+        }
+        myURI.pass = envValue;
+      }
+      // Transform s3 url to HTTP request:
+      httpHeaderOverride = s3TransformToHttp(myURI);
+      bool errorInSignatureCalculation = !httpHeaderOverride.continueOperation;
+      if(errorInSignatureCalculation) return false;
+      // Do not return, continue to HTTP case
+    }
+#endif // ifndef NOSSL
+
     // HTTP, stream or regular download?
     if (myURI.protocol == "http" || myURI.protocol == "https"){
-      stateType = HTTP::HTTP;
-
-      // Send HEAD request to determine range request is supported, and get total length
+      stateType = HTTP;
       downer.clearHeaders();
+
+      // One set of headers specified for HEAD request
+      httpHeaderOverride.prepareHeadHeaders(downer);
+      // Send HEAD request to determine range request is supported, and get total length
       if (userAgentOverride.size()){downer.setHeader("User-Agent", userAgentOverride);}
       if (!downer.head(myURI) || !downer.isOk()){
         FAIL_MSG("Error getting URI info for '%s': %" PRIu32 " %s", myURI.getUrl().c_str(),
@@ -120,6 +228,8 @@ namespace HTTP{
         myURI = downer.lastURL();
       }
 
+      // Other set of headers specified for GET request
+      httpHeaderOverride.prepareGetHeaders(downer);
       // streaming mode when size is unknown
       if (!supportRangeRequest){
         MEDIUM_MSG("URI get without range request: %s, totalsize: %zu", myURI.getUrl().c_str(), totalSize);