Fixes to Downloader and URIReader classes, fix S3 support not liking range requests anymore after 15 minutes
This commit is contained in:
Thulinma
parent
8f0e0c8eba
commit
d9c7e0f3ca
3 changed files with 51 additions and 63 deletions
|
|
@ -188,6 +188,9 @@ namespace HTTP{
|
|||
if (!progressCallback()){
|
||||
WARN_MSG("Download aborted by callback");
|
||||
H.headerOnly = false;
|
||||
H.Clean();
|
||||
getSocket().close();
|
||||
getSocket().Received().clear();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -231,7 +234,9 @@ namespace HTTP{
|
|||
if (!progressCallback()){
|
||||
WARN_MSG("Download aborted by callback");
|
||||
H.headerOnly = false;
|
||||
H.Clean();
|
||||
getSocket().close();
|
||||
getSocket().Received().clear();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -245,7 +250,6 @@ namespace HTTP{
|
|||
|
||||
if (getSocket()){
|
||||
FAIL_MSG("Timeout while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount);
|
||||
H.Clean();
|
||||
getSocket().close();
|
||||
}else{
|
||||
if (loop > 1){
|
||||
|
|
@ -253,8 +257,9 @@ namespace HTTP{
|
|||
}else{
|
||||
MEDIUM_MSG("Lost connection while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount);
|
||||
}
|
||||
H.Clean();
|
||||
}
|
||||
H.Clean();
|
||||
getSocket().Received().clear();
|
||||
Util::sleep(100); // wait a bit before retrying
|
||||
}
|
||||
FAIL_MSG("Could not retrieve %s", link.getUrl().c_str());
|
||||
|
|
|
|||
|
|
@ -11,27 +11,6 @@
|
|||
|
||||
namespace HTTP{
|
||||
|
||||
// When another protocol needs this, rename struct to HeaderOverride or similar
|
||||
struct HTTPHeadThenGet {
|
||||
bool continueOperation;
|
||||
std::string date, headAuthorization, getAuthorization;
|
||||
|
||||
HTTPHeadThenGet() : continueOperation(false) {}
|
||||
|
||||
void prepareHeadHeaders(HTTP::Downloader& downloader) {
|
||||
if(!continueOperation) return;
|
||||
downloader.setHeader("Date", date);
|
||||
downloader.setHeader("Authorization", headAuthorization);
|
||||
}
|
||||
|
||||
void prepareGetHeaders(HTTP::Downloader& downloader) {
|
||||
if(!continueOperation) return;
|
||||
// .setHeader() overwrites existing header value
|
||||
downloader.setHeader("Date", date);
|
||||
downloader.setHeader("Authorization", getAuthorization);
|
||||
}
|
||||
};
|
||||
|
||||
#ifndef NOSSL
|
||||
inline bool s3CalculateSignature(std::string& signature, const std::string method, const std::string date, const std::string& requestPath, const std::string& accessKey, const std::string& secret) {
|
||||
std::string toSign = method + "\n\n\n" + date + "\n" + requestPath;
|
||||
|
|
@ -53,35 +32,39 @@ namespace HTTP{
|
|||
signature = "AWS " + accessKey + ":" + base64encoded;
|
||||
return true;
|
||||
}
|
||||
#endif // ifndef NOSSL
|
||||
|
||||
|
||||
inline HTTP::URL injectHeaders(const HTTP::URL& url, const std::string & method, HTTP::Downloader & downer) {
|
||||
#ifndef NOSSL
|
||||
// Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
|
||||
// Transform to:
|
||||
// url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
|
||||
// header Date: ${Util::getDateString(()}
|
||||
// header Authorization: AWS ${url.user}:${signature}
|
||||
inline HTTPHeadThenGet s3TransformToHttp(HTTP::URL& url) {
|
||||
HTTPHeadThenGet result;
|
||||
result.date = Util::getDateString();
|
||||
if (url.protocol.size() > 3 && url.protocol.substr(0, 3) == "s3+"){
|
||||
std::string date = Util::getDateString();
|
||||
HTTP::URL newUrl = url;
|
||||
// remove "s3+" prefix
|
||||
url.protocol = url.protocol.erase(0, 3);
|
||||
newUrl.protocol = newUrl.protocol.erase(0, 3);
|
||||
// Use user and pass to create signature and remove from HTTP request
|
||||
std::string accessKey(url.user), secret(url.pass);
|
||||
url.user = "";
|
||||
url.pass = "";
|
||||
newUrl.user = "";
|
||||
newUrl.pass = "";
|
||||
std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&");
|
||||
if(url.args.size()) requestPath += "?" + url.args;
|
||||
std::string authLine;
|
||||
// Calculate Authorization data
|
||||
if(!s3CalculateSignature(result.headAuthorization, "HEAD", result.date, requestPath, accessKey, secret)) {
|
||||
result.continueOperation = false;
|
||||
return result;
|
||||
if(method.size() && s3CalculateSignature(authLine, method, date, requestPath, accessKey, secret)) {
|
||||
downer.setHeader("Date", date);
|
||||
downer.setHeader("Authorization", authLine);
|
||||
}
|
||||
if(!s3CalculateSignature(result.getAuthorization, "GET", result.date, requestPath, accessKey, secret)) {
|
||||
result.continueOperation = false;
|
||||
return result;
|
||||
}
|
||||
result.continueOperation = true;
|
||||
return result;
|
||||
return newUrl;
|
||||
}
|
||||
#endif // ifndef NOSSL
|
||||
return url;
|
||||
}
|
||||
|
||||
|
||||
void URIReader::init(){
|
||||
handle = -1;
|
||||
|
|
@ -120,13 +103,11 @@ namespace HTTP{
|
|||
void URIReader::dataCallback(const char *ptr, size_t size){allData.append(ptr, size);}
|
||||
|
||||
bool URIReader::open(const HTTP::URL &uri){
|
||||
close();
|
||||
myURI = uri;
|
||||
curPos = 0;
|
||||
allData.truncate(0);
|
||||
bufPos = 0;
|
||||
originalUrl = myURI;
|
||||
|
||||
if (!myURI.protocol.size() || myURI.protocol == "file"){
|
||||
close();
|
||||
if (!myURI.path.size() || myURI.path == "-"){
|
||||
downer.getSocket().open(-1, fileno(stdin));
|
||||
stateType = HTTP::Stream;
|
||||
|
|
@ -173,8 +154,6 @@ namespace HTTP{
|
|||
}
|
||||
|
||||
// prepare for s3 and http
|
||||
HTTPHeadThenGet httpHeaderOverride;
|
||||
|
||||
#ifndef NOSSL
|
||||
// In case of s3 URI we prepare HTTP request with AWS authorization and rely on HTTP logic below
|
||||
if (myURI.protocol == "s3+https" || myURI.protocol == "s3+http"){
|
||||
|
|
@ -195,10 +174,7 @@ namespace HTTP{
|
|||
}
|
||||
myURI.pass = envValue;
|
||||
}
|
||||
// Transform s3 url to HTTP request:
|
||||
httpHeaderOverride = s3TransformToHttp(myURI);
|
||||
bool errorInSignatureCalculation = !httpHeaderOverride.continueOperation;
|
||||
if(errorInSignatureCalculation) return false;
|
||||
myURI = injectHeaders(originalUrl, "", downer);
|
||||
// Do not return, continue to HTTP case
|
||||
}
|
||||
#endif // ifndef NOSSL
|
||||
|
|
@ -209,7 +185,7 @@ namespace HTTP{
|
|||
downer.clearHeaders();
|
||||
|
||||
// One set of headers specified for HEAD request
|
||||
httpHeaderOverride.prepareHeadHeaders(downer);
|
||||
injectHeaders(originalUrl, "HEAD", downer);
|
||||
// Send HEAD request to determine range request is supported, and get total length
|
||||
if (userAgentOverride.size()){downer.setHeader("User-Agent", userAgentOverride);}
|
||||
if (!downer.head(myURI) || !downer.isOk()){
|
||||
|
|
@ -218,6 +194,7 @@ namespace HTTP{
|
|||
// Close the socket, and clean up the buffer
|
||||
downer.getSocket().close();
|
||||
downer.getSocket().Received().clear();
|
||||
allData.truncate(0);
|
||||
if (!downer.isOk()){return false;}
|
||||
supportRangeRequest = false;
|
||||
totalSize = std::string::npos;
|
||||
|
|
@ -229,7 +206,7 @@ namespace HTTP{
|
|||
}
|
||||
|
||||
// Other set of headers specified for GET request
|
||||
httpHeaderOverride.prepareGetHeaders(downer);
|
||||
injectHeaders(originalUrl, "GET", downer);
|
||||
// streaming mode when size is unknown
|
||||
if (!supportRangeRequest){
|
||||
MEDIUM_MSG("URI get without range request: %s, totalsize: %zu", myURI.getUrl().c_str(), totalSize);
|
||||
|
|
@ -272,6 +249,7 @@ namespace HTTP{
|
|||
if (stateType == HTTP::HTTP && supportRangeRequest){
|
||||
downer.getSocket().close();
|
||||
downer.getSocket().Received().clear();
|
||||
injectHeaders(originalUrl, "GET", downer);
|
||||
if (!downer.getRangeNonBlocking(myURI.getUrl(), pos, 0)){
|
||||
FAIL_MSG("Error making range request");
|
||||
return false;
|
||||
|
|
@ -371,6 +349,10 @@ namespace HTTP{
|
|||
}
|
||||
|
||||
void URIReader::close(){
|
||||
//Wipe internal state
|
||||
curPos = 0;
|
||||
allData.truncate(0);
|
||||
bufPos = 0;
|
||||
// Close downloader socket if open
|
||||
downer.getSocket().close();
|
||||
downer.getSocket().Received().clear();
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ namespace HTTP{
|
|||
size_t bufPos; ///< Current read position in buffer
|
||||
int handle; ///< Open file handle, if file-based.
|
||||
char *mapped; ///< Memory-map of open file handle, if file-based.
|
||||
HTTP::URL originalUrl;
|
||||
bool supportRangeRequest;
|
||||
Util::ResizeablePointer rPtr;
|
||||
Util::ResizeablePointer allData;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue