hydrar/mistserver
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixes to Downloader and URIReader classes, fix S3 support not liking range requests anymore after 15 minutes

Browse source
This commit is contained in:
Thulinma 2022-09-01 10:05:40 +02:00
parent 8f0e0c8eba
commit d9c7e0f3ca
3 changed files with 51 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
lib/downloader.cpp
View file

@ -188,6 +188,9 @@ namespace HTTP{
if (!progressCallback()){
WARN_MSG("Download aborted by callback");
H.headerOnly = false;
H.Clean();
getSocket().close();
getSocket().Received().clear();
return false;
}
}
@ -231,7 +234,9 @@ namespace HTTP{
if (!progressCallback()){
WARN_MSG("Download aborted by callback");
H.headerOnly = false;
H.Clean();
getSocket().close();
getSocket().Received().clear();
return false;
}
}
@ -245,7 +250,6 @@ namespace HTTP{
if (getSocket()){
FAIL_MSG("Timeout while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount);
H.Clean();
getSocket().close();
}else{
if (loop > 1){
@ -253,8 +257,9 @@ namespace HTTP{
}else{
MEDIUM_MSG("Lost connection while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount);
}
H.Clean();
}
H.Clean();
getSocket().Received().clear();
Util::sleep(100); // wait a bit before retrying
}
FAIL_MSG("Could not retrieve %s", link.getUrl().c_str());

104
lib/urireader.cpp
View file

@ -11,27 +11,6 @@
namespace HTTP{
// When another protocol needs this, rename struct to HeaderOverride or similar
struct HTTPHeadThenGet {
bool continueOperation;
std::string date, headAuthorization, getAuthorization;
HTTPHeadThenGet() : continueOperation(false) {}
void prepareHeadHeaders(HTTP::Downloader& downloader) {
if(!continueOperation) return;
downloader.setHeader("Date", date);
downloader.setHeader("Authorization", headAuthorization);
}
void prepareGetHeaders(HTTP::Downloader& downloader) {
if(!continueOperation) return;
// .setHeader() overwrites existing header value
downloader.setHeader("Date", date);
downloader.setHeader("Authorization", getAuthorization);
}
};
#ifndef NOSSL
inline bool s3CalculateSignature(std::string& signature, const std::string method, const std::string date, const std::string& requestPath, const std::string& accessKey, const std::string& secret) {
std::string toSign = method + "\n\n\n" + date + "\n" + requestPath;
@ -53,36 +32,40 @@ namespace HTTP{
signature = "AWS " + accessKey + ":" + base64encoded;
return true;
}
// Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
// Transform to:
// url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
// header Date: ${Util::getDateString(()}
// header Authorization: AWS ${url.user}:${signature}
inline HTTPHeadThenGet s3TransformToHttp(HTTP::URL& url) {
HTTPHeadThenGet result;
result.date = Util::getDateString();
// remove "s3+" prefix
url.protocol = url.protocol.erase(0, 3);
// Use user and pass to create signature and remove from HTTP request
std::string accessKey(url.user), secret(url.pass);
url.user = "";
url.pass = "";
std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&");
if(url.args.size()) requestPath += "?" + url.args;
// Calculate Authorization data
if(!s3CalculateSignature(result.headAuthorization, "HEAD", result.date, requestPath, accessKey, secret)) {
result.continueOperation = false;
return result;
}
if(!s3CalculateSignature(result.getAuthorization, "GET", result.date, requestPath, accessKey, secret)) {
result.continueOperation = false;
return result;
}
result.continueOperation = true;
return result;
}
#endif // ifndef NOSSL
inline HTTP::URL injectHeaders(const HTTP::URL& url, const std::string & method, HTTP::Downloader & downer) {
#ifndef NOSSL
// Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
// Transform to:
// url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts
// header Date: ${Util::getDateString(()}
// header Authorization: AWS ${url.user}:${signature}
if (url.protocol.size() > 3 && url.protocol.substr(0, 3) == "s3+"){
std::string date = Util::getDateString();
HTTP::URL newUrl = url;
// remove "s3+" prefix
newUrl.protocol = newUrl.protocol.erase(0, 3);
// Use user and pass to create signature and remove from HTTP request
std::string accessKey(url.user), secret(url.pass);
newUrl.user = "";
newUrl.pass = "";
std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&");
if(url.args.size()) requestPath += "?" + url.args;
std::string authLine;
// Calculate Authorization data
if(method.size() && s3CalculateSignature(authLine, method, date, requestPath, accessKey, secret)) {
downer.setHeader("Date", date);
downer.setHeader("Authorization", authLine);
}
return newUrl;
}
#endif // ifndef NOSSL
return url;
}
void URIReader::init(){
handle = -1;
mapped = 0;
@ -120,13 +103,11 @@ namespace HTTP{
void URIReader::dataCallback(const char *ptr, size_t size){allData.append(ptr, size);}
bool URIReader::open(const HTTP::URL &uri){
close();
myURI = uri;
curPos = 0;
allData.truncate(0);
bufPos = 0;
originalUrl = myURI;
if (!myURI.protocol.size() || myURI.protocol == "file"){
close();
if (!myURI.path.size() || myURI.path == "-"){
downer.getSocket().open(-1, fileno(stdin));
stateType = HTTP::Stream;
@ -173,8 +154,6 @@ namespace HTTP{
}
// prepare for s3 and http
HTTPHeadThenGet httpHeaderOverride;
#ifndef NOSSL
// In case of s3 URI we prepare HTTP request with AWS authorization and rely on HTTP logic below
if (myURI.protocol == "s3+https" || myURI.protocol == "s3+http"){
@ -195,10 +174,7 @@ namespace HTTP{
}
myURI.pass = envValue;
}
// Transform s3 url to HTTP request:
httpHeaderOverride = s3TransformToHttp(myURI);
bool errorInSignatureCalculation = !httpHeaderOverride.continueOperation;
if(errorInSignatureCalculation) return false;
myURI = injectHeaders(originalUrl, "", downer);
// Do not return, continue to HTTP case
}
#endif // ifndef NOSSL
@ -209,7 +185,7 @@ namespace HTTP{
downer.clearHeaders();
// One set of headers specified for HEAD request
httpHeaderOverride.prepareHeadHeaders(downer);
injectHeaders(originalUrl, "HEAD", downer);
// Send HEAD request to determine range request is supported, and get total length
if (userAgentOverride.size()){downer.setHeader("User-Agent", userAgentOverride);}
if (!downer.head(myURI) || !downer.isOk()){
@ -218,6 +194,7 @@ namespace HTTP{
// Close the socket, and clean up the buffer
downer.getSocket().close();
downer.getSocket().Received().clear();
allData.truncate(0);
if (!downer.isOk()){return false;}
supportRangeRequest = false;
totalSize = std::string::npos;
@ -229,7 +206,7 @@ namespace HTTP{
}
// Other set of headers specified for GET request
httpHeaderOverride.prepareGetHeaders(downer);
injectHeaders(originalUrl, "GET", downer);
// streaming mode when size is unknown
if (!supportRangeRequest){
MEDIUM_MSG("URI get without range request: %s, totalsize: %zu", myURI.getUrl().c_str(), totalSize);
@ -272,6 +249,7 @@ namespace HTTP{
if (stateType == HTTP::HTTP && supportRangeRequest){
downer.getSocket().close();
downer.getSocket().Received().clear();
injectHeaders(originalUrl, "GET", downer);
if (!downer.getRangeNonBlocking(myURI.getUrl(), pos, 0)){
FAIL_MSG("Error making range request");
return false;
@ -371,6 +349,10 @@ namespace HTTP{
}
void URIReader::close(){
//Wipe internal state
curPos = 0;
allData.truncate(0);
bufPos = 0;
// Close downloader socket if open
downer.getSocket().close();
downer.getSocket().Received().clear();

1
lib/urireader.h
View file

@ -77,6 +77,7 @@ namespace HTTP{
size_t bufPos; ///< Current read position in buffer
int handle; ///< Open file handle, if file-based.
char *mapped; ///< Memory-map of open file handle, if file-based.
HTTP::URL originalUrl;
bool supportRangeRequest;
Util::ResizeablePointer rPtr;
Util::ResizeablePointer allData;