hydrar/mistserver
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixing all of the things. 😩

Browse source
This commit is contained in:
Thulinma 2023-03-15 03:43:19 +01:00
parent 53f941449f
commit c979acff52
14 changed files with 171 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

83
lib/downloader.cpp
View file

@ -71,6 +71,14 @@ namespace HTTP{
return S;
}
void Downloader::clean(){
H.headerOnly = false;
H.Clean();
getSocket().close();
getSocket().Received().clear();
extraHeaders.clear();
}
///Sets an override to use the given socket
void Downloader::setSocket(Socket::Connection * socketPtr){
sPtr = socketPtr;
@ -187,10 +195,7 @@ namespace HTTP{
if (progressCallback != 0){
if (!progressCallback()){
WARN_MSG("Download aborted by callback");
H.headerOnly = false;
H.Clean();
getSocket().close();
getSocket().Received().clear();
clean();
return false;
}
}
@ -203,10 +208,7 @@ namespace HTTP{
// If the return status code is invalid, close the socket, wipe all buffers, and return false
if(!getStatusCode()){
H.headerOnly = false;
getSocket().close();
getSocket().Received().clear();
H.Clean();
clean();
return false;
}
@ -233,10 +235,7 @@ namespace HTTP{
if (progressCallback != 0){
if (!progressCallback()){
WARN_MSG("Download aborted by callback");
H.headerOnly = false;
H.Clean();
getSocket().close();
getSocket().Received().clear();
clean();
return false;
}
}
@ -246,11 +245,9 @@ namespace HTTP{
}
}
}
H.headerOnly = false;
if (getSocket()){
FAIL_MSG("Timeout while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount);
getSocket().close();
}else{
if (loop > 1){
INFO_MSG("Lost connection while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount);
@ -258,8 +255,7 @@ namespace HTTP{
MEDIUM_MSG("Lost connection while retrieving %s (%zu/%" PRIu32 ")", link.getUrl().c_str(), loop, retryCount);
}
}
H.Clean();
getSocket().Received().clear();
clean();
Util::sleep(100); // wait a bit before retrying
}
FAIL_MSG("Could not retrieve %s", link.getUrl().c_str());
@ -309,6 +305,7 @@ namespace HTTP{
nbMaxRecursiveDepth = maxRecursiveDepth;
nbLoop = retryCount + 1; // max 5 attempts
isComplete = false;
extraHeaders.erase("Range");
doRequest(nbLink);
nbReqTime = Util::bootSecs();
nbLastOff = getSocket().dataDown();
@ -343,28 +340,17 @@ namespace HTTP{
}
if (H.hasHeader("Accept-Ranges") && getHeader("Accept-Ranges").size() > 0){
getRangeNonBlocking(nbLink, H.currentLength, 0, cb);
return true;
getRangeNonBlocking(nbLink, cb.getDataCallbackPos(), 0, cb);
continue;
}else{
doRequest(nbLink);
}
if (!getSocket()){
WARN_MSG("Aborting download: could not open connection");
return true;
}
nbReqTime = Util::bootSecs();
nbLastOff = getSocket().dataDown();
}
if (Util::bootSecs() >= nbReqTime + dataTimeout){
FAIL_MSG("Timeout while retrieving %s (%zu/%" PRIu32 ")", nbLink.getUrl().c_str(),
retryCount - nbLoop + 1, retryCount);
getSocket().close();
return false; // because we may have retries left
}
// No data? Wait for a second or so.
// No data? Return false to indicate retry later.
if (!getSocket().spool() && getSocket()){
if (progressCallback != 0){
if (!progressCallback()){
@ -372,9 +358,30 @@ namespace HTTP{
return true;
}
}
if (Util::bootSecs() >= nbReqTime + dataTimeout){
FAIL_MSG("Timeout while retrieving %s (%zu/%" PRIu32 ")", nbLink.getUrl().c_str(),
retryCount - nbLoop + 1, retryCount);
getSocket().close();
return false; // Retry on next attempt
}
return false;
}
// Data! Check if we can parse it...
// Reset the data timeout
if (nbReqTime != Util::bootSecs()){
if (progressCallback != 0){
if (!progressCallback()){
WARN_MSG("Download aborted by callback");
return true;
}
}
if (getSocket().dataDown() > nbLastOff + 25600){
nbReqTime = Util::bootSecs();
nbLastOff = getSocket().dataDown();
}
}
//Attempt to parse the data we received
if (H.Read(getSocket(), cb)){
if (shouldContinue()){
if (nbMaxRecursiveDepth == 0){
@ -393,21 +400,7 @@ namespace HTTP{
isComplete = true; // Success
return true;
}
// reset the data timeout
if (nbReqTime != Util::bootSecs()){
if (progressCallback != 0){
if (!progressCallback()){
WARN_MSG("Download aborted by callback");
return true;
}
}
if (getSocket().dataDown() > nbLastOff + 25600){
nbReqTime = Util::bootSecs();
nbLastOff = getSocket().dataDown();
}
}
}
WARN_MSG("Invalid connection state for HTTP request");
return false; // we should never get here
}

1
lib/downloader.h
View file

@ -45,6 +45,7 @@ namespace HTTP{
Parser &getHTTP();
Socket::Connection &getSocket();
const Socket::Connection &getSocket() const;
void clean();
void setSocket(Socket::Connection * socketPtr);
uint32_t retryCount, dataTimeout;
bool isProxied() const;

2
lib/http_parser.h
View file

@ -17,7 +17,7 @@ namespace HTTP{
void parseVars(const std::string &data, std::map<std::string, std::string> &storage, const std::string & separator = "&");
/// Simple class for reading and writing HTTP 1.0 and 1.1.
class Parser : public Util::DataCallback{
class Parser{
public:
Parser();
bool Read(Socket::Connection &conn, Util::DataCallback &cb = Util::defaultDataCallback);

27
lib/urireader.cpp
View file

@ -65,13 +65,17 @@ namespace HTTP{
return url;
}
HTTP::URL localURIResolver(){
char workDir[512];
getcwd(workDir, 512);
return HTTP::URL(std::string("file://") + workDir + "/");
}
void URIReader::init(){
handle = -1;
mapped = 0;
char workDir[512];
getcwd(workDir, 512);
myURI = HTTP::URL(std::string("file://") + workDir + "/");
myURI = localURIResolver();
originalUrl = myURI;
cbProgress = 0;
minLen = 1;
maxLen = std::string::npos;
@ -97,11 +101,13 @@ namespace HTTP{
open(reluri);
}
bool URIReader::open(const std::string &reluri){return open(myURI.link(reluri));}
bool URIReader::open(const std::string &reluri){return open(originalUrl.link(reluri));}
/// Internal callback function, used to buffer data.
void URIReader::dataCallback(const char *ptr, size_t size){allData.append(ptr, size);}
size_t URIReader::getDataCallbackPos() const{return allData.size();}
bool URIReader::open(const HTTP::URL &uri){
close();
myURI = uri;
@ -247,14 +253,13 @@ namespace HTTP{
//HTTP-based needs to do a range request
if (stateType == HTTP::HTTP && supportRangeRequest){
downer.getSocket().close();
downer.getSocket().Received().clear();
downer.clean();
curPos = pos;
injectHeaders(originalUrl, "GET", downer);
if (!downer.getRangeNonBlocking(myURI.getUrl(), pos, 0)){
if (!downer.getRangeNonBlocking(myURI, pos, 0)){
FAIL_MSG("Error making range request");
return false;
}
curPos = pos;
return true;
}
return false;
@ -354,9 +359,7 @@ namespace HTTP{
allData.truncate(0);
bufPos = 0;
// Close downloader socket if open
downer.getSocket().close();
downer.getSocket().Received().clear();
downer.getHTTP().Clean();
downer.clean();
// Unmap file if mapped
if (mapped){
munmap(mapped, totalSize);
@ -408,7 +411,7 @@ namespace HTTP{
uint64_t URIReader::getPos(){return curPos;}
const HTTP::URL &URIReader::getURI() const{return myURI;}
const HTTP::URL &URIReader::getURI() const{return originalUrl;}
size_t URIReader::getSize() const{return totalSize;}

5
lib/urireader.h
View file

@ -60,7 +60,8 @@ namespace HTTP{
size_t getSize() const; ///< Returns the size of the currently open URI, if known. Returns std::string::npos if unknown size.
void (*httpBodyCallback)(const char *ptr, size_t size);
void dataCallback(const char *ptr, size_t size);
virtual void dataCallback(const char *ptr, size_t size);
virtual size_t getDataCallbackPos() const;
std::string userAgentOverride;
@ -86,4 +87,6 @@ namespace HTTP{
HTTP::Downloader downer; ///< For HTTP(S)-based URIs, the Downloader instance used for the download.
void init();
};
HTTP::URL localURIResolver();
}// namespace HTTP

10
lib/url.cpp
View file

@ -345,3 +345,13 @@ HTTP::URL HTTP::URL::link(const std::string &l) const{
DONTEVEN_MSG("Relative link: %s+%s", base.c_str(), l.c_str());
return URL(base + l);
}
/// Returns true if the URL matches, ignoring username, password and fragment
bool HTTP::URL::operator==(const URL& rhs) const{
return (host == rhs.host && getPort() == rhs.getPort() && protocol == rhs.protocol && path == rhs.path && args == rhs.args);
}
/// Returns false if the URL matches, ignoring username, password and fragment.
/// Simply calls == internally, and negates the result.
bool HTTP::URL::operator!=(const URL& rhs) const{return !(*this == rhs);}

2
lib/url.h
View file

@ -34,6 +34,8 @@ namespace HTTP{
std::string pass; ///< Password, if it was present
URL link(const std::string &l) const;
bool IPv6Addr;
bool operator==(const URL& rhs) const;
bool operator!=(const URL& rhs) const;
};
}// namespace HTTP

1
lib/util.h
View file

@ -30,6 +30,7 @@ namespace Util{
virtual void dataCallback(const char *ptr, size_t size){
INFO_MSG("default callback, size: %zu", size);
}
virtual size_t getDataCallbackPos() const{return 0;}
virtual ~DataCallback(){};
};