SSL socket class, downloadertest application, HTTP::Downloader support for HTTPS connections, authentication, proxies and POST requests
This commit is contained in:
Thulinma
parent
ce9aae3095
commit
5e3df09831
6 changed files with 467 additions and 44 deletions
|
|
@ -4,6 +4,19 @@
|
|||
|
||||
namespace HTTP{
|
||||
|
||||
Downloader::Downloader(){
|
||||
progressCallback = 0;
|
||||
connectedPort = 0;
|
||||
ssl = false;
|
||||
proxied = false;
|
||||
char *p = getenv("http_proxy");
|
||||
if (p){
|
||||
proxyUrl = HTTP::URL(p);
|
||||
proxied = true;
|
||||
INFO_MSG("Proxying through %s", proxyUrl.getUrl().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a reference to the internal HTTP::Parser body element
|
||||
std::string &Downloader::data(){return H.body;}
|
||||
|
||||
|
|
@ -39,58 +52,98 @@ namespace HTTP{
|
|||
Parser &Downloader::getHTTP(){return H;}
|
||||
|
||||
/// Returns a reference to the internal Socket::Connection class instance.
|
||||
Socket::Connection &Downloader::getSocket(){return S;}
|
||||
Socket::Connection &Downloader::getSocket(){
|
||||
#ifdef SSL
|
||||
if (ssl){return S_SSL;}
|
||||
#endif
|
||||
return S;
|
||||
}
|
||||
|
||||
/// Sends a request for the given URL, does no waiting.
|
||||
void Downloader::doRequest(const HTTP::URL &link){
|
||||
if (link.protocol != "http"){
|
||||
FAIL_MSG("Protocol not supported: %s", link.protocol.c_str());
|
||||
return;
|
||||
}
|
||||
void Downloader::doRequest(const HTTP::URL &link, const std::string &method, const std::string &body){
|
||||
if (!canRequest(link)){return;}
|
||||
bool needSSL = (link.protocol == "https");
|
||||
INFO_MSG("Retrieving %s", link.getUrl().c_str());
|
||||
H.Clean();
|
||||
// Reconnect if needed
|
||||
if (!S || link.host != connectedHost || link.getPort() != connectedPort){
|
||||
S.close();
|
||||
connectedHost = link.host;
|
||||
connectedPort = link.getPort();
|
||||
S = Socket::Connection(connectedHost, connectedPort, true);
|
||||
}
|
||||
H.url = "/" + link.path;
|
||||
if (link.args.size()){H.url += "?" + link.args;}
|
||||
if (link.port.size()){
|
||||
H.SetHeader("Host", link.host + ":" + link.port);
|
||||
if (!proxied || needSSL){
|
||||
if (!getSocket() || link.host != connectedHost || link.getPort() != connectedPort ||
|
||||
needSSL != ssl){
|
||||
getSocket().close();
|
||||
connectedHost = link.host;
|
||||
connectedPort = link.getPort();
|
||||
#ifdef SSL
|
||||
if (needSSL){
|
||||
S_SSL = Socket::SSLConnection(connectedHost, connectedPort, true);
|
||||
}else{
|
||||
S = Socket::Connection(connectedHost, connectedPort, true);
|
||||
}
|
||||
#else
|
||||
S = Socket::Connection(connectedHost, connectedPort, true);
|
||||
#endif
|
||||
}
|
||||
}else{
|
||||
H.SetHeader("Host", link.host);
|
||||
if (!getSocket() || proxyUrl.host != connectedHost || proxyUrl.getPort() != connectedPort ||
|
||||
needSSL != ssl){
|
||||
getSocket().close();
|
||||
connectedHost = proxyUrl.host;
|
||||
connectedPort = proxyUrl.getPort();
|
||||
S = Socket::Connection(connectedHost, connectedPort, true);
|
||||
}
|
||||
}
|
||||
ssl = needSSL;
|
||||
if (!getSocket()){
|
||||
return; // socket is closed
|
||||
}
|
||||
if (proxied && !ssl){
|
||||
H.url = link.getProxyUrl();
|
||||
if (proxyUrl.port.size()){
|
||||
H.SetHeader("Host", proxyUrl.host + ":" + proxyUrl.port);
|
||||
}else{
|
||||
H.SetHeader("Host", proxyUrl.host);
|
||||
}
|
||||
}else{
|
||||
H.url = "/" + link.path;
|
||||
if (link.args.size()){H.url += "?" + link.args;}
|
||||
if (link.port.size()){
|
||||
H.SetHeader("Host", link.host + ":" + link.port);
|
||||
}else{
|
||||
H.SetHeader("Host", link.host);
|
||||
}
|
||||
}
|
||||
if (method.size()){
|
||||
H.method = method;
|
||||
}
|
||||
H.SetHeader("User-Agent", "MistServer " PACKAGE_VERSION);
|
||||
H.SetHeader("X-Version", PACKAGE_VERSION);
|
||||
H.SetHeader("Accept", "*/*");
|
||||
if (authStr.size() && (link.user.size() || link.pass.size())){
|
||||
H.auth(link.user, link.pass, authStr);
|
||||
}
|
||||
if (proxyAuthStr.size() && (proxyUrl.user.size() || proxyUrl.pass.size())){
|
||||
H.auth(proxyUrl.user, proxyUrl.pass, proxyAuthStr, "Proxy-Authorization");
|
||||
}
|
||||
if (extraHeaders.size()){
|
||||
for (std::map<std::string, std::string>::iterator it = extraHeaders.begin();
|
||||
it != extraHeaders.end(); ++it){
|
||||
H.SetHeader(it->first, it->second);
|
||||
}
|
||||
}
|
||||
H.SendRequest(S);
|
||||
H.SendRequest(getSocket(), body);
|
||||
H.Clean();
|
||||
}
|
||||
|
||||
/// Downloads the given URL into 'H', returns true on success.
|
||||
/// Makes at most 5 attempts, and will wait no longer than 5 seconds without receiving data.
|
||||
bool Downloader::get(const HTTP::URL &link, uint8_t maxRecursiveDepth){
|
||||
if (!link.host.size()){return false;}
|
||||
if (link.protocol != "http"){
|
||||
FAIL_MSG("Protocol not supported: %s", link.protocol.c_str());
|
||||
return false;
|
||||
}
|
||||
if (!canRequest(link)){return false;}
|
||||
unsigned int loop = 6; // max 5 attempts
|
||||
while (--loop){// loop while we are unsuccessful
|
||||
doRequest(link);
|
||||
uint64_t reqTime = Util::bootSecs();
|
||||
while (S && Util::bootSecs() < reqTime + 5){
|
||||
while (getSocket() && Util::bootSecs() < reqTime + 5){
|
||||
// No data? Wait for a second or so.
|
||||
if (!S.spool()){
|
||||
if (!getSocket().spool()){
|
||||
if (progressCallback != 0){
|
||||
if (!progressCallback()){
|
||||
WARN_MSG("Download aborted by callback");
|
||||
|
|
@ -101,16 +154,17 @@ namespace HTTP{
|
|||
continue;
|
||||
}
|
||||
// Data! Check if we can parse it...
|
||||
if (H.Read(S)){
|
||||
if (getStatusCode() >= 300 && getStatusCode() < 400){
|
||||
// follow redirect
|
||||
std::string location = getHeader("Location");
|
||||
if (H.Read(getSocket())){
|
||||
if (shouldContinue()){
|
||||
if (maxRecursiveDepth == 0){
|
||||
FAIL_MSG("Maximum redirect depth reached: %s", location.c_str());
|
||||
FAIL_MSG("Maximum recursion depth reached");
|
||||
return false;
|
||||
}
|
||||
if (!canContinue(link)){return false;}
|
||||
if (getStatusCode() >= 300 && getStatusCode() < 400){
|
||||
return get(link.link(getHeader("Location")), --maxRecursiveDepth);
|
||||
}else{
|
||||
FAIL_MSG("Following redirect to %s", location.c_str());
|
||||
return get(link.link(location), maxRecursiveDepth--);
|
||||
return get(link, --maxRecursiveDepth);
|
||||
}
|
||||
}
|
||||
return true; // Success!
|
||||
|
|
@ -118,7 +172,56 @@ namespace HTTP{
|
|||
// reset the 5 second timeout
|
||||
reqTime = Util::bootSecs();
|
||||
}
|
||||
if (S){
|
||||
if (getSocket()){
|
||||
FAIL_MSG("Timeout while retrieving %s", link.getUrl().c_str());
|
||||
return false;
|
||||
}
|
||||
Util::sleep(500); // wait a bit before retrying
|
||||
}
|
||||
FAIL_MSG("Could not retrieve %s", link.getUrl().c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Downloader::post(const HTTP::URL &link, const std::string &payload, bool sync, uint8_t maxRecursiveDepth){
|
||||
if (!canRequest(link)){return false;}
|
||||
unsigned int loop = 6; // max 5 attempts
|
||||
while (--loop){// loop while we are unsuccessful
|
||||
doRequest(link, "POST", payload);
|
||||
//Not synced? Ignore the response and immediately return false.
|
||||
if (!sync){return false;}
|
||||
uint64_t reqTime = Util::bootSecs();
|
||||
while (getSocket() && Util::bootSecs() < reqTime + 5){
|
||||
// No data? Wait for a second or so.
|
||||
if (!getSocket().spool()){
|
||||
if (progressCallback != 0){
|
||||
if (!progressCallback()){
|
||||
WARN_MSG("Download aborted by callback");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
Util::sleep(250);
|
||||
continue;
|
||||
}
|
||||
// Data! Check if we can parse it...
|
||||
if (H.Read(getSocket())){
|
||||
if (shouldContinue()){
|
||||
if (maxRecursiveDepth == 0){
|
||||
FAIL_MSG("Maximum recursion depth reached");
|
||||
return false;
|
||||
}
|
||||
if (!canContinue(link)){return false;}
|
||||
if (getStatusCode() >= 300 && getStatusCode() < 400){
|
||||
return post(link.link(getHeader("Location")), payload, sync, --maxRecursiveDepth);
|
||||
}else{
|
||||
return post(link, payload, sync, --maxRecursiveDepth);
|
||||
}
|
||||
}
|
||||
return true; // Success!
|
||||
}
|
||||
// reset the 5 second timeout
|
||||
reqTime = Util::bootSecs();
|
||||
}
|
||||
if (getSocket()){
|
||||
FAIL_MSG("Timeout while retrieving %s", link.getUrl().c_str());
|
||||
return false;
|
||||
}
|
||||
|
|
@ -127,5 +230,75 @@ namespace HTTP{
|
|||
FAIL_MSG("Could not retrieve %s", link.getUrl().c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool Downloader::canRequest(const HTTP::URL &link){
|
||||
if (!link.host.size()){return false;}
|
||||
if (link.protocol != "http" && link.protocol != "https"){
|
||||
FAIL_MSG("Protocol not supported: %s", link.protocol.c_str());
|
||||
return false;
|
||||
}
|
||||
#ifndef SSL
|
||||
if (link.protocol == "https"){
|
||||
FAIL_MSG("Protocol not supported: %s", link.protocol.c_str());
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Downloader::shouldContinue(){
|
||||
if (H.hasHeader("Set-Cookie")){
|
||||
std::string cookie = H.GetHeader("Set-Cookie");
|
||||
setHeader("Cookie", cookie.substr(0, cookie.find(';')));
|
||||
}
|
||||
uint32_t sCode = getStatusCode();
|
||||
if (sCode == 401 || sCode == 407 || (sCode >= 300 && sCode < 400)){
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Downloader::canContinue(const HTTP::URL &link){
|
||||
if (getStatusCode() == 401){
|
||||
// retry with authentication
|
||||
if (H.hasHeader("WWW-Authenticate")){authStr = H.GetHeader("WWW-Authenticate");}
|
||||
if (H.hasHeader("Www-Authenticate")){authStr = H.GetHeader("Www-Authenticate");}
|
||||
if (!authStr.size()){
|
||||
FAIL_MSG("Authentication required but no WWW-Authenticate header present");
|
||||
return false;
|
||||
}
|
||||
if (!link.user.size() && !link.pass.size()){
|
||||
FAIL_MSG("Authentication required but not included in URL");
|
||||
return false;
|
||||
}
|
||||
FAIL_MSG("Authenticating...");
|
||||
return true;
|
||||
}
|
||||
if (getStatusCode() == 407){
|
||||
// retry with authentication
|
||||
if (H.hasHeader("Proxy-Authenticate")){
|
||||
proxyAuthStr = H.GetHeader("Proxy-Authenticate");
|
||||
}
|
||||
if (!proxyAuthStr.size()){
|
||||
FAIL_MSG("Proxy authentication required but no Proxy-Authenticate header present");
|
||||
return false;
|
||||
}
|
||||
if (!proxyUrl.user.size() && !proxyUrl.pass.size()){
|
||||
FAIL_MSG("Proxy authentication required but not included in URL");
|
||||
return false;
|
||||
}
|
||||
FAIL_MSG("Authenticating proxy...");
|
||||
return true;
|
||||
}
|
||||
if (getStatusCode() >= 300 && getStatusCode() < 400){
|
||||
// follow redirect
|
||||
std::string location = getHeader("Location");
|
||||
if (!location.size()){return false;}
|
||||
INFO_MSG("Following redirect to %s", location.c_str());
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}// namespace HTTP
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue