Added callback support to HTTP parser

lib/http_parser.cpp

@@ -1,19 +1,20 @@
 /// \file http_parser.cpp
 /// Holds all code for the HTTP namespace.
 
-#include "http_parser.h"
-#include "util.h"
 #include "auth.h"
 #include "defines.h"
 #include "encode.h"
+#include "http_parser.h"
 #include "timing.h"
 #include "url.h"
+#include "util.h"
 #include <iomanip>
 
 /// This constructor creates an empty HTTP::Parser, ready for use for either reading or writing.
 /// All this constructor does is call HTTP::Parser::Clean().
 HTTP::Parser::Parser(){
   headerOnly = false;
+  bodyCallback = 0;
   Clean();
   std::stringstream nStr;
   nStr << std::hex << std::setw(16) << std::setfill('0') << (uint64_t)(Util::bootMS());
@@ -267,8 +268,7 @@ void HTTP::Parser::SendResponse(std::string code, std::string message, Socket::C
 void HTTP::Parser::StartResponse(std::string code, std::string message, HTTP::Parser &request,
                                  Socket::Connection &conn, bool bufferAllChunks){
   std::string prot = request.protocol;
-  sendingChunks =
+  sendingChunks = (!bufferAllChunks && protocol == "HTTP/1.1" && request.GetHeader("Connection") != "close");
-      (!bufferAllChunks && protocol == "HTTP/1.1" && request.GetHeader("Connection") != "close");
   CleanPreserveHeaders();
   protocol = prot;
   if (sendingChunks){
@@ -286,8 +286,7 @@ void HTTP::Parser::StartResponse(std::string code, std::string message, HTTP::Pa
 /// a zero-content-length HTTP/1.0 response. This call simply calls StartResponse("200", "OK",
 /// request, conn) \param request The HTTP request to respond to. \param conn The connection to send
 /// over.
-void HTTP::Parser::StartResponse(HTTP::Parser &request, Socket::Connection &conn,
+void HTTP::Parser::StartResponse(HTTP::Parser &request, Socket::Connection &conn, bool bufferAllChunks){
-                                 bool bufferAllChunks){
   StartResponse("200", "OK", request, conn, bufferAllChunks);
 }
 
@@ -300,8 +299,7 @@ void HTTP::Parser::Proxy(Socket::Connection &from, Socket::Connection &to){
   if (getChunks){
     unsigned int proxyingChunk = 0;
     while (to.connected() && from.connected()){
-      if ((from.Received().size() &&
+      if ((from.Received().size() && (from.Received().size() > 1 || *(from.Received().get().rbegin()) == '\n')) ||
-           (from.Received().size() > 1 || *(from.Received().get().rbegin()) == '\n')) ||
           from.spool()){
         if (proxyingChunk){
           while (proxyingChunk && from.Received().size()){
@@ -483,26 +481,30 @@ void HTTP::Parser::SetVar(std::string i, std::string v){
 /// If a whole request could be read, it is removed from the front of the socket buffer and true
 /// returned. If not, as much as can be interpreted is removed and false returned. \param conn The
 /// socket to read from. \return True if a whole request or response was read, false otherwise.
-bool HTTP::Parser::Read(Socket::Connection &conn){
+bool HTTP::Parser::Read(Socket::Connection &conn, Util::DataCallback &cb){
-  // Make sure the received data ends in a newline (\n).
+  while (conn.Received().size()){
-  while ((!seenHeaders || (getChunks && !doingChunk)) && conn.Received().get().size() &&
+    // Make sure the received data ends in a newline (\n).
-         *(conn.Received().get().rbegin()) != '\n'){
+    while ((!seenHeaders || (getChunks && !doingChunk)) && conn.Received().get().size() &&
-    if (conn.Received().size() > 1){
+           *(conn.Received().get().rbegin()) != '\n'){
-      // make a copy of the first part
+      if (conn.Received().size() > 1){
-      std::string tmp = conn.Received().get();
+        // make a copy of the first part
-      // clear the first part, wiping it from the partlist
+        std::string tmp = conn.Received().get();
-      conn.Received().get().clear();
+        // clear the first part, wiping it from the partlist
-      conn.Received().size();
+        conn.Received().get().clear();
-      // take the now first (was second) part, insert the stored part in front of it
+        conn.Received().size();
-      conn.Received().get().insert(0, tmp);
+        // take the now first (was second) part, insert the stored part in front of it
-    }else{
+        conn.Received().get().insert(0, tmp);
-      return false;
+      }else{
+        return false;
+      }
+    }
+
+    // return true if a parse succeeds, and is not a request
+    if (parse(conn.Received().get(), cb) && (!JSON::Value(url).isInt() || headerOnly || length ||
+                                             getChunks || (!conn && !conn.Received().size()))){
+      return true;
     }
   }
-  // if a parse succeeds, simply return true
-  if (parse(conn.Received().get())){return true;}
-  // otherwise, if we have parts left, call ourselves recursively
-  if (conn.Received().size()){return Read(conn);}
   return false;
 }// HTTPReader::Read
 
@@ -526,7 +528,7 @@ uint8_t HTTP::Parser::getPercentage() const{
 /// from the data buffer.
 /// \param HTTPbuffer The data buffer to read from.
 /// \return True on success, false otherwise.
-bool HTTP::Parser::parse(std::string &HTTPbuffer){
+bool HTTP::Parser::parse(std::string &HTTPbuffer, Util::DataCallback &cb){
   size_t f;
   std::string tmpA, tmpB, tmpC;
   /// \todo Make this not resize HTTPbuffer in parts, but read all at once and then remove the
@@ -588,11 +590,15 @@ bool HTTP::Parser::parse(std::string &HTTPbuffer){
           body.clear();
           if (GetHeader("Content-Length") != ""){
             length = atoi(GetHeader("Content-Length").c_str());
-            if (body.capacity() < length){body.reserve(length);}
+            if (!bodyCallback && (&cb == &Util::defaultDataCallback) && body.capacity() < length){
+              body.reserve(length);
+            }
           }
           if (GetHeader("Content-length") != ""){
             length = atoi(GetHeader("Content-length").c_str());
-            if (body.capacity() < length){body.reserve(length);}
+            if (!bodyCallback && (&cb == &Util::defaultDataCallback) && body.capacity() < length){
+              body.reserve(length);
+            }
           }
           if (GetHeader("Transfer-Encoding") == "chunked"){
             getChunks = true;
@@ -611,12 +617,33 @@ bool HTTP::Parser::parse(std::string &HTTPbuffer){
       if (length > 0){
         if (headerOnly){return true;}
         unsigned int toappend = length - body.length();
+
+        // limit the amount of bytes that will be appended to the amount there
+        // is available
+        if (toappend > HTTPbuffer.size()){toappend = HTTPbuffer.size();}
+
         if (toappend > 0){
-          body.append(HTTPbuffer, 0, toappend);
+          bool shouldAppend = true;
+          // check if pointer callback function is set and run callback. remove partial data from buffer
+          if (bodyCallback){
+            bodyCallback(HTTPbuffer.data(), toappend);
+            length -= toappend;
+            shouldAppend = false;
+          }
+
+          // check if reference callback function is set and run callback. remove partial data from buffer
+          if (&cb != &Util::defaultDataCallback){
+            cb.dataCallback(HTTPbuffer.data(), toappend);
+            length -= toappend;
+            shouldAppend = false;
+          }
+
+          if (shouldAppend){body.append(HTTPbuffer, 0, toappend);}
           HTTPbuffer.erase(0, toappend);
+          currentLength += toappend;
         }
         if (length == body.length()){
-          //parse POST variables
+          // parse POST variables
           if (method == "POST"){parseVars(body, vars);}
           return true;
         }else{
@@ -624,16 +651,32 @@ bool HTTP::Parser::parse(std::string &HTTPbuffer){
         }
       }else{
         if (getChunks){
+
+          // toappend
+          currentLength += HTTPbuffer.size();
+
           if (headerOnly){return true;}
           if (doingChunk){
             unsigned int toappend = HTTPbuffer.size();
             if (toappend > doingChunk){toappend = doingChunk;}
-            body.append(HTTPbuffer, 0, toappend);
+
+            bool shouldAppend = true;
+            if (bodyCallback){
+              bodyCallback(HTTPbuffer.data(), toappend);
+              shouldAppend = false;
+            }
+
+            if (&cb != &Util::defaultDataCallback){
+              cb.dataCallback(HTTPbuffer.data(), toappend);
+              shouldAppend = false;
+            }
+
+            if (shouldAppend){body.append(HTTPbuffer, 0, toappend);}
             HTTPbuffer.erase(0, toappend);
             doingChunk -= toappend;
           }else{
             f = HTTPbuffer.find('\n');
-            if (f == std::string::npos) return false;
+            if (f == std::string::npos){return false;}
             tmpA = HTTPbuffer.substr(0, f);
             while (tmpA.find('\r') != std::string::npos){tmpA.erase(tmpA.find('\r'));}
             unsigned int chunkLen = 0;
@@ -655,7 +698,23 @@ bool HTTP::Parser::parse(std::string &HTTPbuffer){
           }
           return false;
         }else{
-          return true;
+          unsigned int toappend = HTTPbuffer.size();
+          bool shouldAppend = true;
+          if (bodyCallback){
+            bodyCallback(HTTPbuffer.data(), toappend);
+            shouldAppend = false;
+          }
+
+          if (&cb != &Util::defaultDataCallback){
+            cb.dataCallback(HTTPbuffer.data(), toappend);
+            shouldAppend = false;
+          }
+
+          if (shouldAppend){body.append(HTTPbuffer, 0, toappend);}
+          HTTPbuffer.erase(0, toappend);
+
+          // return false when callbacks are used.
+          return shouldAppend;
         }
       }
     }
@@ -743,4 +802,3 @@ void HTTP::Parser::Chunkify(const char *data, unsigned int size, Socket::Connect
     }
   }
 }
-

lib/http_parser.h

@@ -3,6 +3,7 @@
 
 #pragma once
 #include "socket.h"
+#include "util.h"
 #include <map>
 #include <stdio.h>
 #include <stdlib.h>
@@ -16,10 +17,10 @@ namespace HTTP{
   void parseVars(const std::string &data, std::map<std::string, std::string> &storage);
 
   /// Simple class for reading and writing HTTP 1.0 and 1.1.
-  class Parser{
+  class Parser : public Util::DataCallback{
   public:
     Parser();
-    bool Read(Socket::Connection &conn);
+    bool Read(Socket::Connection &conn, Util::DataCallback &cb = Util::defaultDataCallback);
     bool Read(std::string &strbuf);
     const std::string &GetHeader(const std::string &i) const;
     bool hasHeader(const std::string &i) const;
@@ -54,10 +55,12 @@ namespace HTTP{
     std::string url;
     std::string protocol;
     unsigned int length;
+    unsigned int currentLength;
     bool headerOnly; ///< If true, do not parse body if the length is a known size.
     bool bufferChunks;
     // this bool was private
     bool sendingChunks;
+    void (*bodyCallback)(const char *, size_t);
 
   private:
     std::string cnonce;
@@ -65,7 +68,7 @@ namespace HTTP{
     bool seenReq;
     bool getChunks;
     unsigned int doingChunk;
-    bool parse(std::string &HTTPbuffer);
+    bool parse(std::string &HTTPbuffer, Util::DataCallback &cb = Util::defaultDataCallback);
     std::string builder;
     std::string read_buffer;
     std::map<std::string, std::string> headers;
@@ -74,4 +77,3 @@ namespace HTTP{
   };
 
 }// namespace HTTP
-

lib/util.cpp

@@ -29,6 +29,8 @@
 #define RAX_REQDFIELDS_LEN 36
 
 namespace Util{
+    Util::DataCallback defaultDataCallback;
+
   /// Helper function that cross-platform checks if a given directory exists.
   bool isDirectory(const std::string &path){
 #if defined(_WIN32)

lib/util.h

@@ -6,6 +6,7 @@
 #include <map>
 #include <stdint.h>
 #include <stdlib.h>
+#include "defines.h"
 
 namespace Util{
   bool isDirectory(const std::string &path);
@@ -18,6 +19,15 @@ namespace Util{
   uint64_t ftell(FILE *stream);
   uint64_t fseek(FILE *stream, uint64_t offset, int whence);
 
+  class DataCallback{
+    public:
+    virtual void dataCallback(const char * ptr, size_t size){
+      INFO_MSG("default callback, size: %llu", size);
+    }
+  };
+
+  extern Util::DataCallback defaultDataCallback;
+
  //Forward declaration
   class FieldAccX;