Nieuwe awesome parser des doods voor RAW tcpflow logs van F4M streams, en hopelijk fix voor F4M streams zelf, nieuwe dataparser voor FLV streams, en awesomeheid in het algemeen

util/MP4/box.cpp

@@ -42,10 +42,10 @@ class Box {
     BoxHeader GetHeader( );
     void ResetPayload( );
     void Parse( std::string PrintOffset = "" );
-  private:
-    BoxHeader header;
     uint8_t * Payload;
     uint32_t PayloadSize;
+  private:
+    BoxHeader header;
 };//Box Class
 
 Box::Box() {
@@ -385,6 +385,8 @@ void Box::Parse( std::string PrintOffset ) {
         std::cerr << PrintOffset << "      DiscontinuityIndicator: " << (int)FragmentRunEntryTable[i].DiscontinuityIndicator << "\n";
       }
     }
+  } else if ( header.BoxType == 0x6D646174 ) {
+    std::cerr << "mdat box containing " << PayloadSize << " bytes of payload" << std::endl;
   } else {
     std::cerr << "BoxType '"
               << (char)(header.BoxType >> 24)

util/ddv_socket.cpp

@@ -1,3 +1,4 @@
+#pragma once
 #include <string>
 #include <sys/types.h>
 #include <sys/socket.h>

util/flv_data.cpp

@@ -0,0 +1,90 @@
+#include <unistd.h> //for read()
+#include <fcntl.h>
+
+struct FLV_Pack {
+  int len;
+  int buf;
+  bool isKeyframe;
+  char * data;
+};//FLV_Pack
+
+char FLVHeader[13];
+bool All_Hell_Broke_Loose = false;
+
+//checks FLV Header for correctness
+//returns true if everything is alright, false otherwise
+bool FLV_Checkheader(char * header){
+  if (header[0] != 'F') return false;
+  if (header[1] != 'L') return false;
+  if (header[2] != 'V') return false;
+  if (header[8] != 0x09) return false;
+  if (header[9] != 0) return false;
+  if (header[10] != 0) return false;
+  if (header[11] != 0) return false;
+  if (header[12] != 0) return false;
+  return true;
+}//FLV_Checkheader
+
+//returns true if header is an FLV header
+bool FLV_Isheader(char * header){
+  if (header[0] != 'F') return false;
+  if (header[1] != 'L') return false;
+  if (header[2] != 'V') return false;
+  return true;
+}//FLV_Isheader
+
+bool ReadUntil(char * buffer, unsigned int count, unsigned int & sofar, char * D, unsigned int S, unsigned int & P){
+  if (sofar >= count){return true;}
+  int r = 0;
+  if (P+(count-sofar) > S){r = S-P;}else{r = count-sofar;}
+  memcpy(buffer+sofar, D+P, r);
+  P += r;
+  sofar += r;
+  if (sofar >= count){return true;}
+  return false;
+}
+
+//gets a packet, storing in given FLV_Pack pointer.
+//will assign pointer if null
+//resizes FLV_Pack data field bigger if data doesn't fit
+// (does not auto-shrink for speed!)
+bool FLV_GetPacket(FLV_Pack *& p, char * D, unsigned int S, unsigned int & P){
+  static bool done = true;
+  static unsigned int sofar = 0;
+  if (!p){p = (FLV_Pack*)calloc(1, sizeof(FLV_Pack));}
+  if (p->buf < 15){p->data = (char*)realloc(p->data, 15000000); p->buf = 15000000;}
+
+  if (done){
+    //read a header
+    if (ReadUntil(p->data, 11, sofar, D, S, P)){
+      //if its a correct FLV header, throw away and read tag header
+      if (FLV_Isheader(p->data)){
+        if (ReadUntil(p->data, 13, sofar, D, S, P)){
+          if (FLV_Checkheader(p->data)){
+            sofar = 0;
+            memcpy(FLVHeader, p->data, 13);
+          }else{All_Hell_Broke_Loose = true;}
+        }
+      }else{
+        //if a tag header, calculate length and read tag body
+        p->len = p->data[3] + 15;
+        p->len += (p->data[2] << 8);
+        p->len += (p->data[1] << 16);
+        //if (p->buf < p->len){p->data = (char*)realloc(p->data, p->len);p->buf = p->len;}
+        done = false;
+      }
+    }
+  }else{
+    //read tag body
+    if (ReadUntil(p->data, p->len, sofar, D, S, P)){
+      //calculate keyframeness, next time read header again, return true
+      p->isKeyframe = false;
+      if ((p->data[0] == 0x09) && (((p->data[11] & 0xf0) >> 4) == 1)){p->isKeyframe = true;}
+      done = true;
+      sofar = 0;
+      return true;
+    }
+  }
+  return false;
+}//FLV_GetPacket
+

util/http_parser.cpp

@@ -1,10 +1,14 @@
+#pragma once
+#include "ddv_socket.cpp"
 #include <map>
 #include <stdlib.h>
+#include <stdio.h>
 
 class HTTPReader{
   public:
     HTTPReader();
     bool ReadSocket(int CONN_fd);
+    bool ReadSocket(FILE * F);
     std::string GetHeader(std::string i);
     std::string GetVar(std::string i);
     void SetHeader(std::string i, std::string v);
@@ -18,6 +22,8 @@ class HTTPReader{
     void SendBodyPart(int conn, char * buffer, int len);
     void SendBodyPart(int conn, std::string bodypart);
     void Clean();
+    bool CleanForNext();
+    std::string body;
     std::string method;
     std::string url;
     std::string protocol;
@@ -39,12 +45,26 @@ void HTTPReader::Clean(){
   method = "GET";
   url = "/";
   protocol = "HTTP/1.1";
+  body = "";
   length = 0;
   HTTPbuffer = "";
   headers.erase(headers.begin(), headers.end());
   vars.erase(vars.begin(), vars.end());
 }
 
+bool HTTPReader::CleanForNext(){
+  seenHeaders = false;
+  seenReq = false;
+  method = "GET";
+  url = "/";
+  protocol = "HTTP/1.1";
+  body = "";
+  length = 0;
+  headers.erase(headers.begin(), headers.end());
+  vars.erase(vars.begin(), vars.end());
+  return parse();
+}
+
 std::string HTTPReader::BuildRequest(){
   std::map<std::string, std::string>::iterator it;
   std::string tmp = method+" "+url+" "+protocol+"\n";
@@ -129,6 +149,16 @@ bool HTTPReader::ReadSocket(int CONN_fd){
   return false;
 }//HTTPReader::ReadSocket
 
+bool HTTPReader::ReadSocket(FILE * F){
+  //returned true als hele http packet gelezen is
+  int b = 1;
+  char buffer[500];
+  while (b > 0){
+    b = fread(buffer, 1, 500, F);
+    HTTPbuffer.append(buffer, b);
+  }
+  return false;
+}//HTTPReader::ReadSocket
 
 bool HTTPReader::parse(){
   size_t f;
@@ -165,7 +195,13 @@ bool HTTPReader::parse(){
     if (seenHeaders){
       if (length > 0){
         //TODO: POST variable parsing
-        return (HTTPbuffer.length() >= length);
+        if (HTTPbuffer.length() >= length){
+          body = HTTPbuffer.substr(0, length);
+          HTTPbuffer.erase(0, length);
+          return true;
+        }else{
+          return false;
+        }
       }else{
         return true;
       }