Nieuwe awesome parser des doods voor RAW tcpflow logs van F4M streams, en hopelijk fix voor F4M streams zelf, nieuwe dataparser voor FLV streams, en awesomeheid in het algemeen

Connector_HTTP/main.cpp

@@ -130,6 +130,10 @@ int mainHandler(int CONN_fd){
   int Flash_RequestPending = 0;
   std::queue<std::string> Flash_FragBuffer;
   FLV_Pack * tag = 0;
+  char * Video_Init_Data = 0;
+  int Video_Init_Len = 0;
+  char * Audio_Init_Data = 0;
+  int Audio_Init_Len = 0;
   HTTPReader HTTP_R, HTTP_S;//HTTP Receiver en HTTP Sender.
 
   int retval;
@@ -250,9 +254,29 @@ int mainHandler(int CONN_fd){
           break;
         case -1: break;//not ready yet
         default:
-          if (FLV_GetPacket(tag, ss)){//able to read a full packet?
+          if (FLV_GetPacket(tag, ss)){//able to read a full packet?f
             if (handler == HANDLER_FLASH){
               if(tag->data[0] != 0x12 ) {
+                if ((tag->isKeyframe) && (Video_Init_Data == 0)){
+                  if (((tag->data[11] & 0x0f) == 7) && (tag->data[12] == 0)){
+                    tag->data[4] = 0;//timestamp to zero
+                    tag->data[5] = 0;//timestamp to zero
+                    tag->data[6] = 0;//timestamp to zero
+                    Video_Init_Data = (char*)malloc(tag->len);
+                    Video_Init_Len = tag->len;
+                    memcpy(Video_Init_Data, tag->data, tag->len);
+                  }
+                }
+                if ((tag->data[0] == 0x08) && (Audio_Init_Data == 0)){
+                  if (((tag->data[11] & 0xf0) >> 4) == 10){//aac packet
+                    tag->data[4] = 0;//timestamp to zero
+                    tag->data[5] = 0;//timestamp to zero
+                    tag->data[6] = 0;//timestamp to zero
+                    Audio_Init_Data = (char*)malloc(tag->len);
+                    Audio_Init_Len = tag->len;
+                    memcpy(Audio_Init_Data, tag->data, tag->len);
+                  }
+                }
                 if (tag->isKeyframe){
                   if (FlashBuf != ""){
                     Flash_FragBuffer.push(FlashBuf);
@@ -260,7 +284,9 @@ int mainHandler(int CONN_fd){
                     fprintf(stderr, "Received a fragment. Now %i in buffer.\n", (int)Flash_FragBuffer.size());
                     #endif
                   }
-		  FlashBuf = "";
+                  FlashBuf.clear();
+                  if (Video_Init_Len > 0) FlashBuf.append(Video_Init_Data, Video_Init_Len);
+                  if (Audio_Init_Len > 0) FlashBuf.append(Audio_Init_Data, Audio_Init_Len);
                 }
                 FlashBuf.append(tag->data,tag->len);
               } else {
@@ -295,6 +321,8 @@ int mainHandler(int CONN_fd){
     }
   }
   close(CONN_fd);
+  if (Video_Init_Data){free(Video_Init_Data);}
+  if (Audio_Init_Data){free(Audio_Init_Data);}
   if (inited) close(ss);
   #if DEBUG >= 1
   if (All_Hell_Broke_Loose){fprintf(stderr, "All Hell Broke Loose\n");}

HTTP_Box_Parser/Makefile

@@ -0,0 +1,19 @@
+SRC = main.cpp
+OBJ = $(SRC:.cpp=.o)
+OUT = Box_Parser
+INCLUDES = 
+CCFLAGS = -Wall -Wextra -funsigned-char -g
+CC = $(CROSS)g++
+LD = $(CROSS)ld
+AR = $(CROSS)ar
+LIBS = -lssl -lcrypto
+.SUFFIXES: .cpp 
+.PHONY: clean default
+default: $(OUT)
+.cpp.o:
+	$(CC) $(INCLUDES) $(CCFLAGS) -c $< -o $@
+$(OUT): $(OBJ)
+	$(CC) $(LIBS) -o $(OUT) $(OBJ)
+clean:
+	rm -rf $(OBJ) $(OUT) Makefile.bak *~
+

HTTP_Box_Parser/main.cpp

@@ -0,0 +1,46 @@
+#include <stdint.h>
+#include <iostream>
+#include <string>
+#include "../util/http_parser.cpp"
+#include "../util/MP4/box_includes.h"
+#include "../util/flv_data.cpp"
+
+std::string tagType(FLV_Pack * F){
+  switch (F->data[0]){
+    case 0x09:
+      if (F->isKeyframe){
+        return "video keyframe";
+      }else{
+        return "video";
+      }
+      break;
+    case 0x08: return "audio"; break;
+    case 0x12: return "data"; break;
+  }
+  return "unknown";
+}
+
+int main( ) {
+  std::string temp;
+
+  HTTPReader H;
+  FLV_Pack * F = 0;
+  unsigned int P = 0;
+  
+  while (H.ReadSocket(stdin) || H.CleanForNext()){
+    if (H.body.size() > 10000){
+      Box * TestBox = new Box((uint8_t*)H.body.c_str(), H.body.size());
+      TestBox->Parse();
+      P = 0;
+      while (TestBox->PayloadSize > P){
+        if (FLV_GetPacket(F, (char*)TestBox->Payload, TestBox->PayloadSize, P)){
+          std::cout << "Got a " << F->len << " bytes " << tagType(F) << " FLV tag." << std::endl;
+        }
+      }
+      delete TestBox;
+    }else{
+      std::cout << "Skipped too small fragment" << std::endl;
+    }
+  }
+
+}

util/MP4/box.cpp

@@ -42,10 +42,10 @@ class Box {
     BoxHeader GetHeader( );
     void ResetPayload( );
     void Parse( std::string PrintOffset = "" );
-  private:
-    BoxHeader header;
     uint8_t * Payload;
     uint32_t PayloadSize;
+  private:
+    BoxHeader header;
 };//Box Class
 
 Box::Box() {
@@ -385,6 +385,8 @@ void Box::Parse( std::string PrintOffset ) {
         std::cerr << PrintOffset << "      DiscontinuityIndicator: " << (int)FragmentRunEntryTable[i].DiscontinuityIndicator << "\n";
       }
     }
+  } else if ( header.BoxType == 0x6D646174 ) {
+    std::cerr << "mdat box containing " << PayloadSize << " bytes of payload" << std::endl;
   } else {
     std::cerr << "BoxType '"
               << (char)(header.BoxType >> 24)

util/ddv_socket.cpp

@@ -1,3 +1,4 @@
+#pragma once
 #include <string>
 #include <sys/types.h>
 #include <sys/socket.h>

util/flv_data.cpp

@@ -0,0 +1,90 @@
+#include <unistd.h> //for read()
+#include <fcntl.h>
+
+struct FLV_Pack {
+  int len;
+  int buf;
+  bool isKeyframe;
+  char * data;
+};//FLV_Pack
+
+char FLVHeader[13];
+bool All_Hell_Broke_Loose = false;
+
+//checks FLV Header for correctness
+//returns true if everything is alright, false otherwise
+bool FLV_Checkheader(char * header){
+  if (header[0] != 'F') return false;
+  if (header[1] != 'L') return false;
+  if (header[2] != 'V') return false;
+  if (header[8] != 0x09) return false;
+  if (header[9] != 0) return false;
+  if (header[10] != 0) return false;
+  if (header[11] != 0) return false;
+  if (header[12] != 0) return false;
+  return true;
+}//FLV_Checkheader
+
+//returns true if header is an FLV header
+bool FLV_Isheader(char * header){
+  if (header[0] != 'F') return false;
+  if (header[1] != 'L') return false;
+  if (header[2] != 'V') return false;
+  return true;
+}//FLV_Isheader
+
+bool ReadUntil(char * buffer, unsigned int count, unsigned int & sofar, char * D, unsigned int S, unsigned int & P){
+  if (sofar >= count){return true;}
+  int r = 0;
+  if (P+(count-sofar) > S){r = S-P;}else{r = count-sofar;}
+  memcpy(buffer+sofar, D+P, r);
+  P += r;
+  sofar += r;
+  if (sofar >= count){return true;}
+  return false;
+}
+
+//gets a packet, storing in given FLV_Pack pointer.
+//will assign pointer if null
+//resizes FLV_Pack data field bigger if data doesn't fit
+// (does not auto-shrink for speed!)
+bool FLV_GetPacket(FLV_Pack *& p, char * D, unsigned int S, unsigned int & P){
+  static bool done = true;
+  static unsigned int sofar = 0;
+  if (!p){p = (FLV_Pack*)calloc(1, sizeof(FLV_Pack));}
+  if (p->buf < 15){p->data = (char*)realloc(p->data, 15000000); p->buf = 15000000;}
+
+  if (done){
+    //read a header
+    if (ReadUntil(p->data, 11, sofar, D, S, P)){
+      //if its a correct FLV header, throw away and read tag header
+      if (FLV_Isheader(p->data)){
+        if (ReadUntil(p->data, 13, sofar, D, S, P)){
+          if (FLV_Checkheader(p->data)){
+            sofar = 0;
+            memcpy(FLVHeader, p->data, 13);
+          }else{All_Hell_Broke_Loose = true;}
+        }
+      }else{
+        //if a tag header, calculate length and read tag body
+        p->len = p->data[3] + 15;
+        p->len += (p->data[2] << 8);
+        p->len += (p->data[1] << 16);
+        //if (p->buf < p->len){p->data = (char*)realloc(p->data, p->len);p->buf = p->len;}
+        done = false;
+      }
+    }
+  }else{
+    //read tag body
+    if (ReadUntil(p->data, p->len, sofar, D, S, P)){
+      //calculate keyframeness, next time read header again, return true
+      p->isKeyframe = false;
+      if ((p->data[0] == 0x09) && (((p->data[11] & 0xf0) >> 4) == 1)){p->isKeyframe = true;}
+      done = true;
+      sofar = 0;
+      return true;
+    }
+  }
+  return false;
+}//FLV_GetPacket
+

util/http_parser.cpp

@@ -1,10 +1,14 @@
+#pragma once
+#include "ddv_socket.cpp"
 #include <map>
 #include <stdlib.h>
+#include <stdio.h>
 
 class HTTPReader{
   public:
     HTTPReader();
     bool ReadSocket(int CONN_fd);
+    bool ReadSocket(FILE * F);
     std::string GetHeader(std::string i);
     std::string GetVar(std::string i);
     void SetHeader(std::string i, std::string v);
@@ -18,6 +22,8 @@ class HTTPReader{
     void SendBodyPart(int conn, char * buffer, int len);
     void SendBodyPart(int conn, std::string bodypart);
     void Clean();
+    bool CleanForNext();
+    std::string body;
     std::string method;
     std::string url;
     std::string protocol;
@@ -39,12 +45,26 @@ void HTTPReader::Clean(){
   method = "GET";
   url = "/";
   protocol = "HTTP/1.1";
+  body = "";
   length = 0;
   HTTPbuffer = "";
   headers.erase(headers.begin(), headers.end());
   vars.erase(vars.begin(), vars.end());
 }
 
+bool HTTPReader::CleanForNext(){
+  seenHeaders = false;
+  seenReq = false;
+  method = "GET";
+  url = "/";
+  protocol = "HTTP/1.1";
+  body = "";
+  length = 0;
+  headers.erase(headers.begin(), headers.end());
+  vars.erase(vars.begin(), vars.end());
+  return parse();
+}
+
 std::string HTTPReader::BuildRequest(){
   std::map<std::string, std::string>::iterator it;
   std::string tmp = method+" "+url+" "+protocol+"\n";
@@ -129,6 +149,16 @@ bool HTTPReader::ReadSocket(int CONN_fd){
   return false;
 }//HTTPReader::ReadSocket
 
+bool HTTPReader::ReadSocket(FILE * F){
+  //returned true als hele http packet gelezen is
+  int b = 1;
+  char buffer[500];
+  while (b > 0){
+    b = fread(buffer, 1, 500, F);
+    HTTPbuffer.append(buffer, b);
+  }
+  return false;
+}//HTTPReader::ReadSocket
 
 bool HTTPReader::parse(){
   size_t f;
@@ -165,7 +195,13 @@ bool HTTPReader::parse(){
     if (seenHeaders){
       if (length > 0){
         //TODO: POST variable parsing
-        return (HTTPbuffer.length() >= length);
+        if (HTTPbuffer.length() >= length){
+          body = HTTPbuffer.substr(0, length);
+          HTTPbuffer.erase(0, length);
+          return true;
+        }else{
+          return false;
+        }
       }else{
         return true;
       }