From 4613a1d306e2b461333337c06ac230e1e3a49727 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Mon, 21 Mar 2011 00:40:49 +0100 Subject: [PATCH] Nieuwe awesome parser des doods voor RAW tcpflow logs van F4M streams, en hopelijk fix voor F4M streams zelf, nieuwe dataparser voor FLV streams, en awesomeheid in het algemeen --- Connector_HTTP/main.cpp | 44 ++++++++++++++++---- HTTP_Box_Parser/Makefile | 19 +++++++++ HTTP_Box_Parser/main.cpp | 46 ++++++++++++++++++++ util/MP4/box.cpp | 6 ++- util/ddv_socket.cpp | 1 + util/flv_data.cpp | 90 ++++++++++++++++++++++++++++++++++++++++ util/http_parser.cpp | 38 ++++++++++++++++- 7 files changed, 233 insertions(+), 11 deletions(-) create mode 100644 HTTP_Box_Parser/Makefile create mode 100644 HTTP_Box_Parser/main.cpp create mode 100644 util/flv_data.cpp diff --git a/Connector_HTTP/main.cpp b/Connector_HTTP/main.cpp index 3912698b..0b535dc1 100644 --- a/Connector_HTTP/main.cpp +++ b/Connector_HTTP/main.cpp @@ -130,6 +130,10 @@ int mainHandler(int CONN_fd){ int Flash_RequestPending = 0; std::queue Flash_FragBuffer; FLV_Pack * tag = 0; + char * Video_Init_Data = 0; + int Video_Init_Len = 0; + char * Audio_Init_Data = 0; + int Audio_Init_Len = 0; HTTPReader HTTP_R, HTTP_S;//HTTP Receiver en HTTP Sender. int retval; @@ -250,18 +254,40 @@ int mainHandler(int CONN_fd){ break; case -1: break;//not ready yet default: - if (FLV_GetPacket(tag, ss)){//able to read a full packet? + if (FLV_GetPacket(tag, ss)){//able to read a full packet?f if (handler == HANDLER_FLASH){ if(tag->data[0] != 0x12 ) { - if (tag->isKeyframe){ - if (FlashBuf != ""){ - Flash_FragBuffer.push(FlashBuf); + if ((tag->isKeyframe) && (Video_Init_Data == 0)){ + if (((tag->data[11] & 0x0f) == 7) && (tag->data[12] == 0)){ + tag->data[4] = 0;//timestamp to zero + tag->data[5] = 0;//timestamp to zero + tag->data[6] = 0;//timestamp to zero + Video_Init_Data = (char*)malloc(tag->len); + Video_Init_Len = tag->len; + memcpy(Video_Init_Data, tag->data, tag->len); + } + } + if ((tag->data[0] == 0x08) && (Audio_Init_Data == 0)){ + if (((tag->data[11] & 0xf0) >> 4) == 10){//aac packet + tag->data[4] = 0;//timestamp to zero + tag->data[5] = 0;//timestamp to zero + tag->data[6] = 0;//timestamp to zero + Audio_Init_Data = (char*)malloc(tag->len); + Audio_Init_Len = tag->len; + memcpy(Audio_Init_Data, tag->data, tag->len); + } + } + if (tag->isKeyframe){ + if (FlashBuf != ""){ + Flash_FragBuffer.push(FlashBuf); #if DEBUG >= 4 - fprintf(stderr, "Received a fragment. Now %i in buffer.\n", (int)Flash_FragBuffer.size()); + fprintf(stderr, "Received a fragment. Now %i in buffer.\n", (int)Flash_FragBuffer.size()); #endif - } - FlashBuf = ""; - } + } + FlashBuf.clear(); + if (Video_Init_Len > 0) FlashBuf.append(Video_Init_Data, Video_Init_Len); + if (Audio_Init_Len > 0) FlashBuf.append(Audio_Init_Data, Audio_Init_Len); + } FlashBuf.append(tag->data,tag->len); } else { FlashMeta = ""; @@ -295,6 +321,8 @@ int mainHandler(int CONN_fd){ } } close(CONN_fd); + if (Video_Init_Data){free(Video_Init_Data);} + if (Audio_Init_Data){free(Audio_Init_Data);} if (inited) close(ss); #if DEBUG >= 1 if (All_Hell_Broke_Loose){fprintf(stderr, "All Hell Broke Loose\n");} diff --git a/HTTP_Box_Parser/Makefile b/HTTP_Box_Parser/Makefile new file mode 100644 index 00000000..6dd89381 --- /dev/null +++ b/HTTP_Box_Parser/Makefile @@ -0,0 +1,19 @@ +SRC = main.cpp +OBJ = $(SRC:.cpp=.o) +OUT = Box_Parser +INCLUDES = +CCFLAGS = -Wall -Wextra -funsigned-char -g +CC = $(CROSS)g++ +LD = $(CROSS)ld +AR = $(CROSS)ar +LIBS = -lssl -lcrypto +.SUFFIXES: .cpp +.PHONY: clean default +default: $(OUT) +.cpp.o: + $(CC) $(INCLUDES) $(CCFLAGS) -c $< -o $@ +$(OUT): $(OBJ) + $(CC) $(LIBS) -o $(OUT) $(OBJ) +clean: + rm -rf $(OBJ) $(OUT) Makefile.bak *~ + diff --git a/HTTP_Box_Parser/main.cpp b/HTTP_Box_Parser/main.cpp new file mode 100644 index 00000000..1d06741e --- /dev/null +++ b/HTTP_Box_Parser/main.cpp @@ -0,0 +1,46 @@ +#include +#include +#include +#include "../util/http_parser.cpp" +#include "../util/MP4/box_includes.h" +#include "../util/flv_data.cpp" + +std::string tagType(FLV_Pack * F){ + switch (F->data[0]){ + case 0x09: + if (F->isKeyframe){ + return "video keyframe"; + }else{ + return "video"; + } + break; + case 0x08: return "audio"; break; + case 0x12: return "data"; break; + } + return "unknown"; +} + +int main( ) { + std::string temp; + + HTTPReader H; + FLV_Pack * F = 0; + unsigned int P = 0; + + while (H.ReadSocket(stdin) || H.CleanForNext()){ + if (H.body.size() > 10000){ + Box * TestBox = new Box((uint8_t*)H.body.c_str(), H.body.size()); + TestBox->Parse(); + P = 0; + while (TestBox->PayloadSize > P){ + if (FLV_GetPacket(F, (char*)TestBox->Payload, TestBox->PayloadSize, P)){ + std::cout << "Got a " << F->len << " bytes " << tagType(F) << " FLV tag." << std::endl; + } + } + delete TestBox; + }else{ + std::cout << "Skipped too small fragment" << std::endl; + } + } + +} diff --git a/util/MP4/box.cpp b/util/MP4/box.cpp index 2e948bf6..e4c221cf 100644 --- a/util/MP4/box.cpp +++ b/util/MP4/box.cpp @@ -42,10 +42,10 @@ class Box { BoxHeader GetHeader( ); void ResetPayload( ); void Parse( std::string PrintOffset = "" ); - private: - BoxHeader header; uint8_t * Payload; uint32_t PayloadSize; + private: + BoxHeader header; };//Box Class Box::Box() { @@ -385,6 +385,8 @@ void Box::Parse( std::string PrintOffset ) { std::cerr << PrintOffset << " DiscontinuityIndicator: " << (int)FragmentRunEntryTable[i].DiscontinuityIndicator << "\n"; } } + } else if ( header.BoxType == 0x6D646174 ) { + std::cerr << "mdat box containing " << PayloadSize << " bytes of payload" << std::endl; } else { std::cerr << "BoxType '" << (char)(header.BoxType >> 24) diff --git a/util/ddv_socket.cpp b/util/ddv_socket.cpp index 0f26d36c..b1bd03e0 100644 --- a/util/ddv_socket.cpp +++ b/util/ddv_socket.cpp @@ -1,3 +1,4 @@ +#pragma once #include #include #include diff --git a/util/flv_data.cpp b/util/flv_data.cpp new file mode 100644 index 00000000..28a18da4 --- /dev/null +++ b/util/flv_data.cpp @@ -0,0 +1,90 @@ +#include //for read() +#include + +struct FLV_Pack { + int len; + int buf; + bool isKeyframe; + char * data; +};//FLV_Pack + +char FLVHeader[13]; +bool All_Hell_Broke_Loose = false; + +//checks FLV Header for correctness +//returns true if everything is alright, false otherwise +bool FLV_Checkheader(char * header){ + if (header[0] != 'F') return false; + if (header[1] != 'L') return false; + if (header[2] != 'V') return false; + if (header[8] != 0x09) return false; + if (header[9] != 0) return false; + if (header[10] != 0) return false; + if (header[11] != 0) return false; + if (header[12] != 0) return false; + return true; +}//FLV_Checkheader + +//returns true if header is an FLV header +bool FLV_Isheader(char * header){ + if (header[0] != 'F') return false; + if (header[1] != 'L') return false; + if (header[2] != 'V') return false; + return true; +}//FLV_Isheader + +bool ReadUntil(char * buffer, unsigned int count, unsigned int & sofar, char * D, unsigned int S, unsigned int & P){ + if (sofar >= count){return true;} + int r = 0; + if (P+(count-sofar) > S){r = S-P;}else{r = count-sofar;} + memcpy(buffer+sofar, D+P, r); + P += r; + sofar += r; + if (sofar >= count){return true;} + return false; +} + +//gets a packet, storing in given FLV_Pack pointer. +//will assign pointer if null +//resizes FLV_Pack data field bigger if data doesn't fit +// (does not auto-shrink for speed!) +bool FLV_GetPacket(FLV_Pack *& p, char * D, unsigned int S, unsigned int & P){ + static bool done = true; + static unsigned int sofar = 0; + if (!p){p = (FLV_Pack*)calloc(1, sizeof(FLV_Pack));} + if (p->buf < 15){p->data = (char*)realloc(p->data, 15000000); p->buf = 15000000;} + + if (done){ + //read a header + if (ReadUntil(p->data, 11, sofar, D, S, P)){ + //if its a correct FLV header, throw away and read tag header + if (FLV_Isheader(p->data)){ + if (ReadUntil(p->data, 13, sofar, D, S, P)){ + if (FLV_Checkheader(p->data)){ + sofar = 0; + memcpy(FLVHeader, p->data, 13); + }else{All_Hell_Broke_Loose = true;} + } + }else{ + //if a tag header, calculate length and read tag body + p->len = p->data[3] + 15; + p->len += (p->data[2] << 8); + p->len += (p->data[1] << 16); + //if (p->buf < p->len){p->data = (char*)realloc(p->data, p->len);p->buf = p->len;} + done = false; + } + } + }else{ + //read tag body + if (ReadUntil(p->data, p->len, sofar, D, S, P)){ + //calculate keyframeness, next time read header again, return true + p->isKeyframe = false; + if ((p->data[0] == 0x09) && (((p->data[11] & 0xf0) >> 4) == 1)){p->isKeyframe = true;} + done = true; + sofar = 0; + return true; + } + } + return false; +}//FLV_GetPacket + diff --git a/util/http_parser.cpp b/util/http_parser.cpp index 6f7d619f..ff32eda3 100644 --- a/util/http_parser.cpp +++ b/util/http_parser.cpp @@ -1,10 +1,14 @@ +#pragma once +#include "ddv_socket.cpp" #include #include +#include class HTTPReader{ public: HTTPReader(); bool ReadSocket(int CONN_fd); + bool ReadSocket(FILE * F); std::string GetHeader(std::string i); std::string GetVar(std::string i); void SetHeader(std::string i, std::string v); @@ -18,6 +22,8 @@ class HTTPReader{ void SendBodyPart(int conn, char * buffer, int len); void SendBodyPart(int conn, std::string bodypart); void Clean(); + bool CleanForNext(); + std::string body; std::string method; std::string url; std::string protocol; @@ -39,12 +45,26 @@ void HTTPReader::Clean(){ method = "GET"; url = "/"; protocol = "HTTP/1.1"; + body = ""; length = 0; HTTPbuffer = ""; headers.erase(headers.begin(), headers.end()); vars.erase(vars.begin(), vars.end()); } +bool HTTPReader::CleanForNext(){ + seenHeaders = false; + seenReq = false; + method = "GET"; + url = "/"; + protocol = "HTTP/1.1"; + body = ""; + length = 0; + headers.erase(headers.begin(), headers.end()); + vars.erase(vars.begin(), vars.end()); + return parse(); +} + std::string HTTPReader::BuildRequest(){ std::map::iterator it; std::string tmp = method+" "+url+" "+protocol+"\n"; @@ -129,6 +149,16 @@ bool HTTPReader::ReadSocket(int CONN_fd){ return false; }//HTTPReader::ReadSocket +bool HTTPReader::ReadSocket(FILE * F){ + //returned true als hele http packet gelezen is + int b = 1; + char buffer[500]; + while (b > 0){ + b = fread(buffer, 1, 500, F); + HTTPbuffer.append(buffer, b); + } + return false; +}//HTTPReader::ReadSocket bool HTTPReader::parse(){ size_t f; @@ -165,7 +195,13 @@ bool HTTPReader::parse(){ if (seenHeaders){ if (length > 0){ //TODO: POST variable parsing - return (HTTPbuffer.length() >= length); + if (HTTPbuffer.length() >= length){ + body = HTTPbuffer.substr(0, length); + HTTPbuffer.erase(0, length); + return true; + }else{ + return false; + } }else{ return true; }