Removed JSON dependency from FLV, sped up FLV input significantly, minor RTMP input speedup

src/input/input_buffer.cpp

@@ -624,11 +624,11 @@ namespace Mist {
   }
 
   void inputBuffer::updateTrackMeta(unsigned long tNum) {
-    VERYHIGH_MSG("Updating meta for track %d", tNum);
     //Store a reference for easier access
     std::map<unsigned long, DTSCPageData> & locations = bufferLocations[tNum];
     char * mappedPointer = nProxy.metaPages[tNum].mapped;
     if (!mappedPointer){return;}
+    VERYHIGH_MSG("Updating meta for track %lu, %lu pages", tNum, locations.size());
 
     //First detect all entries on metaPage
     for (int i = 0; i < 8192; i += 8) {
@@ -637,11 +637,11 @@ namespace Mist {
         continue;
       }
       unsigned long keyNum = ntohl(tmpOffset[0]);
-      INSANE_MSG("Page %d detected, with %d keys", keyNum, ntohl(tmpOffset[1]));
 
       //Add an entry into bufferLocations[tNum] for the pages we haven't handled yet.
       if (!locations.count(keyNum)) {
         locations[keyNum].curOffset = 0;
+        VERYHIGH_MSG("Page %d detected, with %d keys", keyNum, ntohl(tmpOffset[1]));
       }
       locations[keyNum].pageNum = keyNum;
       locations[keyNum].keyNum = ntohl(tmpOffset[1]);

src/input/input_flv.cpp

@@ -5,6 +5,7 @@
 #include <cstdlib>
 #include <cstdio>
 #include <string>
+#include <mist/util.h>
 #include <mist/stream.h>
 #include <mist/defines.h>
 
@@ -53,33 +54,35 @@ namespace Mist {
     //See whether a separate header file exists.
     if (readExistingHeader()){return true;}
     //Create header file from FLV data
-    fseek(inFile, 13, SEEK_SET);
+    Util::fseek(inFile, 13, SEEK_SET);
     AMF::Object amf_storage;
-    JSON::Value lastPack;
     long long int lastBytePos = 13;
+    uint64_t bench = Util::getMicros();
     while (!feof(inFile) && !FLV::Parse_Error){
       if (tmpTag.FileLoader(inFile)){
-        lastPack.null();
-        lastPack = tmpTag.toJSON(myMeta, amf_storage);
-        lastPack["bpos"] = lastBytePos;
-        myMeta.update(lastPack);
-        lastBytePos = ftell(inFile);
+        tmpTag.toMeta(myMeta, amf_storage);
+        if (!tmpTag.getDataLen()){continue;}
+        if (tmpTag.needsInitData() && tmpTag.isInitData()){continue;}
+        myMeta.update(tmpTag.tagTime(), tmpTag.offset(), tmpTag.getTrackID(), tmpTag.getDataLen(), lastBytePos, tmpTag.isKeyframe);
+        lastBytePos = Util::ftell(inFile);
       }
     }
+    bench = Util::getMicros(bench);
+    INFO_MSG("Header generated in %llu ms: @%lld, %s, %s", bench/1000, lastBytePos, myMeta.vod?"VoD":"NOVoD", myMeta.live?"Live":"NOLive");
     if (FLV::Parse_Error){
-      std::cerr << FLV::Error_Str << std::endl;
-      return false;
+      FLV::Parse_Error = false;
+      ERROR_MSG("Stopping at FLV parse error @%lld: %s", lastBytePos, FLV::Error_Str.c_str());
     }
     myMeta.toFile(config->getString("input") + ".dtsh");
     return true;
   }
   
   void inputFLV::getNext(bool smart) {
-    long long int lastBytePos = ftell(inFile);
+    long long int lastBytePos = Util::ftell(inFile);
     while (!feof(inFile) && !FLV::Parse_Error){
       if (tmpTag.FileLoader(inFile)){
         if ( !selectedTracks.count(tmpTag.getTrackID())){
-          lastBytePos = ftell(inFile);
+          lastBytePos = Util::ftell(inFile);
           continue;
         }
         break;
@@ -90,11 +93,12 @@ namespace Mist {
       return;
     }
     if (FLV::Parse_Error){
-      FAIL_MSG("FLV error: %s", FLV::Error_Str.c_str());
+      FLV::Parse_Error = false;
+      FAIL_MSG("FLV error @ %lld: %s", lastBytePos, FLV::Error_Str.c_str());
       thisPacket.null();
       return;
     }
-    if (!tmpTag.getDataLen()){
+    if (!tmpTag.getDataLen() || (tmpTag.needsInitData() && tmpTag.isInitData())){
       return getNext();
     }
     thisPacket.genericFill(tmpTag.tagTime(), tmpTag.offset(), tmpTag.getTrackID(), tmpTag.getData(), tmpTag.getDataLen(), lastBytePos, tmpTag.isKeyframe); //init packet from tmpTags data
@@ -104,14 +108,14 @@ namespace Mist {
     //We will seek to the corresponding keyframe of the video track if selected, otherwise audio keyframe.
     //Flv files are never multi-track, so track 1 is video, track 2 is audio.
     int trackSeek = (selectedTracks.count(1) ? 1 : 2);
-    size_t seekPos = myMeta.tracks[trackSeek].keys[0].getBpos();
+    uint64_t seekPos = myMeta.tracks[trackSeek].keys[0].getBpos();
     for (unsigned int i = 0; i < myMeta.tracks[trackSeek].keys.size(); i++){
       if (myMeta.tracks[trackSeek].keys[i].getTime() > seekTime){
         break;
       }
       seekPos = myMeta.tracks[trackSeek].keys[i].getBpos();
     }
-    fseek(inFile, seekPos, SEEK_SET);
+    Util::fseek(inFile, seekPos, SEEK_SET);
   }
 
   void inputFLV::trackSelect(std::string trackSpec) {

src/io.cpp

@@ -417,16 +417,14 @@ namespace Mist {
     unsigned long tid = packet.getTrackId();
     //Do nothing if the trackid is invalid
     if (!tid) {
-      INFO_MSG("Packet without trackid");
+      WARN_MSG("Packet without trackid!");
       return;
     }
-    //If the track is not negotiated yet, start the negotiation
-    if (!trackState.count(tid)) {
-      continueNegotiate(tid, myMeta);
-    }
+    //negotiate track ID if needed
+    continueNegotiate(tid, myMeta);
     //If the track is declined, stop here
     if (trackState[tid] == FILL_DEC) {
-      INFO_MSG("Track %lu Declined", tid);
+      INFO_MSG("Track %lu declined", tid);
       preBuffer[tid].clear();
       return;
     }
@@ -434,9 +432,12 @@ namespace Mist {
     if (trackState[tid] != FILL_ACC) {
       preBuffer[tid].push_back(packet);
     }else{
-      while (preBuffer[tid].size()){
-        bufferSinglePacket(preBuffer[tid].front(), myMeta);
-        preBuffer[tid].pop_front();
+      if (preBuffer[tid].size()){
+        INFO_MSG("Track %lu accepted", tid);
+        while (preBuffer[tid].size()){
+          bufferSinglePacket(preBuffer[tid].front(), myMeta);
+          preBuffer[tid].pop_front();
+        }
       }
       bufferSinglePacket(packet, myMeta);
     }
@@ -448,9 +449,7 @@ namespace Mist {
     //This update needs to happen whether the track is accepted or not.
     bool isKeyframe = false;
     if (myMeta.tracks[tid].type == "video") {
-      if (packet.hasMember("keyframe") && packet.getFlag("keyframe")) {
-        isKeyframe = true;
-      }
+      isKeyframe = packet.getFlag("keyframe");
     } else {
       if (!pagesByTrack.count(tid) || pagesByTrack[tid].size() == 0) {
         //Assume this is the first packet on the track
@@ -467,6 +466,7 @@ namespace Mist {
     //This also happens in bufferNext, with the same rules
     if (myMeta.live){
       if (packet.getTime() > 0xFFFF0000 && !myMeta.tracks[tid].lastms){
+        INFO_MSG("Ignoring packet with unexpected timestamp");
         return;//ignore bullshit timestamps
       }
       if (packet.getTime() < myMeta.tracks[tid].lastms){
@@ -509,6 +509,7 @@ namespace Mist {
     }
     //If we have no pages by track, we have not received a starting keyframe yet. Drop this packet.
     if (!pagesByTrack.count(tid) || pagesByTrack[tid].size() == 0){
+      INFO_MSG("Track %lu not starting with a keyframe!", tid);
       return;
     }
 

src/output/output_rtmp.cpp

@@ -936,16 +936,18 @@ namespace Mist {
           }else{
             amf_storage = &(pushMeta.begin()->second);
           }
-          JSON::Value pack_out = F.toJSON(myMeta, *amf_storage, next.cs_id*3 + (F.data[0] == 0x09 ? 0 : (F.data[0] == 0x08 ? 1 : 2) ));
-          if ( !pack_out.isNull()){
+
+          unsigned int reTrack = next.cs_id*3 + (F.data[0] == 0x09 ? 1 : (F.data[0] == 0x08 ? 2 : 3));
+          F.toMeta(myMeta, *amf_storage, reTrack);
+          if (F.getDataLen() && !(F.needsInitData() && F.isInitData())){
+            thisPacket.genericFill(F.tagTime(), F.offset(), reTrack, F.getData(), F.getDataLen(), 0, F.isKeyframe);
             if (!nProxy.userClient.getData()){
               char userPageName[NAME_BUFFER_SIZE];
               snprintf(userPageName, NAME_BUFFER_SIZE, SHM_USERS, streamName.c_str());
               nProxy.userClient = IPC::sharedClient(userPageName, PLAY_EX_SIZE, true);
             }
-            continueNegotiate(pack_out["trackid"].asInt());
             nProxy.streamName = streamName;
-            bufferLivePacket(pack_out);
+            bufferLivePacket(thisPacket);
           }
           break;
         }