Various metadata-related features and improvements:

- Added support for new "NowMs" field that holds up to where no new packets are guaranteed to show up, in order to lower latency.
- Added support for JSON tracks over all TS-based protocols (input and output)
- Added support for AMF metadata conversion to JSON (RTMP/FLV input)
- Fixed MP4 input subtitle tracks
- Generalized websocket-based outputs to all support the same commands and run the same core logic
- Added new "JSONLine" protocol that allows for generic direct line-by-line ingest of subtitles and/or JSON metadata tracks over a TCP socket or console standard input.

src/input/input_buffer.cpp

@@ -349,6 +349,8 @@ namespace Mist{
       }
       for (std::set<size_t>::iterator idx = tracks.begin(); idx != tracks.end(); idx++){
         size_t i = *idx;
+        //Don't delete idle metadata tracks
+        if (M.getType(i) == "meta"){continue;}
         uint64_t lastUp = M.getLastUpdated(i);
         //Prevent issues when getLastUpdated > current time. This can happen if the second rolls over exactly during this loop.
         if (lastUp >= time){continue;}

src/input/input_mp4.cpp

@@ -478,7 +478,7 @@ namespace Mist{
             packSendSize = 24 + (BsetPart.timeOffset ? 17 : 0) + (BsetPart.bpos ? 15 : 0) + 19 +
                            stszBox.getEntrySize(stszIndex) + 11 - 2 + 19;
             meta.update(BsetPart.time, BsetPart.timeOffset, tNumber,
-                        stszBox.getEntrySize(stszIndex) - 2, BsetPart.bpos, true, packSendSize);
+                        stszBox.getEntrySize(stszIndex) - 2, BsetPart.bpos+2, true, packSendSize);
           }
         }else{
           meta.update(BsetPart.time, BsetPart.timeOffset, tNumber,
@@ -558,16 +558,11 @@ namespace Mist{
     }
 
     if (M.getCodec(curPart.trackID) == "subtitle"){
-      unsigned int txtLen = Bit::btohs(readBuffer + (curPart.bpos-readPos));
-      if (!txtLen && false){
-        curPart.index++;
-        return getNext(idx);
-      }
       static JSON::Value thisPack;
       thisPack.null();
       thisPack["trackid"] = (uint64_t)curPart.trackID;
       thisPack["bpos"] = curPart.bpos; //(long long)fileSource.tellg();
-      thisPack["data"] = std::string(readBuffer + (curPart.bpos-readPos) + 2, txtLen);
+      thisPack["data"] = std::string(readBuffer + (curPart.bpos-readPos), curPart.size);
       thisPack["time"] = curPart.time;
       if (curPart.duration){thisPack["duration"] = curPart.duration;}
       thisPack["keyframe"] = true;
@@ -583,6 +578,7 @@ namespace Mist{
     curPart.index++;
     if (curPart.index < headerData(M.getID(curPart.trackID)).size()){
       headerData(M.getID(curPart.trackID)).getPart(curPart.index, curPart.bpos);
+      if (M.getCodec(curPart.trackID) == "subtitle"){curPart.bpos += 2;}
       curPart.size = parts.getSize(curPart.index);
       curPart.offset = parts.getOffset(curPart.index);
       curPart.time = M.getPartTime(curPart.index, thisIdx);
@@ -616,6 +612,7 @@ namespace Mist{
     for (size_t i = 0; i < headerDataSize; i++){
 
       thisHeader.getPart(i, addPart.bpos);
+      if (M.getCodec(idx) == "subtitle"){addPart.bpos += 2;}
       addPart.size = parts.getSize(i);
       addPart.offset = parts.getOffset(i);
       addPart.time = M.getPartTime(i, idx);

src/input/input_ts.cpp

@@ -243,6 +243,25 @@ namespace Mist{
     capa["optional"]["segmentsize"]["type"] = "uint";
     capa["optional"]["segmentsize"]["default"] = 1900;
 
+    capa["optional"]["datatrack"]["name"] = "MPEG Data track parser";
+    capa["optional"]["datatrack"]["help"] = "Which parser to use for data tracks";
+    capa["optional"]["datatrack"]["type"] = "select";
+    capa["optional"]["datatrack"]["option"] = "--datatrack";
+    capa["optional"]["datatrack"]["short"] = "D";
+    capa["optional"]["datatrack"]["default"] = "";
+    capa["optional"]["datatrack"]["select"][0u][0u] = "";
+    capa["optional"]["datatrack"]["select"][0u][1u] = "None / disabled";
+    capa["optional"]["datatrack"]["select"][1u][0u] = "json";
+    capa["optional"]["datatrack"]["select"][1u][1u] = "2b size-prepended JSON";
+
+    JSON::Value option;
+    option["long"] = "datatrack";
+    option["short"] = "D";
+    option["arg"] = "string";
+    option["default"] = "";
+    option["help"] = "Which parser to use for data tracks";
+    config->addOption("datatrack", option);
+
     capa["optional"]["fallback_stream"]["name"] = "Fallback stream";
     capa["optional"]["fallback_stream"]["help"] =
         "Alternative stream to load for playback when there is no active broadcast";
@@ -253,7 +272,7 @@ namespace Mist{
     capa["optional"]["raw"]["help"] = "Enable raw MPEG-TS passthrough mode";
     capa["optional"]["raw"]["option"] = "--raw";
 
-    JSON::Value option;
+    option.null();
     option["long"] = "raw";
     option["short"] = "R";
     option["help"] = "Enable raw MPEG-TS passthrough mode";
@@ -277,6 +296,11 @@ namespace Mist{
       config->getOption("input", true).append("ts-exec:srt-live-transmit " + srtUrl.getUrl() + " file://con");
       INFO_MSG("Rewriting SRT source '%s' to '%s'", source.c_str(), config->getString("input").c_str());
     }
+    if (config->getString("datatrack") == "json"){
+      liveStream.setRawDataParser(TS::JSON);
+      tsStream.setRawDataParser(TS::JSON);
+    }
+
     // We call preRun early and, if successful, close the opened reader.
     // This is to ensure we have udpMode/rawMode/standAlone all set properly before the first call to needsLock.
     // The reader must be closed so that the angel process does not have a reader open.

src/input/input_tsrist.cpp

@@ -146,6 +146,25 @@ namespace Mist{
     option["help"] = "Enable raw MPEG-TS passthrough mode";
     config->addOption("raw", option);
 
+    capa["optional"]["datatrack"]["name"] = "MPEG Data track parser";
+    capa["optional"]["datatrack"]["help"] = "Which parser to use for data tracks";
+    capa["optional"]["datatrack"]["type"] = "select";
+    capa["optional"]["datatrack"]["option"] = "--datatrack";
+    capa["optional"]["datatrack"]["short"] = "D";
+    capa["optional"]["datatrack"]["default"] = "";
+    capa["optional"]["datatrack"]["select"][0u][0u] = "";
+    capa["optional"]["datatrack"]["select"][0u][1u] = "None / disabled";
+    capa["optional"]["datatrack"]["select"][1u][0u] = "json";
+    capa["optional"]["datatrack"]["select"][1u][1u] = "2b size-prepended JSON";
+
+    option.null();
+    option["long"] = "datatrack";
+    option["short"] = "D";
+    option["arg"] = "string";
+    option["default"] = "";
+    option["help"] = "Which parser to use for data tracks";
+    config->addOption("datatrack", option);
+
     lastTimeStamp = 0;
     timeStampOffset = 0;
     receiver_ctx = 0;
@@ -156,7 +175,12 @@ namespace Mist{
     rist_destroy(receiver_ctx);
   }
 
-  bool inputTSRIST::checkArguments(){return true;}
+  bool inputTSRIST::checkArguments(){
+    if (config->getString("datatrack") == "json"){
+      tsStream.setRawDataParser(TS::JSON);
+    }
+    return true;
+  }
 
   /// Live Setup of SRT Input. Runs only if we are the "main" thread
   bool inputTSRIST::preRun(){

src/input/input_tssrt.cpp

@@ -117,6 +117,25 @@ namespace Mist{
     option["help"] = "Enable raw MPEG-TS passthrough mode";
     config->addOption("raw", option);
     
+    capa["optional"]["datatrack"]["name"] = "MPEG Data track parser";
+    capa["optional"]["datatrack"]["help"] = "Which parser to use for data tracks";
+    capa["optional"]["datatrack"]["type"] = "select";
+    capa["optional"]["datatrack"]["option"] = "--datatrack";
+    capa["optional"]["datatrack"]["short"] = "D";
+    capa["optional"]["datatrack"]["default"] = "";
+    capa["optional"]["datatrack"]["select"][0u][0u] = "";
+    capa["optional"]["datatrack"]["select"][0u][1u] = "None / disabled";
+    capa["optional"]["datatrack"]["select"][1u][0u] = "json";
+    capa["optional"]["datatrack"]["select"][1u][1u] = "2b size-prepended JSON";
+
+    option.null();
+    option["long"] = "datatrack";
+    option["short"] = "D";
+    option["arg"] = "string";
+    option["default"] = "";
+    option["help"] = "Which parser to use for data tracks";
+    config->addOption("datatrack", option);
+
     // Setup if we are called form with a thread for push-based input.
     if (s.connected()){
       srtConn = s;
@@ -140,7 +159,12 @@ namespace Mist{
 
   inputTSSRT::~inputTSSRT(){}
 
-  bool inputTSSRT::checkArguments(){return true;}
+  bool inputTSSRT::checkArguments(){
+    if (config->getString("datatrack") == "json"){
+      tsStream.setRawDataParser(TS::JSON);
+    }
+    return true;
+  }
 
   /// Live Setup of SRT Input. Runs only if we are the "main" thread
   bool inputTSSRT::preRun(){