Added Util::packetSorter with async/sync modes, set DTSC outputs to use async mode

lib/stream.cpp

@@ -201,6 +201,133 @@ void Util::sanitizeName(std::string &streamname){
   }
 }
 
+/// Initalizes the packetSorter in sync mode.
+Util::packetSorter::packetSorter(){
+  dequeMode = false;
+}
+
+/// Sets sync mode on if true (sync), off if false (async).
+void Util::packetSorter::setSyncMode(bool synced){
+  if (dequeMode != !synced){
+    dequeMode = !synced;
+    if (!dequeMode){
+      //we've switched away from deque
+      for (std::deque<Util::sortedPageInfo>::iterator it = dequeBuffer.begin(); it != dequeBuffer.end(); ++it){
+        insert(*it);
+      }
+      dequeBuffer.clear();
+    }else{
+      //we've switched away from set
+      for (std::set<Util::sortedPageInfo>::iterator it = setBuffer.begin(); it != setBuffer.end(); ++it){
+        insert(*it);
+      }
+      setBuffer.clear();
+    }
+  }
+}
+
+/// Returns true if we're synced, false if async.
+bool Util::packetSorter::getSyncMode() const{return !dequeMode;}
+
+/// Returns the amount of packets currently in the sorter.
+size_t Util::packetSorter::size() const{
+  if (dequeMode){return dequeBuffer.size();}else{return setBuffer.size();}
+}
+
+/// Clears all packets from the sorter; does not reset mode.
+void Util::packetSorter::clear(){
+  dequeBuffer.clear();
+  setBuffer.clear();
+}
+
+/// Returns a pointer to the first packet in the sorter.
+const Util::sortedPageInfo * Util::packetSorter::begin() const{
+  if (dequeMode){
+    return &*dequeBuffer.begin();
+  }else{
+    return &*setBuffer.begin();
+  }
+}
+
+/// Inserts a new packet in the sorter.
+void Util::packetSorter::insert(const sortedPageInfo &pInfo){
+  if (dequeMode){
+    dequeBuffer.push_back(pInfo);
+  }else{
+    setBuffer.insert(pInfo);
+  }
+}
+
+/// Removes the given track ID packet from the sorter. Removes at most one packet, make sure to prevent duplicates elsewhere!
+void Util::packetSorter::dropTrack(size_t tid){
+  if (dequeMode){
+    for (std::deque<Util::sortedPageInfo>::iterator it = dequeBuffer.begin(); it != dequeBuffer.end(); ++it){
+      if (it->tid == tid){
+        dequeBuffer.erase(it);
+        return;
+      }
+    }
+  }else{
+    for (std::set<Util::sortedPageInfo>::iterator it = setBuffer.begin(); it != setBuffer.end(); ++it){
+      if (it->tid == tid){
+        setBuffer.erase(it);
+        return;
+      }
+    }
+  }
+}
+
+/// Removes the first packet from the sorter and inserts the given packet.
+void Util::packetSorter::replaceFirst(const sortedPageInfo &pInfo){
+  if (dequeMode){
+    dequeBuffer.pop_front();
+    if (dequeBuffer.size() && dequeBuffer.front().time > pInfo.time){
+      dequeBuffer.push_front(pInfo);
+    }else{
+      dequeBuffer.push_back(pInfo);
+    }
+  }else{
+    setBuffer.erase(setBuffer.begin());
+    setBuffer.insert(pInfo);
+  }
+}
+
+/// Removes the first packet from the sorter and inserts it back at the end. No-op for sync mode.
+void Util::packetSorter::moveFirstToEnd(){
+  if (dequeMode){
+    dequeBuffer.push_back(dequeBuffer.front());
+    dequeBuffer.pop_front();
+  }
+}
+
+/// Returns true if there is an entry in the sorter for the given track ID.
+bool Util::packetSorter::hasEntry(size_t tid) const{
+  if (dequeMode){
+    for (std::deque<Util::sortedPageInfo>::const_iterator it = dequeBuffer.begin(); it != dequeBuffer.end(); ++it){
+      if (it->tid == tid){return true;}
+    }
+  }else{
+    for (std::set<Util::sortedPageInfo>::const_iterator it = setBuffer.begin(); it != setBuffer.end(); ++it){
+      if (it->tid == tid){return true;}
+    }
+  }
+  return false;
+}
+
+/// Fills toFill with track IDs of tracks that are in the sorter.
+void Util::packetSorter::getTrackList(std::set<size_t> &toFill) const{
+  toFill.clear();
+  if (dequeMode){
+    for (std::deque<Util::sortedPageInfo>::const_iterator it = dequeBuffer.begin(); it != dequeBuffer.end(); ++it){
+      toFill.insert(it->tid);
+    }
+  }else{
+    for (std::set<Util::sortedPageInfo>::const_iterator it = setBuffer.begin(); it != setBuffer.end(); ++it){
+      toFill.insert(it->tid);
+    }
+  }
+}
+
 JSON::Value Util::getStreamConfig(const std::string &streamname){
   JSON::Value result;
   if (streamname.size() > 100){

lib/stream.h

@@ -50,6 +50,39 @@ namespace Util{
   extern trackSortOrder defaultTrackSortOrder;
   void sortTracks(std::set<size_t> & validTracks, const DTSC::Meta & M, trackSortOrder sorting, std::list<size_t> & srtTrks);
 
+  /// This struct keeps packet information sorted in playback order
+  struct sortedPageInfo{
+    bool operator<(const sortedPageInfo &rhs) const{
+      if (time < rhs.time){return true;}
+      return (time == rhs.time && tid < rhs.tid);
+    }
+    size_t tid;
+    uint64_t time;
+    uint64_t offset;
+    size_t partIndex;
+  };
+
+  /// Packet sorter used to determine which packet should be output next
+  class packetSorter{
+    public:
+      packetSorter();
+      size_t size() const;
+      void clear();
+      const sortedPageInfo * begin() const;
+      void insert(const sortedPageInfo &pInfo);
+      void dropTrack(size_t tid);
+      void replaceFirst(const sortedPageInfo &pInfo);
+      void moveFirstToEnd();
+      bool hasEntry(size_t tid) const;
+      void getTrackList(std::set<size_t> &toFill) const;
+      void setSyncMode(bool synced);
+      bool getSyncMode() const;
+    private:
+      bool dequeMode;
+      std::deque<sortedPageInfo> dequeBuffer;
+      std::set<sortedPageInfo> setBuffer;
+  };
+
 
   class DTSCShmReader{
   public:

src/output/output.cpp

@@ -788,7 +788,7 @@ namespace Mist{
       userSelect.erase(tid);
       return false;
     }
-    sortedPageInfo tmp;
+    Util::sortedPageInfo tmp;
     tmp.tid = tid;
     tmp.offset = 0;
     tmp.partIndex = 0;
@@ -1456,12 +1456,7 @@ namespace Mist{
               streamName.c_str(), meta.getCodec(trackId).c_str(), trackId, usr.getKeyNum() + 1,
               pageNumForKey(trackId, usr.getKeyNum() + 1), pageNumMax(trackId), reason.c_str());
     // now actually drop the track from the buffer
-    for (std::set<sortedPageInfo>::iterator it = buffer.begin(); it != buffer.end(); ++it){
+    buffer.dropTrack(trackId);
-      if (it->tid == trackId){
-        buffer.erase(it);
-        break;
-      }
-    }
     userSelect.erase(trackId);
   }
 
@@ -1471,7 +1466,7 @@ namespace Mist{
   /// prepareNext continues as if this function was never called.
   bool Output::getKeyFrame(){
     // store copy of current state
-    std::set<sortedPageInfo> tmp_buffer = buffer;
+    Util::packetSorter tmp_buffer = buffer;
     std::map<size_t, Comms::Users> tmp_userSelect = userSelect;
     std::map<size_t, uint32_t> tmp_currentPage = currentPage;
 
@@ -1528,26 +1523,13 @@ namespace Mist{
       if (buffer.size() < userSelect.size()){
         // prepare to drop any selectedTrack without buffer entry
         for (std::map<size_t, Comms::Users>::iterator it = userSelect.begin(); it != userSelect.end(); ++it){
-          bool found = false;
+          if (!buffer.hasEntry(it->first)){dropTracks.insert(it->first);}
-          for (std::set<sortedPageInfo>::iterator bi = buffer.begin(); bi != buffer.end(); ++bi){
-            if (bi->tid == it->first){
-              found = true;
-              break;
-            }
-          }
-          if (!found){dropTracks.insert(it->first);}
         }
       }else{
-        std::set<size_t> seen;
         // prepare to drop any buffer entry without selectedTrack
-        for (std::set<sortedPageInfo>::iterator bi = buffer.begin(); bi != buffer.end(); ++bi){
+        buffer.getTrackList(dropTracks);
-          if (!userSelect.count(bi->tid)){dropTracks.insert(bi->tid);}
+        for (std::map<size_t, Comms::Users>::iterator it = userSelect.begin(); it != userSelect.end(); ++it){
-          if (seen.count(bi->tid)){
+          dropTracks.erase(it->first);
-            INFO_MSG("Dropping duplicate buffer entry for track %zu", bi->tid);
-            buffer.erase(bi);
-            return false;
-          }
-          seen.insert(bi->tid);
         }
       }
       if (!dropTracks.size()){
@@ -1565,7 +1547,12 @@ namespace Mist{
       return false;
     }
 
-    sortedPageInfo nxt = *(buffer.begin());
+    Util::sortedPageInfo nxt;
+    uint64_t nextTime = 0;
+    //In case we're not in sync mode, we might have to retry a few times
+    for (size_t trackTries = 0; trackTries < buffer.size(); ++trackTries){
+
+      nxt = *(buffer.begin());
 
       if (meta.reloadReplacedPagesIfNeeded()){return false;}
       if (!M.getValidTracks().count(nxt.tid)){
@@ -1588,10 +1575,10 @@ namespace Mist{
           if (curPage[nxt.tid].mapped && curPage[nxt.tid].mapped[0] == 'D'){
             nxt.time = getDTSCTime(curPage[nxt.tid].mapped, 0);
           }
-        buffer.erase(buffer.begin());
+          buffer.replaceFirst(nxt);
-        buffer.insert(nxt);
           return false;
         }
+        INFO_MSG("Invalid packet: no data @%" PRIu64 " for time %" PRIu64 " on track %zu", nxt.offset, nxt.time, nxt.tid);
         dropTrack(nxt.tid, "VoD page load failure");
         return false;
       }
@@ -1599,7 +1586,7 @@ namespace Mist{
       // We know this packet will be valid, pre-load it so we know its length
       DTSC::Packet preLoad(curPage[nxt.tid].mapped + nxt.offset, 0, true);
 
-    uint64_t nextTime = 0;
+      nextTime = 0;
 
       // Check if we have a next valid packet
       if (curPage[nxt.tid].len > nxt.offset+preLoad.getDataLen()+20 && memcmp(curPage[nxt.tid].mapped + nxt.offset + preLoad.getDataLen(), "\000\000\000\000", 4)){
@@ -1644,6 +1631,11 @@ namespace Mist{
             return false;
           }
         }else{
+          if (!buffer.getSyncMode() && trackTries < buffer.size()-1){
+            //We shuffle the just-tried packet back to the end of the queue, then retry up to buffer.size() times
+            buffer.moveFirstToEnd();
+            continue;
+          }
           //Okay, there's no next page yet, and no next packet on this page either.
           //That means we're waiting for data to show up, somewhere.
           // after ~25 seconds, give up and drop the track.
@@ -1679,6 +1671,7 @@ namespace Mist{
           return false;
         }
       }
+    }
 
     // we've handled all special cases - at this point the packet should exist
     // let's load it
@@ -1693,7 +1686,7 @@ namespace Mist{
     emptyCount = 0; // valid packet - reset empty counter
 
     if (!userSelect[nxt.tid]){
-      INFO_MSG("Track %zu is not alive!", nxt.tid);
+      dropTrack(nxt.tid, "track is not alive!");
       return false;
     }
 
@@ -1710,9 +1703,7 @@ namespace Mist{
     ++nxt.partIndex;
 
     // exchange the current packet in the buffer for the next one
-    buffer.erase(buffer.begin());
+    buffer.replaceFirst(nxt);
-    buffer.insert(nxt);
-
     return true;
   }
 

src/output/output.h

@@ -10,23 +10,11 @@
 #include <mist/shared_memory.h>
 #include <mist/socket.h>
 #include <mist/timing.h>
+#include <mist/stream.h>
 #include <set>
 
 namespace Mist{
 
-  /// This struct keeps packet information sorted in playback order, so the
-  /// Mist::Output class knows when to buffer which packet.
-  struct sortedPageInfo{
-    bool operator<(const sortedPageInfo &rhs) const{
-      if (time < rhs.time){return true;}
-      return (time == rhs.time && tid < rhs.tid);
-    }
-    size_t tid;
-    uint64_t time;
-    uint64_t offset;
-    size_t partIndex;
-  };
-
   /// The output class is intended to be inherited by MistOut process classes.
   /// It contains all generic code and logic, while the child classes implement
   /// anything specific to particular protocols or containers.
@@ -85,6 +73,9 @@ namespace Mist{
 
     void selectAllTracks();
 
+    /// Accessor for buffer.setSyncMode.
+    void setSyncMode(bool synced){buffer.setSyncMode(synced);}
+
   private: // these *should* not be messed with in child classes.
     /*LTS-START*/
     void Log(std::string type, std::string message);
@@ -102,7 +93,7 @@ namespace Mist{
     bool isRecordingToFile;
     uint64_t lastStats; ///< Time of last sending of stats.
 
-    std::set<sortedPageInfo> buffer; ///< A sorted list of next-to-be-loaded packets.
+    Util::packetSorter buffer; ///< A sorted list of next-to-be-loaded packets.
     bool sought;          ///< If a seek has been done, this is set to true. Used for seeking on
                           ///< prepareNext().
     std::string prevHost; ///< Old value for getConnectedBinHost, for caching

src/output/output_dtsc.cpp

@@ -12,6 +12,7 @@
 namespace Mist{
   OutDTSC::OutDTSC(Socket::Connection &conn) : Output(conn){
     JSON::Value prep;
+    setSyncMode(false);
     if (config->getString("target").size()){
       streamName = config->getString("streamname");
       pushUrl = HTTP::URL(config->getString("target"));