Support limiting output range for most outputs and outgoing pushes

lib/defines.h

@@ -18,7 +18,7 @@
 #define PRETTY_ARG_TIME(t)                                                                         \
   (int)(t) / 86400, ((int)(t) % 86400) / 3600, ((int)(t) % 3600) / 60, (int)(t) % 60
 #define PRETTY_PRINT_MSTIME "%ud%.2uh%.2um%.2us.%.3u"
-#define PRETTY_ARG_MSTIME(t) PRETTY_ARG_TIME(t / 1000), (int)(t % 1000)
+#define PRETTY_ARG_MSTIME(t) PRETTY_ARG_TIME((t) / 1000), (int)((t) % 1000)
 #if DEBUG > -1
 
 #define APPIDENT APPNAME "/" PACKAGE_VERSION

lib/dtsc.cpp

@@ -896,6 +896,7 @@ namespace DTSC{
     streamMemBuf = 0;
     isMemBuf = false;
     isMaster = true;
+    removeLimiter();
     reInit(_streamName, src);
   }
 
@@ -907,6 +908,7 @@ namespace DTSC{
     streamMemBuf = 0;
     isMemBuf = false;
     isMaster = master;
+    removeLimiter();
     reInit(_streamName, master, autoBackOff);
   }
 
@@ -916,6 +918,7 @@ namespace DTSC{
     streamMemBuf = 0;
     isMemBuf = false;
     isMaster = true;
+    removeLimiter();
     reInit(_streamName, fileName);
   }
 
@@ -989,6 +992,7 @@ namespace DTSC{
     // Unix Time at zero point of a stream
     if (src.hasMember("unixzero")){
       setBootMsOffset(src.getMember("unixzero").asInt() - Util::unixMS() + Util::bootMS());
+      setUTCOffset(src.getMember("unixzero").asInt());
     }else{
       MEDIUM_MSG("No member \'unixzero\' found in DTSC::Scan. Calculating locally.");
       int64_t nowMs = 0;
@@ -2001,6 +2005,7 @@ namespace DTSC{
   }
   uint64_t Meta::getFirstms(size_t trackIdx) const{
     const DTSC::Track &t = tracks.at(trackIdx);
+    if (isLimited && limitMin > t.track.getInt(t.trackFirstmsField)){return limitMin;}
     return t.track.getInt(t.trackFirstmsField);
   }
 
@@ -2010,6 +2015,7 @@ namespace DTSC{
   }
   uint64_t Meta::getLastms(size_t trackIdx) const{
     const DTSC::Track &t = tracks.find(trackIdx)->second;
+    if (isLimited && limitMax < t.track.getInt(t.trackLastmsField)){return limitMax;}
     return t.track.getInt(t.trackLastmsField);
   }
 
@@ -2023,6 +2029,7 @@ namespace DTSC{
   }
 
   uint64_t Meta::getDuration(size_t trackIdx) const{
+    if (isLimited){return getLastms(trackIdx) - getFirstms(trackIdx);}
     const DTSC::Track &t = tracks.at(trackIdx);
     return t.track.getInt(t.trackLastmsField) - t.track.getInt(t.trackFirstmsField);
   }
@@ -2117,12 +2124,16 @@ namespace DTSC{
   void Meta::setVod(bool vod){
     stream.setInt(streamVodField, vod ? 1 : 0);
   }
-  bool Meta::getVod() const{return stream.getInt(streamVodField);}
+  bool Meta::getVod() const{
+    return isLimited || stream.getInt(streamVodField);
+  }
 
   void Meta::setLive(bool live){
     stream.setInt(streamLiveField, live ? 1 : 0);
   }
-  bool Meta::getLive() const{return stream.getInt(streamLiveField);}
+  bool Meta::getLive() const{
+    return (!isLimited || limitMax == 0xFFFFFFFFFFFFFFFFull) && stream.getInt(streamLiveField);
+  }
 
   bool Meta::hasBFrames(size_t idx) const{
     std::set<size_t> vTracks = getValidTracks();
@@ -2272,7 +2283,7 @@ namespace DTSC{
       char thisPageName[NAME_BUFFER_SIZE];
       snprintf(thisPageName, NAME_BUFFER_SIZE, SHM_TRACK_DATA, streamName.c_str(), trackIdx,
                (uint32_t)t.pages.getInt("firstkey", t.pages.getDeleted()));
-      IPC::sharedPage p(thisPageName, 20971520);
+      IPC::sharedPage p(thisPageName, 20971520, false, false);
       p.master = true;
 
       // Then delete the page entry
@@ -2554,6 +2565,13 @@ namespace DTSC{
   const Util::RelAccX &Meta::parts(size_t idx) const{return tracks.at(idx).parts;}
   Util::RelAccX &Meta::keys(size_t idx){return tracks.at(idx).keys;}
   const Util::RelAccX &Meta::keys(size_t idx) const{return tracks.at(idx).keys;}
+
+  const Keys Meta::getKeys(size_t trackIdx) const{
+    DTSC::Keys k(keys(trackIdx));
+    if (isLimited){k.applyLimiter(limitMin, limitMax, DTSC::Parts(parts(trackIdx)));}
+    return k;
+  }
+
   const Util::RelAccX &Meta::fragments(size_t idx) const{return tracks.at(idx).fragments;}
   const Util::RelAccX &Meta::pages(size_t idx) const{return tracks.at(idx).pages;}
   Util::RelAccX &Meta::pages(size_t idx){return tracks.at(idx).pages;}
@@ -2652,7 +2670,8 @@ namespace DTSC{
   uint64_t Meta::getSendLen(bool skipDynamic, std::set<size_t> selectedTracks) const{
     uint64_t dataLen = 34; // + (merged ? 17 : 0) + (bufferWindow ? 24 : 0) + 21;
     if (getVod()){dataLen += 14;}
-    if (getLive()){dataLen += 15 + 19;} // 19 for unixzero
+    if (getLive()){dataLen += 15;}
+    if (getLive() || getUTCOffset()){dataLen += 19;} // unixzero field
     for (std::map<size_t, Track>::const_iterator it = tracks.begin(); it != tracks.end(); it++){
       if (!it->second.parts.getPresent()){continue;}
       if (!selectedTracks.size() || selectedTracks.count(it->first)){
@@ -2804,9 +2823,13 @@ namespace DTSC{
     if (getLive()){conn.SendNow("\000\004live\001\000\000\000\000\000\000\000\001", 15);}
     conn.SendNow("\000\007version\001", 10);
     conn.SendNow(c64(DTSH_VERSION), 8);
-    if (getLive()){
+    if (getLive() || getUTCOffset()){
       conn.SendNow("\000\010unixzero\001", 11);
+      if (getLive()){
         conn.SendNow(c64(Util::unixMS() - Util::bootMS() + getBootMsOffset()), 8);
+      }else{
+        conn.SendNow(c64(getUTCOffset()), 8);
+      }
     }
     if (lVarSize){
       conn.SendNow("\000\016inputLocalVars\002", 17);
@@ -3272,6 +3295,19 @@ namespace DTSC{
     // return is by reference
   }
 
+  void Meta::removeLimiter(){
+    isLimited = false;
+    limitMin = 0;
+    limitMax = 0;
+  }
+
+  void Meta::applyLimiter(uint64_t min, uint64_t max){
+    isLimited = true;
+    limitMin = min;
+    limitMax = max;
+    INFO_MSG("Applied limiter from %" PRIu64 " to %" PRIu64, min, max);
+  }
+
   /// Returns true if the tracks idx1 and idx2 are keyframe aligned
   bool Meta::keyTimingsMatch(size_t idx1, size_t idx2) const {
     const DTSC::Track &t1 = tracks.at(idx1);
@@ -3325,6 +3361,7 @@ namespace DTSC{
     partsField = cKeys.getFieldData("parts");
     timeField = cKeys.getFieldData("time");
     sizeField = cKeys.getFieldData("size");
+    isLimited = false;
   }
 
   Keys::Keys(const Util::RelAccX &_keys) : isConst(true), keys(empty), cKeys(_keys){
@@ -3335,23 +3372,143 @@ namespace DTSC{
     partsField = cKeys.getFieldData("parts");
     timeField = cKeys.getFieldData("time");
     sizeField = cKeys.getFieldData("size");
+    isLimited = false;
   }
 
-  size_t Keys::getFirstValid() const{return cKeys.getDeleted();}
+  size_t Keys::getFirstValid() const{
-  size_t Keys::getEndValid() const{return cKeys.getEndPos();}
+    return isLimited ? limMin : cKeys.getDeleted();
+  }
+  size_t Keys::getEndValid() const{
+    return isLimited ? limMax : cKeys.getEndPos();
+  }
   size_t Keys::getValidCount() const{return getEndValid() - getFirstValid();}
 
-  size_t Keys::getFirstPart(size_t idx) const{return cKeys.getInt(firstPartField, idx);}
+  size_t Keys::getFirstPart(size_t idx) const{
+    if (isLimited && idx == limMin){return limMinFirstPart;}
+    return cKeys.getInt(firstPartField, idx);
+  }
   size_t Keys::getBpos(size_t idx) const{return cKeys.getInt(bposField, idx);}
-  uint64_t Keys::getDuration(size_t idx) const{return cKeys.getInt(durationField, idx);}
+  uint64_t Keys::getDuration(size_t idx) const{
+    if (isLimited && idx + 1 == limMax){return limMaxDuration;}
+    if (isLimited && idx == limMin){return limMinDuration;}
+    return cKeys.getInt(durationField, idx);
+  }
   size_t Keys::getNumber(size_t idx) const{return cKeys.getInt(numberField, idx);}
-  size_t Keys::getParts(size_t idx) const{return cKeys.getInt(partsField, idx);}
+  size_t Keys::getParts(size_t idx) const{
-  uint64_t Keys::getTime(size_t idx) const{return cKeys.getInt(timeField, idx);}
+    if (isLimited && idx + 1 == limMax){return limMaxParts;}
+    if (isLimited && idx == limMin){return limMinParts;}
+    return cKeys.getInt(partsField, idx);
+  }
+  uint64_t Keys::getTime(size_t idx) const{
+    if (isLimited && idx == limMin){return limMinTime;}
+    return cKeys.getInt(timeField, idx);
+  }
   void Keys::setSize(size_t idx, size_t _size){
     if (isConst){return;}
     keys.setInt(sizeField, _size, idx);
   }
-  size_t Keys::getSize(size_t idx) const{return cKeys.getInt(sizeField, idx);}
+  size_t Keys::getSize(size_t idx) const{
+    if (isLimited && idx + 1 == limMax){return limMaxSize;}
+    if (isLimited && idx == limMin){return limMinSize;}
+    return cKeys.getInt(sizeField, idx);
+  }
+
+  uint64_t Keys::getTotalPartCount(){
+    return getParts(getEndValid()-1) + getFirstPart(getEndValid()-1) - getFirstPart(getFirstValid());
+  }
+  
+  uint32_t Keys::getIndexForTime(uint64_t timestamp){
+    uint32_t firstKey = getFirstValid();
+    uint32_t endKey = getEndValid();
+
+    for (size_t i = firstKey; i < endKey; i++){
+      if (getTime(i) + getDuration(i) > timestamp){return i;}
+    }
+    return endKey;
+  }
+
+  void Keys::applyLimiter(uint64_t _min, uint64_t _max, DTSC::Parts _p){
+
+    // Determine first and last key available within the limits
+    // Note: limMax replaces getEndValid(), and is thus one _past_ the end key index!
+    limMin = getFirstValid();
+    limMax = getEndValid();
+    for (size_t i = limMin; i < limMax; i++){
+      if (getTime(i) <= _min){limMin = i;}
+      if (getTime(i) >= _max){
+        limMax = i;
+        break;
+      }
+    }
+    // We can't have 0 keys, so force at least 1 key in cases where min >= max.
+    if (limMin >= limMax){limMax = limMin + 1;}
+
+    // If the first key is the last key, the override calculation is a little trickier
+    if (limMin + 1 == limMax){
+      //Calculate combined first/last key override
+      {
+        limMinDuration = 0;
+        limMinParts = 0;
+        limMinSize = 0;
+        limMinFirstPart = getFirstPart(limMin);
+        limMinTime = getTime(limMin);
+        size_t partNo = limMinFirstPart;
+        size_t truePartEnd = partNo + getParts(limMin);
+        while (partNo < truePartEnd){
+          if (limMinTime >= _min){
+            if (limMinTime + limMinDuration >= _max){break;}
+            ++limMinParts;
+            limMinDuration += _p.getDuration(partNo);
+            limMinSize += _p.getSize(partNo);
+          }else{
+            ++limMinFirstPart;
+            limMinTime += _p.getDuration(partNo);
+          }
+          ++partNo;
+        }
+        limMaxSize = limMinSize;
+        limMaxParts = limMinParts;
+        limMaxDuration = limMinDuration;
+      }
+    }else{
+      //Calculate first key overrides
+      {
+        limMinDuration = getDuration(limMin);
+        limMinParts = getParts(limMin);
+        limMinSize = getSize(limMin);
+        limMinFirstPart = getFirstPart(limMin);
+        limMinTime = getTime(limMin);
+        size_t partNo = limMinFirstPart;
+        size_t truePartEnd = partNo + limMinParts;
+        while (partNo < truePartEnd){
+          if (limMinTime >= _min){break;}
+          --limMinParts;
+          limMinDuration -= _p.getDuration(partNo);
+          limMinSize -= _p.getSize(partNo);
+          ++limMinFirstPart;
+          limMinTime += _p.getDuration(partNo);
+          ++partNo;
+        }
+      }
+      //Calculate last key overrides
+      {
+        limMaxDuration = limMaxParts = limMaxSize = 0;
+        size_t partNo = getFirstPart(limMax-1);
+        size_t truePartEnd = partNo + getParts(limMax-1);
+        uint64_t endTime = getTime(limMax-1);
+        while (partNo < truePartEnd){
+          if (endTime + limMaxDuration >= _max){break;}
+          ++limMaxParts;
+          limMaxDuration += _p.getDuration(partNo);
+          limMaxSize += _p.getSize(partNo);
+          ++partNo;
+        }
+      }
+    }
+
+    HIGH_MSG("Key limiter applied from %" PRIu64 " to %" PRIu64 ", key times %" PRIu64 " to %" PRIu64 ", %lld parts, %lld parts", _min, _max, getTime(limMin), getTime(limMax-1), (long long)limMinParts-(long long)getParts(limMin), (long long)limMaxParts-(long long)getParts(limMax-1));
+    isLimited = true;
+  }
 
   Fragments::Fragments(const Util::RelAccX &_fragments) : fragments(_fragments){}
   size_t Fragments::getFirstValid() const{return fragments.getDeleted();}

lib/dtsc.h

@@ -191,8 +191,27 @@ namespace DTSC{
     void setSize(size_t idx, size_t _size);
     size_t getSize(size_t idx) const;
 
+    uint64_t getTotalPartCount();
+    uint32_t getIndexForTime(uint64_t timestamp);
+
+    void applyLimiter(uint64_t _min, uint64_t _max, DTSC::Parts _p);
+
   private:
     bool isConst;
+    bool isLimited;
+    size_t limMin;
+    size_t limMax;
+    //Overrides for max key
+    size_t limMaxParts;
+    uint64_t limMaxDuration;
+    size_t limMaxSize;
+    //Overrides for min key
+    size_t limMinParts;
+    size_t limMinFirstPart;
+    uint64_t limMinDuration;
+    uint64_t limMinTime;
+    size_t limMinSize;
+
     Util::RelAccX empty;
 
     Util::RelAccX &keys;
@@ -477,6 +496,8 @@ namespace DTSC{
     Util::RelAccX &pages(size_t idx);
     const Util::RelAccX &pages(size_t idx) const;
 
+    const Keys getKeys(size_t trackIdx) const;
+
     std::string toPrettyString() const;
 
     void remap(const std::string &_streamName = "");
@@ -495,6 +516,9 @@ namespace DTSC{
 
     void getHealthJSON(JSON::Value & returnReference) const;
 
+    void removeLimiter();
+    void applyLimiter(uint64_t min, uint64_t max);
+
   protected:
     void sBufMem(size_t trackCount = DEFAULT_TRACK_COUNT);
     void sBufShm(const std::string &_streamName, size_t trackCount = DEFAULT_TRACK_COUNT, bool master = true, bool autoBackOff = true);
@@ -509,6 +533,9 @@ namespace DTSC{
     std::map<size_t, IPC::sharedPage> tM;
 
     bool isMaster;
+    uint64_t limitMin;
+    uint64_t limitMax;
+    bool isLimited;
 
     char *streamMemBuf;
     bool isMemBuf;

lib/flv_tag.cpp

@@ -502,13 +502,11 @@ bool FLV::Tag::DTSCMetaInit(const DTSC::Meta &M, std::set<size_t> &selTracks){
   int i = 0;
   uint64_t mediaLen = 0;
   for (std::set<size_t>::iterator it = selTracks.begin(); it != selTracks.end(); it++){
-    if (M.getLastms(*it) - M.getFirstms(*it) > mediaLen){
+    if (M.getDuration(*it) > mediaLen){mediaLen = M.getDuration(*it);}
-      mediaLen = M.getLastms(*it) - M.getFirstms(*it);
-    }
     if (M.getType(*it) == "video"){
       trinfo.addContent(AMF::Object("", AMF::AMF0_OBJECT));
       trinfo.getContentP(i)->addContent(AMF::Object(
-          "length", ((double)M.getLastms(*it) / 1000) * ((double)M.getFpks(*it) / 1000.0), AMF::AMF0_NUMBER));
+          "length", ((double)M.getDuration(*it) / 1000) * ((double)M.getFpks(*it) / 1000.0), AMF::AMF0_NUMBER));
       trinfo.getContentP(i)->addContent(AMF::Object("timescale", ((double)M.getFpks(*it) / 1000), AMF::AMF0_NUMBER));
       trinfo.getContentP(i)->addContent(AMF::Object("sampledescription", AMF::AMF0_STRICT_ARRAY));
       amfdata.getContentP(1)->addContent(AMF::Object("hasVideo", 1, AMF::AMF0_BOOL));
@@ -552,7 +550,7 @@ bool FLV::Tag::DTSCMetaInit(const DTSC::Meta &M, std::set<size_t> &selTracks){
     if (M.getType(*it) == "audio"){
       trinfo.addContent(AMF::Object("", AMF::AMF0_OBJECT));
       trinfo.getContentP(i)->addContent(
-          AMF::Object("length", (double)(M.getLastms(*it) * M.getRate(*it)), AMF::AMF0_NUMBER));
+          AMF::Object("length", (double)(M.getDuration(*it) * M.getRate(*it)), AMF::AMF0_NUMBER));
       trinfo.getContentP(i)->addContent(AMF::Object("timescale", M.getRate(*it), AMF::AMF0_NUMBER));
       trinfo.getContentP(i)->addContent(AMF::Object("sampledescription", AMF::AMF0_STRICT_ARRAY));
       amfdata.getContentP(1)->addContent(AMF::Object("hasAudio", 1, AMF::AMF0_BOOL));
@@ -575,7 +573,7 @@ bool FLV::Tag::DTSCMetaInit(const DTSC::Meta &M, std::set<size_t> &selTracks){
     }
   }
   if (M.getVod()){
-    amfdata.getContentP(1)->addContent(AMF::Object("duration", mediaLen / 1000, AMF::AMF0_NUMBER));
+    amfdata.getContentP(1)->addContent(AMF::Object("duration", mediaLen / 1000.0, AMF::AMF0_NUMBER));
   }
   amfdata.getContentP(1)->addContent(trinfo);
 

src/input/input.cpp

@@ -1656,7 +1656,7 @@ namespace Mist{
     }
     bufferFinalize(idx, page);
     bufferTimer = Util::bootMS() - bufferTimer;
-    if (packCounter != tPages.getInt("parts", pageIdx)){
+    if (packCounter < tPages.getInt("parts", pageIdx)){
       FAIL_MSG("Track %zu, page %" PRIu32 " (" PRETTY_PRINT_MSTIME " - " PRETTY_PRINT_MSTIME ") NOT FULLY buffered in %" PRIu64 "ms - erasing for later retry",
                idx, pageNumber, PRETTY_ARG_MSTIME(tPages.getInt("firsttime", pageIdx)), PRETTY_ARG_MSTIME(thisTime), bufferTimer);
       INFO_MSG("  (%" PRIu32 "/%" PRIu64 " parts, %" PRIu64 " bytes)", packCounter,

src/input/input_hls.cpp

@@ -762,7 +762,10 @@ namespace Mist{
       Util::logExitReason(ER_UNKNOWN, "Failed to load HLS playlist, aborting");
       return;
     }
+    uint64_t oldBootMsOffset = M.getBootMsOffset();
     meta.reInit(isSingular() ? streamName : "", false);
+    meta.setUTCOffset(zUTC);
+    meta.setBootMsOffset(oldBootMsOffset);
     INFO_MSG("Parsing live stream to create header...");
     TS::Packet packet; // to analyse and extract data
     int pidCounter = 1;
@@ -831,7 +834,7 @@ namespace Mist{
       INFO_MSG("Could not read existing header, regenerating");
       return false;
     }
-    if (!M.inputLocalVars.isMember("version") || M.inputLocalVars["version"].asInt() < 3){
+    if (!M.inputLocalVars.isMember("version") || M.inputLocalVars["version"].asInt() < 4){
       INFO_MSG("Header needs update, regenerating");
       return false;
     }
@@ -888,9 +891,9 @@ namespace Mist{
       pidMapping[val] = key;
     }
     // Set bootMsOffset in order to display the program time correctly in the player
-    streamOffset = M.inputLocalVars["streamoffset"].asInt();
+    zUTC = M.inputLocalVars["zUTC"].asInt();
-    if (meta.getLive()){meta.setUTCOffset(streamOffset + (Util::unixMS() - Util::bootMS()));}
+    meta.setUTCOffset(zUTC);
-    meta.setBootMsOffset(streamOffset);
+    if (M.getLive()){meta.setBootMsOffset(streamOffset);}
     return true;
   }
 
@@ -999,12 +1002,12 @@ namespace Mist{
     if (!config->is_active){return false;}
 
     // set bootMsOffset in order to display the program time correctly in the player
-    if (meta.getLive()){meta.setUTCOffset(streamOffset + (Util::unixMS() - Util::bootMS()));}
+    meta.setUTCOffset(zUTC);
-    meta.setBootMsOffset(streamOffset);
+    if (M.getLive()){meta.setBootMsOffset(streamOffset);}
     if (streamIsLive || isLiveDVR){return true;}
 
     // Set local vars used for parsing existing headers
-    meta.inputLocalVars["version"] = 3;
+    meta.inputLocalVars["version"] = 4;
 
     // Write playlist entry info
     JSON::Value allEntries;
@@ -1029,7 +1032,7 @@ namespace Mist{
     }
     meta.inputLocalVars["playlist_urls"] = playlist_urls;
     meta.inputLocalVars["playlistEntries"] = allEntries;
-    meta.inputLocalVars["streamoffset"] = streamOffset;
+    meta.inputLocalVars["zUTC"] = zUTC;
 
     // Write packet ID mappings
     JSON::Value thisMappingsR;
@@ -1599,6 +1602,8 @@ namespace Mist{
         INFO_MSG("Setting program unix start time to '%s' (%" PRIu64 ")", line.substr(pos + 1).c_str(), zUTC);
         // store offset so that we can set it after reading the header
         streamOffset = zUTC - (Util::unixMS() - Util::bootMS());
+        meta.setUTCOffset(zUTC);
+        if (M.getLive()){meta.setBootMsOffset(streamOffset);}
       }else{
         // ignore wrong lines
         VERYHIGH_MSG("ignore wrong line: %s", line.c_str());

src/output/output.cpp

@@ -917,8 +917,9 @@ namespace Mist{
   /// For live, it seeks to the last sync'ed keyframe of the main track, no closer than
   /// needsLookAhead+minKeepAway ms from the end. Unless lastms < 5000, then it seeks to the first
   /// keyframe of the main track. Aborts if there is no main track or it has no keyframes.
-  void Output::initialSeek(){
+  void Output::initialSeek(bool dryRun){
     if (!meta){return;}
+    meta.removeLimiter();
     uint64_t seekPos = 0;
     if (meta.getLive() && buffer.getSyncMode()){
       size_t mainTrack = getMainSelectedTrack();
@@ -967,28 +968,59 @@ namespace Mist{
         MEDIUM_MSG("Stream currently contains data from %" PRIu64 " ms to %" PRIu64 " ms", startTime(), endTime());
       }
       // Overwrite recstart/recstop with recstartunix/recstopunix if set
-      if (M.getLive() &&
+      if (M.getLive() && (
-          (targetParams.count("recstartunix") || targetParams.count("recstopunix"))){
+          targetParams.count("recstartunix") || targetParams.count("recstopunix") ||
-        uint64_t unixStreamBegin = Util::epoch() - endTime()/1000;
+          targetParams.count("startunix") || targetParams.count("stopunix") ||
-        if (targetParams.count("recstartunix")){
+          targetParams.count("unixstart") || targetParams.count("unixstop")
-          uint64_t startUnix = atoll(targetParams["recstartunix"].c_str());
+      )){
-          if (startUnix < unixStreamBegin){
+        uint64_t zUTC = M.getUTCOffset();
-            WARN_MSG("Recording start time is earlier than stream begin - starting earliest possible");
+        if (!zUTC){
-            targetParams["recstart"] = "-1";
+          if (!M.getLive()){
+            WARN_MSG("Attempting to set unix-based start/stop time for a VoD asset without known UTC timestamp! This will likely not work as you expect, since we have nothing to base the timestamps on");
           }else{
-            targetParams["recstart"] = JSON::Value((startUnix - unixStreamBegin) * 1000).asString();
+            zUTC = M.getBootMsOffset() + Util::getGlobalConfig("systemBoot").asInt();
           }
         }
+        if (targetParams.count("recstartunix")){
+          int64_t startUnix = atoll(targetParams["recstartunix"].c_str()) * 1000;
+          // If the time is before the first 10 hours of unix epoch, assume relative time
+          if (startUnix <= 36000000){startUnix += Util::unixMS();}
+          targetParams["recstart"] = JSON::Value(startUnix - zUTC).asString();
+        }
         if (targetParams.count("recstopunix")){
-          uint64_t stopUnix = atoll(targetParams["recstopunix"].c_str());
+          int64_t stopUnix = atoll(targetParams["recstopunix"].c_str()) * 1000;
-          if (stopUnix < unixStreamBegin){
+          // If the time is before the first 10 hours of unix epoch, assume relative time
-            onFail("Recording stop time is earlier than stream begin - aborting", true);
+          if (stopUnix <= 36000000){stopUnix += Util::unixMS();}
-            return;
+          targetParams["recstop"] = JSON::Value(stopUnix - zUTC).asString();
-          }else{
+        }
-            targetParams["recstop"] = JSON::Value((stopUnix - unixStreamBegin) * 1000).asString();
+        if (targetParams.count("unixstart")){
-          }
+          int64_t startUnix = atoll(targetParams["unixstart"].c_str()) * 1000;
+          // If the time is before the first 10 hours of unix epoch, assume relative time
+          if (startUnix <= 36000000){startUnix += Util::unixMS();}
+          targetParams["recstart"] = JSON::Value(startUnix - zUTC).asString();
+        }
+        if (targetParams.count("unixstop")){
+          int64_t stopUnix = atoll(targetParams["unixstop"].c_str()) * 1000;
+          // If the time is before the first 10 hours of unix epoch, assume relative time
+          if (stopUnix <= 36000000){stopUnix += Util::unixMS();}
+          targetParams["recstop"] = JSON::Value(stopUnix - zUTC).asString();
+        }
+        if (targetParams.count("startunix")){
+          int64_t startUnix = atoll(targetParams["startunix"].c_str()) * 1000;
+          // If the time is before the first 10 hours of unix epoch, assume relative time
+          if (startUnix <= 36000000){startUnix += Util::unixMS();}
+          targetParams["recstart"] = JSON::Value(startUnix - zUTC).asString();
+        }
+        if (targetParams.count("stopunix")){
+          int64_t stopUnix = atoll(targetParams["stopunix"].c_str()) * 1000;
+          // If the time is before the first 10 hours of unix epoch, assume relative time
+          if (stopUnix <= 36000000){stopUnix += Util::unixMS();}
+          targetParams["recstop"] = JSON::Value(stopUnix - zUTC).asString();
         }
       }
+      //Autoconvert start/stop to recstart/recstop to improve usability
+      if (targetParams.count("start") && !targetParams.count("recstart")){targetParams["recstart"] = targetParams["start"];}
+      if (targetParams.count("stop") && !targetParams.count("recstop")){targetParams["recstop"] = targetParams["stop"];}
       // Check recstart/recstop for correctness
       if (targetParams.count("recstop")){
         uint64_t endRec = atoll(targetParams["recstop"].c_str());
@@ -1012,18 +1044,31 @@ namespace Mist{
                    " ms instead", atoll(targetParams["recstart"].c_str()), startRec);
           targetParams["recstart"] = JSON::Value(startRec).asString();
         }
+        size_t mainTrack = getMainSelectedTrack();
+        if (M.getType(mainTrack) == "video"){
+          seekPos = M.getTimeForKeyIndex(mainTrack, M.getKeyIndexForTime(mainTrack, startRec));
+          if (seekPos != startRec){
+            INFO_MSG("Shifting recording start from %" PRIu64 " to %" PRIu64 " so that it starts with a keyframe", startRec, seekPos);
+          }
+        }else{
           seekPos = startRec;
         }
+      }
       
       if (targetParams.count("split")){
         long long endRec = atoll(targetParams["split"].c_str()) * 1000;
         INFO_MSG("Will split recording every %lld seconds", atoll(targetParams["split"].c_str()));
         targetParams["nxt-split"] = JSON::Value((int64_t)(seekPos + endRec)).asString();
       }
-      // Duration to record in seconds. Oversides recstop.
+      // Duration to record in seconds. Overrides recstop.
       if (targetParams.count("duration")){
-        long long endRec = atoll(targetParams["duration"].c_str()) * 1000;
+        int64_t endRec;
-        targetParams["recstop"] = JSON::Value((int64_t)(seekPos + endRec)).asString();
+        if (targetParams.count("recstart")){
+          endRec = atoll(targetParams["recstart"].c_str()) + atoll(targetParams["duration"].c_str()) * 1000;
+        }else{
+          endRec = seekPos + atoll(targetParams["duration"].c_str()) * 1000;
+        }
+        targetParams["recstop"] = JSON::Value(endRec).asString();
         // Recheck recording end time
         endRec = atoll(targetParams["recstop"].c_str());
         if (endRec < 0 || endRec < startTime()){
@@ -1034,8 +1079,7 @@ namespace Mist{
       // Print calculated start and stop time
       if (targetParams.count("recstart")){
         INFO_MSG("Recording will start at timestamp %llu ms", atoll(targetParams["recstart"].c_str()));
-      }
+      } else{
-      else{
         INFO_MSG("Recording will start at timestamp %" PRIu64 " ms", endTime()); 
       }
       if (targetParams.count("recstop")){
@@ -1056,6 +1100,14 @@ namespace Mist{
           }
         }
       }
+      // If we have a stop position and it's within available range,
+      // apply a limiter to the stream to make it appear like a VoD asset
+      if (targetParams.count("recstop") || !M.getLive()){
+        size_t mainTrack = getMainSelectedTrack();
+        uint64_t stopPos = M.getLastms(mainTrack);
+        if (targetParams.count("recstop")){stopPos = atoll(targetParams["recstop"].c_str());}
+        if (!M.getLive() || stopPos <= M.getLastms(mainTrack)){meta.applyLimiter(seekPos, stopPos);}
+      }
     }else{
       if (M.getLive() && targetParams.count("pushdelay")){
         INFO_MSG("Converting pushdelay syntax into corresponding recstart+realtime options");
@@ -1082,51 +1134,28 @@ namespace Mist{
         targetParams["realtime"] = "1"; //force real-time speed
         maxSkipAhead = 1;
       }
-      if (M.getLive() && (targetParams.count("startunix") || targetParams.count("stopunix"))){
+      if (targetParams.count("startunix") || targetParams.count("stopunix")){
-        uint64_t unixStreamBegin = Util::epoch() - endTime()/1000;
+        uint64_t zUTC = M.getUTCOffset();
-        size_t mainTrack = getMainSelectedTrack();
+        if (!zUTC){
-        int64_t streamAvail = M.getNowms(mainTrack);
+          if (!M.getLive()){
-        if (targetParams.count("startunix")){
+            WARN_MSG("Attempting to set unix-based start/stop time for a VoD asset without known UTC timestamp! This will likely not work as you expect, since we have nothing to base the timestamps on");
-          int64_t startUnix = atoll(targetParams["startunix"].c_str());
-          if (startUnix < 0){
-            int64_t origStartUnix = startUnix;
-            startUnix += Util::epoch();
-            if (startUnix < unixStreamBegin){
-              INFO_MSG("Waiting for stream to reach playback starting point. Current last ms is '%" PRIu64 "'", streamAvail);
-              while (startUnix < Util::epoch() - (endTime() / 1000) && keepGoing()){
-                Util::wait(1000);
-                stats();
-                startUnix = origStartUnix + Util::epoch();
-                HIGH_MSG("Waiting for stream to reach playback starting point. Current last ms is '%" PRIu64 "'", streamAvail);
-              }
-            }
-          }
-          if (startUnix < unixStreamBegin){
-            WARN_MSG("Start time (%" PRId64 ") is earlier than stream begin (%" PRId64 ") - starting earliest possible", startUnix, unixStreamBegin);
-            targetParams["start"] = "-1";
           }else{
-            targetParams["start"] = JSON::Value((startUnix - unixStreamBegin) * 1000).asString();
+            zUTC = M.getBootMsOffset() + Util::getGlobalConfig("systemBoot").asInt();
           }
         }
+        if (targetParams.count("startunix")){
+          int64_t startUnix = atoll(targetParams["startunix"].c_str()) * 1000;
+          // If the time is before the first 10 hours of unix epoch, assume relative time
+          if (startUnix <= 36000000){startUnix += Util::unixMS();}
+          targetParams["start"] = JSON::Value(startUnix - zUTC).asString();
+        }
         if (targetParams.count("stopunix")){
-          int64_t stopUnix = atoll(targetParams["stopunix"].c_str());
+          int64_t stopUnix = atoll(targetParams["stopunix"].c_str()) * 1000;
-          if (stopUnix < 0){stopUnix += Util::epoch();}
+          // If the time is before the first 10 hours of unix epoch, assume relative time
-          if (stopUnix < unixStreamBegin){
+          if (stopUnix <= 36000000){stopUnix += Util::unixMS();}
-            onFail("Stop time is earlier than stream begin - aborting", true);
+          targetParams["stop"] = JSON::Value(stopUnix - zUTC).asString();
-            return;
-          }else{
-            targetParams["stop"] = JSON::Value((stopUnix - unixStreamBegin) * 1000).asString();
         }
       }
-      }
-      if (targetParams.count("stop")){
-        int64_t endRec = atoll(targetParams["stop"].c_str());
-        if (endRec < 0 || endRec < startTime()){
-          onFail("Entire range is in the past", true);
-          return;
-        }
-        INFO_MSG("Playback will stop at %" PRIu64, endRec);
-      }
       if (targetParams.count("start") && atoll(targetParams["start"].c_str()) != 0){
         size_t mainTrack = getMainSelectedTrack();
         int64_t startRec = atoll(targetParams["start"].c_str());
@@ -1137,11 +1166,11 @@ namespace Mist{
           }
           int64_t streamAvail = M.getNowms(mainTrack);
           int64_t lastUpdated = Util::getMS();
-          INFO_MSG("Waiting for stream to reach playback starting point. Current last ms is '%" PRIu64 "'", streamAvail);
+          INFO_MSG("Waiting for stream to reach playback starting point (%" PRIu64 " -> %" PRIu64 "). Time left: " PRETTY_PRINT_MSTIME, startRec, streamAvail, PRETTY_ARG_MSTIME(startRec - streamAvail));
           while (Util::getMS() - lastUpdated < 5000 && startRec > streamAvail && keepGoing()){
             Util::sleep(500);
             if (M.getNowms(mainTrack) > streamAvail){
-              HIGH_MSG("Waiting for stream to reach playback starting point. Current last ms is '%" PRIu64 "'", streamAvail);
+              HIGH_MSG("Waiting for stream to reach playback starting point (%" PRIu64 " -> %" PRIu64 "). Time left: " PRETTY_PRINT_MSTIME, startRec, streamAvail, PRETTY_ARG_MSTIME(startRec - streamAvail));
               stats();
               streamAvail = M.getNowms(mainTrack);
               lastUpdated = Util::getMS();
@@ -1154,10 +1183,54 @@ namespace Mist{
                    startRec, startTime());
           startRec = startTime();
         }
-        INFO_MSG("Playback will start at %" PRIu64, startRec);
+        if (M.getType(mainTrack) == "video"){
+          seekPos = M.getTimeForKeyIndex(mainTrack, M.getKeyIndexForTime(mainTrack, startRec));
+          if (seekPos != startRec){
+            INFO_MSG("Shifting recording start from %" PRIu64 " to %" PRIu64 " so that it starts with a keyframe", startRec, seekPos);
+          }
+        }else{
           seekPos = startRec;
         }
+        INFO_MSG("Playback will start at %" PRIu64, seekPos);
       }
+      // Duration to record in seconds. Overrides stop.
+      if (targetParams.count("duration")){
+        int64_t endRec;
+        if (targetParams.count("start")){
+          endRec = atoll(targetParams["start"].c_str()) + atoll(targetParams["duration"].c_str()) * 1000;
+        }else{
+          endRec = seekPos + atoll(targetParams["duration"].c_str()) * 1000;
+        }
+        targetParams["stop"] = JSON::Value(endRec).asString();
+      }
+      if (targetParams.count("stop")){
+        int64_t endRec = atoll(targetParams["stop"].c_str());
+        if (endRec < 0 || endRec < startTime()){
+          onFail("Entire range is in the past", true);
+          return;
+        }
+        INFO_MSG("Playback will stop at %" PRIu64, endRec);
+      }
+      // If we have a stop position and it's within available range,
+      // apply a limiter to the stream to make it appear like a VoD asset
+      if (targetParams.count("stop") || !M.getLive()){
+        size_t mainTrack = getMainSelectedTrack();
+        uint64_t stopPos = M.getLastms(mainTrack);
+        if (targetParams.count("stop")){stopPos = atoll(targetParams["stop"].c_str());}
+        if (!M.getLive() || stopPos <= M.getLastms(mainTrack)){
+          meta.applyLimiter(seekPos, stopPos);
+        }else{
+          // End point past end of track? Don't limit the end point.
+          meta.applyLimiter(seekPos, 0xFFFFFFFFFFFFFFFFull);
+        }
+      }else{
+        // No stop point, only apply limiter if a start point is set, and never limit the end point.
+        if (targetParams.count("start")){
+          meta.applyLimiter(seekPos, 0xFFFFFFFFFFFFFFFFull);
+        }
+      }
+    }
+    if (dryRun){return;}
     /*LTS-END*/
     if (!keepGoing()){
       ERROR_MSG("Aborting seek to %" PRIu64 " since the stream is no longer active", seekPos);
@@ -1581,11 +1654,11 @@ namespace Mist{
             break;
           }
         }
+        if (!sought){initialSeek();}
         if (!sentHeader && keepGoing()){
           DONTEVEN_MSG("sendHeader");
           sendHeader();
         }
-        if (!sought){initialSeek();}
         if (prepareNext()){
           if (thisPacket){
             lastPacketTime = thisTime;

src/output/output.h

@@ -60,7 +60,7 @@ namespace Mist{
     virtual void dropTrack(size_t trackId, const std::string &reason, bool probablyBad = true);
     virtual void onRequest();
     static void listener(Util::Config &conf, int (*callback)(Socket::Connection &S));
-    virtual void initialSeek();
+    virtual void initialSeek(bool dryRun = false);
     uint64_t getMinKeepAway();
     virtual bool liveSeek(bool rateOnly = false);
     virtual bool onFinish(){return false;}

src/output/output_aac.cpp

@@ -26,13 +26,14 @@ namespace Mist{
     cfg->addOption("target", opt);
   }
 
-  void OutAAC::initialSeek(){
+  void OutAAC::initialSeek(bool dryRun){
     if (!meta){return;}
     maxSkipAhead = 30000;
     if (targetParams.count("buffer")){
       maxSkipAhead = atof(targetParams["buffer"].c_str())*1000;
     }
-    Output::initialSeek();
+    Output::initialSeek(dryRun);
+    if (dryRun){return;}
     uint64_t cTime = currentTime();
     if (M.getLive() && cTime > maxSkipAhead){
       seek(cTime-maxSkipAhead);

src/output/output_aac.h

@@ -7,7 +7,7 @@ namespace Mist{
     static void init(Util::Config *cfg);
     void respondHTTP(const HTTP::Parser & req, bool headersOnly);
     void sendNext();
-    void initialSeek();
+    void initialSeek(bool dryRun = false);
 
   private:
     virtual bool inlineRestartCapable() const{return true;}

src/output/output_flv.cpp

@@ -64,9 +64,11 @@ namespace Mist{
             for (std::map<size_t, Comms::Users>::iterator it = userSelect.begin();
                  it != userSelect.end(); it++){
               if (M.getType(it->first) == "video" && tag.DTSCVideoInit(meta, it->first)){
+                tag.tagTime(thisTime);
                 myConn.SendNow(tag.data, tag.len);
               }
               if (M.getType(it->first) == "audio" && tag.DTSCAudioInit(meta.getCodec(it->first), meta.getRate(it->first), meta.getSize(it->first), meta.getChannels(it->first), meta.getInit(it->first))){
+                tag.tagTime(thisTime);
                 myConn.SendNow(tag.data, tag.len);
               }
             }
@@ -114,12 +116,15 @@ namespace Mist{
       selectedTracks.insert(it->first);
     }
     tag.DTSCMetaInit(M, selectedTracks);
+    tag.tagTime(startTime());
     myConn.SendNow(tag.data, tag.len);
     for (std::set<size_t>::iterator it = selectedTracks.begin(); it != selectedTracks.end(); it++){
       if (M.getType(*it) == "video" && tag.DTSCVideoInit(meta, *it)){
+        tag.tagTime(startTime());
         myConn.SendNow(tag.data, tag.len);
       }
       if (M.getType(*it) == "audio" && tag.DTSCAudioInit(meta.getCodec(*it), meta.getRate(*it), meta.getSize(*it), meta.getChannels(*it), meta.getInit(*it))){
+        tag.tagTime(startTime());
         myConn.SendNow(tag.data, tag.len);
       }
     }

src/output/output_hls.cpp

@@ -30,6 +30,14 @@ namespace Mist{
     }
     std::string tknStr;
     if (tkn.size() && Comms::tknMode & 0x04){tknStr = "?tkn=" + tkn;}
+    if (targetParams.count("start")){
+      if (tknStr.size()){ tknStr += "&"; }else{ tknStr = "?"; }
+      tknStr += "start="+targetParams["start"];
+    }
+    if (targetParams.count("stop")){
+      if (tknStr.size()){ tknStr += "&"; }else{ tknStr = "?"; }
+      tknStr += "stop="+targetParams["stop"];
+    }
     for (std::map<size_t, Comms::Users>::iterator it = userSelect.begin(); it != userSelect.end(); ++it){
       if (M.getType(it->first) == "video"){
         ++vidTracks;
@@ -102,6 +110,9 @@ namespace Mist{
       size_t keyNumber = fragments.getFirstKey(i);
       uint64_t startTime = keys.getTime(keyNumber);
       if (!duration){duration = M.getLastms(timingTid) - startTime;}
+      if (startTime + duration < M.getFirstms(timingTid)){continue;}
+      if (startTime >= M.getLastms(timingTid)){continue;}
+      if (startTime + duration > M.getLastms(timingTid)){duration = M.getLastms(timingTid) - startTime;}
       double floatDur = (double)duration / 1000;
       char lineBuf[400];
 
@@ -373,6 +384,7 @@ namespace Mist{
       ts_from = from;
     }else{
       initialize();
+      initialSeek(true);
       std::string request = H.url.substr(H.url.find("/", 5) + 1);
       H.setCORSHeaders();
       H.SetHeader("Content-Type", "application/vnd.apple.mpegurl");

src/output/output_http.cpp

@@ -352,6 +352,7 @@ namespace Mist{
       if (H.GetVar("stop") != ""){targetParams["stop"] = H.GetVar("stop");}
       if (H.GetVar("startunix") != ""){targetParams["startunix"] = H.GetVar("startunix");}
       if (H.GetVar("stopunix") != ""){targetParams["stopunix"] = H.GetVar("stopunix");}
+      if (H.GetVar("duration") != ""){targetParams["duration"] = H.GetVar("duration");}
       // allow setting of max lead time through buffer variable.
       // max lead time is set in MS, but the variable is in integer seconds for simplicity.
       if (H.GetVar("buffer") != ""){

src/output/output_httpts.cpp

@@ -52,10 +52,10 @@ namespace Mist{
 
   OutHTTPTS::~OutHTTPTS(){}
 
-  void OutHTTPTS::initialSeek(){
+  void OutHTTPTS::initialSeek(bool dryRun){
     // Adds passthrough support to the regular initialSeek function
     if (targetParams.count("passthrough")){selectAllTracks();}
-    Output::initialSeek();
+    Output::initialSeek(dryRun);
   }
 
   void OutHTTPTS::init(Util::Config *cfg){

src/output/output_httpts.h

@@ -9,7 +9,7 @@ namespace Mist{
     static void init(Util::Config *cfg);
     void onHTTP();
     void sendTS(const char *tsData, size_t len = 188);
-    void initialSeek();
+    void initialSeek(bool dryRun = false);
 
   private:
     bool isRecording();

src/output/output_jpg.cpp

@@ -60,7 +60,7 @@ namespace Mist{
   /// Pretends the stream is always ready to play - we don't care about waiting times or whatever
   bool OutJPG::isReadyForPlay(){return true;}
 
-  void OutJPG::initialSeek(){
+  void OutJPG::initialSeek(bool dryRun){
     size_t mainTrack = getMainSelectedTrack();
     if (mainTrack == INVALID_TRACK_ID){return;}
     INFO_MSG("Doing initial seek");

src/output/output_jpg.h

@@ -10,7 +10,7 @@ namespace Mist{
 
   private:
     void generate();
-    void initialSeek();
+    void initialSeek(bool dryRun = false);
     void NoFFMPEG();
     std::string cachedir;
     uint64_t cachetime;

src/output/output_mp4.cpp

@@ -152,7 +152,7 @@ namespace Mist{
   uint64_t OutMP4::estimateFileSize() const{
     uint64_t retVal = 0;
     for (std::map<size_t, Comms::Users>::const_iterator it = userSelect.begin(); it != userSelect.end(); it++){
-      DTSC::Keys keys(M.keys(it->first));
+      DTSC::Keys keys = M.getKeys(it->first);
       size_t endKey = keys.getEndValid();
       for (size_t i = 0; i < endKey; i++){
         retVal += keys.getSize(i); // Handle number as index, faster for VoD
@@ -178,9 +178,8 @@ namespace Mist{
          subIt != userSelect.end(); subIt++){
       tmpRes += 8 + 20; // TRAF + TFHD Box
 
-      DTSC::Keys keys(M.keys(subIt->first));
+      DTSC::Keys keys = M.getKeys(subIt->first);
       DTSC::Parts parts(M.parts(subIt->first));
-      DTSC::Fragments fragments(M.fragments(subIt->first));
 
       uint32_t startKey = M.getKeyIndexForTime(subIt->first, startFragmentTime);
       uint32_t endKey = M.getKeyIndexForTime(subIt->first, endFragmentTime) + 1;
@@ -233,8 +232,9 @@ namespace Mist{
     for (std::map<size_t, Comms::Users>::const_iterator it = userSelect.begin(); it != userSelect.end(); it++){
       const std::string tType = M.getType(it->first);
       uint64_t tmpRes = 0;
+      DTSC::Keys keys = M.getKeys(it->first);
       DTSC::Parts parts(M.parts(it->first));
-      uint64_t partCount = parts.getValidCount();
+      uint64_t partCount = keys.getTotalPartCount();
 
       tmpRes += 8 + 92                                        // TRAK + TKHD Boxes
                 + 36                                          // EDTS Box
@@ -260,7 +260,6 @@ namespace Mist{
                   + 16                               // PASP
                   + 8 + M.getInit(it->first).size(); // avcC
         if (!fragmented){
-          DTSC::Keys keys(M.keys(it->first));
           tmpRes += 16 + (keys.getValidCount() * 4); // STSS
         }
       }
@@ -283,16 +282,17 @@ namespace Mist{
       if (!fragmented){
         // Unfortunately, for our STTS and CTTS boxes, we need to loop through all parts of the
         // track
+        size_t firstPart = keys.getFirstPart(keys.getFirstValid());
         uint64_t sttsCount = 1;
-        uint64_t prevDur = parts.getDuration(0);
+        uint64_t prevDur = parts.getDuration(firstPart);
-        uint64_t prevOffset = parts.getOffset(0);
+        uint64_t prevOffset = parts.getOffset(firstPart);
         uint64_t cttsCount = 1;
-        fileSize += parts.getSize(0);
+        fileSize += parts.getSize(firstPart);
         bool isMeta = (tType == "meta");
         for (unsigned int part = 1; part < partCount; ++part){
-          uint64_t partDur = parts.getDuration(part);
+          uint64_t partDur = parts.getDuration(firstPart + part);
-          uint64_t partOffset = parts.getOffset(part);
+          uint64_t partOffset = parts.getOffset(firstPart + part);
-          uint64_t partSize = parts.getSize(part)+(isMeta?2:0);
+          uint64_t partSize = parts.getSize(firstPart + part)+(isMeta?2:0);
           if (prevDur != partDur){
             prevDur = partDur;
             ++sttsCount;
@@ -397,7 +397,8 @@ namespace Mist{
     for (std::map<size_t, Comms::Users>::const_iterator it = userSelect.begin(); it != userSelect.end(); it++){
       if (prevVidTrack != INVALID_TRACK_ID && it->first == prevVidTrack){continue;}
       DTSC::Parts parts(M.parts(it->first));
-      size_t partCount = parts.getValidCount();
+      DTSC::Keys keys = M.getKeys(it->first);
+      size_t partCount = keys.getTotalPartCount();
       uint64_t tDuration = M.getLastms(it->first) - M.getFirstms(it->first);
       std::string tType = M.getType(it->first);
 
@@ -522,26 +523,28 @@ namespace Mist{
         MP4::CTTS cttsBox;
         cttsBox.setVersion(0);
 
+        size_t firstPart = keys.getFirstPart(keys.getFirstValid());
+
         size_t totalEntries = 0;
         MP4::CTTSEntry tmpEntry;
         tmpEntry.sampleCount = 0;
-        tmpEntry.sampleOffset = parts.getOffset(0);
+        tmpEntry.sampleOffset = parts.getOffset(firstPart);
 
         size_t sttsCounter = 0;
         MP4::STTSEntry sttsEntry;
         sttsEntry.sampleCount = 0;
-        sttsEntry.sampleDelta = parts.getSize(0);;
+        sttsEntry.sampleDelta = parts.getSize(firstPart);
 
         //Calculate amount of entries for CTTS/STTS boxes so we can set the last entry first
         //Our MP4 box implementations dynamically reallocate to fit the data you put inside them,
         //Which means setting the last entry first prevents constant reallocs and slowness.
         for (size_t part = 0; part < partCount; ++part){
-          uint64_t partOffset = parts.getOffset(part);
+          uint64_t partOffset = parts.getOffset(firstPart + part);
           if (partOffset != tmpEntry.sampleOffset){
             ++totalEntries;
             tmpEntry.sampleOffset = partOffset;
           }
-          uint64_t partDur = parts.getDuration(part);
+          uint64_t partDur = parts.getDuration(firstPart + part);
           if (partDur != sttsEntry.sampleDelta){
             ++sttsCounter;
             sttsEntry.sampleDelta = partDur;
@@ -562,14 +565,14 @@ namespace Mist{
         //Reset the values we just used, first.
         totalEntries = 0;
         tmpEntry.sampleCount = 0;
-        tmpEntry.sampleOffset = parts.getOffset(0);
+        tmpEntry.sampleOffset = parts.getOffset(firstPart);
         sttsCounter = 0;
         sttsEntry.sampleCount = 0;
-        sttsEntry.sampleDelta = parts.getDuration(0);
+        sttsEntry.sampleDelta = parts.getDuration(firstPart);
         
         bool isMeta = (tType == "meta");
         for (size_t part = 0; part < partCount; ++part){
-          uint64_t partDur = parts.getDuration(part);
+          uint64_t partDur = parts.getDuration(firstPart + part);
           if (sttsEntry.sampleDelta != partDur){
             // If the duration of this and previous part differ, write current values and reset
             sttsBox.setSTTSEntry(sttsEntry, sttsCounter++);
@@ -578,12 +581,12 @@ namespace Mist{
           }
           sttsEntry.sampleCount++;
 
-          uint64_t partSize = parts.getSize(part)+(isMeta?2:0);
+          uint64_t partSize = parts.getSize(firstPart + part)+(isMeta?2:0);
           stszBox.setEntrySize(partSize, part);
           size += partSize;
           
           if (hasCTTS){
-            uint64_t partOffset = parts.getOffset(part);
+            uint64_t partOffset = parts.getOffset(firstPart + part);
             if (partOffset != tmpEntry.sampleOffset){
               // If the offset of this and previous part differ, write current values and reset
               cttsBox.setCTTSEntry(tmpEntry, totalEntries++);
@@ -610,11 +613,10 @@ namespace Mist{
       if (tType == "video" && !fragmented){
         MP4::STSS stssBox(0);
         size_t tmpCount = 0;
-        DTSC::Keys keys(M.keys(it->first));
+        size_t firstKey = keys.getFirstValid();
-        uint32_t firstKey = keys.getFirstValid();
+        size_t endKey = keys.getEndValid();
-        uint32_t endKey = keys.getEndValid();
         for (size_t i = firstKey; i < endKey; ++i){
-          stssBox.setSampleNumber(tmpCount + 1, i);
+          stssBox.setSampleNumber(tmpCount + 1, i-firstKey);
           tmpCount += keys.getParts(i);
         }
         stblBox.setContent(stssBox, stblOffset++);
@@ -714,8 +716,11 @@ namespace Mist{
         if (prevVidTrack != INVALID_TRACK_ID && subIt->first == prevVidTrack){continue;}
         keyPart temp;
         temp.trackID = subIt->first;
-        temp.time = M.getFirstms(subIt->first);
+
-        temp.index = 0;
+        DTSC::Keys keys = M.getKeys(subIt->first);
+        temp.time = keys.getTime(keys.getFirstValid());
+        temp.index = keys.getFirstPart(keys.getFirstValid());
+        temp.firstIndex = temp.index;
         sortSet.insert(temp);
       }
       while (!sortSet.empty()){
@@ -727,16 +732,15 @@ namespace Mist{
         DTSC::Parts & parts = *tL.parts;
         // setting the right STCO size in the STCO box
         if (useLargeBoxes){// Re-using the previously defined boolean for speedup
-          tL.co64Box.setChunkOffset(dataOffset + dataSize, temp.index);
+          tL.co64Box.setChunkOffset(dataOffset + dataSize, temp.index - temp.firstIndex);
         }else{
-          tL.stcoBox.setChunkOffset(dataOffset + dataSize, temp.index);
+          tL.stcoBox.setChunkOffset(dataOffset + dataSize, temp.index - temp.firstIndex);
         }
         dataSize += parts.getSize(temp.index);
 
         if (M.getType(temp.trackID) == "meta"){dataSize += 2;}
-        // add next keyPart to sortSet
+        // add next keyPart to sortSet, if we have not yet reached the end time
-        if (temp.index + 1 < parts.getEndValid()){// Only create new element, when there are new
+        if (temp.time + parts.getDuration(temp.index) < M.getLastms(temp.trackID)){
-                                                    // elements to be added
           temp.time += parts.getDuration(temp.index);
           ++temp.index;
           sortSet.insert(temp);
@@ -762,7 +766,6 @@ namespace Mist{
   /// Calculate a seekPoint, based on byteStart, metadata, tracks and headerSize.
   /// The seekPoint will be set to the timestamp of the first packet to send.
   void OutMP4::findSeekPoint(uint64_t byteStart, uint64_t &seekPoint, uint64_t headerSize){
-    seekPoint = 0;
     // if we're starting in the header, seekPoint is always zero.
     if (byteStart <= headerSize){return;}
     // okay, we're past the header. Substract the headersize from the starting postion.
@@ -792,7 +795,7 @@ namespace Mist{
       // otherwise, set currPos to where we are now and continue
       currPos += partSize;
 
-      if (temp.index + 1 < parts.getEndValid()){// only insert when there are parts left
+      if (temp.time + parts.getDuration(temp.index) < M.getLastms(temp.trackID)){// only insert when there are parts left
         temp.time += parts.getDuration(temp.index);
         ++temp.index;
         sortSet.insert(temp);
@@ -924,9 +927,8 @@ namespace Mist{
 
     for (std::map<size_t, Comms::Users>::const_iterator subIt = userSelect.begin();
          subIt != userSelect.end(); subIt++){
-      DTSC::Keys keys(M.keys(subIt->first));
+      DTSC::Keys keys = M.getKeys(subIt->first);
       DTSC::Parts parts(M.parts(subIt->first));
-      DTSC::Fragments fragments(M.fragments(subIt->first));
 
       uint32_t startKey = M.getKeyIndexForTime(subIt->first, startFragmentTime);
       uint32_t endKey = M.getKeyIndexForTime(subIt->first, endFragmentTime) + 1;
@@ -1063,6 +1065,7 @@ namespace Mist{
   void OutMP4::respondHTTP(const HTTP::Parser & req, bool headersOnly){
     //Set global defaults, first
     HTTPOutput::respondHTTP(req, headersOnly);
+    initialSeek();
 
     H.SetHeader("Content-Type", "video/MP4");
     if (!M.getLive()){H.SetHeader("Accept-Ranges", "bytes, parsec");}
@@ -1077,50 +1080,13 @@ namespace Mist{
       return;
     }
 
-    DTSC::Fragments fragments(M.fragments(mainTrack));
-
-    if (req.GetVar("startfrag") != ""){
-      realTime = 0;
-      size_t startFrag = JSON::Value(req.GetVar("startfrag")).asInt();
-      if (startFrag >= fragments.getFirstValid() && startFrag < fragments.getEndValid()){
-        startTime = M.getTimeForFragmentIndex(mainTrack, startFrag);
-
-        // Set endTime to one fragment further, can receive override from next parameter check
-        if (startFrag + 1 < fragments.getEndValid()){
-          endTime = M.getTimeForFragmentIndex(mainTrack, startFrag + 1);
-        }else{
-          endTime = M.getLastms(mainTrack);
-        }
-
-      }else{
-        startTime = M.getLastms(mainTrack);
-      }
-    }
-
-    if (req.GetVar("endfrag") != ""){
-      size_t endFrag = JSON::Value(req.GetVar("endfrag")).asInt();
-      if (endFrag < fragments.getEndValid()){
-        endTime = M.getTimeForFragmentIndex(mainTrack, endFrag);
-      }else{
-        endTime = M.getLastms(mainTrack);
-      }
-    }
-
-    if (req.GetVar("starttime") != ""){
-      startTime = std::max((uint64_t)JSON::Value(req.GetVar("starttime")).asInt(), M.getFirstms(mainTrack));
-    }
-
-    if (req.GetVar("endtime") != ""){
-      endTime = std::min((uint64_t)JSON::Value(req.GetVar("endtime")).asInt(), M.getLastms(mainTrack));
-    }
-
     // Check if the url contains .3gp --> if yes, we will send a 3gp header
-    sending3GP = (H.url.find(".3gp") != std::string::npos);
+    sending3GP = (req.url.find(".3gp") != std::string::npos);
 
     fileSize = 0;
     headerSize = mp4HeaderSize(fileSize, M.getLive());
 
-    seekPoint = 0;
+    seekPoint = Output::startTime();
     // for live we use fragmented mode
     if (M.getLive()){fragSeqNum = 0;}
 
@@ -1129,8 +1095,9 @@ namespace Mist{
          subIt != userSelect.end(); subIt++){
       keyPart temp;
       temp.trackID = subIt->first;
-      temp.time = M.getFirstms(subIt->first);
+      DTSC::Keys keys = M.getKeys(subIt->first);
-      temp.index = 0;
+      temp.time = keys.getTime(keys.getFirstValid());
+      temp.index = keys.getFirstPart(keys.getFirstValid());
       sortSet.insert(temp);
     }
 
@@ -1314,7 +1281,7 @@ namespace Mist{
       keyPart temp = *sortSet.begin();
       sortSet.erase(sortSet.begin());
       currPos += parts.getSize(temp.index);
-      if (temp.index + 1 < parts.getEndValid()){// only insert when there are parts left
+      if (temp.time + parts.getDuration(temp.index) < M.getLastms(temp.trackID)){// only insert when there are parts left
         temp.time += parts.getDuration(temp.index);
         ++temp.index;
         sortSet.insert(temp);

src/output/output_mp4.h

@@ -15,6 +15,7 @@ namespace Mist{
     uint64_t time;
     uint64_t byteOffset; // Stores relative bpos for fragmented MP4
     uint64_t index;
+    uint64_t firstIndex;
     size_t sampleSize;
     uint16_t sampleDuration;
     uint16_t sampleOffset;

src/output/output_ts.cpp

@@ -216,10 +216,10 @@ namespace Mist{
     config->addOption("datatrack", opt);
   }
 
-  void OutTS::initialSeek(){
+  void OutTS::initialSeek(bool dryRun){
     // Adds passthrough support to the regular initialSeek function
     if (targetParams.count("passthrough")){selectAllTracks();}
-    Output::initialSeek();
+    Output::initialSeek(dryRun);
   }
 
   void OutTS::sendTS(const char *tsData, size_t len){

src/output/output_ts.h

@@ -9,7 +9,7 @@ namespace Mist{
     static void init(Util::Config *cfg);
     void sendTS(const char *tsData, size_t len = 188);
     static bool listenMode();
-    virtual void initialSeek();
+    virtual void initialSeek(bool dryRun = false);
     bool isReadyForPlay();
     void onRequest();
     std::string getConnectedHost();