diff --git a/lib/dtsc.h b/lib/dtsc.h
index e09d44ee..7f464545 100644
--- a/lib/dtsc.h
+++ b/lib/dtsc.h
@@ -271,6 +271,7 @@ namespace DTSC {
       std::deque<unsigned long> keySizes;
       std::deque<Part> parts;
       Key & getKey(unsigned int keyNum);
+      Fragment & getFrag(unsigned int fragNum);
       unsigned int timeToKeynum(unsigned int timestamp);
       unsigned int timeToFragnum(unsigned int timestamp);
       void reset();
@@ -297,8 +298,11 @@ namespace DTSC {
       int width;
       int height;
       int fpks;
+      void removeFirstKey();
+      uint32_t secsSinceFirstFragmentInsert();
     private:
       std::string cachedIdent;
+      std::deque<uint32_t> fragInsertTime;
   };
 
   ///\brief Class for storage of meta data
diff --git a/lib/dtscmeta.cpp b/lib/dtscmeta.cpp
index d2c1b371..4071cf9b 100644
--- a/lib/dtscmeta.cpp
+++ b/lib/dtscmeta.cpp
@@ -1179,6 +1179,8 @@ namespace DTSC {
         newFrag.setDuration(0);
         newFrag.setSize(0);
         fragments.push_back(newFrag);
+        //We set the insert time lastms-firstms in the future, to prevent unstable playback
+        fragInsertTime.push_back(Util::bootSecs() + ((lastms - firstms)/1000));
       } else {
         Fragment & lastFrag = fragments[fragments.size() - 1];
         lastFrag.setLength(lastFrag.getLength() + 1);
@@ -1188,6 +1190,41 @@ namespace DTSC {
     (*keySizes.rbegin()) += packSendSize;
     fragments.rbegin()->setSize(fragments.rbegin()->getSize() + packDataSize);
   }
+
+  /// Removes the first buffered key, including any fragments it was part of
+  void Track::removeFirstKey(){
+    HIGH_MSG("Erasing key %d:%lu", trackID, keys[0].getNumber());
+    //remove all parts of this key
+    for (int i = 0; i < keys[0].getParts(); i++) {
+      parts.pop_front();
+    }
+    //remove the key itself
+    keys.pop_front();
+    keySizes.pop_front();
+    //update firstms
+    firstms = keys[0].getTime();
+    //delete any fragments no longer fully buffered
+    while (fragments[0].getNumber() < keys[0].getNumber()) {
+      fragments.pop_front();
+      fragInsertTime.pop_front();
+      //and update the missed fragment counter
+      ++missedFrags;
+    }
+  }
+
+  /// Returns the amount of whole seconds since the first fragment was inserted into the buffer.
+  /// This assumes playback from the start of the buffer at time of insert, meaning that
+  /// the time is offset by that difference. E.g.: if a buffer is 50s long, the newest fragment
+  /// will have a value of 0 until 50s have passed, after which it will increase at a rate of
+  /// 1 per second.
+  uint32_t Track::secsSinceFirstFragmentInsert(){
+    uint32_t bs = Util::bootSecs();
+    if (bs > fragInsertTime.front()){
+      return bs - fragInsertTime.front();
+    }else{
+      return 0;
+    }
+  }
   
   void Track::finalize(){
     keys.rbegin()->setLength(lastms - keys.rbegin()->getTime() + parts.rbegin()->getDuration());
@@ -1216,6 +1253,7 @@ namespace DTSC {
     return keys[keyNum - keys[0].getNumber()];
   }
 
+  /// Returns the number of the key containing timestamp, or last key if nowhere.
   unsigned int Track::timeToKeynum(unsigned int timestamp){
     unsigned int result = 0;
     for (std::deque<Key>::iterator it = keys.begin(); it != keys.end(); it++){
@@ -1227,13 +1265,12 @@ namespace DTSC {
     return result;
   }
 
+  /// Gets indice of the fragment containing timestamp, or last fragment if nowhere.
   unsigned int Track::timeToFragnum(unsigned int timestamp){
-    unsigned long long int totalTime = firstms;
     for (unsigned int i = 0; i<fragments.size(); i++){
-      if (timestamp <= totalTime){
+      if (timestamp <= getKey(fragments[i].getNumber()).getTime() + fragments[i].getDuration()){
         return i;
       }
-      totalTime += fragments[i].getDuration();
     }
     return fragments.size()-1;
   }
@@ -1241,6 +1278,7 @@ namespace DTSC {
   ///\brief Resets a track, clears all meta values
   void Track::reset() {
     fragments.clear();
+    fragInsertTime.clear();
     parts.clear();
     keySizes.clear();
     keys.clear();
diff --git a/src/input/input_buffer.cpp b/src/input/input_buffer.cpp
index 574198e1..c70e83f2 100644
--- a/src/input/input_buffer.cpp
+++ b/src/input/input_buffer.cpp
@@ -207,48 +207,50 @@ namespace Mist {
     liveMeta.post();
   }
 
+  ///Checks if removing a key from this track is allowed/safe, and if so, removes it.
+  ///Returns true if a key was actually removed, false otherwise
+  ///Aborts if any of the following conditions are true (while active):
+  /// * no keys present
+  /// * not at least 4 whole fragments present
+  /// * first fragment hasn't been at least lastms-firstms ms in buffer
+  /// * less than 8 times the biggest fragment duration is buffered
+  /// If a key was deleted and the first buffered data page is no longer used, it is deleted also.
   bool inputBuffer::removeKey(unsigned int tid) {
     DTSC::Track & Trk = myMeta.tracks[tid];
-    //Make sure we have at least 3 whole fragments at all times,
-    //unless we're shutting down the whole buffer right now
-    if (Trk.fragments.size() < 5 && config->is_active) {
+    //If this track is empty, abort
+    if (!Trk.keys.size()) {
       return false;
     }
-    //If we're shutting down, and this track is empty, abort
-    if (!myMeta.tracks[tid].keys.size()) {
-      return false;
-    }
-    if (config->is_active && Trk.fragments.size() > 2){
-      ///Make sure we have at least 8X the target duration.
-      //The target duration is the biggest fragment, rounded up to whole seconds.
-      uint32_t targetDuration = (Trk.biggestFragment() / 1000 + 1) * 1000;
-      //The start is the third fragment's begin
-      uint32_t fragStart = Trk.getKey((++(++Trk.fragments.begin()))->getNumber()).getTime();
-      //The end is the last fragment's begin
-      uint32_t fragEnd = Trk.getKey(Trk.fragments.rbegin()->getNumber()).getTime();
-      if ((fragEnd - fragStart) < targetDuration * 8){
+    //the following checks only run if we're not shutting down
+    if (config->is_active){
+      //Make sure we have at least 4 whole fragments at all times,
+      if (Trk.fragments.size() < 5) {
         return false;
       }
+      //ensure we have each fragment buffered for at least the whole bufferTime
+      if (!Trk.secsSinceFirstFragmentInsert() || (Trk.lastms - Trk.firstms) < bufferTime){
+        return false;
+      }
+      if (Trk.fragments.size() > 2){
+        ///Make sure we have at least 8X the target duration.
+        //The target duration is the biggest fragment, rounded up to whole seconds.
+        uint32_t targetDuration = (Trk.biggestFragment() / 1000 + 1) * 1000;
+        //The start is the third fragment's begin
+        uint32_t fragStart = Trk.getKey((++(++Trk.fragments.begin()))->getNumber()).getTime();
+        //The end is the last fragment's begin
+        uint32_t fragEnd = Trk.getKey(Trk.fragments.rbegin()->getNumber()).getTime();
+        if ((fragEnd - fragStart) < targetDuration * 8){
+          return false;
+        }
+      }
     }
-    HIGH_MSG("Erasing key %d:%lu", tid, myMeta.tracks[tid].keys[0].getNumber());
-    //remove all parts of this key
-    for (int i = 0; i < myMeta.tracks[tid].keys[0].getParts(); i++) {
-      myMeta.tracks[tid].parts.pop_front();
-    }
-    //remove the key itself
-    myMeta.tracks[tid].keys.pop_front();
-    myMeta.tracks[tid].keySizes.pop_front();
-    //re-calculate firstms
-    myMeta.tracks[tid].firstms = myMeta.tracks[tid].keys[0].getTime();
-    //delete the fragment if it's no longer fully buffered
-    if (myMeta.tracks[tid].fragments[0].getNumber() < myMeta.tracks[tid].keys[0].getNumber()) {
-      myMeta.tracks[tid].fragments.pop_front();
-      myMeta.tracks[tid].missedFrags ++;
-    }
+    //Alright, everything looks good, let's delete the key and possibly also fragment
+    Trk.removeFirstKey();
     //if there is more than one page buffered for this track...
     if (bufferLocations[tid].size() > 1) {
       //Check if the first key starts on the second page or higher
-      if (myMeta.tracks[tid].keys[0].getNumber() >= (++(bufferLocations[tid].begin()))->first || !config->is_active){
+      if (Trk.keys[0].getNumber() >= (++(bufferLocations[tid].begin()))->first || !config->is_active){
+        //If so, we can delete the first page entirely
         HIGH_MSG("Erasing track %d, keys %lu-%lu from buffer", tid, bufferLocations[tid].begin()->first, bufferLocations[tid].begin()->first + bufferLocations[tid].begin()->second.keyNum - 1);
         bufferRemove(tid, bufferLocations[tid].begin()->first);
 
@@ -382,6 +384,7 @@ namespace Mist {
         }
       }
       //Buffer size management
+      /// \TODO Make sure data has been in the buffer for at least bufferTime after it goes in
       while (it->second.keys.size() > 1 && (it->second.lastms - it->second.keys[1].getTime()) > bufferTime) {
         if (!removeKey(it->first)) {
           break;