Speed optimizes:

- MP4 output optimizations
- DTSC::Meta::getPageNumberForTime speedup
- RelAccX class speedup
- Generic output optimizations

lib/dtsc.cpp

@@ -2964,10 +2964,13 @@ namespace DTSC{
   /// If the timestamp is not available, returns the closest page number that is.
   size_t Meta::getPageNumberForTime(uint32_t idx, uint64_t time) const{
     const Util::RelAccX &pages = tracks.at(idx).pages;
-    size_t res = pages.getStartPos();
+    Util::RelAccXFieldData avail = pages.getFieldData("avail");
-    for (size_t i = pages.getStartPos(); i < pages.getEndPos(); ++i){
+    Util::RelAccXFieldData firsttime = pages.getFieldData("firsttime");
-      if (pages.getInt("avail", i) == 0){continue;}
+    uint32_t res = pages.getStartPos();
-      if (pages.getInt("firsttime", i) > time){break;}
+    uint64_t endPos = pages.getEndPos();
+    for (uint64_t i = res; i < endPos; ++i){
+      if (pages.getInt(avail, i) == 0){continue;}
+      if (pages.getInt(firsttime, i) > time){break;}
       res = i;
     }
     return pages.getInt("firstkey", res);

lib/mp4_generic.cpp

@@ -2359,14 +2359,9 @@ namespace MP4{
   uint32_t CTTS::getEntryCount(){return getInt32(4);}
 
   void CTTS::setCTTSEntry(CTTSEntry newCTTSEntry, uint32_t no){
-    if (no + 1 > getEntryCount()){
+    if (no + 1 > getEntryCount()){setEntryCount(no + 1);}
-      for (unsigned int i = getEntryCount(); i < no; i++){
-        setInt64(0, 8 + (i * 8)); // filling up undefined entries of 64 bits
-      }
-      setEntryCount(no + 1);
-    }
-    setInt32(newCTTSEntry.sampleCount, 8 + no * 8);
     setInt32(*(reinterpret_cast<uint32_t *>(&newCTTSEntry.sampleOffset)), 8 + (no * 8) + 4);
+    setInt32(newCTTSEntry.sampleCount, 8 + no * 8);
   }
 
   CTTSEntry CTTS::getCTTSEntry(uint32_t no){

lib/util.cpp

@@ -17,17 +17,17 @@
 #endif
 #include <stdlib.h>
 
-#define RECORD_POINTER p + getOffset() + (getRecordPosition(recordNo) * getRSize()) + fd.offset
 #define RAXHDR_FIELDOFFSET p[1]
-#define RAXHDR_RECORDCNT *(uint32_t *)(p + 2)
-#define RAXHDR_RECORDSIZE *(uint32_t *)(p + 6)
-#define RAXHDR_STARTPOS *(uint32_t *)(p + 10)
-#define RAXHDR_DELETED *(uint64_t *)(p + 14)
-#define RAXHDR_PRESENT *(uint32_t *)(p + 22)
-#define RAXHDR_OFFSET *(uint16_t *)(p + 26)
-#define RAXHDR_ENDPOS *(uint64_t *)(p + 28)
 #define RAX_REQDFIELDS_LEN 36
 
+// Converts the given record number into an offset of records after getOffset()'s offset.
+// Does no bounds checking whatsoever, allowing access to not-yet-created or already-deleted
+// records.
+// This access method is stable with changing start/end positions and present record counts,
+// because it only
+// depends on the record count, which may not change for ring buffers.
+#define RECORD_POINTER p + *hdrOffset + (((*hdrRecordCnt)?(recordNo % *hdrRecordCnt) : recordNo) * *hdrRecordSize) + fd.offset
+
 namespace Util{
   Util::DataCallback defaultDataCallback;
 
@@ -410,6 +410,13 @@ namespace Util{
       return;
     }
     p = data;
+    hdrRecordCnt = (uint32_t*)(p+2);
+    hdrRecordSize = (uint32_t*)(p+6);
+    hdrStartPos = (uint32_t*)(p+10);
+    hdrDeleted = (uint64_t*)(p+14);
+    hdrPresent = (uint32_t*)(p+22);
+    hdrOffset = (uint16_t*)(p+26);
+    hdrEndPos = (uint64_t*)(p+28);
     if (waitReady){
       while (!isReady()){Util::sleep(50);}
     }
@@ -459,28 +466,28 @@ namespace Util{
   }
 
   /// Gets the amount of records present in the structure.
-  uint32_t RelAccX::getRCount() const{return RAXHDR_RECORDCNT;}
+  uint32_t RelAccX::getRCount() const{return *hdrRecordCnt;}
 
   /// Gets the size in bytes of a single record in the structure.
-  uint32_t RelAccX::getRSize() const{return RAXHDR_RECORDSIZE;}
+  uint32_t RelAccX::getRSize() const{return *hdrRecordSize;}
 
   /// Gets the position in the records where the entries start
-  uint32_t RelAccX::getStartPos() const{return RAXHDR_STARTPOS;}
+  uint32_t RelAccX::getStartPos() const{return *hdrStartPos;}
 
   /// Gets the number of deleted records
-  uint64_t RelAccX::getDeleted() const{return RAXHDR_DELETED;}
+  uint64_t RelAccX::getDeleted() const{return *hdrDeleted;}
 
   /// Gets the number of records present
-  size_t RelAccX::getPresent() const{return RAXHDR_PRESENT;}
+  size_t RelAccX::getPresent() const{return *hdrPresent;}
 
   /// Gets the number of the last valid index
-  uint64_t RelAccX::getEndPos() const{return RAXHDR_ENDPOS;}
+  uint64_t RelAccX::getEndPos() const{return *hdrEndPos;}
 
   /// Gets the number of fields per recrd
   uint32_t RelAccX::getFieldCount() const{return fields.size();}
 
   /// Gets the offset from the structure start where records begin.
-  uint16_t RelAccX::getOffset() const{return *(uint16_t *)(p + 26);}
+  uint16_t RelAccX::getOffset() const{return *hdrOffset;}
 
   /// Returns true if the structure is ready for read operations.
   bool RelAccX::isReady() const{return p && (p[0] & 1);}
@@ -500,20 +507,6 @@ namespace Util{
     return true;
   }
 
-  /// Converts the given record number into an offset of records after getOffset()'s offset.
-  /// Does no bounds checking whatsoever, allowing access to not-yet-created or already-deleted
-  /// records.
-  /// This access method is stable with changing start/end positions and present record counts,
-  /// because it only
-  /// depends on the record count, which may not change for ring buffers.
-  uint32_t RelAccX::getRecordPosition(uint64_t recordNo) const{
-    if (getRCount()){
-      return recordNo % getRCount();
-    }else{
-      return recordNo;
-    }
-  }
-
   /// Returns the (max) size of the given field.
   /// For string types, returns the exact size excluding terminating null byte.
   /// For other types, returns the maximum size possible.
@@ -693,13 +686,11 @@ namespace Util{
     }
     // We now know for sure fLen is set
     // Get current offset and record size
-    uint16_t &offset = RAXHDR_OFFSET;
-    uint32_t &recSize = RAXHDR_RECORDSIZE;
     // The first field initializes the offset and record size.
     if (!fields.size()){
-      recSize = 0;                 // Nothing yet, this is the first data field.
+      *hdrRecordSize = 0;                 // Nothing yet, this is the first data field.
-      offset = RAX_REQDFIELDS_LEN; // All mandatory fields are first - so we start there.
+      *hdrOffset = RAX_REQDFIELDS_LEN; // All mandatory fields are first - so we start there.
-      RAXHDR_FIELDOFFSET = offset; // store the field_offset
+      RAXHDR_FIELDOFFSET = *hdrOffset; // store the field_offset
     }
     uint8_t typeLen = 1;
     // Check if fLen is a non-default value
@@ -711,36 +702,36 @@ namespace Util{
     }
     // store the details for internal use
     // recSize is the field offset, since we haven't updated it yet
-    fields[name] = RelAccXFieldData(fType, fLen, recSize);
+    fields[name] = RelAccXFieldData(fType, fLen, *hdrRecordSize);
 
     // write the data to memory
-    p[offset] = (name.size() << 3) | (typeLen & 0x7);
+    p[*hdrOffset] = (name.size() << 3) | (typeLen & 0x7);
-    memcpy(p + offset + 1, name.data(), name.size());
+    memcpy(p + (*hdrOffset) + 1, name.data(), name.size());
-    p[offset + 1 + name.size()] = fType;
+    p[(*hdrOffset) + 1 + name.size()] = fType;
-    if (typeLen == 2){*(uint8_t *)(p + offset + 2 + name.size()) = fLen;}
+    if (typeLen == 2){*(uint8_t *)(p + (*hdrOffset) + 2 + name.size()) = fLen;}
-    if (typeLen == 3){*(uint16_t *)(p + offset + 2 + name.size()) = fLen;}
+    if (typeLen == 3){*(uint16_t *)(p + (*hdrOffset) + 2 + name.size()) = fLen;}
-    if (typeLen == 5){*(uint32_t *)(p + offset + 2 + name.size()) = fLen;}
+    if (typeLen == 5){*(uint32_t *)(p + (*hdrOffset) + 2 + name.size()) = fLen;}
 
     // Calculate new offset and record size
-    offset += 1 + name.size() + typeLen;
+    *hdrOffset += 1 + name.size() + typeLen;
-    recSize += fLen;
+    *hdrRecordSize += fLen;
   }
 
   /// Sets the record counter to the given value.
-  void RelAccX::setRCount(uint32_t count){RAXHDR_RECORDCNT = count;}
+  void RelAccX::setRCount(uint32_t count){*hdrRecordCnt = count;}
 
   /// Sets the position in the records where the entries start
-  void RelAccX::setStartPos(uint32_t n){RAXHDR_STARTPOS = n;}
+  void RelAccX::setStartPos(uint32_t n){*hdrStartPos = n;}
 
   /// Sets the number of deleted records
-  void RelAccX::setDeleted(uint64_t n){RAXHDR_DELETED = n;}
+  void RelAccX::setDeleted(uint64_t n){*hdrDeleted = n;}
 
   /// Sets the number of records present
   /// Defaults to the record count if set to zero.
-  void RelAccX::setPresent(uint32_t n){RAXHDR_PRESENT = n;}
+  void RelAccX::setPresent(uint32_t n){*hdrPresent = n;}
 
   /// Sets the number of the last valid index
-  void RelAccX::setEndPos(uint64_t n){RAXHDR_ENDPOS = n;}
+  void RelAccX::setEndPos(uint64_t n){*hdrEndPos = n;}
 
   /// Sets the ready flag.
   /// After calling this function, addField() may no longer be called.
@@ -836,33 +827,27 @@ namespace Util{
   /// Updates the deleted record counter, the start position and the present record counter,
   /// shifting the ring buffer start position forward without moving the ring buffer end position.
   void RelAccX::deleteRecords(uint32_t amount){
-    uint32_t &startPos = RAXHDR_STARTPOS;
+    *hdrStartPos += amount;    // update start position
-    uint64_t &deletedRecs = RAXHDR_DELETED;
+    *hdrDeleted += amount; // update deleted record counter
-    uint32_t &recsPresent = RAXHDR_PRESENT;
+    if (*hdrPresent >= amount){
-    startPos += amount;    // update start position
+      *hdrPresent -= amount; // decrease records present
-    deletedRecs += amount; // update deleted record counter
-    if (recsPresent >= amount){
-      recsPresent -= amount; // decrease records present
     }else{
       WARN_MSG("Depleting recordCount!");
-      recsPresent = 0;
+      *hdrPresent = 0;
     }
   }
 
   /// Updates the present record counter, shifting the ring buffer end position forward without
   /// moving the ring buffer start position.
   void RelAccX::addRecords(uint32_t amount){
-    uint32_t &recsPresent = RAXHDR_PRESENT;
+    if ((*hdrPresent) + amount > *hdrRecordCnt){
-    uint32_t &recordsCount = RAXHDR_RECORDCNT;
-    uint64_t &recordEndPos = RAXHDR_ENDPOS;
-    if (recsPresent + amount > recordsCount){
       BACKTRACE;
-      WARN_MSG("Exceeding recordCount (%d [%d + %d] > %d)", recsPresent + amount, recsPresent, amount, recordsCount);
+      WARN_MSG("Exceeding recordCount (%d [%d + %d] > %d)", (*hdrPresent) + amount, *hdrPresent, amount, *hdrRecordCnt);
-      recsPresent = 0;
+      *hdrPresent = 0;
     }else{
-      recsPresent += amount;
+      *hdrPresent += amount;
     }
-    recordEndPos += amount;
+    *hdrEndPos += amount;
   }
 
   void RelAccX::minimalFrom(const RelAccX &src){

lib/util.h

@@ -145,7 +145,6 @@ namespace Util{
     bool isExit() const;
     bool isReload() const;
     bool isRecordAvailable(uint64_t recordNo) const;
-    uint32_t getRecordPosition(uint64_t recordNo) const;
     uint32_t getSize(const std::string &name, uint64_t recordNo = 0) const;
 
     char *getPointer(const std::string &name, uint64_t recordNo = 0) const;
@@ -186,6 +185,13 @@ namespace Util{
     std::map<std::string, RelAccXFieldData> fields;
 
   private:
+    uint32_t * hdrRecordCnt;
+    uint32_t * hdrRecordSize;
+    uint32_t * hdrStartPos;
+    uint64_t * hdrDeleted;
+    uint32_t * hdrPresent;
+    uint16_t * hdrOffset;
+    uint64_t * hdrEndPos;
     char *p;
   };
 

src/io.cpp

@@ -207,8 +207,10 @@ namespace Mist{
 
     Util::RelAccX &tPages = meta.pages(packTrack);
     size_t pageIdx = 0;
+    size_t currPagNum = curPageNum[packTrack];
+    Util::RelAccXFieldData firstkey = tPages.getFieldData("firstkey");
     for (uint64_t i = tPages.getDeleted(); i < tPages.getEndPos(); i++){
-      if (tPages.getInt("firstkey", i) == curPageNum[packTrack]){
+      if (tPages.getInt(firstkey, i) == currPagNum){
         pageIdx = i;
         break;
       }
@@ -219,7 +221,7 @@ namespace Mist{
     // Do nothing when there is not enough free space on the page to add the packet.
     if (pageSize - pageOffset < packDataLen){
       FAIL_MSG("Track %" PRIu32 "p%zu : Pack %" PRIu64 "ms of %" PRIu64 "b exceeds size %" PRIu64 " @ bpos %" PRIu64,
-               packTrack, curPageNum[packTrack], packTime, packDataLen, pageSize, pageOffset);
+               packTrack, currPagNum, packTime, packDataLen, pageSize, pageOffset);
       return;
     }
 

src/output/output.cpp

@@ -1482,18 +1482,6 @@ namespace Mist{
       return false;
     }
 
-    if (M.getPageNumberForTime(nxt.tid, nxt.time) != currentPage[nxt.tid]){
-      loadPageForKey(nxt.tid, M.getPageNumberForTime(nxt.tid, nxt.time));
-      nxt.offset = 0;
-      //Only read the next time if the page load succeeded and there is a packet to read from
-      if (curPage[nxt.tid].mapped && curPage[nxt.tid].mapped[0] == 'D'){
-        nxt.time = getDTSCTime(curPage[nxt.tid].mapped, 0);
-      }
-      buffer.erase(buffer.begin());
-      buffer.insert(nxt);
-      return false;
-    }
-
     // if we're going to read past the end of the data page, load the next page
     // this only happens for VoD
     if (nxt.offset >= curPage[nxt.tid].len ||
@@ -1502,6 +1490,17 @@ namespace Mist{
         dropTrack(nxt.tid, "end of VoD track reached", false);
         return false;
       }
+      if (M.getPageNumberForTime(nxt.tid, nxt.time) != currentPage[nxt.tid]){
+        loadPageForKey(nxt.tid, M.getPageNumberForTime(nxt.tid, nxt.time));
+        nxt.offset = 0;
+        //Only read the next time if the page load succeeded and there is a packet to read from
+        if (curPage[nxt.tid].mapped && curPage[nxt.tid].mapped[0] == 'D'){
+          nxt.time = getDTSCTime(curPage[nxt.tid].mapped, 0);
+        }
+        buffer.erase(buffer.begin());
+        buffer.insert(nxt);
+        return false;
+      }
       dropTrack(nxt.tid, "VoD page load failure");
       return false;
     }
@@ -1511,9 +1510,6 @@ namespace Mist{
 
     uint64_t nextTime = 0;
 
-    DTSC::Keys keys(M.keys(nxt.tid));
-    size_t thisKey = keys.getNumForTime(nxt.time);
-
     // Check if we have a next valid packet
     if (curPage[nxt.tid].len > nxt.offset+preLoad.getDataLen()+20 && memcmp(curPage[nxt.tid].mapped + nxt.offset + preLoad.getDataLen(), "\000\000\000\000", 4)){
       nextTime = getDTSCTime(curPage[nxt.tid].mapped, nxt.offset + preLoad.getDataLen());
@@ -1531,6 +1527,8 @@ namespace Mist{
         dropTrack(nxt.tid, "end of VoD track reached", false);
         return true;
       }
+      DTSC::Keys keys(M.keys(nxt.tid));
+      size_t thisKey = keys.getNumForTime(nxt.time);
       //Check if there exists a different page for the next key
       size_t nextKeyPage = M.getPageNumberForKey(nxt.tid, thisKey + 1);
       if (nextKeyPage != INVALID_KEY_NUM && nextKeyPage != currentPage[nxt.tid]){
@@ -1594,13 +1592,19 @@ namespace Mist{
       return false;
     }
 
+    //Update keynum only when the second flips over in the timestamp
+    //We do this because DTSC::Keys is pretty CPU-heavy
+    if (nxt.time / 1000 < nextTime/1000){
+      DTSC::Keys keys(M.keys(nxt.tid));
+      size_t thisKey = keys.getNumForTime(nxt.time);
+      userSelect[nxt.tid].setKeyNum(thisKey);
+    }
+
     // we assume the next packet is the next on this same page
     nxt.offset += thisPacket.getDataLen();
     nxt.time = nextTime;
     ++nxt.partIndex;
 
-    userSelect[nxt.tid].setKeyNum(thisKey);
-
     // exchange the current packet in the buffer for the next one
     buffer.erase(buffer.begin());
     buffer.insert(nxt);

src/output/output_mp4.cpp

@@ -20,6 +20,70 @@ namespace Mist{
     return result.str();
   }
 
+  SortSet::SortSet(){
+    entries = 0;
+    currBegin = 0;
+    hasBegin = false;
+  }
+
+  /// Finds the current beginning of the SortSet
+  void SortSet::findBegin(){
+    if (!entries){return;}
+    currBegin = 0;
+    hasBegin = false;
+    for (size_t i = 0; i < entries; ++i){
+      if (!hasBegin && avail[i]){
+        currBegin = i;
+        hasBegin = true;
+        continue;
+      }
+      if (avail[i] && ((keyPart*)(void*)ptr)[i] < ((keyPart*)(void*)ptr)[currBegin]){
+        currBegin = i;
+      }
+    }
+  }
+
+  /// Returns a reference to the current beginning of the SortSet
+  const keyPart & SortSet::begin(){
+    static const keyPart blank = {0, 0, 0, 0};
+    if (!hasBegin){return blank;}
+    return ((keyPart*)(void*)ptr)[currBegin];
+  }
+
+  /// Marks the current beginning of the SortSet as erased
+  void SortSet::erase(){
+    if (!hasBegin){return;}
+    avail[currBegin] = 0;
+    findBegin();
+  }
+
+  bool SortSet::empty(){return !hasBegin;}
+
+  void SortSet::insert(const keyPart & part){
+    size_t i = 0;
+    for (i = 0; i < entries; ++i){
+      if (!avail[i] || ((keyPart*)(void*)ptr)[i].trackID == part.trackID){
+        ((keyPart*)(void*)ptr)[i] = part;
+        avail[i] = 1;
+        if (!hasBegin || part < begin()){
+          currBegin = i;
+          hasBegin = true;
+        }
+        return;
+      }
+    }
+    entries = i+1;
+    ptr.allocate(sizeof(keyPart)*entries);
+    ((keyPart*)(void*)ptr)[i] = part;
+    avail.append("\001", 1);
+    if (!hasBegin || part < begin()){
+      currBegin = i;
+      hasBegin = true;
+    }
+  }
+
+
+
   std::string OutMP4::protectionHeader(size_t idx){
     std::string tmp = toUTF16(M.getPlayReady(idx));
     tmp.erase(0, 2); // remove first 2 characters
@@ -152,6 +216,7 @@ namespace Mist{
       firstms = std::min(firstms, M.getFirstms(it->first));
     }
     for (std::map<size_t, Comms::Users>::const_iterator it = userSelect.begin(); it != userSelect.end(); it++){
+      const std::string tType = M.getType(it->first);
       uint64_t tmpRes = 0;
       DTSC::Parts parts(M.parts(it->first));
       uint64_t partCount = parts.getValidCount();
@@ -173,7 +238,7 @@ namespace Mist{
       tmpRes += 16 + (fragmented ? 0 : (1 * 12)); // STSC <-- Currently 1 entry, but might become more complex in near future
 
       // Type-specific boxes
-      if (M.getType(it->first) == "video"){
+      if (tType == "video"){
         tmpRes += 20                                 // VMHD Box
                   + 16                               // STSD
                   + 86                               // AVC1
@@ -184,7 +249,7 @@ namespace Mist{
           tmpRes += 16 + (keys.getValidCount() * 4); // STSS
         }
       }
-      if (M.getType(it->first) == "audio"){
+      if (tType == "audio"){
         tmpRes += 16   // SMHD Box
                   + 16 // STSD
                   + 36 // MP4A
@@ -194,7 +259,7 @@ namespace Mist{
         }
       }
 
-      if (M.getType(it->first) == "meta"){
+      if (tType == "meta"){
         tmpRes += 12    // NMHD Box
                   + 16  // STSD
                   + 64; // tx3g Box
@@ -208,10 +273,11 @@ namespace Mist{
         uint64_t prevOffset = parts.getOffset(0);
         uint64_t cttsCount = 1;
         fileSize += parts.getSize(0);
+        bool isMeta = (tType == "meta");
         for (unsigned int part = 1; part < partCount; ++part){
           uint64_t partDur = parts.getDuration(part);
           uint64_t partOffset = parts.getOffset(part);
-          uint64_t partSize = parts.getSize(part);
+          uint64_t partSize = parts.getSize(part)+(isMeta?2:0);
           if (prevDur != partDur){
             prevDur = partDur;
             ++sttsCount;
@@ -245,31 +311,47 @@ namespace Mist{
     return res;
   }
 
-  ///\todo This function does not indicate errors anywhere... maybe fix this...
+  class trackLookup{
-  std::string OutMP4::mp4Header(uint64_t &size, int fragmented){
+    public:
+      trackLookup(){
+        parts = 0;
+      }
+      ~trackLookup(){
+        if (parts){delete parts;}
+      }
+      void init(const Util::RelAccX & relParts, MP4::STCO box){
+        parts = new DTSC::Parts(relParts);
+        stcoBox = box;
+      }
+      void init(const Util::RelAccX & relParts, MP4::CO64 box){
+        parts = new DTSC::Parts(relParts);
+        co64Box = box;
+      }
+      DTSC::Parts * parts;
+      MP4::STCO stcoBox;
+      MP4::CO64 co64Box;
+  };
+
+
+  bool OutMP4::mp4Header(Util::ResizeablePointer & headOut, uint64_t &size, int fragmented){
     uint32_t mainTrack = M.mainTrack();
-    if (mainTrack == INVALID_TRACK_ID){return "";}
+    if (mainTrack == INVALID_TRACK_ID){return false;}
     if (M.getLive()){needsLookAhead = 100;}
-    // Make sure we have a proper being value for the size...
+    // Clear size if it was set before the function was called, just in case
     size = 0;
-    // Stores the result of the function
-    std::stringstream header;
     // Determines whether the outputfile is larger than 4GB, in which case we need to use 64-bit
     // boxes for offsets
     bool useLargeBoxes = !fragmented && (estimateFileSize() > 0xFFFFFFFFull);
     // Keeps track of the total size of the mdat box
     uint64_t mdatSize = 0;
 
-    // Start actually creating the header
-
     // MP4 Files always start with an FTYP box. Constructor sets default values
     MP4::FTYP ftypBox;
     if (sending3GP){
       ftypBox.setMajorBrand("3gp6");
       ftypBox.setCompatibleBrands("3gp6", 3);
     }
-
+    headOut.append(ftypBox.asBox(), ftypBox.boxedSize());
-    header.write(ftypBox.asBox(), ftypBox.boxedSize());
 
     // Start building the moov box. This is the metadata box for an mp4 file, and will contain all
     // metadata.
@@ -422,55 +504,84 @@ namespace Mist{
         MP4::CTTS cttsBox;
         cttsBox.setVersion(0);
 
+        size_t totalEntries = 0;
         MP4::CTTSEntry tmpEntry;
         tmpEntry.sampleCount = 0;
         tmpEntry.sampleOffset = parts.getOffset(0);
 
-        std::deque<std::pair<size_t, size_t> > sttsCounter;
+        size_t sttsCounter = 0;
-        stszBox.setEntrySize(0, partCount - 1); // Speed up allocation
+        MP4::STTSEntry sttsEntry;
-        size_t totalEntries = 0;
+        sttsEntry.sampleCount = 0;
+        sttsEntry.sampleDelta = parts.getSize(0);;
 
+        //Calculate amount of entries for CTTS/STTS boxes so we can set the last entry first
+        //Our MP4 box implementations dynamically reallocate to fit the data you put inside them,
+        //Which means setting the last entry first prevents constant reallocs and slowness.
         for (size_t part = 0; part < partCount; ++part){
-          stats();
-
-          uint64_t partDur = parts.getDuration(part);
-          uint64_t partSize = parts.getSize(part);
           uint64_t partOffset = parts.getOffset(part);
-
+          if (partOffset != tmpEntry.sampleOffset){
-          // Create a new entry with current duration if EITHER there is no entry yet, or this parts
+            ++totalEntries;
-          // duration differs from the previous
+            tmpEntry.sampleOffset = partOffset;
-          if (!sttsCounter.size() || sttsCounter.rbegin()->second != partDur){
-            sttsCounter.push_back(std::pair<size_t, size_t>(0, partDur));
           }
-          // Update the counter
+          uint64_t partDur = parts.getDuration(part);
-          sttsCounter.rbegin()->first++;
+          if (partDur != sttsEntry.sampleDelta){
+            ++sttsCounter;
+            sttsEntry.sampleDelta = partDur;
+          }
+        }
 
-          if (M.getType(it->first) == "meta"){partSize += 2;}
+        //Set temporary last entry for CTTS box
+        bool hasCTTS = (totalEntries || tmpEntry.sampleOffset);
+        if (hasCTTS){
+          cttsBox.setCTTSEntry(tmpEntry, totalEntries);
+        }
+        //Set temporary last entry for STTS box
+        sttsBox.setSTTSEntry(sttsEntry, sttsCounter-1);
+        //Set temporary last entry for STSZ box
+        stszBox.setEntrySize(0, partCount - 1);
 
+        //All set! Now we can do everything for real.
+        //Reset the values we just used, first.
+        totalEntries = 0;
+        tmpEntry.sampleCount = 0;
+        tmpEntry.sampleOffset = parts.getOffset(0);
+        sttsCounter = 0;
+        sttsEntry.sampleCount = 0;
+        sttsEntry.sampleDelta = parts.getDuration(0);
+        
+        bool isMeta = (tType == "meta");
+        for (size_t part = 0; part < partCount; ++part){
+          uint64_t partDur = parts.getDuration(part);
+          if (sttsEntry.sampleDelta != partDur){
+            // If the duration of this and previous part differ, write current values and reset
+            sttsBox.setSTTSEntry(sttsEntry, sttsCounter++);
+            sttsEntry.sampleCount = 0;
+            sttsEntry.sampleDelta = partDur;
+          }
+          sttsEntry.sampleCount++;
+
+          uint64_t partSize = parts.getSize(part)+(isMeta?2:0);
           stszBox.setEntrySize(partSize, part);
           size += partSize;
           
-          if (partOffset != tmpEntry.sampleOffset){
+          if (hasCTTS){
-            // If the offset of this and previous part differ, write current values and reset
+            uint64_t partOffset = parts.getOffset(part);
-            cttsBox.setCTTSEntry(tmpEntry,
+            if (partOffset != tmpEntry.sampleOffset){
-                                 totalEntries++); // Rewrite for sanity. index FIRST, value SECOND
+              // If the offset of this and previous part differ, write current values and reset
-            tmpEntry.sampleCount = 0;
+              cttsBox.setCTTSEntry(tmpEntry, totalEntries++);
-            tmpEntry.sampleOffset = partOffset;
+              tmpEntry.sampleCount = 0;
+              tmpEntry.sampleOffset = partOffset;
+            }
+            tmpEntry.sampleCount++;
           }
-          tmpEntry.sampleCount++;
         }
+        //Write last entry for STTS
+        sttsBox.setSTTSEntry(sttsEntry, sttsCounter);
 
-        MP4::STTSEntry sttsEntry;
+        //Only add the CTTS box to the STBL box if we had any entries in it
-        sttsBox.setSTTSEntry(sttsEntry, sttsCounter.size() - 1);
+        if (hasCTTS){
-        size_t sttsIdx = 0;
+          //Write last entry for CTTS
-        for (std::deque<std::pair<size_t, size_t> >::iterator it2 = sttsCounter.begin();
+          cttsBox.setCTTSEntry(tmpEntry, totalEntries);
-             it2 != sttsCounter.end(); it2++){
-          sttsEntry.sampleCount = it2->first;
-          sttsEntry.sampleDelta = it2->second;
-          sttsBox.setSTTSEntry(sttsEntry, sttsIdx++);
-        }
-        if (totalEntries || tmpEntry.sampleOffset){
-          cttsBox.setCTTSEntry(tmpEntry, totalEntries++);
           stblBox.setContent(cttsBox, stblOffset++);
         }
       }
@@ -485,7 +596,7 @@ namespace Mist{
         uint32_t firstKey = keys.getFirstValid();
         uint32_t endKey = keys.getEndValid();
         for (size_t i = firstKey; i < endKey; ++i){
-          stssBox.setSampleNumber(tmpCount + 1, i); /// rewrite this for sanity.... SHOULD be: index FIRST, value SECOND
+          stssBox.setSampleNumber(tmpCount + 1, i);
           tmpCount += keys.getParts(i);
         }
         stblBox.setContent(stssBox, stblOffset++);
@@ -557,17 +668,18 @@ namespace Mist{
       // initial offset length ftyp, length moov + 8
       uint64_t dataOffset = ftypBox.boxedSize() + moovBox.boxedSize() + 8;
 
-      std::map<size_t, MP4::STCO> checkStcoBoxes;
+
-      std::map<size_t, MP4::CO64> checkCO64Boxes;
+      QuickMap<trackLookup> trackMap;
 
       std::deque<MP4::TRAK> trak = moovBox.getChildren<MP4::TRAK>();
       for (std::deque<MP4::TRAK>::iterator trakIt = trak.begin(); trakIt != trak.end(); trakIt++){
         size_t idx = trakIt->getChild<MP4::TKHD>().getTrackID() - 1;
         MP4::STBL stblBox = trakIt->getChild<MP4::MDIA>().getChild<MP4::MINF>().getChild<MP4::STBL>();
+        trackMap.insert(idx, trackLookup());
         if (useLargeBoxes){
-          checkCO64Boxes.insert(std::pair<size_t, MP4::CO64>(idx, stblBox.getChild<MP4::CO64>()));
+          trackMap.get(idx).init(M.parts(idx), stblBox.getChild<MP4::CO64>());
         }else{
-          checkStcoBoxes.insert(std::pair<size_t, MP4::STCO>(idx, stblBox.getChild<MP4::STCO>()));
+          trackMap.get(idx).init(M.parts(idx), stblBox.getChild<MP4::STCO>());
         }
       }
 
@@ -576,27 +688,27 @@ namespace Mist{
       // Keep track of the current size of the data within the mdat
       uint64_t dataSize = 0;
       // Current values are actual byte offset without header-sized offset
-      std::set<keyPart> sortSet; // filling sortset for interleaving parts
+      SortSet sortSet; // filling sortset for interleaving parts
       for (std::map<size_t, Comms::Users>::const_iterator subIt = userSelect.begin();
            subIt != userSelect.end(); subIt++){
         keyPart temp;
         temp.trackID = subIt->first;
         temp.time = M.getFirstms(subIt->first);
         temp.index = 0;
-        INFO_MSG("adding to sortSet: tid %zu time %" PRIu64, subIt->first, temp.time);
         sortSet.insert(temp);
       }
       while (!sortSet.empty()){
         stats();
-        keyPart temp = *sortSet.begin();
+        keyPart temp = sortSet.begin();
-        sortSet.erase(sortSet.begin());
+        trackLookup & tL = trackMap.get(temp.trackID);
+        sortSet.erase();
 
-        DTSC::Parts parts(M.parts(temp.trackID));
+        DTSC::Parts & parts = *tL.parts;
         // setting the right STCO size in the STCO box
         if (useLargeBoxes){// Re-using the previously defined boolean for speedup
-          checkCO64Boxes[temp.trackID].setChunkOffset(dataOffset + dataSize, temp.index);
+          tL.co64Box.setChunkOffset(dataOffset + dataSize, temp.index);
         }else{
-          checkStcoBoxes[temp.trackID].setChunkOffset(dataOffset + dataSize, temp.index);
+          tL.stcoBox.setChunkOffset(dataOffset + dataSize, temp.index);
         }
         dataSize += parts.getSize(temp.index);
 
@@ -613,17 +725,17 @@ namespace Mist{
       ///\todo Update for when mdat box exceeds 4GB
       mdatSize = dataSize + 8; //+8 for mp4 header
     }
-    header << std::string(moovBox.asBox(), moovBox.boxedSize());
+    headOut.append(moovBox.asBox(), moovBox.boxedSize());
 
     if (!fragmented){// if we are making a non fragmented MP4 and there are parts
       char mdatHeader[8] ={0x00, 0x00, 0x00, 0x00, 'm', 'd', 'a', 't'};
 
       if (mdatSize < 0xFFFFFFFF){Bit::htobl(mdatHeader, mdatSize);}
-      header.write(mdatHeader, 8);
+      headOut.append(mdatHeader, 8);
     }
-    size += header.str().size();
+    size += headOut.size();
-    if (fragmented){realBaseOffset = header.str().size();}
+    if (fragmented){realBaseOffset = headOut.size();}
-    return header.str();
+    return true;
   }
 
   /// Calculate a seekPoint, based on byteStart, metadata, tracks and headerSize.
@@ -970,10 +1082,14 @@ namespace Mist{
         if (rangeType == 'p'){
           H.SetBody("Starsystem not in communications range");
           H.SendResponse("416", "Starsystem not in communications range", myConn);
+          parseData = false;
+          wantRequest = true;
           return;
         }else{
           H.SetBody("Requested Range Not Satisfiable");
           H.SendResponse("416", "Requested Range Not Satisfiable", myConn);
+          parseData = false;
+          wantRequest = true;
           return;
         }
       }else{
@@ -992,10 +1108,15 @@ namespace Mist{
     if (byteStart < headerSize){
       // For storing the header.
       if ((!startTime && endTime == 0xffffffffffffffffull) || (endTime == 0)){
-        std::string headerData = mp4Header(fileSize, M.getLive());
+        Util::ResizeablePointer headerData;
+        if (!mp4Header(headerData, fileSize, M.getLive())){
+          FAIL_MSG("Could not generate MP4 header!");
+          H.SetBody("Error while generating MP4 header");
+          H.SendResponse("500", "Error generating MP4 header", myConn);
+          return;
+        }
         INFO_MSG("Have %zu bytes, sending %zu bytes", headerData.size(), std::min(headerSize, byteEnd) - byteStart);
-        H.Chunkify(headerData.data() + byteStart, std::min(headerSize, byteEnd) - byteStart,
+        H.Chunkify(headerData + byteStart, std::min(headerSize, byteEnd) - byteStart, myConn);
-                   myConn); // send MP4 header
         leftOver -= std::min(headerSize, byteEnd) - byteStart;
       }
     }
@@ -1097,9 +1218,6 @@ namespace Mist{
     if (!sortSet.empty()){
       keyPart temp = *sortSet.begin();
       sortSet.erase(sortSet.begin());
-
-      DTSC::Parts parts(M.parts(temp.trackID));
-
       currPos += parts.getSize(temp.index);
       if (temp.index + 1 < parts.getEndValid()){// only insert when there are parts left
         temp.time += parts.getDuration(temp.index);

src/output/output_mp4.h

@@ -17,6 +17,63 @@ namespace Mist{
     uint64_t index;
   };
 
+  class SortSet{
+  private:
+    Util::ResizeablePointer ptr;
+    Util::ResizeablePointer avail;
+    size_t entries;
+    size_t currBegin;
+    void findBegin();
+    bool hasBegin;
+  public:
+    SortSet();
+    const keyPart & begin();
+    void erase();
+    bool empty();
+    void insert(const keyPart & part);
+  };
+
+  /// Class that implements a tiny subset of std::map, optimized for speed for our type of usage.
+  template <class T> class QuickMap{
+  private:
+    Util::ResizeablePointer ptr;
+    size_t entries;
+  public:
+    QuickMap(){
+      entries = 0;
+    }
+    ~QuickMap(){
+      size_t len = 8 + sizeof(T*);
+      for (size_t i = 0; i < entries; ++i){
+        delete *(T**)(void*)(ptr+len*i+8);
+      }
+    }
+    T & get(uint64_t idx){
+      static T blank;
+      size_t len = 8 + sizeof(T*);
+      for (size_t i = 0; i < entries; ++i){
+        if (*((uint64_t*)(void*)(ptr+len*i)) == idx){
+          return **(T**)(void*)(ptr+len*i+8);
+        }
+      }
+      return blank;
+    }
+    void insert(uint64_t idx, T elem){
+      size_t i = 0;
+      size_t len = 8 + sizeof(T*);
+      for (i = 0; i < entries; ++i){
+        if (*((uint64_t*)(void*)(ptr+len*i)) == idx){
+          *(T**)(void*)(ptr+len*i+8) = new T(elem);
+          return;
+        }
+      }
+      entries = i+1;
+      ptr.allocate(len*entries);
+      *(T**)(void*)(ptr+len*i+8) = new T(elem);
+      *((uint64_t*)(void*)(ptr+len*i)) = idx;
+    }
+  };
+
   struct fragSet{
     uint64_t firstPart;
     uint64_t lastPart;
@@ -31,7 +88,7 @@ namespace Mist{
     static void init(Util::Config *cfg);
 
     uint64_t mp4HeaderSize(uint64_t &fileSize, int fragmented = 0) const;
-    std::string mp4Header(uint64_t &size, int fragmented = 0);
+    bool mp4Header(Util::ResizeablePointer & headOut, uint64_t &size, int fragmented = 0);
 
     uint64_t mp4moofSize(uint64_t startFragmentTime, uint64_t endFragmentTime, uint64_t &mdatSize) const;
     virtual void sendFragmentHeaderTime(uint64_t startFragmentTime,