diff --git a/lib/dtsc.cpp b/lib/dtsc.cpp index 86cb700c..764c07b5 100644 --- a/lib/dtsc.cpp +++ b/lib/dtsc.cpp @@ -2964,10 +2964,13 @@ namespace DTSC{ /// If the timestamp is not available, returns the closest page number that is. size_t Meta::getPageNumberForTime(uint32_t idx, uint64_t time) const{ const Util::RelAccX &pages = tracks.at(idx).pages; - size_t res = pages.getStartPos(); - for (size_t i = pages.getStartPos(); i < pages.getEndPos(); ++i){ - if (pages.getInt("avail", i) == 0){continue;} - if (pages.getInt("firsttime", i) > time){break;} + Util::RelAccXFieldData avail = pages.getFieldData("avail"); + Util::RelAccXFieldData firsttime = pages.getFieldData("firsttime"); + uint32_t res = pages.getStartPos(); + uint64_t endPos = pages.getEndPos(); + for (uint64_t i = res; i < endPos; ++i){ + if (pages.getInt(avail, i) == 0){continue;} + if (pages.getInt(firsttime, i) > time){break;} res = i; } return pages.getInt("firstkey", res); diff --git a/lib/mp4_generic.cpp b/lib/mp4_generic.cpp index 62bc9714..774c7460 100644 --- a/lib/mp4_generic.cpp +++ b/lib/mp4_generic.cpp @@ -2359,14 +2359,9 @@ namespace MP4{ uint32_t CTTS::getEntryCount(){return getInt32(4);} void CTTS::setCTTSEntry(CTTSEntry newCTTSEntry, uint32_t no){ - if (no + 1 > getEntryCount()){ - for (unsigned int i = getEntryCount(); i < no; i++){ - setInt64(0, 8 + (i * 8)); // filling up undefined entries of 64 bits - } - setEntryCount(no + 1); - } - setInt32(newCTTSEntry.sampleCount, 8 + no * 8); + if (no + 1 > getEntryCount()){setEntryCount(no + 1);} setInt32(*(reinterpret_cast(&newCTTSEntry.sampleOffset)), 8 + (no * 8) + 4); + setInt32(newCTTSEntry.sampleCount, 8 + no * 8); } CTTSEntry CTTS::getCTTSEntry(uint32_t no){ diff --git a/lib/util.cpp b/lib/util.cpp index 6116fa9e..4d1114d6 100644 --- a/lib/util.cpp +++ b/lib/util.cpp @@ -17,17 +17,17 @@ #endif #include -#define RECORD_POINTER p + getOffset() + (getRecordPosition(recordNo) * getRSize()) + fd.offset #define RAXHDR_FIELDOFFSET p[1] -#define RAXHDR_RECORDCNT *(uint32_t *)(p + 2) -#define RAXHDR_RECORDSIZE *(uint32_t *)(p + 6) -#define RAXHDR_STARTPOS *(uint32_t *)(p + 10) -#define RAXHDR_DELETED *(uint64_t *)(p + 14) -#define RAXHDR_PRESENT *(uint32_t *)(p + 22) -#define RAXHDR_OFFSET *(uint16_t *)(p + 26) -#define RAXHDR_ENDPOS *(uint64_t *)(p + 28) #define RAX_REQDFIELDS_LEN 36 +// Converts the given record number into an offset of records after getOffset()'s offset. +// Does no bounds checking whatsoever, allowing access to not-yet-created or already-deleted +// records. +// This access method is stable with changing start/end positions and present record counts, +// because it only +// depends on the record count, which may not change for ring buffers. +#define RECORD_POINTER p + *hdrOffset + (((*hdrRecordCnt)?(recordNo % *hdrRecordCnt) : recordNo) * *hdrRecordSize) + fd.offset + namespace Util{ Util::DataCallback defaultDataCallback; @@ -410,6 +410,13 @@ namespace Util{ return; } p = data; + hdrRecordCnt = (uint32_t*)(p+2); + hdrRecordSize = (uint32_t*)(p+6); + hdrStartPos = (uint32_t*)(p+10); + hdrDeleted = (uint64_t*)(p+14); + hdrPresent = (uint32_t*)(p+22); + hdrOffset = (uint16_t*)(p+26); + hdrEndPos = (uint64_t*)(p+28); if (waitReady){ while (!isReady()){Util::sleep(50);} } @@ -459,28 +466,28 @@ namespace Util{ } /// Gets the amount of records present in the structure. - uint32_t RelAccX::getRCount() const{return RAXHDR_RECORDCNT;} + uint32_t RelAccX::getRCount() const{return *hdrRecordCnt;} /// Gets the size in bytes of a single record in the structure. - uint32_t RelAccX::getRSize() const{return RAXHDR_RECORDSIZE;} + uint32_t RelAccX::getRSize() const{return *hdrRecordSize;} /// Gets the position in the records where the entries start - uint32_t RelAccX::getStartPos() const{return RAXHDR_STARTPOS;} + uint32_t RelAccX::getStartPos() const{return *hdrStartPos;} /// Gets the number of deleted records - uint64_t RelAccX::getDeleted() const{return RAXHDR_DELETED;} + uint64_t RelAccX::getDeleted() const{return *hdrDeleted;} /// Gets the number of records present - size_t RelAccX::getPresent() const{return RAXHDR_PRESENT;} + size_t RelAccX::getPresent() const{return *hdrPresent;} /// Gets the number of the last valid index - uint64_t RelAccX::getEndPos() const{return RAXHDR_ENDPOS;} + uint64_t RelAccX::getEndPos() const{return *hdrEndPos;} /// Gets the number of fields per recrd uint32_t RelAccX::getFieldCount() const{return fields.size();} /// Gets the offset from the structure start where records begin. - uint16_t RelAccX::getOffset() const{return *(uint16_t *)(p + 26);} + uint16_t RelAccX::getOffset() const{return *hdrOffset;} /// Returns true if the structure is ready for read operations. bool RelAccX::isReady() const{return p && (p[0] & 1);} @@ -500,20 +507,6 @@ namespace Util{ return true; } - /// Converts the given record number into an offset of records after getOffset()'s offset. - /// Does no bounds checking whatsoever, allowing access to not-yet-created or already-deleted - /// records. - /// This access method is stable with changing start/end positions and present record counts, - /// because it only - /// depends on the record count, which may not change for ring buffers. - uint32_t RelAccX::getRecordPosition(uint64_t recordNo) const{ - if (getRCount()){ - return recordNo % getRCount(); - }else{ - return recordNo; - } - } - /// Returns the (max) size of the given field. /// For string types, returns the exact size excluding terminating null byte. /// For other types, returns the maximum size possible. @@ -693,13 +686,11 @@ namespace Util{ } // We now know for sure fLen is set // Get current offset and record size - uint16_t &offset = RAXHDR_OFFSET; - uint32_t &recSize = RAXHDR_RECORDSIZE; // The first field initializes the offset and record size. if (!fields.size()){ - recSize = 0; // Nothing yet, this is the first data field. - offset = RAX_REQDFIELDS_LEN; // All mandatory fields are first - so we start there. - RAXHDR_FIELDOFFSET = offset; // store the field_offset + *hdrRecordSize = 0; // Nothing yet, this is the first data field. + *hdrOffset = RAX_REQDFIELDS_LEN; // All mandatory fields are first - so we start there. + RAXHDR_FIELDOFFSET = *hdrOffset; // store the field_offset } uint8_t typeLen = 1; // Check if fLen is a non-default value @@ -711,36 +702,36 @@ namespace Util{ } // store the details for internal use // recSize is the field offset, since we haven't updated it yet - fields[name] = RelAccXFieldData(fType, fLen, recSize); + fields[name] = RelAccXFieldData(fType, fLen, *hdrRecordSize); // write the data to memory - p[offset] = (name.size() << 3) | (typeLen & 0x7); - memcpy(p + offset + 1, name.data(), name.size()); - p[offset + 1 + name.size()] = fType; - if (typeLen == 2){*(uint8_t *)(p + offset + 2 + name.size()) = fLen;} - if (typeLen == 3){*(uint16_t *)(p + offset + 2 + name.size()) = fLen;} - if (typeLen == 5){*(uint32_t *)(p + offset + 2 + name.size()) = fLen;} + p[*hdrOffset] = (name.size() << 3) | (typeLen & 0x7); + memcpy(p + (*hdrOffset) + 1, name.data(), name.size()); + p[(*hdrOffset) + 1 + name.size()] = fType; + if (typeLen == 2){*(uint8_t *)(p + (*hdrOffset) + 2 + name.size()) = fLen;} + if (typeLen == 3){*(uint16_t *)(p + (*hdrOffset) + 2 + name.size()) = fLen;} + if (typeLen == 5){*(uint32_t *)(p + (*hdrOffset) + 2 + name.size()) = fLen;} // Calculate new offset and record size - offset += 1 + name.size() + typeLen; - recSize += fLen; + *hdrOffset += 1 + name.size() + typeLen; + *hdrRecordSize += fLen; } /// Sets the record counter to the given value. - void RelAccX::setRCount(uint32_t count){RAXHDR_RECORDCNT = count;} + void RelAccX::setRCount(uint32_t count){*hdrRecordCnt = count;} /// Sets the position in the records where the entries start - void RelAccX::setStartPos(uint32_t n){RAXHDR_STARTPOS = n;} + void RelAccX::setStartPos(uint32_t n){*hdrStartPos = n;} /// Sets the number of deleted records - void RelAccX::setDeleted(uint64_t n){RAXHDR_DELETED = n;} + void RelAccX::setDeleted(uint64_t n){*hdrDeleted = n;} /// Sets the number of records present /// Defaults to the record count if set to zero. - void RelAccX::setPresent(uint32_t n){RAXHDR_PRESENT = n;} + void RelAccX::setPresent(uint32_t n){*hdrPresent = n;} /// Sets the number of the last valid index - void RelAccX::setEndPos(uint64_t n){RAXHDR_ENDPOS = n;} + void RelAccX::setEndPos(uint64_t n){*hdrEndPos = n;} /// Sets the ready flag. /// After calling this function, addField() may no longer be called. @@ -836,33 +827,27 @@ namespace Util{ /// Updates the deleted record counter, the start position and the present record counter, /// shifting the ring buffer start position forward without moving the ring buffer end position. void RelAccX::deleteRecords(uint32_t amount){ - uint32_t &startPos = RAXHDR_STARTPOS; - uint64_t &deletedRecs = RAXHDR_DELETED; - uint32_t &recsPresent = RAXHDR_PRESENT; - startPos += amount; // update start position - deletedRecs += amount; // update deleted record counter - if (recsPresent >= amount){ - recsPresent -= amount; // decrease records present + *hdrStartPos += amount; // update start position + *hdrDeleted += amount; // update deleted record counter + if (*hdrPresent >= amount){ + *hdrPresent -= amount; // decrease records present }else{ WARN_MSG("Depleting recordCount!"); - recsPresent = 0; + *hdrPresent = 0; } } /// Updates the present record counter, shifting the ring buffer end position forward without /// moving the ring buffer start position. void RelAccX::addRecords(uint32_t amount){ - uint32_t &recsPresent = RAXHDR_PRESENT; - uint32_t &recordsCount = RAXHDR_RECORDCNT; - uint64_t &recordEndPos = RAXHDR_ENDPOS; - if (recsPresent + amount > recordsCount){ + if ((*hdrPresent) + amount > *hdrRecordCnt){ BACKTRACE; - WARN_MSG("Exceeding recordCount (%d [%d + %d] > %d)", recsPresent + amount, recsPresent, amount, recordsCount); - recsPresent = 0; + WARN_MSG("Exceeding recordCount (%d [%d + %d] > %d)", (*hdrPresent) + amount, *hdrPresent, amount, *hdrRecordCnt); + *hdrPresent = 0; }else{ - recsPresent += amount; + *hdrPresent += amount; } - recordEndPos += amount; + *hdrEndPos += amount; } void RelAccX::minimalFrom(const RelAccX &src){ diff --git a/lib/util.h b/lib/util.h index 4b4a9f7f..edfc9442 100644 --- a/lib/util.h +++ b/lib/util.h @@ -145,7 +145,6 @@ namespace Util{ bool isExit() const; bool isReload() const; bool isRecordAvailable(uint64_t recordNo) const; - uint32_t getRecordPosition(uint64_t recordNo) const; uint32_t getSize(const std::string &name, uint64_t recordNo = 0) const; char *getPointer(const std::string &name, uint64_t recordNo = 0) const; @@ -186,6 +185,13 @@ namespace Util{ std::map fields; private: + uint32_t * hdrRecordCnt; + uint32_t * hdrRecordSize; + uint32_t * hdrStartPos; + uint64_t * hdrDeleted; + uint32_t * hdrPresent; + uint16_t * hdrOffset; + uint64_t * hdrEndPos; char *p; }; diff --git a/src/io.cpp b/src/io.cpp index 56988814..10c1c7ac 100644 --- a/src/io.cpp +++ b/src/io.cpp @@ -207,8 +207,10 @@ namespace Mist{ Util::RelAccX &tPages = meta.pages(packTrack); size_t pageIdx = 0; + size_t currPagNum = curPageNum[packTrack]; + Util::RelAccXFieldData firstkey = tPages.getFieldData("firstkey"); for (uint64_t i = tPages.getDeleted(); i < tPages.getEndPos(); i++){ - if (tPages.getInt("firstkey", i) == curPageNum[packTrack]){ + if (tPages.getInt(firstkey, i) == currPagNum){ pageIdx = i; break; } @@ -219,7 +221,7 @@ namespace Mist{ // Do nothing when there is not enough free space on the page to add the packet. if (pageSize - pageOffset < packDataLen){ FAIL_MSG("Track %" PRIu32 "p%zu : Pack %" PRIu64 "ms of %" PRIu64 "b exceeds size %" PRIu64 " @ bpos %" PRIu64, - packTrack, curPageNum[packTrack], packTime, packDataLen, pageSize, pageOffset); + packTrack, currPagNum, packTime, packDataLen, pageSize, pageOffset); return; } diff --git a/src/output/output.cpp b/src/output/output.cpp index a087d17b..8c34e053 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -1482,18 +1482,6 @@ namespace Mist{ return false; } - if (M.getPageNumberForTime(nxt.tid, nxt.time) != currentPage[nxt.tid]){ - loadPageForKey(nxt.tid, M.getPageNumberForTime(nxt.tid, nxt.time)); - nxt.offset = 0; - //Only read the next time if the page load succeeded and there is a packet to read from - if (curPage[nxt.tid].mapped && curPage[nxt.tid].mapped[0] == 'D'){ - nxt.time = getDTSCTime(curPage[nxt.tid].mapped, 0); - } - buffer.erase(buffer.begin()); - buffer.insert(nxt); - return false; - } - // if we're going to read past the end of the data page, load the next page // this only happens for VoD if (nxt.offset >= curPage[nxt.tid].len || @@ -1502,6 +1490,17 @@ namespace Mist{ dropTrack(nxt.tid, "end of VoD track reached", false); return false; } + if (M.getPageNumberForTime(nxt.tid, nxt.time) != currentPage[nxt.tid]){ + loadPageForKey(nxt.tid, M.getPageNumberForTime(nxt.tid, nxt.time)); + nxt.offset = 0; + //Only read the next time if the page load succeeded and there is a packet to read from + if (curPage[nxt.tid].mapped && curPage[nxt.tid].mapped[0] == 'D'){ + nxt.time = getDTSCTime(curPage[nxt.tid].mapped, 0); + } + buffer.erase(buffer.begin()); + buffer.insert(nxt); + return false; + } dropTrack(nxt.tid, "VoD page load failure"); return false; } @@ -1511,9 +1510,6 @@ namespace Mist{ uint64_t nextTime = 0; - DTSC::Keys keys(M.keys(nxt.tid)); - size_t thisKey = keys.getNumForTime(nxt.time); - // Check if we have a next valid packet if (curPage[nxt.tid].len > nxt.offset+preLoad.getDataLen()+20 && memcmp(curPage[nxt.tid].mapped + nxt.offset + preLoad.getDataLen(), "\000\000\000\000", 4)){ nextTime = getDTSCTime(curPage[nxt.tid].mapped, nxt.offset + preLoad.getDataLen()); @@ -1531,6 +1527,8 @@ namespace Mist{ dropTrack(nxt.tid, "end of VoD track reached", false); return true; } + DTSC::Keys keys(M.keys(nxt.tid)); + size_t thisKey = keys.getNumForTime(nxt.time); //Check if there exists a different page for the next key size_t nextKeyPage = M.getPageNumberForKey(nxt.tid, thisKey + 1); if (nextKeyPage != INVALID_KEY_NUM && nextKeyPage != currentPage[nxt.tid]){ @@ -1594,13 +1592,19 @@ namespace Mist{ return false; } + //Update keynum only when the second flips over in the timestamp + //We do this because DTSC::Keys is pretty CPU-heavy + if (nxt.time / 1000 < nextTime/1000){ + DTSC::Keys keys(M.keys(nxt.tid)); + size_t thisKey = keys.getNumForTime(nxt.time); + userSelect[nxt.tid].setKeyNum(thisKey); + } + // we assume the next packet is the next on this same page nxt.offset += thisPacket.getDataLen(); nxt.time = nextTime; ++nxt.partIndex; - userSelect[nxt.tid].setKeyNum(thisKey); - // exchange the current packet in the buffer for the next one buffer.erase(buffer.begin()); buffer.insert(nxt); diff --git a/src/output/output_mp4.cpp b/src/output/output_mp4.cpp index 1a97c7c6..ea576232 100644 --- a/src/output/output_mp4.cpp +++ b/src/output/output_mp4.cpp @@ -20,6 +20,70 @@ namespace Mist{ return result.str(); } + SortSet::SortSet(){ + entries = 0; + currBegin = 0; + hasBegin = false; + } + + /// Finds the current beginning of the SortSet + void SortSet::findBegin(){ + if (!entries){return;} + currBegin = 0; + hasBegin = false; + for (size_t i = 0; i < entries; ++i){ + if (!hasBegin && avail[i]){ + currBegin = i; + hasBegin = true; + continue; + } + if (avail[i] && ((keyPart*)(void*)ptr)[i] < ((keyPart*)(void*)ptr)[currBegin]){ + currBegin = i; + } + } + } + + /// Returns a reference to the current beginning of the SortSet + const keyPart & SortSet::begin(){ + static const keyPart blank = {0, 0, 0, 0}; + if (!hasBegin){return blank;} + return ((keyPart*)(void*)ptr)[currBegin]; + } + + /// Marks the current beginning of the SortSet as erased + void SortSet::erase(){ + if (!hasBegin){return;} + avail[currBegin] = 0; + findBegin(); + } + + bool SortSet::empty(){return !hasBegin;} + + void SortSet::insert(const keyPart & part){ + size_t i = 0; + for (i = 0; i < entries; ++i){ + if (!avail[i] || ((keyPart*)(void*)ptr)[i].trackID == part.trackID){ + ((keyPart*)(void*)ptr)[i] = part; + avail[i] = 1; + if (!hasBegin || part < begin()){ + currBegin = i; + hasBegin = true; + } + return; + } + } + entries = i+1; + ptr.allocate(sizeof(keyPart)*entries); + ((keyPart*)(void*)ptr)[i] = part; + avail.append("\001", 1); + if (!hasBegin || part < begin()){ + currBegin = i; + hasBegin = true; + } + } + + + std::string OutMP4::protectionHeader(size_t idx){ std::string tmp = toUTF16(M.getPlayReady(idx)); tmp.erase(0, 2); // remove first 2 characters @@ -152,6 +216,7 @@ namespace Mist{ firstms = std::min(firstms, M.getFirstms(it->first)); } for (std::map::const_iterator it = userSelect.begin(); it != userSelect.end(); it++){ + const std::string tType = M.getType(it->first); uint64_t tmpRes = 0; DTSC::Parts parts(M.parts(it->first)); uint64_t partCount = parts.getValidCount(); @@ -173,7 +238,7 @@ namespace Mist{ tmpRes += 16 + (fragmented ? 0 : (1 * 12)); // STSC <-- Currently 1 entry, but might become more complex in near future // Type-specific boxes - if (M.getType(it->first) == "video"){ + if (tType == "video"){ tmpRes += 20 // VMHD Box + 16 // STSD + 86 // AVC1 @@ -184,7 +249,7 @@ namespace Mist{ tmpRes += 16 + (keys.getValidCount() * 4); // STSS } } - if (M.getType(it->first) == "audio"){ + if (tType == "audio"){ tmpRes += 16 // SMHD Box + 16 // STSD + 36 // MP4A @@ -194,7 +259,7 @@ namespace Mist{ } } - if (M.getType(it->first) == "meta"){ + if (tType == "meta"){ tmpRes += 12 // NMHD Box + 16 // STSD + 64; // tx3g Box @@ -208,10 +273,11 @@ namespace Mist{ uint64_t prevOffset = parts.getOffset(0); uint64_t cttsCount = 1; fileSize += parts.getSize(0); + bool isMeta = (tType == "meta"); for (unsigned int part = 1; part < partCount; ++part){ uint64_t partDur = parts.getDuration(part); uint64_t partOffset = parts.getOffset(part); - uint64_t partSize = parts.getSize(part); + uint64_t partSize = parts.getSize(part)+(isMeta?2:0); if (prevDur != partDur){ prevDur = partDur; ++sttsCount; @@ -245,31 +311,47 @@ namespace Mist{ return res; } - ///\todo This function does not indicate errors anywhere... maybe fix this... - std::string OutMP4::mp4Header(uint64_t &size, int fragmented){ + class trackLookup{ + public: + trackLookup(){ + parts = 0; + } + ~trackLookup(){ + if (parts){delete parts;} + } + void init(const Util::RelAccX & relParts, MP4::STCO box){ + parts = new DTSC::Parts(relParts); + stcoBox = box; + } + void init(const Util::RelAccX & relParts, MP4::CO64 box){ + parts = new DTSC::Parts(relParts); + co64Box = box; + } + DTSC::Parts * parts; + MP4::STCO stcoBox; + MP4::CO64 co64Box; + }; + + + bool OutMP4::mp4Header(Util::ResizeablePointer & headOut, uint64_t &size, int fragmented){ uint32_t mainTrack = M.mainTrack(); - if (mainTrack == INVALID_TRACK_ID){return "";} + if (mainTrack == INVALID_TRACK_ID){return false;} if (M.getLive()){needsLookAhead = 100;} - // Make sure we have a proper being value for the size... + // Clear size if it was set before the function was called, just in case size = 0; - // Stores the result of the function - std::stringstream header; // Determines whether the outputfile is larger than 4GB, in which case we need to use 64-bit // boxes for offsets bool useLargeBoxes = !fragmented && (estimateFileSize() > 0xFFFFFFFFull); // Keeps track of the total size of the mdat box uint64_t mdatSize = 0; - // Start actually creating the header - // MP4 Files always start with an FTYP box. Constructor sets default values MP4::FTYP ftypBox; if (sending3GP){ ftypBox.setMajorBrand("3gp6"); ftypBox.setCompatibleBrands("3gp6", 3); } - - header.write(ftypBox.asBox(), ftypBox.boxedSize()); + headOut.append(ftypBox.asBox(), ftypBox.boxedSize()); // Start building the moov box. This is the metadata box for an mp4 file, and will contain all // metadata. @@ -422,55 +504,84 @@ namespace Mist{ MP4::CTTS cttsBox; cttsBox.setVersion(0); + size_t totalEntries = 0; MP4::CTTSEntry tmpEntry; tmpEntry.sampleCount = 0; tmpEntry.sampleOffset = parts.getOffset(0); - std::deque > sttsCounter; - stszBox.setEntrySize(0, partCount - 1); // Speed up allocation - size_t totalEntries = 0; + size_t sttsCounter = 0; + MP4::STTSEntry sttsEntry; + sttsEntry.sampleCount = 0; + sttsEntry.sampleDelta = parts.getSize(0);; + //Calculate amount of entries for CTTS/STTS boxes so we can set the last entry first + //Our MP4 box implementations dynamically reallocate to fit the data you put inside them, + //Which means setting the last entry first prevents constant reallocs and slowness. for (size_t part = 0; part < partCount; ++part){ - stats(); - - uint64_t partDur = parts.getDuration(part); - uint64_t partSize = parts.getSize(part); uint64_t partOffset = parts.getOffset(part); - - // Create a new entry with current duration if EITHER there is no entry yet, or this parts - // duration differs from the previous - if (!sttsCounter.size() || sttsCounter.rbegin()->second != partDur){ - sttsCounter.push_back(std::pair(0, partDur)); - } - // Update the counter - sttsCounter.rbegin()->first++; - - if (M.getType(it->first) == "meta"){partSize += 2;} - - stszBox.setEntrySize(partSize, part); - size += partSize; - if (partOffset != tmpEntry.sampleOffset){ - // If the offset of this and previous part differ, write current values and reset - cttsBox.setCTTSEntry(tmpEntry, - totalEntries++); // Rewrite for sanity. index FIRST, value SECOND - tmpEntry.sampleCount = 0; + ++totalEntries; tmpEntry.sampleOffset = partOffset; } - tmpEntry.sampleCount++; + uint64_t partDur = parts.getDuration(part); + if (partDur != sttsEntry.sampleDelta){ + ++sttsCounter; + sttsEntry.sampleDelta = partDur; + } } - MP4::STTSEntry sttsEntry; - sttsBox.setSTTSEntry(sttsEntry, sttsCounter.size() - 1); - size_t sttsIdx = 0; - for (std::deque >::iterator it2 = sttsCounter.begin(); - it2 != sttsCounter.end(); it2++){ - sttsEntry.sampleCount = it2->first; - sttsEntry.sampleDelta = it2->second; - sttsBox.setSTTSEntry(sttsEntry, sttsIdx++); + //Set temporary last entry for CTTS box + bool hasCTTS = (totalEntries || tmpEntry.sampleOffset); + if (hasCTTS){ + cttsBox.setCTTSEntry(tmpEntry, totalEntries); } - if (totalEntries || tmpEntry.sampleOffset){ - cttsBox.setCTTSEntry(tmpEntry, totalEntries++); + //Set temporary last entry for STTS box + sttsBox.setSTTSEntry(sttsEntry, sttsCounter-1); + //Set temporary last entry for STSZ box + stszBox.setEntrySize(0, partCount - 1); + + //All set! Now we can do everything for real. + //Reset the values we just used, first. + totalEntries = 0; + tmpEntry.sampleCount = 0; + tmpEntry.sampleOffset = parts.getOffset(0); + sttsCounter = 0; + sttsEntry.sampleCount = 0; + sttsEntry.sampleDelta = parts.getDuration(0); + + bool isMeta = (tType == "meta"); + for (size_t part = 0; part < partCount; ++part){ + uint64_t partDur = parts.getDuration(part); + if (sttsEntry.sampleDelta != partDur){ + // If the duration of this and previous part differ, write current values and reset + sttsBox.setSTTSEntry(sttsEntry, sttsCounter++); + sttsEntry.sampleCount = 0; + sttsEntry.sampleDelta = partDur; + } + sttsEntry.sampleCount++; + + uint64_t partSize = parts.getSize(part)+(isMeta?2:0); + stszBox.setEntrySize(partSize, part); + size += partSize; + + if (hasCTTS){ + uint64_t partOffset = parts.getOffset(part); + if (partOffset != tmpEntry.sampleOffset){ + // If the offset of this and previous part differ, write current values and reset + cttsBox.setCTTSEntry(tmpEntry, totalEntries++); + tmpEntry.sampleCount = 0; + tmpEntry.sampleOffset = partOffset; + } + tmpEntry.sampleCount++; + } + } + //Write last entry for STTS + sttsBox.setSTTSEntry(sttsEntry, sttsCounter); + + //Only add the CTTS box to the STBL box if we had any entries in it + if (hasCTTS){ + //Write last entry for CTTS + cttsBox.setCTTSEntry(tmpEntry, totalEntries); stblBox.setContent(cttsBox, stblOffset++); } } @@ -485,7 +596,7 @@ namespace Mist{ uint32_t firstKey = keys.getFirstValid(); uint32_t endKey = keys.getEndValid(); for (size_t i = firstKey; i < endKey; ++i){ - stssBox.setSampleNumber(tmpCount + 1, i); /// rewrite this for sanity.... SHOULD be: index FIRST, value SECOND + stssBox.setSampleNumber(tmpCount + 1, i); tmpCount += keys.getParts(i); } stblBox.setContent(stssBox, stblOffset++); @@ -557,17 +668,18 @@ namespace Mist{ // initial offset length ftyp, length moov + 8 uint64_t dataOffset = ftypBox.boxedSize() + moovBox.boxedSize() + 8; - std::map checkStcoBoxes; - std::map checkCO64Boxes; + + QuickMap trackMap; std::deque trak = moovBox.getChildren(); for (std::deque::iterator trakIt = trak.begin(); trakIt != trak.end(); trakIt++){ size_t idx = trakIt->getChild().getTrackID() - 1; MP4::STBL stblBox = trakIt->getChild().getChild().getChild(); + trackMap.insert(idx, trackLookup()); if (useLargeBoxes){ - checkCO64Boxes.insert(std::pair(idx, stblBox.getChild())); + trackMap.get(idx).init(M.parts(idx), stblBox.getChild()); }else{ - checkStcoBoxes.insert(std::pair(idx, stblBox.getChild())); + trackMap.get(idx).init(M.parts(idx), stblBox.getChild()); } } @@ -576,27 +688,27 @@ namespace Mist{ // Keep track of the current size of the data within the mdat uint64_t dataSize = 0; // Current values are actual byte offset without header-sized offset - std::set sortSet; // filling sortset for interleaving parts + SortSet sortSet; // filling sortset for interleaving parts for (std::map::const_iterator subIt = userSelect.begin(); subIt != userSelect.end(); subIt++){ keyPart temp; temp.trackID = subIt->first; temp.time = M.getFirstms(subIt->first); temp.index = 0; - INFO_MSG("adding to sortSet: tid %zu time %" PRIu64, subIt->first, temp.time); sortSet.insert(temp); } while (!sortSet.empty()){ stats(); - keyPart temp = *sortSet.begin(); - sortSet.erase(sortSet.begin()); + keyPart temp = sortSet.begin(); + trackLookup & tL = trackMap.get(temp.trackID); + sortSet.erase(); - DTSC::Parts parts(M.parts(temp.trackID)); + DTSC::Parts & parts = *tL.parts; // setting the right STCO size in the STCO box if (useLargeBoxes){// Re-using the previously defined boolean for speedup - checkCO64Boxes[temp.trackID].setChunkOffset(dataOffset + dataSize, temp.index); + tL.co64Box.setChunkOffset(dataOffset + dataSize, temp.index); }else{ - checkStcoBoxes[temp.trackID].setChunkOffset(dataOffset + dataSize, temp.index); + tL.stcoBox.setChunkOffset(dataOffset + dataSize, temp.index); } dataSize += parts.getSize(temp.index); @@ -613,17 +725,17 @@ namespace Mist{ ///\todo Update for when mdat box exceeds 4GB mdatSize = dataSize + 8; //+8 for mp4 header } - header << std::string(moovBox.asBox(), moovBox.boxedSize()); + headOut.append(moovBox.asBox(), moovBox.boxedSize()); if (!fragmented){// if we are making a non fragmented MP4 and there are parts char mdatHeader[8] ={0x00, 0x00, 0x00, 0x00, 'm', 'd', 'a', 't'}; if (mdatSize < 0xFFFFFFFF){Bit::htobl(mdatHeader, mdatSize);} - header.write(mdatHeader, 8); + headOut.append(mdatHeader, 8); } - size += header.str().size(); - if (fragmented){realBaseOffset = header.str().size();} - return header.str(); + size += headOut.size(); + if (fragmented){realBaseOffset = headOut.size();} + return true; } /// Calculate a seekPoint, based on byteStart, metadata, tracks and headerSize. @@ -970,10 +1082,14 @@ namespace Mist{ if (rangeType == 'p'){ H.SetBody("Starsystem not in communications range"); H.SendResponse("416", "Starsystem not in communications range", myConn); + parseData = false; + wantRequest = true; return; }else{ H.SetBody("Requested Range Not Satisfiable"); H.SendResponse("416", "Requested Range Not Satisfiable", myConn); + parseData = false; + wantRequest = true; return; } }else{ @@ -992,10 +1108,15 @@ namespace Mist{ if (byteStart < headerSize){ // For storing the header. if ((!startTime && endTime == 0xffffffffffffffffull) || (endTime == 0)){ - std::string headerData = mp4Header(fileSize, M.getLive()); + Util::ResizeablePointer headerData; + if (!mp4Header(headerData, fileSize, M.getLive())){ + FAIL_MSG("Could not generate MP4 header!"); + H.SetBody("Error while generating MP4 header"); + H.SendResponse("500", "Error generating MP4 header", myConn); + return; + } INFO_MSG("Have %zu bytes, sending %zu bytes", headerData.size(), std::min(headerSize, byteEnd) - byteStart); - H.Chunkify(headerData.data() + byteStart, std::min(headerSize, byteEnd) - byteStart, - myConn); // send MP4 header + H.Chunkify(headerData + byteStart, std::min(headerSize, byteEnd) - byteStart, myConn); leftOver -= std::min(headerSize, byteEnd) - byteStart; } } @@ -1097,9 +1218,6 @@ namespace Mist{ if (!sortSet.empty()){ keyPart temp = *sortSet.begin(); sortSet.erase(sortSet.begin()); - - DTSC::Parts parts(M.parts(temp.trackID)); - currPos += parts.getSize(temp.index); if (temp.index + 1 < parts.getEndValid()){// only insert when there are parts left temp.time += parts.getDuration(temp.index); diff --git a/src/output/output_mp4.h b/src/output/output_mp4.h index 1c6146c7..f09bd263 100644 --- a/src/output/output_mp4.h +++ b/src/output/output_mp4.h @@ -17,6 +17,63 @@ namespace Mist{ uint64_t index; }; + class SortSet{ + private: + Util::ResizeablePointer ptr; + Util::ResizeablePointer avail; + size_t entries; + size_t currBegin; + void findBegin(); + bool hasBegin; + public: + SortSet(); + const keyPart & begin(); + void erase(); + bool empty(); + void insert(const keyPart & part); + }; + + /// Class that implements a tiny subset of std::map, optimized for speed for our type of usage. + template class QuickMap{ + private: + Util::ResizeablePointer ptr; + size_t entries; + public: + QuickMap(){ + entries = 0; + } + ~QuickMap(){ + size_t len = 8 + sizeof(T*); + for (size_t i = 0; i < entries; ++i){ + delete *(T**)(void*)(ptr+len*i+8); + } + } + T & get(uint64_t idx){ + static T blank; + size_t len = 8 + sizeof(T*); + for (size_t i = 0; i < entries; ++i){ + if (*((uint64_t*)(void*)(ptr+len*i)) == idx){ + return **(T**)(void*)(ptr+len*i+8); + } + } + return blank; + } + void insert(uint64_t idx, T elem){ + size_t i = 0; + size_t len = 8 + sizeof(T*); + for (i = 0; i < entries; ++i){ + if (*((uint64_t*)(void*)(ptr+len*i)) == idx){ + *(T**)(void*)(ptr+len*i+8) = new T(elem); + return; + } + } + entries = i+1; + ptr.allocate(len*entries); + *(T**)(void*)(ptr+len*i+8) = new T(elem); + *((uint64_t*)(void*)(ptr+len*i)) = idx; + } + }; + struct fragSet{ uint64_t firstPart; uint64_t lastPart; @@ -31,7 +88,7 @@ namespace Mist{ static void init(Util::Config *cfg); uint64_t mp4HeaderSize(uint64_t &fileSize, int fragmented = 0) const; - std::string mp4Header(uint64_t &size, int fragmented = 0); + bool mp4Header(Util::ResizeablePointer & headOut, uint64_t &size, int fragmented = 0); uint64_t mp4moofSize(uint64_t startFragmentTime, uint64_t endFragmentTime, uint64_t &mdatSize) const; virtual void sendFragmentHeaderTime(uint64_t startFragmentTime,