From c25a53372917a4d25540ce48b0d33da4a038553f Mon Sep 17 00:00:00 2001 From: Erik Zandvliet Date: Thu, 28 Jan 2016 15:00:25 +0100 Subject: [PATCH] Recording, HLS Push, UDP (Multicast) Input, Threaded TS --- CMakeLists.txt | 11 +- lib/CMakeLists.txt | 69 ----- lib/dtsc.h | 2 +- lib/dtscmeta.cpp | 72 ++--- lib/http_parser.cpp | 4 +- lib/shared_memory.cpp | 193 +++++++----- lib/shared_memory.h | 2 + lib/ts_packet.cpp | 110 +++---- lib/ts_packet.h | 13 +- lib/ts_stream.cpp | 429 +++++++++++++++++++++++--- lib/ts_stream.h | 24 +- src/CMakeLists.txt | 101 ------ src/controller/controller_streams.cpp | 3 + src/input/input.cpp | 99 ++---- src/input/input_buffer.cpp | 129 ++++---- src/input/input_ts.cpp | 310 +++++++++++++++---- src/input/input_ts.h | 11 + src/io.cpp | 95 +++--- src/io.h | 64 ++-- src/output/mist_out.cpp | 1 + src/output/output.cpp | 91 +++--- src/output/output_hls.cpp | 392 +++++++++++++++++------ src/output/output_hls.h | 7 + src/output/output_hss.cpp | 16 +- src/output/output_http.cpp | 4 +- src/output/output_push.cpp | 346 +++++++++++++++++++++ src/output/output_push.h | 19 ++ src/output/output_rtmp.cpp | 5 +- src/output/output_ts_base.cpp | 2 +- 29 files changed, 1809 insertions(+), 815 deletions(-) delete mode 100644 lib/CMakeLists.txt delete mode 100644 src/CMakeLists.txt create mode 100644 src/output/output_push.cpp create mode 100644 src/output/output_push.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 63dd42bb..7384b099 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -251,7 +251,7 @@ makeAnalyser(DTSC dtsc) makeAnalyser(AMF amf) makeAnalyser(MP4 mp4) makeAnalyser(OGG ogg) -makeAnalyser(RTP rtp) #LTS +#makeAnalyser(RTP rtp) #LTS makeAnalyser(RTSP rtsp_rtp) #LTS makeAnalyser(TS ts) #LTS makeAnalyser(TSStream tsstream) #LTS @@ -278,8 +278,8 @@ macro(makeInput inputName format) if (";${ARGN};" MATCHES ";nolock;")#Currently only used in TSStream list(APPEND my_definitions "INPUT_NOLOCK") endif() - if (";${ARGN};" MATCHES ";live;")#Currently only used in TSStream - list(APPEND my_definitions "INPUT_LIVE") + if (";${ARGN};" MATCHES ";tslive;") + list(APPEND my_definitions "TSLIVE_INPUT") endif() list(APPEND my_definitions "INPUTTYPE=\"input_${format}.h\"") @@ -305,7 +305,7 @@ makeInput(Buffer buffer) makeInput(ISMV ismv)#LTS makeInput(MP4 mp4)#LTS makeInput(TS ts)#LTS -makeInput(TSStream ts nolock live)#LTS +makeInput(TSStream ts nolock tslive)#LTS makeInput(Folder folder folder)#LTS ######################################## @@ -313,9 +313,9 @@ makeInput(Folder folder folder)#LTS ######################################## macro(makeOutput outputName format) #Parse all extra arguments, for http and ts flags + SET (tsBaseClass Output) if (";${ARGN};" MATCHES ";http;") SET(httpOutput src/output/output_http.cpp) - SET(tsBaseClass Output) if (";${ARGN};" MATCHES ";ts;") SET(tsBaseClass HTTPOutput) endif() @@ -356,6 +356,7 @@ makeOutput(JSON json http) makeOutput(TS ts ts) makeOutput(HTTPTS httpts http ts) makeOutput(HLS hls http ts) +makeOutput(Push push)#LTS makeOutput(RTSP rtsp)#LTS makeOutput(TSPush ts_push ts)#LTS makeOutput(DASH dash_mp4 http)#LTS diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt deleted file mode 100644 index 4b84051d..00000000 --- a/lib/CMakeLists.txt +++ /dev/null @@ -1,69 +0,0 @@ -add_library ( mist SHARED - amf.cpp - amf.h - auth.cpp - auth.h - base64.cpp - base64.h - bitfields.cpp - bitfields.h - bitstream.cpp - bitstream.h - checksum.h - CMakeLists.txt - config.cpp - config.h - converter.cpp - converter.h - defines.h - dtsc.cpp - dtsc.h - dtscmeta.cpp - filesystem.cpp - filesystem.h - flv_tag.cpp - flv_tag.h - ftp.cpp - ftp.h - http_parser.cpp - http_parser.h - json.cpp - json.h - mp4_adobe.cpp - mp4_adobe.h - mp4.cpp - mp4_generic.cpp - mp4_generic.h - mp4.h - mp4_ms.cpp - mp4_ms.h - nal.cpp - nal.h - ogg.cpp - ogg.h - procs.cpp - procs.h - rtmpchunks.cpp - rtmpchunks.h - shared_memory.cpp - shared_memory.h - socket.cpp - socket.h - stream.cpp - stream.h - theora.cpp - theora.h - timing.cpp - timing.h - tinythread.cpp - tinythread.h - ts_packet.cpp - ts_packet.h - vorbis.cpp - vorbis.h -) - -target_link_libraries( mist - -lpthread - -lrt -) diff --git a/lib/dtsc.h b/lib/dtsc.h index 217ab3a1..0a1a39e2 100644 --- a/lib/dtsc.h +++ b/lib/dtsc.h @@ -298,7 +298,7 @@ namespace DTSC { int getSendLen(); void send(Socket::Connection & conn); void writeTo(char *& p); - JSON::Value toJSON(); + JSON::Value toJSON(bool skipBinary = false); std::deque fragments; std::deque keys; std::deque keySizes; diff --git a/lib/dtscmeta.cpp b/lib/dtscmeta.cpp index 5194c691..2b6cc43c 100644 --- a/lib/dtscmeta.cpp +++ b/lib/dtscmeta.cpp @@ -1772,43 +1772,46 @@ namespace DTSC { } ///\brief Converts a track to a JSON::Value - JSON::Value Track::toJSON() { + JSON::Value Track::toJSON(bool skipBinary) { JSON::Value result; std::string tmp; - tmp.reserve(fragments.size() * PACKED_FRAGMENT_SIZE); - for (std::deque::iterator it = fragments.begin(); it != fragments.end(); it++) { - tmp.append(it->getData(), PACKED_FRAGMENT_SIZE); + if (!skipBinary) { + tmp.reserve(fragments.size() * PACKED_FRAGMENT_SIZE); + for (std::deque::iterator it = fragments.begin(); it != fragments.end(); it++) { + tmp.append(it->getData(), PACKED_FRAGMENT_SIZE); + } + result["fragments"] = tmp; + tmp = ""; + tmp.reserve(keys.size() * PACKED_KEY_SIZE); + for (std::deque::iterator it = keys.begin(); it != keys.end(); it++) { + tmp.append(it->getData(), PACKED_KEY_SIZE); + } + result["keys"] = tmp; + tmp = ""; + tmp.reserve(keySizes.size() * 4); + for (unsigned int i = 0; i < keySizes.size(); i++){ + tmp += (char)((keySizes[i] >> 24)); + tmp += (char)((keySizes[i] >> 16)); + tmp += (char)((keySizes[i] >> 8)); + tmp += (char)(keySizes[i]); + } + result["keysizes"] = tmp; + tmp = ""; + tmp.reserve(parts.size() * 9); + for (std::deque::iterator it = parts.begin(); it != parts.end(); it++) { + tmp.append(it->getData(), 9); + } + result["parts"] = tmp; + /*LTS-START*/ + tmp = ""; + tmp.reserve(ivecs.size() * 8); + for (std::deque::iterator it = ivecs.begin(); it != ivecs.end(); it++) { + tmp.append(it->getData(), 8); + } + result["ivecs"] = tmp; + /*LTS-END*/ + result["init"] = init; } - result["fragments"] = tmp; - tmp = ""; - tmp.reserve(keys.size() * PACKED_KEY_SIZE); - for (std::deque::iterator it = keys.begin(); it != keys.end(); it++) { - tmp.append(it->getData(), PACKED_KEY_SIZE); - } - result["keys"] = tmp; - tmp = ""; - tmp.reserve(keySizes.size() * 4); - for (unsigned int i = 0; i < keySizes.size(); i++){ - tmp += (char)((keySizes[i] >> 24)); - tmp += (char)((keySizes[i] >> 16)); - tmp += (char)((keySizes[i] >> 8)); - tmp += (char)(keySizes[i]); - } - result["keysizes"] = tmp; - tmp = ""; - tmp.reserve(parts.size() * 9); - for (std::deque::iterator it = parts.begin(); it != parts.end(); it++) { - tmp.append(it->getData(), 9); - } - result["parts"] = tmp; - /*LTS-START*/ - tmp = ""; - tmp.reserve(ivecs.size() * 8); - for (std::deque::iterator it = ivecs.begin(); it != ivecs.end(); it++) { - tmp.append(it->getData(), 8); - } - result["ivecs"] = tmp; - /*LTS-END*/ result["trackid"] = trackID; result["firstms"] = (long long)firstms; result["lastms"] = (long long)lastms; @@ -1818,7 +1821,6 @@ namespace DTSC { } result["codec"] = codec; result["type"] = type; - result["init"] = init; if (type == "audio") { result["rate"] = rate; result["size"] = size; diff --git a/lib/http_parser.cpp b/lib/http_parser.cpp index d9742f7e..784aa59b 100644 --- a/lib/http_parser.cpp +++ b/lib/http_parser.cpp @@ -180,13 +180,11 @@ void HTTP::Parser::StartResponse(HTTP::Parser & request, Socket::Connection & co StartResponse("200", "OK", request, conn, bufferAllChunks); } -/// After receiving a header with this object, this function call will: -/// - Forward the headers to the 'to' Socket::Connection. +/// After receiving a header with this object, and after a call with SendResponse/SendRequest with this object, this function call will: /// - Retrieve all the body from the 'from' Socket::Connection. /// - Forward those contents as-is to the 'to' Socket::Connection. /// It blocks until completed or either of the connections reaches an error state. void HTTP::Parser::Proxy(Socket::Connection & from, Socket::Connection & to) { - SendResponse(url, method, to); if (getChunks) { unsigned int proxyingChunk = 0; while (to.connected() && from.connected()) { diff --git a/lib/shared_memory.cpp b/lib/shared_memory.cpp index c0773dbe..d23fd443 100644 --- a/lib/shared_memory.cpp +++ b/lib/shared_memory.cpp @@ -15,14 +15,21 @@ #include "bitfields.h" #include "timing.h" +#if defined(__CYGWIN__) || defined(_WIN32) +#include +#include +#include +#endif + + namespace IPC { #if defined(__CYGWIN__) || defined(_WIN32) static std::map preservedPages; - void preservePage(std::string p){ + void preservePage(std::string p) { preservedPages[p].init(p, 0, false, false); } - void releasePage(std::string p){ + void releasePage(std::string p) { preservedPages.erase(p); } #endif @@ -68,7 +75,7 @@ namespace IPC { static void btohl(char * p, unsigned int & val) { val = ((long)p[0] << 24) | ((long)p[1] << 16) | ((long)p[2] << 8) | p[3]; } - + /// Reads a long long value of p in host order to val. static void btohll(char * p, long long & val) { val = ((long long)p[0] << 56) | ((long long)p[1] << 48) | ((long long)p[2] << 40) | ((long long)p[3] << 32) | ((long long)p[4] << 24) | ((long long)p[5] << 16) | ((long long)p[6] << 8) | p[7]; @@ -108,7 +115,7 @@ namespace IPC { #if defined(__CYGWIN__) || defined(_WIN32) return mySem != 0; #else - return mySem != SEM_FAILED; + return mySem && mySem != SEM_FAILED; #endif } @@ -122,23 +129,34 @@ namespace IPC { void semaphore::open(const char * name, int oflag, mode_t mode, unsigned int value) { close(); int timer = 0; - while (!(*this) && timer++ < 10){ + while (!(*this) && timer++ < 10) { #if defined(__CYGWIN__) || defined(_WIN32) std::string semaName = "Global\\"; semaName += name; - if (oflag & O_CREAT){ - if (oflag & O_EXCL){ + if (oflag & O_CREAT) { + if (oflag & O_EXCL) { //attempt opening, if succes, close handle and return false; - HANDLE tmpSem = OpenSemaphore(0, false, semaName.c_str()); - if (tmpSem){ + HANDLE tmpSem = OpenMutex(SYNCHRONIZE, false, semaName.c_str()); + if (tmpSem) { CloseHandle(tmpSem); mySem = 0; break; } } - mySem = CreateSemaphore(0, value, 1 , semaName.c_str()); - }else{ - mySem = OpenSemaphore(0, false, semaName.c_str()); + SECURITY_ATTRIBUTES security = getSecurityAttributes(); + mySem = CreateMutex(&security, true, semaName.c_str()); + if (value){ + ReleaseMutex(mySem); + } + } else { + mySem = OpenMutex(SYNCHRONIZE, false, semaName.c_str()); + } + if (!(*this)) { + if (GetLastError() == ERROR_FILE_NOT_FOUND){//Error code 2 + Util::wait(500); + } else { + break; + } } #else if (oflag & O_CREAT) { @@ -146,17 +164,16 @@ namespace IPC { } else { mySem = sem_open(name, oflag); } -#endif - if (!(*this)){ - if (errno == ENOENT){ + if (!(*this)) { + if (errno == ENOENT) { Util::wait(500); - }else{ + } else { break; } } +#endif } - if (!(*this)){ - DEBUG_MSG(DLVL_VERYHIGH, "Attempt to open semaphore %s: %s", name, strerror(errno)); + if (!(*this)) { } myName = (char *)name; } @@ -177,7 +194,7 @@ namespace IPC { void semaphore::post() { if (*this) { #if defined(__CYGWIN__) || defined(_WIN32) - ReleaseSemaphore(mySem, 1, 0); + ReleaseMutex(mySem); #else sem_post(mySem); #endif @@ -203,7 +220,7 @@ namespace IPC { int result; #if defined(__CYGWIN__) || defined(_WIN32) result = WaitForSingleObject(mySem, 0);//wait at most 1ms - if (result == 0x80){ + if (result == 0x80) { WARN_MSG("Consistency error caught on semaphore %s", myName.c_str()); result = 0; } @@ -212,13 +229,13 @@ namespace IPC { #endif return (result == 0); } - + ///\brief Tries to wait for the semaphore for a single second, returns true if successful, false otherwise bool semaphore::tryWaitOneSecond() { int result; #if defined(__CYGWIN__) || defined(_WIN32) result = WaitForSingleObject(mySem, 1000);//wait at most 1s - if (result == 0x80){ + if (result == 0x80) { WARN_MSG("Consistency error caught on semaphore %s", myName.c_str()); result = 0; } @@ -268,12 +285,38 @@ namespace IPC { } +#if defined(__CYGWIN__) || defined(_WIN32) + SECURITY_ATTRIBUTES semaphore::getSecurityAttributes() { + ///\todo We really should clean this up sometime probably + ///We currently have everything static, because the result basically depends on everything + static SECURITY_ATTRIBUTES result; + static bool resultValid = false; + static SECURITY_DESCRIPTOR securityDescriptor; + if (resultValid) { + return result; + } + + InitializeSecurityDescriptor(&securityDescriptor, SECURITY_DESCRIPTOR_REVISION); + if (!SetSecurityDescriptorDacl(&securityDescriptor, TRUE, NULL, FALSE)){ + FAIL_MSG("Failed to set pSecurityDescriptor: %u", GetLastError()); + return result; + } + + result.nLength = sizeof(SECURITY_ATTRIBUTES); + result.lpSecurityDescriptor = &securityDescriptor; + result.bInheritHandle = FALSE; + + resultValid = true; + return result; + } +#endif + ///brief Creates a shared page ///\param name_ The name of the page to be created ///\param len_ The size to make the page ///\param master_ Whether to create or merely open the page ///\param autoBackoff When only opening the page, wait for it to appear or fail - sharedPage::sharedPage(std::string name_, unsigned int len_, bool master_, bool autoBackoff){ + sharedPage::sharedPage(std::string name_, unsigned int len_, bool master_, bool autoBackoff) { handle = 0; len = 0; master = false; @@ -302,7 +345,7 @@ namespace IPC { if (mapped && len) { #if defined(__CYGWIN__) || defined(_WIN32) //under Cygwin, the mapped location is shifted by 4 to contain the page size. - UnmapViewOfFile(mapped-4); + UnmapViewOfFile(mapped - 4); #else munmap(mapped, len); #endif @@ -315,7 +358,7 @@ namespace IPC { void sharedPage::close() { unmap(); if (handle > 0) { - INSANE_MSG("Closing page %s in %s mode", name.c_str(), master?"master":"client"); + INSANE_MSG("Closing page %s in %s mode", name.c_str(), master ? "master" : "client"); #if defined(__CYGWIN__) || defined(_WIN32) CloseHandle(handle); #else @@ -353,11 +396,11 @@ namespace IPC { master = master_; mapped = 0; if (name.size()) { - INSANE_MSG("Opening page %s in %s mode %s auto-backoff", name.c_str(), master?"master":"client", autoBackoff?"with":"without"); + INSANE_MSG("Opening page %s in %s mode %s auto-backoff", name.c_str(), master ? "master" : "client", autoBackoff ? "with" : "without"); #if defined(__CYGWIN__) || defined(_WIN32) if (master) { //Under cygwin, all pages are 4 bytes longer than claimed. - handle = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, len+4, name.c_str()); + handle = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, len + 4, name.c_str()); } else { int i = 0; do { @@ -378,10 +421,10 @@ namespace IPC { return; } //Under cygwin, the extra 4 bytes contain the real size of the page. - if (master){ - ((unsigned int*)mapped)[0] = len_; - }else{ - len = ((unsigned int*)mapped)[0]; + if (master) { + ((unsigned int *)mapped)[0] = len_; + } else { + len = ((unsigned int *)mapped)[0]; } //Now shift by those 4 bytes. mapped += 4; @@ -401,7 +444,7 @@ namespace IPC { } } if (handle == -1) { - if (!master_ && autoBackoff){ + if (!master_ && autoBackoff) { FAIL_MSG("shm_open for page %s failed: %s", name.c_str(), strerror(errno)); } return; @@ -480,7 +523,7 @@ namespace IPC { len = 0; } } - + /// Unmaps, closes and unlinks (if master and name is set) the shared file. void sharedFile::close() { unmap(); @@ -616,8 +659,8 @@ namespace IPC { ///\brief Sets the host of this connection void statExchange::host(std::string name) { - if (name.size() < 16){ - memset(data+32, 0, 16); + if (name.size() < 16) { + memset(data + 32, 0, 16); } memcpy(data + 32, name.c_str(), std::min((int)name.size(), 16)); } @@ -630,7 +673,7 @@ namespace IPC { ///\brief Sets the name of the stream this user is viewing void statExchange::streamName(std::string name) { size_t splitChar = name.find_first_of("+ "); - if (splitChar != std::string::npos){ + if (splitChar != std::string::npos) { name[splitChar] = '+'; } memcpy(data + 48, name.c_str(), std::min((int)name.size(), 100)); @@ -730,7 +773,7 @@ namespace IPC { ///\brief Creates the next page with the correct size void sharedServer::newPage() { - sharedPage tmp(std::string(baseName.substr(1) + (char)(myPages.size() + (int)'A')), std::min(((8192 * 2)<< myPages.size()), (32 * 1024 * 1024)), true); + sharedPage tmp(std::string(baseName.substr(1) + (char)(myPages.size() + (int)'A')), std::min(((8192 * 2) << myPages.size()), (32 * 1024 * 1024)), true); myPages.insert(tmp); tmp.master = false; DEBUG_MSG(DLVL_VERYHIGH, "Created a new page: %s", tmp.name.c_str()); @@ -784,7 +827,7 @@ namespace IPC { } semGuard tmpGuard(&mySemaphore); unsigned int id = 0; - unsigned int userCount=0; + unsigned int userCount = 0; unsigned int emptyCount = 0; for (std::set::iterator it = myPages.begin(); it != myPages.end(); it++) { if (!it->mapped || !it->len) { @@ -796,16 +839,16 @@ namespace IPC { while (offset + payLen + (hasCounter ? 1 : 0) <= it->len) { if (hasCounter) { if (it->mapped[offset] != 0) { - char * counter = it->mapped+offset; + char * counter = it->mapped + offset; //increase the count if needed ++userCount; if (id >= amount) { amount = id + 1; DEBUG_MSG(DLVL_VERYHIGH, "Shared memory %s is now at count %u", baseName.c_str(), amount); - } - unsigned short tmpPID = *((unsigned short *)(it->mapped+1+offset+payLen-2)); - if(!Util::Procs::isRunning(tmpPID) && !(*counter == 126 || *counter == 127 || *counter == 254 || *counter == 255)){ - WARN_MSG("process disappeared, timing out. (pid %d)", tmpPID); + } + unsigned short tmpPID = *((unsigned short *)(it->mapped + 1 + offset + payLen - 2)); + if (!Util::Procs::isRunning(tmpPID) && !(*counter == 126 || *counter == 127 || *counter == 254 || *counter == 255)) { + WARN_MSG("process disappeared, timing out. (pid %d)", tmpPID); *counter = 126; //if process is already dead, instant timeout. } callback(it->mapped + offset + 1, payLen, id); @@ -823,21 +866,21 @@ namespace IPC { DEBUG_MSG(DLVL_WARN, "Client %u disconnect timed out", id); break; default: - #ifndef NOCRASHCHECK - if (tmpPID){ - if(*counter > 10 && *counter < 126 ){ - if(*counter < 30){ - if (*counter > 15){ - WARN_MSG("Process %d is unresponsive",tmpPID); +#ifndef NOCRASHCHECK + if (tmpPID) { + if (*counter > 10 && *counter < 126) { + if (*counter < 30) { + if (*counter > 15) { + WARN_MSG("Process %d is unresponsive", tmpPID); } - Util::Procs::Stop(tmpPID); //soft kill - } else { + Util::Procs::Stop(tmpPID); //soft kill + } else { ERROR_MSG("Killing unresponsive process %d", tmpPID); - Util::Procs::Murder(tmpPID); //improved kill + Util::Procs::Murder(tmpPID); //improved kill } } } - #endif +#endif break; } if (*counter == 127 || *counter == 126 || *counter == 255 || *counter == 254) { @@ -885,20 +928,20 @@ namespace IPC { } offset += payLen + (hasCounter ? 1 : 0); id ++; - } - if(userCount==0) { + } + if (userCount == 0) { ++emptyCount; } else { - emptyCount=0; + emptyCount = 0; } } - - if( emptyCount > 1){ + + if (emptyCount > 1) { deletePage(); - } else if( !emptyCount ){ + } else if (!emptyCount) { newPage(); } - + if (empty) { free(empty); } @@ -911,6 +954,7 @@ namespace IPC { offsetOnPage = 0; } + ///\brief Copy constructor for sharedClients ///\param rhs The client ro copy sharedClient::sharedClient(const sharedClient & rhs) { @@ -956,7 +1000,7 @@ namespace IPC { ///\param name The basename of the server to connect to ///\param len The size of the payload to allocate ///\param withCounter Whether or not this payload has a counter - sharedClient::sharedClient(std::string name, int len, bool withCounter) : baseName("/"+name), payLen(len), offsetOnPage(-1), hasCounter(withCounter) { + sharedClient::sharedClient(std::string name, int len, bool withCounter) : baseName("/" + name), payLen(len), offsetOnPage(-1), hasCounter(withCounter) { #ifdef __APPLE__ //note: O_CREAT is only needed for mac, probably mySemaphore.open(baseName.c_str(), O_RDWR | O_CREAT, 0); @@ -967,6 +1011,7 @@ namespace IPC { DEBUG_MSG(DLVL_FAIL, "Creating semaphore %s failed: %s", baseName.c_str(), strerror(errno)); return; } + //Empty is used to compare for emptyness. This is not needed when the page uses a counter char * empty = 0; if (!hasCounter) { empty = (char *)malloc(payLen * sizeof(char)); @@ -976,12 +1021,12 @@ namespace IPC { } memset(empty, 0, payLen); } - while (offsetOnPage == -1){ + while (offsetOnPage == -1) { { semGuard tmpGuard(&mySemaphore); for (char i = 'A'; i <= 'Z'; i++) { myPage.init(baseName.substr(1) + i, (4096 << (i - 'A')), false, false); - if (!myPage.mapped){ + if (!myPage.mapped) { break; } int offset = 0; @@ -990,7 +1035,7 @@ namespace IPC { offsetOnPage = offset; if (hasCounter) { myPage.mapped[offset] = 1; - *((unsigned short *)(myPage.mapped+1+offset+len-2))=getpid(); + *((unsigned short *)(myPage.mapped + 1 + offset + len - 2)) = getpid(); } break; } @@ -1001,11 +1046,13 @@ namespace IPC { } } } - if (offsetOnPage == -1){ + if (offsetOnPage == -1) { Util::wait(500); } } - free(empty); + if (empty) { + free(empty); + } } ///\brief The deconstructor @@ -1058,11 +1105,11 @@ namespace IPC { } userConnection::userConnection(char * _data) { - data = _data; + data = _data; } unsigned long userConnection::getTrackId(size_t offset) const { - if (offset >= SIMUL_TRACKS){ + if (offset >= SIMUL_TRACKS) { WARN_MSG("Trying to get track id for entry %lu, while there are only %d entries allowed", offset, SIMUL_TRACKS); return 0; } @@ -1070,16 +1117,16 @@ namespace IPC { } void userConnection::setTrackId(size_t offset, unsigned long trackId) const { - if (offset >= SIMUL_TRACKS){ + if (offset >= SIMUL_TRACKS) { WARN_MSG("Trying to set track id for entry %lu, while there are only %d entries allowed", offset, SIMUL_TRACKS); return; } Bit::htobl(data + (offset * 6), trackId); - + } unsigned long userConnection::getKeynum(size_t offset) const { - if (offset >= SIMUL_TRACKS){ + if (offset >= SIMUL_TRACKS) { WARN_MSG("Trying to get keynum for entry %lu, while there are only %d entries allowed", offset, SIMUL_TRACKS); return 0; } @@ -1087,12 +1134,12 @@ namespace IPC { } void userConnection::setKeynum(size_t offset, unsigned long keynum) { - if (offset >= SIMUL_TRACKS){ + if (offset >= SIMUL_TRACKS) { WARN_MSG("Trying to set keynum for entry %lu, while there are only %d entries allowed", offset, SIMUL_TRACKS); return; } Bit::htobs(data + (offset * 6) + 4, keynum); - + } } diff --git a/lib/shared_memory.h b/lib/shared_memory.h index 5171e577..dd189670 100644 --- a/lib/shared_memory.h +++ b/lib/shared_memory.h @@ -69,6 +69,8 @@ namespace IPC { void unlink(); private: #if defined(__CYGWIN__) || defined(_WIN32) + ///\todo Maybe sometime implement anything else than 777 + static SECURITY_ATTRIBUTES getSecurityAttributes(); HANDLE mySem; #else sem_t * mySem; diff --git a/lib/ts_packet.cpp b/lib/ts_packet.cpp index 3bcc8f18..237e4b4a 100644 --- a/lib/ts_packet.cpp +++ b/lib/ts_packet.cpp @@ -690,6 +690,10 @@ namespace TS { return data[0]; } + void ProgramMappingEntry::setStreamType(int newType){ + data[0] = newType; + } + std::string ProgramMappingEntry::getCodec() const{ switch (getStreamType()){ case 0x01: @@ -740,10 +744,25 @@ namespace TS { return ((data[1] << 8) | data[2]) & 0x1FFF; } + void ProgramMappingEntry::setElementaryPid(int newElementaryPid) { + data[1] = newElementaryPid >> 8 & 0x1F; + data[2] = newElementaryPid & 0xFF; + } + int ProgramMappingEntry::getESInfoLength() const{ return ((data[3] << 8) | data[4]) & 0x0FFF; } + const char * ProgramMappingEntry::getESInfo() const{ + return data + 5; + } + + void ProgramMappingEntry::setESInfo(const std::string & newInfo){ + data[3] = (newInfo.size() >> 8) & 0x0F; + data[4] = newInfo.size() & 0xFF; + memcpy(data + 5, newInfo.data(), newInfo.size()); + } + void ProgramMappingEntry::advance(){ if (!(*this)) { return; @@ -884,14 +903,6 @@ namespace TS { strBuf[loc+1] = (char)newVal; } - short ProgramMappingTable::getProgramCount() const{ - return (getSectionLength() - 13) / 5; - } - - void ProgramMappingTable::setProgramCount(short newVal) { - setSectionLength(newVal * 5 + 13); - } - ProgramMappingEntry ProgramMappingTable::getEntry(int index) const{ int dataOffset = 4 + (getAdaptationField() > 1 ? getAdaptationFieldLen() + 1 : 0) + getOffset(); ProgramMappingEntry res((char*)(strBuf + dataOffset + 13 + getProgramInfoLength()), (char*)(strBuf + dataOffset + getSectionLength()) ); @@ -901,59 +912,6 @@ namespace TS { return res; } - char ProgramMappingTable::getStreamType(short index) const{ - if (index > getProgramCount()) { - return 0; - } - unsigned int loc = 4 + (getAdaptationField() > 1 ? getAdaptationFieldLen() + 1 : 0) + getOffset() + 13 + getProgramInfoLength(); - return strBuf[loc + (index * 5)]; - } - - void ProgramMappingTable::setStreamType(char newVal, short index) { - if (index > getProgramCount()) { - return; - } - unsigned int loc = 4 + (getAdaptationField() > 1 ? getAdaptationFieldLen() + 1 : 0) + getOffset() + 13 + getProgramInfoLength(); //TODO - updPos(loc+(index*5)+1); - strBuf[loc + (index * 5)] = newVal; - } - - short ProgramMappingTable::getElementaryPID(short index) const{ - if (index > getProgramCount()) { - return 0; - } - unsigned int loc = 4 + (getAdaptationField() > 1 ? getAdaptationFieldLen() + 1 : 0) + getOffset() + 13 + getProgramInfoLength(); - return (((short)strBuf[loc + (index * 5) + 1] & 0x1F) << 8) | strBuf[loc + (index * 5) + 2]; - } - - void ProgramMappingTable::setElementaryPID(short newVal, short index) { - if (index > getProgramCount()) { - return; - } - unsigned int loc = 4 + (getAdaptationField() > 1 ? getAdaptationFieldLen() + 1 : 0) + getOffset() + 13 + getProgramInfoLength(); - updPos(loc+(index*5)+3); - strBuf[loc + (index * 5)+1] = ((newVal >> 8) & 0x1F )| 0xE0; - strBuf[loc + (index * 5)+2] = (char)newVal; - } - - short ProgramMappingTable::getESInfoLength(short index) const{ - if (index > getProgramCount()) { - return 0; - } - unsigned int loc = 4 + (getAdaptationField() > 1 ? getAdaptationFieldLen() + 1 : 0) + getOffset() + 13 + getProgramInfoLength(); - return (((short)strBuf[loc + (index * 5) + 3] & 0x0F) << 8) | strBuf[loc + (index * 5) + 4]; - } - - void ProgramMappingTable::setESInfoLength(short newVal, short index) { - if (index > getProgramCount()) { - return; - } - unsigned int loc = 4 + (getAdaptationField() > 1 ? getAdaptationFieldLen() + 1 : 0) + getOffset() + 13 + getProgramInfoLength(); - updPos(loc+(index*5)+5); - strBuf[loc + (index * 5)+3] = ((newVal >> 8) & 0x0F) | 0xF0; - strBuf[loc + (index * 5)+4] = (char)newVal; - } - int ProgramMappingTable::getCRC() const{ unsigned int loc = 4 + (getAdaptationField() > 1 ? getAdaptationFieldLen() + 1 : 0) + getOffset() + getSectionLength(); return ((int)(strBuf[loc]) << 24) | ((int)(strBuf[loc + 1]) << 16) | ((int)(strBuf[loc + 2]) << 8) | strBuf[loc + 3]; @@ -1011,7 +969,14 @@ namespace TS { PMT.setPID(4096); PMT.setTableId(2); //section length met 2 tracks: 0xB017 - PMT.setSectionLength(0xB00D + (selectedTracks.size() * 5)); + int sectionLen = 0; + for (std::set::iterator it = selectedTracks.begin(); it != selectedTracks.end(); it++){ + sectionLen += 5; + if (myMeta.tracks[*it].codec == "ID3"){ + sectionLen += myMeta.tracks[*it].init.size(); + } + } + PMT.setSectionLength(0xB00D + sectionLen); PMT.setProgramNumber(1); PMT.setVersionNumber(0); PMT.setCurrentNextIndicator(0); @@ -1028,24 +993,27 @@ namespace TS { if (vidTrack == -1){ vidTrack = *(selectedTracks.begin()); } - PMT.setPCRPID(0x100 + vidTrack - 1); + PMT.setPCRPID(vidTrack); PMT.setProgramInfoLength(0); short id = 0; + ProgramMappingEntry entry = PMT.getEntry(0); for (std::set::iterator it = selectedTracks.begin(); it != selectedTracks.end(); it++){ + entry.setElementaryPid(*it); if (myMeta.tracks[*it].codec == "H264"){ - PMT.setStreamType(0x1B,id); + entry.setStreamType(0x1B); }else if (myMeta.tracks[*it].codec == "HEVC"){ - PMT.setStreamType(0x24,id); + entry.setStreamType(0x24); }else if (myMeta.tracks[*it].codec == "AAC"){ - PMT.setStreamType(0x0F,id); + entry.setStreamType(0x0F); }else if (myMeta.tracks[*it].codec == "MP3"){ - PMT.setStreamType(0x03,id); + entry.setStreamType(0x03); }else if (myMeta.tracks[*it].codec == "AC3"){ - PMT.setStreamType(0x81,id); + entry.setStreamType(0x81); + }else if (myMeta.tracks[*it].codec == "ID3"){ + entry.setStreamType(0x15); + entry.setESInfo(myMeta.tracks[*it].init); } - PMT.setElementaryPID(0x100 + (*it) - 1, id); - PMT.setESInfoLength(0,id); - id++; + entry.advance(); } PMT.calcCRC(); return PMT.checkAndGetBuffer(); diff --git a/lib/ts_packet.h b/lib/ts_packet.h index 6faa6e9c..7c6e3b4f 100644 --- a/lib/ts_packet.h +++ b/lib/ts_packet.h @@ -107,11 +107,14 @@ namespace TS { operator bool() const; int getStreamType() const; + void setStreamType(int newType); std::string getCodec() const; std::string getStreamTypeString() const; int getElementaryPid() const; + void setElementaryPid(int newElementaryPid); int getESInfoLength() const; - char * getESInfo() const; + const char * getESInfo() const; + void setESInfo(const std::string & newInfo); void advance(); private: char* data; @@ -142,15 +145,7 @@ namespace TS { void setPCRPID(short newVal); short getProgramInfoLength() const; void setProgramInfoLength(short newVal); - short getProgramCount() const; - void setProgramCount(short newVal); ProgramMappingEntry getEntry(int index) const; - void setStreamType(char newVal, short index); - char getStreamType(short index) const; - void setElementaryPID(short newVal, short index); - short getElementaryPID(short index) const; - void setESInfoLength(short newVal,short index); - short getESInfoLength(short index) const; int getCRC() const; void calcCRC(); std::string toPrettyString(size_t indent) const; diff --git a/lib/ts_stream.cpp b/lib/ts_stream.cpp index fede4f3d..05ce7157 100644 --- a/lib/ts_stream.cpp +++ b/lib/ts_stream.cpp @@ -4,48 +4,148 @@ #include "h265.h" #include "nal.h" #include "mp4_generic.h" +#include namespace TS { + Stream::Stream(bool _threaded){ + threaded = _threaded; + if (threaded){ + globalSem.open("MstTSInputLock", O_CREAT | O_EXCL | O_RDWR, ACCESSPERMS, 1); + if (!globalSem) { + globalSem.open("MstTSInputLock", O_CREAT | O_RDWR, ACCESSPERMS, 1); + } + if (!globalSem) { + FAIL_MSG("Creating semaphore failed: %s", strerror(errno)); + threaded = false; + DEBUG_MSG(DLVL_FAIL, "Creating semaphore failed: %s", strerror(errno)); + return; + } + } + } + void Stream::parse(char * newPack, unsigned long long bytePos) { Packet newPacket; newPacket.FromPointer(newPack); parse(newPacket, bytePos); } - + void Stream::clear(){ + if (threaded){ + globalSem.wait(); + } pesStreams.clear(); pesPositions.clear(); - payloadSize.clear(); outPackets.clear(); + if (threaded){ + globalSem.post(); + } + } + + void Stream::add(char * newPack, unsigned long long bytePos) { + Packet newPacket; + newPacket.FromPointer(newPack); + add(newPacket, bytePos); + } + + void Stream::add(Packet & newPack, unsigned long long bytePos) { + if (threaded){ + globalSem.wait(); + } + + int tid = newPack.getPID(); + pesStreams[tid].push_back(newPack); + pesPositions[tid].push_back(bytePos); + + if (threaded){ + globalSem.post(); + } + } + + bool Stream::isDataTrack(unsigned long tid){ + if (tid == 0){ + return false; + } + if (threaded){ + globalSem.wait(); + } + bool result = !pmtTracks.count(tid); + if (threaded){ + globalSem.post(); + } + return result; } - void Stream::parse(Packet & newPack, unsigned long long bytePos) { - int tid = newPack.getPID(); + void Stream::parse(unsigned long tid) { + if (threaded){ + globalSem.wait(); + } + if (!pesStreams.count(tid) || pesStreams[tid].size() == 0){ + if (threaded){ + globalSem.post(); + } + return; + } + std::deque & trackPackets = pesStreams[tid]; + + if (threaded){ + globalSem.post(); + } + + //Handle PAT packets if (tid == 0){ - associationTable = newPack; - pmtTracks.clear(); + ///\todo Keep track of updates in PAT instead of keeping only the last PAT as a reference + + if (threaded){ + globalSem.wait(); + } + associationTable = trackPackets.back(); + lastPAT = Util::bootSecs(); + if (threaded){ + globalSem.post(); + } + + int pmtCount = associationTable.getProgramCount(); for (int i = 0; i < pmtCount; i++){ pmtTracks.insert(associationTable.getProgramPID(i)); } + + if (threaded){ + globalSem.wait(); + } + pesStreams.erase(0); + pesPositions.erase(0); + if (threaded){ + globalSem.post(); + } return; } - //If we are here, the packet is not a PAT. - //First check if it is listed in the PAT as a PMT track. - int pmtCount = associationTable.getProgramCount(); + + //Handle PMT packets if (pmtTracks.count(tid)){ - mappingTable[tid] = newPack; + ///\todo Keep track of updates in PMT instead of keeping only the last PMT per program as a reference + if (threaded){ + globalSem.wait(); + } + mappingTable[tid] = trackPackets.back(); + lastPMT[tid] = Util::bootSecs(); + if (threaded){ + globalSem.post(); + } ProgramMappingEntry entry = mappingTable[tid].getEntry(0); while (entry){ unsigned long pid = entry.getElementaryPid(); - switch(entry.getStreamType()){ + unsigned long sType = entry.getStreamType(); + switch(sType){ case H264: case AAC: case HEVC: case H265: case AC3: - if (!pidToCodec.count(pid)){ - pidToCodec[pid] = entry.getStreamType(); + case ID3: + pidToCodec[pid] = sType; + if (sType == ID3){ + metaInit[pid] = std::string(entry.getESInfo(), entry.getESInfoLength()); } break; default: @@ -53,57 +153,107 @@ namespace TS { } entry.advance(); } + + if (threaded){ + globalSem.wait(); + } + pesStreams.erase(tid); + pesPositions.erase(tid); + if (threaded){ + globalSem.post(); + } + return; } - //If it is not a PMT, check the list of all PMTs to see if this is a new PES track. - bool inPMT = false; - for (std::map::iterator it = mappingTable.begin(); it!= mappingTable.end(); it++){ - ProgramMappingEntry entry = it->second.getEntry(0); - while (entry){ - if (tid == entry.getElementaryPid()){ - inPMT = true; - break; - } - entry.advance(); - } - if (inPMT){ - break; - } + + if (threaded){ + globalSem.wait(); } - if (!inPMT){ - HIGH_MSG("Encountered a packet on track %d, but the track is not registered in any PMT", tid); - return; + + bool parsePes = false; + + int packNum = 1; + std::deque & inStream = pesStreams[tid]; + std::deque::iterator curPack = inStream.begin(); + curPack++; + while (curPack != inStream.end() && !curPack->getUnitStart()){ + curPack++; + packNum++; } - pesStreams[tid].push_back(newPack); - pesPositions[tid].push_back(bytePos); - if (!newPack.getUnitStart() || pesStreams[tid].size() == 1){ - payloadSize[tid] += newPack.getPayloadLength(); + if (curPack != inStream.end()){ + parsePes = true; } - parsePES(tid); + + if (threaded){ + globalSem.post(); + } + + if (parsePes){ + parsePES(tid); + } + } + + void Stream::parse(Packet & newPack, unsigned long long bytePos) { + add(newPack, bytePos); + + int tid = newPack.getPID(); + parse(tid); } bool Stream::hasPacketOnEachTrack() const { - if (!pidToCodec.size()){ - return false; + if (threaded){ + globalSem.wait(); } - if (outPackets.size() != pidToCodec.size()){ + if (!pidToCodec.size() || pidToCodec.size() != outPackets.size()){ + if (threaded){ + globalSem.post(); + } return false; } for (std::map::const_iterator it = pidToCodec.begin(); it != pidToCodec.end(); it++){ - if (!outPackets.count(it->first) || !outPackets.at(it->first).size()){ + if (!hasPacket(it->first)){ + if (threaded){ + globalSem.post(); + } return false; } } + if (threaded){ + globalSem.post(); + } return true; } bool Stream::hasPacket(unsigned long tid) const { + if (threaded){ + globalSem.wait(); + } if (!pesStreams.count(tid)){ + if (threaded){ + globalSem.post(); + } return false; } if (outPackets.count(tid) && outPackets.at(tid).size()){ + if (threaded){ + globalSem.post(); + } return true; } + std::deque::const_iterator curPack = pesStreams.at(tid).begin(); + curPack++; + while (curPack != pesStreams.at(tid).end() && !curPack->getUnitStart()){ + curPack++; + } + if (curPack != pesStreams.at(tid).end()){ + if (threaded){ + globalSem.post(); + } + return true; + } + if (threaded){ + globalSem.post(); + } return false; } @@ -119,32 +269,63 @@ namespace TS { } void Stream::parsePES(unsigned long tid){ + if (threaded){ + globalSem.wait(); + } std::deque & inStream = pesStreams[tid]; std::deque & inPositions = pesPositions[tid]; if (inStream.size() == 1){ + if (threaded){ + globalSem.post(); + } return; } - if (!inStream.back().getUnitStart()){ + //Find number of packets before unit Start + int packNum = 1; + + std::deque::iterator curPack = inStream.begin(); + curPack++; + while (curPack != inStream.end() && !curPack->getUnitStart()){ + curPack++; + packNum++; + } + if (curPack == inStream.end()){ + if (threaded){ + globalSem.post(); + } return; } unsigned long long bPos = inPositions.front(); //Create a buffer for the current PES, and remove it from the pesStreams buffer. - int paySize = payloadSize[tid]; - char * payload = (char*)malloc(paySize); - int offset = 0; - int packNum = inStream.size() - 1; - std::deque::iterator curPack = inStream.begin(); + int paySize = 0; + + curPack = inStream.begin(); for (int i = 0; i < packNum; i++){ - memcpy(payload + offset, curPack->getPayload(), curPack->getPayloadLength()); - offset += curPack->getPayloadLength(); + paySize += curPack->getPayloadLength(); + curPack++; + } + char * payload = (char*)malloc(paySize); + paySize = 0; + curPack = inStream.begin(); + int lastCtr = curPack->getContinuityCounter() - 1; + for (int i = 0; i < packNum; i++){ + if (curPack->getContinuityCounter() - lastCtr != 1 && curPack->getContinuityCounter()){ + INFO_MSG("Parsing a pes on track %d, missed %d packets", tid, curPack->getContinuityCounter() - lastCtr - 1); + } + lastCtr = curPack->getContinuityCounter(); + memcpy(payload + paySize, curPack->getPayload(), curPack->getPayloadLength()); + paySize += curPack->getPayloadLength(); curPack++; } inStream.erase(inStream.begin(), curPack); inPositions.erase(inPositions.begin(), inPositions.begin() + packNum); + if (threaded){ + globalSem.post(); + } //Parse the PES header - offset = 0; + int offset = 0; while(offset < paySize){ const char * pesHeader = payload + offset; @@ -203,6 +384,9 @@ namespace TS { //Parse all the ADTS packets unsigned long offsetInPes = 0; unsigned long samplesRead = 0; + if (threaded){ + globalSem.wait(); + } while (offsetInPes < realPayloadSize){ outPackets[tid].push_back(DTSC::Packet()); aac::adts adtsPack(pesPayload + offsetInPes, realPayloadSize - offsetInPes); @@ -213,10 +397,19 @@ namespace TS { samplesRead += adtsPack.getSampleCount(); offsetInPes += adtsPack.getHeaderSize() + adtsPack.getPayloadSize(); } + if (threaded){ + globalSem.post(); + } } - if (pidToCodec[tid] == AC3){ + if (pidToCodec[tid] == ID3 || pidToCodec[tid] == AC3){ + if (threaded){ + globalSem.wait(); + } outPackets[tid].push_back(DTSC::Packet()); outPackets[tid].back().genericFill(timeStamp, timeOffset, tid, pesPayload, realPayloadSize, bPos, 0); + if (threaded){ + globalSem.post(); + } } if (pidToCodec[tid] == H264 || pidToCodec[tid] == HEVC || pidToCodec[tid] == H265){ //Convert from annex b @@ -239,11 +432,23 @@ namespace TS { break; } case 0x07: { + if (threaded){ + globalSem.wait(); + } spsInfo[tid] = std::string(parsedData + dataOffset + 4, it->nalSize); + if (threaded){ + globalSem.post(); + } break; } case 0x08: { + if (threaded){ + globalSem.wait(); + } ppsInfo[tid] = std::string(parsedData + dataOffset + 4, it->nalSize); + if (threaded){ + globalSem.post(); + } break; } default: break; @@ -264,7 +469,13 @@ namespace TS { case 32: case 33: case 34: { + if (threaded){ + globalSem.wait(); + } hevcInfo[tid].addUnit(parsedData + dataOffset); + if (threaded){ + globalSem.post(); + } break; } default: break; @@ -272,8 +483,14 @@ namespace TS { } dataOffset += 4 + it->nalSize; } + if (threaded){ + globalSem.wait(); + } outPackets[tid].push_back(DTSC::Packet()); outPackets[tid].back().genericFill(timeStamp, timeOffset, tid, parsedData, parsedSize, bPos, isKeyFrame); + if (threaded){ + globalSem.post(); + } free(parsedData); } //We are done with the realpayload size, reverse calculation so we know the correct offset increase. @@ -285,7 +502,6 @@ namespace TS { offset += realPayloadSize + 6; } free(payload); - payloadSize[tid] = inStream.front().getPayloadLength(); } void Stream::getPacket(unsigned long tid, DTSC::Packet & pack) { @@ -295,17 +511,55 @@ namespace TS { return; } + if (threaded){ + globalSem.wait(); + } + bool packetReady = outPackets.count(tid) && outPackets[tid].size(); + if (threaded){ + globalSem.post(); + } + + if (!packetReady){ + parse(tid); + } + + if (threaded){ + globalSem.wait(); + } + packetReady = outPackets.count(tid) && outPackets[tid].size(); + if (threaded){ + globalSem.post(); + } + + if (!packetReady){ + ERROR_MSG("Obtaining a packet on track %lu failed", tid); + return; + } + + if (threaded){ + globalSem.wait(); + } pack = outPackets[tid].front(); outPackets[tid].pop_front(); if (!outPackets[tid].size()){ outPackets.erase(tid); } + + if (threaded){ + globalSem.post(); + } } void Stream::getEarliestPacket(DTSC::Packet & pack){ + if (threaded){ + globalSem.wait(); + } pack.null(); if (!hasPacketOnEachTrack()){ + if (threaded){ + globalSem.post(); + } return; } @@ -318,12 +572,21 @@ namespace TS { packTime = it->second.front().getTime(); } } + if (threaded){ + globalSem.post(); + } getPacket(packTrack, pack); } - void Stream::initializeMetadata(DTSC::Meta & meta) { + void Stream::initializeMetadata(DTSC::Meta & meta, unsigned long tid) { + if (threaded){ + globalSem.wait(); + } for (std::map::const_iterator it = pidToCodec.begin(); it != pidToCodec.end(); it++){ + if (tid && it->first != tid){ + continue; + } if (!meta.tracks.count(it->first) && it->second == H264){ if (!spsInfo.count(it->first) || !ppsInfo.count(it->first)){ continue; @@ -357,6 +620,12 @@ namespace TS { meta.tracks[it->first].trackID = it->first; meta.tracks[it->first].init = hevcInfo[it->first].generateHVCC(); } + if (!meta.tracks.count(it->first) && it->second == ID3){ + meta.tracks[it->first].type = "meta"; + meta.tracks[it->first].codec = "ID3"; + meta.tracks[it->first].trackID = it->first; + meta.tracks[it->first].init = metaInit[it->first]; + } if (!meta.tracks.count(it->first) && it->second == AC3){ meta.tracks[it->first].type = "audio"; meta.tracks[it->first].codec = "AC3"; @@ -379,5 +648,63 @@ namespace TS { meta.tracks[it->first].init = std::string(audioInit, 2); } } + if (threaded){ + globalSem.post(); + } + } + + std::set Stream::getActiveTracks() { + if (threaded){ + globalSem.wait(); + } + std::set result; + //Track 0 is always active + result.insert(0); + //IF PAT updated in the last 5 seconds, check for contents + if (Util::bootSecs() - lastPAT < 5){ + int pmtCount = associationTable.getProgramCount(); + //For each PMT + for (int i = 0; i < pmtCount; i++){ + int pid = associationTable.getProgramPID(i); + //Add PMT track + result.insert(pid); + //IF PMT updated in last 5 seconds, check for contents + if (Util::bootSecs() - lastPMT[pid] < 5){ + ProgramMappingEntry entry = mappingTable[pid].getEntry(0); + //Add all tracks in PMT + while (entry){ + switch(entry.getStreamType()){ + case H264: + case AAC: + case HEVC: + case H265: + case AC3: + case ID3: + result.insert(entry.getElementaryPid()); + break; + default: + break; + } + entry.advance(); + } + } + } + } + if (threaded){ + globalSem.post(); + } + return result; + } + + void Stream::eraseTrack(unsigned long tid){ + if (threaded){ + globalSem.wait(); + } + pesStreams.erase(tid); + pesPositions.erase(tid); + outPackets.erase(tid); + if (threaded){ + globalSem.post(); + } } } diff --git a/lib/ts_stream.h b/lib/ts_stream.h index c68d709b..7b9ebeec 100644 --- a/lib/ts_stream.h +++ b/lib/ts_stream.h @@ -1,9 +1,12 @@ #include "ts_packet.h" #include "adts.h" #include +#include #include #include "h265.h" +#include "shared_memory.h" + namespace TS { enum codecType { H264 = 0x1B, @@ -11,32 +14,47 @@ namespace TS { AC3 = 0x81, MP3 = 0x03, HEVC = 0x06, - H265 = 0x24 + H265 = 0x24, + ID3 = 0x15 }; class Stream{ public: + Stream(bool _threaded = false); + void add(char * newPack, unsigned long long bytePos = 0); + void add(Packet & newPack, unsigned long long bytePos = 0); void parse(Packet & newPack, unsigned long long bytePos); void parse(char * newPack, unsigned long long bytePos); + void parse(unsigned long tid); bool hasPacketOnEachTrack() const; bool hasPacket(unsigned long tid) const; void getPacket(unsigned long tid, DTSC::Packet & pack); void getEarliestPacket(DTSC::Packet & pack); - void initializeMetadata(DTSC::Meta & meta); + void initializeMetadata(DTSC::Meta & meta, unsigned long tid = 0); void clear(); + void eraseTrack(unsigned long tid); + bool isDataTrack(unsigned long tid); + std::set getActiveTracks(); private: + unsigned long long lastPAT; ProgramAssociationTable associationTable; + + std::map lastPMT; std::map mappingTable; + std::map > pesStreams; std::map > pesPositions; - std::map payloadSize; std::map > outPackets; std::map pidToCodec; std::map adtsInfo; std::map spsInfo; std::map ppsInfo; std::map hevcInfo; + std::map metaInit; + mutable IPC::semaphore globalSem; + + bool threaded; std::set pmtTracks; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt deleted file mode 100644 index 99e57029..00000000 --- a/src/CMakeLists.txt +++ /dev/null @@ -1,101 +0,0 @@ -macro(makeAnalyser analyserName format) - add_executable( MistAnalyser${analyserName} analysers/${format}_analyser.cpp ) - target_link_libraries( MistAnalyser${analyserName} mist ) -endmacro() - -macro(makeInput inputName format) - add_executable( MistIn${inputName} input/mist_in.cpp input/input.cpp input/input_${format}.cpp ) - set_target_properties( MistIn${inputName} PROPERTIES COMPILE_DEFINITIONS INPUTTYPE=\"input_${format}.h\") - target_link_libraries( MistIn${inputName} mist ) -endmacro() - -macro(makeOutput outputName format) -#check if 'http' is one of the argyments, if yes, this is an http output - if (";${ARGN};" MATCHES ";http;") - SET(httpOutput output/output_http.cpp) - if (";${ARGN};" MATCHES ";ts;") - SET(tsBaseClass HTTPOutput) - else() - SET(tsBaseClass Output) - endif() - endif() - if (";${ARGN};" MATCHES ";ts;") - SET(tsOutput output/output_ts_base.cpp) - endif() - add_executable( MistOut${outputName} output/mist_out.cpp output/output.cpp ${httpOutput} ${tsOutput} output/output_${format}.cpp ) - set_target_properties( MistOut${outputName} PROPERTIES COMPILE_DEFINITIONS "OUTPUTTYPE=\"output_${format}.h\";TS_BASECLASS=${tsBaseClass}") - target_link_libraries( MistOut${outputName} mist ) -endmacro() - -makeAnalyser(RTMP rtmp) -makeAnalyser(FLV flv) -makeAnalyser(DTSC dtsc) -makeAnalyser(AMF amf) -makeAnalyser(MP4 mp4) -makeAnalyser(OGG ogg) - -makeInput(DTSC dtsc) -makeInput(MP3 mp3) -makeInput(FLV flv) -makeInput(OGG ogg) -makeInput(Buffer buffer) - -makeOutput(RTMP rtmp) -makeOutput(OGG progressive_ogg http) -makeOutput(FLV progressive_flv http) -makeOutput(MP4 progressive_mp4 http) -makeOutput(MP3 progressive_mp3 http) -makeOutput(HSS hss http) -makeOutput(HDS hds http) -makeOutput(SRT srt http) -makeOutput(JSON json http) -makeOutput(TS ts ts) -makeOutput(HTTPTS httpts http ts) -makeOutput(HLS hls http ts) - -#get the bitlength of this system -execute_process(COMMAND getconf LONG_BIT OUTPUT_VARIABLE RELEASE_RAW ) -#strip off the trailing spaces and newline -string(STRIP ${RELEASE_RAW} RELEASE) -set(RELEASE \"${RELEASE}\" ) - -include_directories(${CMAKE_CURRENT_BINARY_DIR}) -add_executable( sourcery sourcery.cpp ) - -add_custom_target( embedcode - ALL - ./sourcery ${CMAKE_CURRENT_SOURCE_DIR}/embed.js embed_js ${CMAKE_CURRENT_BINARY_DIR}/embed.js.h - DEPENDS sourcery ${CMAKE_CURRENT_SOURCE_DIR}/embed.js - VERBATIM -) - -add_custom_target( localSettingsPage - ALL - ./sourcery ${BINARY_DIR}/lsp/server.html server_html ${CMAKE_CURRENT_BINARY_DIR}/server.html.h - DEPENDS sourcery lsp - VERBATIM -) - -add_executable( MistOutHTTP output/mist_out.cpp output/output.cpp output/output_http.cpp output/output_http_internal.cpp) -set_target_properties( MistOutHTTP PROPERTIES COMPILE_DEFINITIONS "OUTPUTTYPE=\"output_http_internal.h\"") -add_dependencies(MistOutHTTP embedcode) -target_link_libraries( MistOutHTTP mist ) - -add_executable( MistController - controller/controller.cpp - controller/controller_api.h - controller/controller_api.cpp - controller/controller_capabilities.h - controller/controller_capabilities.cpp - controller/controller_connectors.h - controller/controller_connectors.cpp - controller/controller_statistics.h - controller/controller_statistics.cpp - controller/controller_storage.h - controller/controller_storage.cpp - controller/controller_streams.h - controller/controller_streams.cpp -) -set_target_properties( MistController PROPERTIES COMPILE_DEFINITIONS RELEASE=${RELEASE}) -target_link_libraries( MistController mist ) -add_dependencies(MistController localSettingsPage) diff --git a/src/controller/controller_streams.cpp b/src/controller/controller_streams.cpp index 6f74ecd0..fec98303 100644 --- a/src/controller/controller_streams.cpp +++ b/src/controller/controller_streams.cpp @@ -64,6 +64,7 @@ namespace Controller { std::string udpPort = data["udpport"].asString(); //Check running if (!inputProcesses.count(name) || !Util::Procs::isRunning(inputProcesses[name])){ + std::string multicast = data["multicastinterface"].asString(); // False: start TS input INFO_MSG("No TS Input running on port %s for stream %s, starting it", udpPort.c_str(), name.c_str()); std::deque command; @@ -72,6 +73,8 @@ namespace Controller { command.push_back(name); command.push_back("-p"); command.push_back(udpPort); + command.push_back("-M"); + command.push_back(multicast); command.push_back(URL); int stdIn = 0; int stdOut = 1; diff --git a/src/input/input.cpp b/src/input/input.cpp index caa4a6c2..c9878614 100644 --- a/src/input/input.cpp +++ b/src/input/input.cpp @@ -29,11 +29,7 @@ namespace Mist { Input::Input(Util::Config * cfg) : InOutBase() { config = cfg; -#ifdef INPUT_LIVE - standAlone = false; -#else standAlone = true; -#endif JSON::Value option; option["long"] = "json"; @@ -90,7 +86,7 @@ namespace Mist { } void Input::checkHeaderTimes(std::string streamFile) { - if (streamFile == "-") { + if (streamFile == "-" || streamFile == "push://") { return; } std::string headerFile = streamFile + ".dtsh"; @@ -124,6 +120,7 @@ namespace Mist { config->getOption("streamname") = streamName; } streamName = config->getString("streamname"); + nProxy.streamName = streamName; if (config->getBool("json")) { std::cout << capa.toString() << std::endl; return 0; @@ -132,26 +129,19 @@ namespace Mist { std::cerr << config->getString("cmd") << " setup failed." << std::endl; return 0; } -//Do not read the header if this is a live stream -#ifndef INPUT_LIVE + checkHeaderTimes(config->getString("input")); if (!readHeader()) { std::cerr << "Reading header for " << config->getString("input") << " failed." << std::endl; return 0; } parseHeader(); -#endif -//Live inputs only have a serve() mode -#ifndef INPUT_LIVE - if (!config->getString("streamname").size()) { + if (!streamName.size()) { convert(); } else { -#endif serve(); -#ifndef INPUT_LIVE } -#endif return 0; } @@ -203,44 +193,6 @@ namespace Mist { void Input::serve(){ char userPageName[NAME_BUFFER_SIZE]; snprintf(userPageName, NAME_BUFFER_SIZE, SHM_USERS, streamName.c_str()); -#ifdef INPUT_LIVE - unsigned int giveUpCounter = 0; - while (!Util::startInput(streamName) && config->is_active && ++giveUpCounter < 20) { - Util::sleep(500); - } - if (giveUpCounter >= 20){ - FAIL_MSG("Could not start buffer for stream '%s', aborting stream input!", streamName.c_str()); - config->is_active = false; - } - userClient = IPC::sharedClient(userPageName, 30, true); - getNext(); - while (thisPacket || config->is_active) { - unsigned long tid = thisPacket.getTrackId(); - //Check for eligibility of track - IPC::userConnection userConn(userClient.getData()); - if (trackOffset.count(tid) && !userConn.getTrackId(trackOffset[tid])) { - trackOffset.erase(tid); - trackState.erase(tid); - trackMap.erase(tid); - trackBuffer.erase(tid); - pagesByTrack.erase(tid); - metaPages.erase(tid); - curPageNum.erase(tid); - curPage.erase(tid); - INFO_MSG("Erasing track %d", tid); - continue; - } - if (thisPacket) { - continueNegotiate(thisPacket.getTrackId()); - bufferLivePacket(thisPacket); - } else { - Util::sleep(100); - } - getNext(); - userClient.keepAlive(); - } - userClient.finish(); -#else /*LTS-START*/ if(Triggers::shouldTrigger("STREAM_READY", config->getString("streamname"))){ std::string payload = config->getString("streamname")+"\n" +capa["name"].asStringRef()+"\n"; @@ -283,7 +235,6 @@ namespace Mist { Util::sleep(1000); } } -#endif finish(); DEBUG_MSG(DLVL_DEVEL, "Input for stream %s closing clean", streamName.c_str()); //end player functionality @@ -297,7 +248,7 @@ namespace Mist { } removeUnused(); if (standAlone) { - for (std::map::iterator it = metaPages.begin(); it != metaPages.end(); it++) { + for (std::map::iterator it = nProxy.metaPages.begin(); it != nProxy.metaPages.end(); it++) { it->second.master = true; } } @@ -316,9 +267,9 @@ namespace Mist { bufferRemove(it->first, it2->first); pageCounter[it->first].erase(it2->first); for (int i = 0; i < 8192; i += 8) { - unsigned int thisKeyNum = ntohl(((((long long int *)(metaPages[it->first].mapped + i))[0]) >> 32) & 0xFFFFFFFF); + unsigned int thisKeyNum = ntohl(((((long long int *)(nProxy.metaPages[it->first].mapped + i))[0]) >> 32) & 0xFFFFFFFF); if (thisKeyNum == it2->first) { - (((long long int *)(metaPages[it->first].mapped + i))[0]) = 0; + (((long long int *)(nProxy.metaPages[it->first].mapped + i))[0]) = 0; } } change = true; @@ -359,13 +310,13 @@ namespace Mist { for (int i = 0; i < it->second.keys.size(); i++) { if (newData) { //i+1 because keys are 1-indexed - pagesByTrack[it->first][i + 1].firstTime = it->second.keys[i].getTime(); + nProxy.pagesByTrack[it->first][i + 1].firstTime = it->second.keys[i].getTime(); newData = false; } - pagesByTrack[it->first].rbegin()->second.keyNum++; - pagesByTrack[it->first].rbegin()->second.partNum += it->second.keys[i].getParts(); - pagesByTrack[it->first].rbegin()->second.dataSize += it->second.keySizes[i]; - if (pagesByTrack[it->first].rbegin()->second.dataSize > FLIP_DATA_PAGE_SIZE) { + nProxy.pagesByTrack[it->first].rbegin()->second.keyNum++; + nProxy.pagesByTrack[it->first].rbegin()->second.partNum += it->second.keys[i].getParts(); + nProxy.pagesByTrack[it->first].rbegin()->second.dataSize += it->second.keySizes[i]; + if (nProxy.pagesByTrack[it->first].rbegin()->second.dataSize > FLIP_DATA_PAGE_SIZE) { newData = true; } } @@ -398,7 +349,7 @@ namespace Mist { } if (myMeta.tracks[tid].keys[bookKeeping[tid].curKey].getParts() + 1 == curData[tid].partNum) { if (curData[tid].dataSize > FLIP_DATA_PAGE_SIZE) { - pagesByTrack[tid][bookKeeping[tid].first] = curData[tid]; + nProxy.pagesByTrack[tid][bookKeeping[tid].first] = curData[tid]; bookKeeping[tid].first += curData[tid].keyNum; curData[tid].keyNum = 0; curData[tid].dataSize = 0; @@ -415,17 +366,17 @@ namespace Mist { getNext(false); } for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++) { - if (curData.count(it->first) && !pagesByTrack[it->first].count(bookKeeping[it->first].first)) { - pagesByTrack[it->first][bookKeeping[it->first].first] = curData[it->first]; + if (curData.count(it->first) && !nProxy.pagesByTrack[it->first].count(bookKeeping[it->first].first)) { + nProxy.pagesByTrack[it->first][bookKeeping[it->first].first] = curData[it->first]; } } } for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++) { - if (!pagesByTrack.count(it->first)) { + if (!nProxy.pagesByTrack.count(it->first)) { DEBUG_MSG(DLVL_WARN, "No pages for track %d found", it->first); } else { - DEBUG_MSG(DLVL_MEDIUM, "Track %d (%s) split into %lu pages", it->first, myMeta.tracks[it->first].codec.c_str(), pagesByTrack[it->first].size()); - for (std::map::iterator it2 = pagesByTrack[it->first].begin(); it2 != pagesByTrack[it->first].end(); it2++) { + DEBUG_MSG(DLVL_MEDIUM, "Track %d (%s) split into %lu pages", it->first, myMeta.tracks[it->first].codec.c_str(), nProxy.pagesByTrack[it->first].size()); + for (std::map::iterator it2 = nProxy.pagesByTrack[it->first].begin(); it2 != nProxy.pagesByTrack[it->first].end(); it2++) { DEBUG_MSG(DLVL_VERYHIGH, "Page %lu-%lu, (%llu bytes)", it2->first, it2->first + it2->second.keyNum - 1, it2->second.dataSize); } } @@ -443,10 +394,10 @@ namespace Mist { if (keyNum < 1) { keyNum = 1; } - if (isBuffered(track, keyNum)) { + if (nProxy.isBuffered(track, keyNum)) { //get corresponding page number int pageNumber = 0; - for (std::map::iterator it = pagesByTrack[track].begin(); it != pagesByTrack[track].end(); it++) { + for (std::map::iterator it = nProxy.pagesByTrack[track].begin(); it != nProxy.pagesByTrack[track].end(); it++) { if (it->first <= keyNum) { pageNumber = it->first; } else { @@ -457,13 +408,13 @@ namespace Mist { VERYHIGH_MSG("Track %u, key %u is already buffered in page %d. Cancelling bufferFrame", track, keyNum, pageNumber); return true; } - if (!pagesByTrack.count(track)) { + if (!nProxy.pagesByTrack.count(track)) { WARN_MSG("No pages for track %u found! Cancelling bufferFrame", track); return false; } //Update keynum to point to the corresponding page - INFO_MSG("Loading key %u from page %lu", keyNum, (--(pagesByTrack[track].upper_bound(keyNum)))->first); - keyNum = (--(pagesByTrack[track].upper_bound(keyNum)))->first; + INFO_MSG("Loading key %u from page %lu", keyNum, (--(nProxy.pagesByTrack[track].upper_bound(keyNum)))->first); + keyNum = (--(nProxy.pagesByTrack[track].upper_bound(keyNum)))->first; if (!bufferStart(track, keyNum)) { WARN_MSG("bufferStart failed! Cancelling bufferFrame"); return false; @@ -474,8 +425,8 @@ namespace Mist { trackSelect(trackSpec.str()); seek(myMeta.tracks[track].keys[keyNum - 1].getTime()); long long unsigned int stopTime = myMeta.tracks[track].lastms + 1; - if ((int)myMeta.tracks[track].keys.size() > keyNum - 1 + pagesByTrack[track][keyNum].keyNum) { - stopTime = myMeta.tracks[track].keys[keyNum - 1 + pagesByTrack[track][keyNum].keyNum].getTime(); + if ((int)myMeta.tracks[track].keys.size() > keyNum - 1 + nProxy.pagesByTrack[track][keyNum].keyNum) { + stopTime = myMeta.tracks[track].keys[keyNum - 1 + nProxy.pagesByTrack[track][keyNum].keyNum].getTime(); } DEBUG_MSG(DLVL_HIGH, "Playing from %llu to %llu", myMeta.tracks[track].keys[keyNum - 1].getTime(), stopTime); getNext(); diff --git a/src/input/input_buffer.cpp b/src/input/input_buffer.cpp index 3470eba2..7bced564 100644 --- a/src/input/input_buffer.cpp +++ b/src/input/input_buffer.cpp @@ -89,17 +89,34 @@ namespace Mist { capa["optional"]["segmentsize"]["type"] = "uint"; capa["optional"]["segmentsize"]["default"] = 5000LL; option.null(); - option["arg"] = "integer"; + + option["arg"] = "string"; option["long"] = "udp-port"; option["short"] = "U"; option["help"] = "The UDP port on which to listen for TS Packets"; - option["value"].append(0LL); + option["value"].append(""); config->addOption("udpport", option); capa["optional"]["udpport"]["name"] = "TS/UDP port"; - capa["optional"]["udpport"]["help"] = "The UDP port on which to listen for TS Packets, or 0 for disabling TS Input"; + capa["optional"]["udpport"]["help"] = "The UDP port on which to listen for TS Packets, or 0 for disabling TS Input, optionally prefixed with the interface IP to listen on."; capa["optional"]["udpport"]["option"] = "--udp-port"; - capa["optional"]["udpport"]["type"] = "uint"; - capa["optional"]["udpport"]["default"] = 0LL; + capa["optional"]["udpport"]["type"] = "str"; + capa["optional"]["udpport"]["default"] = ""; + option.null(); + + option["arg"] = "string"; + option["long"] = "multicast-interface"; + option["short"] = "M"; + option["help"] = "The interface(s)s on which to listen for UDP Multicast packets, space separated."; + option["value"].append(""); + config->addOption("multicastinterface", option); + capa["optional"]["multicastinterface"]["name"] = "TS Multicast interface"; + capa["optional"]["multicastinterface"]["help"] = "The interface(s) on which to listen for UDP Multicast packets, comma separated."; + capa["optional"]["multicastinterface"]["option"] = "--multicast-interface"; + capa["optional"]["multicastinterface"]["type"] = "str"; + capa["optional"]["multicastinterface"]["default"] = ""; + option.null(); + + /*LTS-end*/ capa["source_match"] = "push://*"; capa["priority"] = 9ll; @@ -130,12 +147,12 @@ namespace Mist { DEBUG_MSG(DLVL_DEVEL, "Cleaning up, removing last keyframes"); for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++){ std::map & locations = bufferLocations[it->first]; - if (!metaPages.count(it->first) || !metaPages[it->first].mapped){ + if (!nProxy.metaPages.count(it->first) || !nProxy.metaPages[it->first].mapped){ continue; } //First detect all entries on metaPage for (int i = 0; i < 8192; i += 8){ - int * tmpOffset = (int *)(metaPages[it->first].mapped + i); + int * tmpOffset = (int *)(nProxy.metaPages[it->first].mapped + i); if (tmpOffset[0] == 0 && tmpOffset[1] == 0){ continue; } @@ -206,14 +223,14 @@ namespace Mist { snprintf(liveSemName, NAME_BUFFER_SIZE, SEM_LIVE, streamName.c_str()); IPC::semaphore liveMeta(liveSemName, O_CREAT | O_RDWR, ACCESSPERMS, 1); liveMeta.wait(); - if (!metaPages.count(0) || !metaPages[0].mapped){ + if (!nProxy.metaPages.count(0) || !nProxy.metaPages[0].mapped){ char pageName[NAME_BUFFER_SIZE]; snprintf(pageName, NAME_BUFFER_SIZE, SHM_STREAM_INDEX, streamName.c_str()); - metaPages[0].init(pageName, DEFAULT_META_PAGE_SIZE, true); - metaPages[0].master = false; + nProxy.metaPages[0].init(pageName, DEFAULT_META_PAGE_SIZE, true); + nProxy.metaPages[0].master = false; } - myMeta.writeTo(metaPages[0].mapped); - memset(metaPages[0].mapped + myMeta.getSendLen(), 0, (metaPages[0].len > myMeta.getSendLen() ? std::min(metaPages[0].len - myMeta.getSendLen(), 4ll) : 0)); + myMeta.writeTo(nProxy.metaPages[0].mapped); + memset(nProxy.metaPages[0].mapped + myMeta.getSendLen(), 0, (nProxy.metaPages[0].len > myMeta.getSendLen() ? std::min(nProxy.metaPages[0].len - myMeta.getSendLen(), 4ll) : 0)); liveMeta.post(); } @@ -296,15 +313,15 @@ namespace Mist { if (myMeta.tracks[tid].keys[0].getNumber() >= (++(bufferLocations[tid].begin()))->first || !config->is_active){ //Find page in indexpage and null it for (int i = 0; i < 8192; i += 8){ - unsigned int thisKeyNum = ((((long long int *)(metaPages[tid].mapped + i))[0]) >> 32) & 0xFFFFFFFF; - if (thisKeyNum == htonl(bufferLocations[tid].begin()->first) && ((((long long int *)(metaPages[tid].mapped + i))[0]) != 0)){ - (((long long int *)(metaPages[tid].mapped + i))[0]) = 0; + unsigned int thisKeyNum = ((((long long int *)(nProxy.metaPages[tid].mapped + i))[0]) >> 32) & 0xFFFFFFFF; + if (thisKeyNum == htonl(bufferLocations[tid].begin()->first) && ((((long long int *)(nProxy.metaPages[tid].mapped + i))[0]) != 0)){ + (((long long int *)(nProxy.metaPages[tid].mapped + i))[0]) = 0; } } DEBUG_MSG(DLVL_DEVEL, "Erasing track %d, keys %lu-%lu from buffer", tid, bufferLocations[tid].begin()->first, bufferLocations[tid].begin()->first + bufferLocations[tid].begin()->second.keyNum - 1); bufferRemove(tid, bufferLocations[tid].begin()->first); for (int i = 0; i < 1024; i++){ - int * tmpOffset = (int *)(metaPages[tid].mapped + (i * 8)); + int * tmpOffset = (int *)(nProxy.metaPages[tid].mapped + (i * 8)); int tmpNum = ntohl(tmpOffset[0]); if (tmpNum == bufferLocations[tid].begin()->first){ tmpOffset[0] = 0; @@ -312,12 +329,12 @@ namespace Mist { } } - curPageNum.erase(tid); + nProxy.curPageNum.erase(tid); char thisPageName[NAME_BUFFER_SIZE]; snprintf(thisPageName, NAME_BUFFER_SIZE, SHM_TRACK_DATA, config->getString("streamname").c_str(), (unsigned long)tid, bufferLocations[tid].begin()->first); - curPage[tid].init(thisPageName, 20971520); - curPage[tid].master = true; - curPage.erase(tid); + nProxy.curPage[tid].init(thisPageName, 20971520); + nProxy.curPage[tid].master = true; + nProxy.curPage.erase(tid); bufferLocations[tid].erase(bufferLocations[tid].begin()); } else { @@ -334,13 +351,13 @@ namespace Mist { for (std::map::iterator it = bufferLocations[tid].begin(); it != bufferLocations[tid].end(); it++){ char thisPageName[NAME_BUFFER_SIZE]; snprintf(thisPageName, NAME_BUFFER_SIZE, SHM_TRACK_DATA, config->getString("streamname").c_str(), tid, it->first); - curPage[tid].init(thisPageName, 20971520, false, false); - curPage[tid].master = true; - curPage.erase(tid); + nProxy.curPage[tid].init(thisPageName, 20971520, false, false); + nProxy.curPage[tid].master = true; + nProxy.curPage.erase(tid); } bufferLocations.erase(tid); - metaPages[tid].master = true; - metaPages.erase(tid); + nProxy.metaPages[tid].master = true; + nProxy.metaPages.erase(tid); } void inputBuffer::finish() { @@ -410,9 +427,9 @@ namespace Mist { while (bufferLocations[tid].size()){ char thisPageName[NAME_BUFFER_SIZE]; snprintf(thisPageName, NAME_BUFFER_SIZE, SHM_TRACK_DATA, config->getString("streamname").c_str(), (unsigned long)tid, bufferLocations[tid].begin()->first); - curPage[tid].init(thisPageName, 20971520); - curPage[tid].master = true; - curPage.erase(tid); + nProxy.curPage[tid].init(thisPageName, 20971520); + nProxy.curPage[tid].master = true; + nProxy.curPage.erase(tid); bufferLocations[tid].erase(bufferLocations[tid].begin()); } if (pushLocation.count(it->first)){ @@ -427,9 +444,9 @@ namespace Mist { } pushLocation.erase(it->first); } - curPageNum.erase(it->first); - metaPages[it->first].master = true; - metaPages.erase(it->first); + nProxy.curPageNum.erase(it->first); + nProxy.metaPages[it->first].master = true; + nProxy.metaPages.erase(it->first); activeTracks.erase(it->first); myMeta.tracks.erase(it->first); changed = true; @@ -522,8 +539,8 @@ namespace Mist { activeTracks.erase(value); bufferLocations.erase(value); } - metaPages[value].master = true; - metaPages.erase(value); + nProxy.metaPages[value].master = true; + nProxy.metaPages.erase(value); continue; } } @@ -546,13 +563,13 @@ namespace Mist { //The track id is set to the value of a track that we are currently negotiating about if (negotiatingTracks.count(value)){ //If the metadata page for this track is not yet registered, initialize it - if (!metaPages.count(value) || !metaPages[value].mapped){ + if (!nProxy.metaPages.count(value) || !nProxy.metaPages[value].mapped){ char tempMetaName[NAME_BUFFER_SIZE]; snprintf(tempMetaName, NAME_BUFFER_SIZE, SHM_TRACK_META, config->getString("streamname").c_str(), value); - metaPages[value].init(tempMetaName, 8388608, false, false); + nProxy.metaPages[value].init(tempMetaName, 8388608, false, false); } //If this tracks metdata page is not initialize, skip the entire element for now. It will be instantiated later - if (!metaPages[value].mapped) { + if (!nProxy.metaPages[value].mapped) { //remove the negotiation if it has timed out if (++negotiationTimeout[value] >= 1000){ negotiatingTracks.erase(value); @@ -564,13 +581,13 @@ namespace Mist { //The page exist, now we try to read in the metadata of the track //Store the size of the dtsc packet to read. - unsigned int len = ntohl(((int *)metaPages[value].mapped)[1]); + unsigned int len = ntohl(((int *)nProxy.metaPages[value].mapped)[1]); //Temporary variable, won't be used again unsigned int tempForReadingMeta = 0; //Read in the metadata through a temporary JSON object ///\todo Optimize this part. Find a way to not have to store the metadata in JSON first, but read it from the page immediately JSON::Value tempJSONForMeta; - JSON::fromDTMI((const unsigned char *)metaPages[value].mapped + 8, len, tempForReadingMeta, tempJSONForMeta); + JSON::fromDTMI((const unsigned char *)nProxy.metaPages[value].mapped + 8, len, tempForReadingMeta, tempJSONForMeta); //Construct a metadata object for the current track DTSC::Meta trackMeta(tempJSONForMeta); //If the track metadata does not contain the negotiated track, assume the metadata is currently being written, and skip the element for now. It will be instantiated in the next call. @@ -579,8 +596,8 @@ namespace Mist { if (++negotiationTimeout[value] >= 1000){ negotiatingTracks.erase(value); //Set master to true before erasing the page, because we are responsible for cleaning up unused pages - metaPages[value].master = true; - metaPages.erase(value); + nProxy.metaPages[value].master = true; + nProxy.metaPages.erase(value); negotiationTimeout.erase(value); } continue; @@ -603,8 +620,8 @@ namespace Mist { //Remove the "negotiate" status in either case negotiatingTracks.erase(value); //Set master to true before erasing the page, because we are responsible for cleaning up unused pages - metaPages[value].master = true; - metaPages.erase(value); + nProxy.metaPages[value].master = true; + nProxy.metaPages.erase(value); //Check if the track collides, and whether the track it collides with is active. if (collidesWith != -1 && activeTracks.count(collidesWith)){/*LTS*/ @@ -639,8 +656,8 @@ namespace Mist { //Set master to true before erasing the page, because we are responsible for cleaning up unused pages updateMeta(); eraseTrackDataPages(value); - metaPages[finalMap].master = true; - metaPages.erase(finalMap); + nProxy.metaPages[finalMap].master = true; + nProxy.metaPages.erase(finalMap); bufferLocations.erase(finalMap); } @@ -666,12 +683,12 @@ namespace Mist { //If the track is active, and this is the element responsible for pushing it if (activeTracks.count(value) && pushLocation[value] == data){ //Open the track index page if we dont have it open yet - if (!metaPages.count(value) || !metaPages[value].mapped){ + if (!nProxy.metaPages.count(value) || !nProxy.metaPages[value].mapped){ char firstPage[NAME_BUFFER_SIZE]; snprintf(firstPage, NAME_BUFFER_SIZE, SHM_TRACK_INDEX, config->getString("streamname").c_str(), value); - metaPages[value].init(firstPage, 8192, false, false); + nProxy.metaPages[value].init(firstPage, 8192, false, false); } - if (metaPages[value].mapped){ + if (nProxy.metaPages[value].mapped){ //Update the metadata for this track updateTrackMeta(value); hasPush = true; @@ -684,7 +701,7 @@ namespace Mist { VERYHIGH_MSG("Updating meta for track %d", tNum); //Store a reference for easier access std::map & locations = bufferLocations[tNum]; - char * mappedPointer = metaPages[tNum].mapped; + char * mappedPointer = nProxy.metaPages[tNum].mapped; //First detect all entries on metaPage for (int i = 0; i < 8192; i += 8) { @@ -725,27 +742,27 @@ namespace Mist { //Otherwise open and parse the page //Open the page if it is not yet open - if (!curPageNum.count(tNum) || curPageNum[tNum] != pageNum || !curPage[tNum].mapped){ + if (!nProxy.curPageNum.count(tNum) || nProxy.curPageNum[tNum] != pageNum || !nProxy.curPage[tNum].mapped){ //DO NOT ERASE THE PAGE HERE, master is not set to true - curPageNum.erase(tNum); + nProxy.curPageNum.erase(tNum); char nextPageName[NAME_BUFFER_SIZE]; snprintf(nextPageName, NAME_BUFFER_SIZE, SHM_TRACK_DATA, config->getString("streamname").c_str(), tNum, pageNum); - curPage[tNum].init(nextPageName, 20971520); + nProxy.curPage[tNum].init(nextPageName, 20971520); //If the page can not be opened, stop here - if (!curPage[tNum].mapped){ + if (!nProxy.curPage[tNum].mapped){ WARN_MSG("Could not open page: %s", nextPageName); return; } - curPageNum[tNum] = pageNum; + nProxy.curPageNum[tNum] = pageNum; } DTSC::Packet tmpPack; - if (!curPage[tNum].mapped[pageData.curOffset]){ + if (!nProxy.curPage[tNum].mapped[pageData.curOffset]){ VERYHIGH_MSG("No packet on page %lu for track %lu, waiting...", pageNum, tNum); return; } - tmpPack.reInit(curPage[tNum].mapped + pageData.curOffset, 0); + tmpPack.reInit(nProxy.curPage[tNum].mapped + pageData.curOffset, 0); //No new data has been written on the page since last update if (!tmpPack){ return; @@ -761,7 +778,7 @@ namespace Mist { //Update the offset on the page with the size of the current packet pageData.curOffset += tmpPack.getDataLen(); //Attempt to read in the next packet - tmpPack.reInit(curPage[tNum].mapped + pageData.curOffset, 0); + tmpPack.reInit(nProxy.curPage[tNum].mapped + pageData.curOffset, 0); } } diff --git a/src/input/input_ts.cpp b/src/input/input_ts.cpp index 08ed2d29..b048243e 100755 --- a/src/input/input_ts.cpp +++ b/src/input/input_ts.cpp @@ -10,9 +10,16 @@ #include #include #include +#include #include #include "input_ts.h" +#include +#include + + + + /// \todo Implement this trigger equivalent... /* if(Triggers::shouldTrigger("STREAM_PUSH", smp)){ @@ -27,10 +34,82 @@ if(Triggers::shouldTrigger("STREAM_PUSH", smp)){ } */ +#ifdef TSLIVE_INPUT +std::string globalStreamName; +TS::Stream liveStream(true); +Util::Config * cfgPointer = NULL; + +#define THREAD_TIMEOUT 15 +std::map threadTimer; + +std::set claimableThreads; + +void parseThread(void * ignored) { + + std::string semName = "MstInTSStreamClaim" + globalStreamName; + IPC::semaphore lock(semName.c_str(), O_CREAT | O_RDWR, ACCESSPERMS, 1); + + int tid = -1; + lock.wait(); + if (claimableThreads.size()) { + tid = *claimableThreads.begin(); + claimableThreads.erase(claimableThreads.begin()); + } + lock.post(); + if (tid == -1) { + return; + } + + if (liveStream.isDataTrack(tid)){ + if (!Util::startInput(globalStreamName)) { + return; + } + } + + Mist::negotiationProxy myProxy; + myProxy.streamName = globalStreamName; + DTSC::Meta myMeta; + + if (liveStream.isDataTrack(tid)){ + char userPageName[NAME_BUFFER_SIZE]; + snprintf(userPageName, NAME_BUFFER_SIZE, SHM_USERS, globalStreamName.c_str()); + myProxy.userClient = IPC::sharedClient(userPageName, PLAY_EX_SIZE, true); + } + + + threadTimer[tid] = Util::bootSecs(); + while (Util::bootSecs() - threadTimer[tid] < THREAD_TIMEOUT && cfgPointer->is_active) { + liveStream.parse(tid); + if (liveStream.hasPacket(tid)){ + liveStream.initializeMetadata(myMeta, tid); + DTSC::Packet pack; + liveStream.getPacket(tid, pack); + if (pack && myMeta.tracks.count(tid)){ + myProxy.bufferLivePacket(pack, myMeta); + } + + lock.wait(); + threadTimer[tid] = Util::bootSecs(); + lock.post(); + } + if (liveStream.isDataTrack(tid)){ + myProxy.userClient.keepAlive(); + } + if (!liveStream.hasPacket(tid)){ + Util::sleep(100); + } + } + lock.wait(); + threadTimer.erase(tid); + lock.post(); + liveStream.eraseTrack(tid); + myProxy.userClient.finish(); +} + +#endif namespace Mist { - - + /// Constructor of TS Input /// \arg cfg Util::Config that contains all current configurations. inputTS::inputTS(Util::Config * cfg) : Input(cfg) { @@ -44,44 +123,80 @@ namespace Mist { capa["codecs"][0u][1u].append("AC3"); capa["optional"]["port"]["name"] = "UDP Port"; - capa["optional"]["port"]["help"] = "The udp port on which to listen for incoming UDP Packets"; - capa["optional"]["port"]["type"] = "uint"; - capa["optional"]["port"]["default"] = 9876; + capa["optional"]["port"]["help"] = "The UDP port on which to listen for incoming UDP Packets, optionally prefixed by the interface IP."; + capa["optional"]["port"]["type"] = "string"; + capa["optional"]["port"]["default"] = "9876"; capa["optional"]["port"]["option"] = "--port"; cfg->addOption("port", - JSON::fromString("{\"arg\":\"integer\",\"value\":9876,\"short\":\"p\",\"long\":\"port\",\"help\":\"The udp port on which to listen for incoming UDP Packets.\"}")); + JSON::fromString("{\"arg\":\"string\",\"value\":9876,\"short\":\"p\",\"long\":\"port\",\"help\":\"The UDP port on which to listen for incoming UDP Packets, optionally prefixed by the interface IP.\"}")); + + capa["optional"]["multicastinterface"]["name"] = "TS Multicast interface"; + capa["optional"]["multicastinterface"]["help"] = "The interface(s) on which to listen for UDP Multicast packets, comma separated."; + capa["optional"]["multicastinterface"]["option"] = "--multicast-interface"; + capa["optional"]["multicastinterface"]["type"] = "str"; + capa["optional"]["multicastinterface"]["default"] = ""; + cfg->addOption("multicastinterface", + JSON::fromString("{\"arg\":\"string\",\"value\":\"\",\"short\":\"M\",\"long\":\"multicast-interface\",\"help\":\"The interfaces on which to listen for UDP Multicast packets, space separatered.\"}")); pushing = false; inFile = NULL; + +#ifdef TSLIVE_INPUT + standAlone = false; +#endif } inputTS::~inputTS() { - if (inFile){ + if (inFile) { fclose(inFile); } +#ifdef TSLIVE_INPUT + std::string semName = "MstInTSStreamClaim" + globalStreamName; + IPC::semaphore lock(semName.c_str(), O_CREAT | O_RDWR, ACCESSPERMS, 1); + lock.wait(); + threadTimer.clear(); + claimableThreads.clear(); + lock.post(); +#endif } - - ///Setup of TS Input + +#ifdef TSLIVE_INPUT + + ///Live Setup of TS Input bool inputTS::setup() { -#ifdef INPUT_LIVE + INFO_MSG("Setup start"); if (config->getString("input") == "-") { inFile = stdin; - }else{ + } else { pushing = true; udpCon.setBlocking(false); - udpCon.bind(config->getInteger("port")); + std::string ipPort = config->getString("port"); + size_t colon = ipPort.rfind(':'); + if (colon != std::string::npos) { + udpCon.bind(JSON::Value(ipPort.substr(colon + 1)).asInt(), ipPort.substr(0, colon), config->getString("multicastinterface")); + } else { + udpCon.bind(JSON::Value(ipPort).asInt(), "", config->getString("multicastinterface")); + } } + INFO_MSG("Setup complete"); + return true; + } + #else - if (config->getString("input") != "-"){ + + ///Setup of TS Input + bool inputTS::setup() { + if (config->getString("input") != "-") { inFile = fopen(config->getString("input").c_str(), "r"); } if (!inFile) { return false; } -#endif return true; } - + +#endif + ///Track selector of TS Input ///\arg trackSpec specifies which tracks are to be selected ///\todo test whether selecting a subset of tracks work @@ -99,33 +214,40 @@ namespace Mist { } } - + +#ifdef TSLIVE_INPUT + //This implementation in used in the live version of TS input, where no header is available in advance. + //Reading the header returns true in this case, to continue parsing the actual stream. + bool inputTS::readHeader() { + return true; + } +#else ///Reads headers from a TS stream, and saves them into metadata ///It works by going through the entire TS stream, and every time ///It encounters a new PES start, it writes the currently found PES data - ///for a specific track to metadata. After the entire stream has been read, + ///for a specific track to metadata. After the entire stream has been read, ///it writes the remaining metadata. ///\todo Find errors, perhaps parts can be made more modular - bool inputTS::readHeader(){ + bool inputTS::readHeader() { if (!inFile) { return false; } DTSC::File tmp(config->getString("input") + ".dtsh"); - if (tmp){ + if (tmp) { myMeta = tmp.getMeta(); return true; - } - + } + TS::Packet packet;//to analyse and extract data fseek(inFile, 0, SEEK_SET);//seek to beginning - + bool first = true; long long int lastBpos = 0; - while (packet.FromFile(inFile) && !feof(inFile)){ + while (packet.FromFile(inFile) && !feof(inFile)) { tsStream.parse(packet, lastBpos); lastBpos = ftell(inFile); - while(tsStream.hasPacketOnEachTrack()){ - if (first){ + while (tsStream.hasPacketOnEachTrack()) { + if (first) { tsStream.initializeMetadata(myMeta); first = false; } @@ -142,106 +264,176 @@ namespace Mist { oFile.close(); return true; } - +#endif - ///Gets the next packet that is to be sent - ///At the moment, the logic of sending the last packet that was finished has been implemented, + ///Gets the next packet that is to be sent + ///At the moment, the logic of sending the last packet that was finished has been implemented, ///but the seeking and finding data is not yet ready. ///\todo Finish the implementation - void inputTS::getNext(bool smart){ + void inputTS::getNext(bool smart) { + INSANE_MSG("Getting next"); thisPacket.null(); bool hasPacket = (selectedTracks.size() == 1 ? tsStream.hasPacket(*selectedTracks.begin()) : tsStream.hasPacketOnEachTrack()); - while (!hasPacket && (pushing || !feof(inFile)) && config->is_active){ + while (!hasPacket && (pushing || !feof(inFile)) && config->is_active) { if (!pushing) { unsigned int bPos = ftell(inFile); tsBuf.FromFile(inFile); - if (selectedTracks.count(tsBuf.getPID())){ + if (selectedTracks.count(tsBuf.getPID())) { tsStream.parse(tsBuf, bPos); } - }else{ - while (udpCon.Receive()){ + } else { + while (udpCon.Receive()) { udpDataBuffer.append(udpCon.data, udpCon.data_len); - while (udpDataBuffer.size() > 188 && (udpDataBuffer[0] != 0x47 || udpDataBuffer[188] != 0x47)){ + while (udpDataBuffer.size() > 188 && (udpDataBuffer[0] != 0x47 || udpDataBuffer[188] != 0x47)) { size_t syncPos = udpDataBuffer.find("\107", 1); udpDataBuffer.erase(0, syncPos); } - while (udpDataBuffer.size() >= 188){ + while (udpDataBuffer.size() >= 188) { tsBuf.FromPointer(udpDataBuffer.data()); tsStream.parse(tsBuf, 0); - udpDataBuffer.erase(0,188); + udpDataBuffer.erase(0, 188); } } - if (userClient.getData()){ - userClient.keepAlive(); - } Util::sleep(500); } - if (userClient.getData()){ - userClient.keepAlive(); - } hasPacket = (selectedTracks.size() == 1 ? tsStream.hasPacket(*selectedTracks.begin()) : tsStream.hasPacketOnEachTrack()); } - if (!hasPacket){ - if(inFile && !feof(inFile)){ + if (!hasPacket) { + if (inFile && !feof(inFile)) { getNext(); } - if (pushing){ + if (pushing) { sleep(500); } return; } - if (selectedTracks.size() == 1){ + if (selectedTracks.size() == 1) { tsStream.getPacket(*selectedTracks.begin(), thisPacket); - }else{ + } else { tsStream.getEarliestPacket(thisPacket); } tsStream.initializeMetadata(myMeta); - if (!myMeta.tracks.count(thisPacket.getTrackId())){ + if (!myMeta.tracks.count(thisPacket.getTrackId())) { getNext(); } } - void inputTS::readPMT(){ + void inputTS::readPMT() { //save current file position int bpos = ftell(inFile); - if (fseek(inFile, 0, SEEK_SET)){ + if (fseek(inFile, 0, SEEK_SET)) { FAIL_MSG("Seek to 0 failed"); return; } TS::Packet tsBuffer; - while (!tsStream.hasPacketOnEachTrack() && tsBuffer.FromFile(inFile)){ + while (!tsStream.hasPacketOnEachTrack() && tsBuffer.FromFile(inFile)) { tsStream.parse(tsBuffer, 0); } - + //Clear leaves the PMT in place tsStream.clear(); //Restore original file position - if (fseek(inFile, bpos, SEEK_SET)){ + if (fseek(inFile, bpos, SEEK_SET)) { return; } } - + ///Seeks to a specific time - void inputTS::seek(int seekTime){ + void inputTS::seek(int seekTime) { tsStream.clear(); readPMT(); unsigned long seekPos = 0xFFFFFFFFull; - for (std::set::iterator it = selectedTracks.begin(); it != selectedTracks.end(); it++){ + for (std::set::iterator it = selectedTracks.begin(); it != selectedTracks.end(); it++) { unsigned long thisBPos = 0; - for (std::deque::iterator keyIt = myMeta.tracks[*it].keys.begin(); keyIt != myMeta.tracks[*it].keys.end(); keyIt++){ - if (keyIt->getTime() > seekTime){ + for (std::deque::iterator keyIt = myMeta.tracks[*it].keys.begin(); keyIt != myMeta.tracks[*it].keys.end(); keyIt++) { + if (keyIt->getTime() > seekTime) { break; } thisBPos = keyIt->getBpos(); } - if (thisBPos < seekPos){ + if (thisBPos < seekPos) { seekPos = thisBPos; } } fseek(inFile, seekPos, SEEK_SET);//seek to the correct position } + +#ifdef TSLIVE_INPUT + void inputTS::serve() { + cfgPointer = config; + globalStreamName = streamName; + unsigned long long threadCheckTimer = Util::bootSecs(); + while (config->is_active) { + if (!pushing) { + unsigned int bPos = ftell(inFile); + int ctr = 0; + while (ctr < 20 && tsBuf.FromFile(inFile)){ + liveStream.add(tsBuf); + ctr++; + } + } else { + while (udpCon.Receive()) { + int offset = 0; + //Try to read full TS Packets + //Assumption here is made that one UDP Datagram consists of complete TS Packets. + //Assumption made because of possible packet loss issues + while ((udpCon.data_len - offset) >= 188) { + //Watch out! We push here to a global, in order for threads to be able to access it. + liveStream.add(udpCon.data + offset); + offset += 188; + } + if (offset < udpCon.data_len) { + WARN_MSG("%d bytes left in datagram", udpCon.data_len - offset); + } + } + } + //Check for and spawn threads here. + if (Util::bootSecs() - threadCheckTimer > 2) { + std::set activeTracks = liveStream.getActiveTracks(); + std::string semName = "MstInTSStreamClaim" + globalStreamName; + IPC::semaphore lock(semName.c_str(), O_CREAT | O_RDWR, ACCESSPERMS, 1); + lock.wait(); + for (std::set::iterator it = activeTracks.begin(); it != activeTracks.end(); it++) { + if (threadTimer.count(*it) && ((Util::bootSecs() - threadTimer[*it]) > (2 * THREAD_TIMEOUT))) { + WARN_MSG("Thread for track %d timed out %d seconds ago without a clean shutdown.", *it, Util::bootSecs() - threadTimer[*it]); + threadTimer.erase(*it); + } + if (!threadTimer.count(*it)) { + + //Add to list of unclaimed threads + claimableThreads.insert(*it); + + //Spawn thread here. + tthread::thread thisThread(parseThread, 0); + thisThread.detach(); + } + } + lock.post(); + threadCheckTimer = Util::bootSecs(); + } + Util::sleep(100); + } + finish(); + INFO_MSG("Input for stream %s closing clean", streamName.c_str()); + } + + void inputTS::finish() { + std::string semName = "MstInTSStreamClaim" + globalStreamName; + IPC::semaphore lock(semName.c_str(), O_CREAT | O_RDWR, ACCESSPERMS, 1); + + + int threadCount = 0; + do { + lock.wait(); + threadCount = threadTimer.size(); + lock.post(); + } while (threadCount); + } +#endif + } + diff --git a/src/input/input_ts.h b/src/input/input_ts.h index 0a347657..fe29da67 100755 --- a/src/input/input_ts.h +++ b/src/input/input_ts.h @@ -22,6 +22,17 @@ namespace Mist { void trackSelect(std::string trackSpec); void readPMT(); +#ifdef TSLIVE_INPUT + //Live tsinput does not have a header, so parseheader should do nothing + void parseHeader() { } + //In case of live TS Input, we override the default serve function + void serve(); + void finish(); +#endif + + + + FILE * inFile;///%lu successful", pageNumber, tid, mapTid); + + if (myMeta.live){ //Register this page on the meta page //NOTE: It is important that this only happens if the stream is live.... @@ -150,7 +158,7 @@ namespace Mist { int * tmpOffset = (int *)(metaPages[tid].mapped + (i * 8)); if ((tmpOffset[0] == 0 && tmpOffset[1] == 0)) { tmpOffset[0] = htonl(curPageNum[tid]); - if (pagesByTrack[tid][pageNumber].dataSize == (25 * 1024 * 1024)){ + if (pagesByTrack[tid][pageNumber].dataSize == DEFAULT_DATA_PAGE_SIZE){ tmpOffset[1] = htonl(1000); } else { tmpOffset[1] = htonl(pagesByTrack[tid][pageNumber].keyNum); @@ -160,8 +168,8 @@ namespace Mist { } } } - - HIGH_MSG("Start buffering page %lu on track %lu~>%lu successful", pageNumber, tid, mapTid); + + ///\return true if everything was successful return true; } @@ -176,13 +184,13 @@ namespace Mist { //A different process will handle this for us return; } - unsigned long mapTid = trackMap[tid]; - if (!pagesByTrack.count(tid)){ + unsigned long mapTid = nProxy.trackMap[tid]; + if (!nProxy.pagesByTrack.count(tid)){ // If there is no pagesByTrack entry, the pages are managed in local code and not through io.cpp (e.g.: MistInBuffer) return; } //If the given pagenumber is not a valid page on this track, do nothing - if (!pagesByTrack[tid].count(pageNumber)){ + if (!nProxy.pagesByTrack[tid].count(pageNumber)){ INFO_MSG("Can't remove page %lu on track %lu~>%lu as it is not a valid page number.", pageNumber, tid, mapTid); return; } @@ -194,7 +202,7 @@ namespace Mist { #ifdef __CYGWIN__ toErase.init(pageName, 26 * 1024 * 1024, false); #else - toErase.init(pageName, pagesByTrack[tid][pageNumber].dataSize, false); + toErase.init(pageName, nProxy.pagesByTrack[tid][pageNumber].dataSize, false); #endif //Set the master flag so that the page will be destroyed once it leaves scope #if defined(__CYGWIN__) || defined(_WIN32) @@ -204,7 +212,7 @@ namespace Mist { //Remove the page from the tracks index page DEBUG_MSG(DLVL_HIGH, "Removing page %lu on track %lu~>%lu from the corresponding metaPage", pageNumber, tid, mapTid); for (int i = 0; i < 1024; i++) { - int * tmpOffset = (int *)(metaPages[tid].mapped + (i * 8)); + int * tmpOffset = (int *)(nProxy.metaPages[tid].mapped + (i * 8)); if (ntohl(tmpOffset[0]) == pageNumber) { tmpOffset[0] = 0; tmpOffset[1] = 0; @@ -217,7 +225,7 @@ namespace Mist { ///Checks whether a key is buffered ///\param tid The trackid on which to locate the key ///\param keyNum The number of the keyframe to find - bool InOutBase::isBuffered(unsigned long tid, unsigned long keyNum) { + bool negotiationProxy::isBuffered(unsigned long tid, unsigned long keyNum) { ///\return The result of bufferedOnPage(tid, keyNum) return bufferedOnPage(tid, keyNum); } @@ -225,7 +233,7 @@ namespace Mist { ///Returns the pagenumber where this key is buffered on ///\param tid The trackid on which to locate the key ///\param keyNum The number of the keyframe to find - unsigned long InOutBase::bufferedOnPage(unsigned long tid, unsigned long keyNum) { + unsigned long negotiationProxy::bufferedOnPage(unsigned long tid, unsigned long keyNum) { //Check whether the track is accepted if (!trackMap.count(tid) || !metaPages.count(tid) || !metaPages[tid].mapped) { ///\return 0 if the page has not been mapped yet @@ -252,12 +260,16 @@ namespace Mist { std::string packData = pack.toNetPacked(); DTSC::Packet newPack(packData.data(), packData.size()); ///\note Internally calls bufferNext(DTSC::Packet & pack) - bufferNext(newPack); + nProxy.bufferNext(newPack, myMeta); } ///Buffers the next packet on the currently opened page ///\param pack The packet to buffer void InOutBase::bufferNext(DTSC::Packet & pack) { + nProxy.bufferNext(pack, myMeta); + } + + void negotiationProxy::bufferNext(DTSC::Packet & pack, DTSC::Meta & myMeta) { //Save the trackid of the track for easier access unsigned long tid = pack.getTrackId(); unsigned long mapTid = trackMap[tid]; @@ -330,6 +342,10 @@ namespace Mist { ///Registers the data page on the track index page as well ///\param tid The trackid of the page to finalize void InOutBase::bufferFinalize(unsigned long tid) { + nProxy.bufferFinalize(tid, myMeta); + } + + void negotiationProxy::bufferFinalize(unsigned long tid, DTSC::Meta & myMeta){ unsigned long mapTid = trackMap[tid]; //If no page is open, do nothing if (!curPage.count(tid)) { @@ -411,6 +427,10 @@ namespace Mist { ///Initiates/continues negotiation with the buffer as well ///\param packet The packet to buffer void InOutBase::bufferLivePacket(DTSC::Packet & packet){ + nProxy.bufferLivePacket(packet, myMeta); + } + + void negotiationProxy::bufferLivePacket(DTSC::Packet & packet, DTSC::Meta & myMeta){ myMeta.vod = false; myMeta.live = true; //Store the trackid for easier access @@ -422,7 +442,7 @@ namespace Mist { } //If the track is not negotiated yet, start the negotiation if (!trackState.count(tid)) { - continueNegotiate(tid); + continueNegotiate(tid, myMeta); } //If the track is declined, stop here if (trackState[tid] == FILL_DEC) { @@ -443,7 +463,7 @@ namespace Mist { if (shouldBlock) { while (trackState[tid] != FILL_DEC && trackState[tid] != FILL_ACC) { INFO_MSG("Blocking on track %lu", tid); - continueNegotiate(tid); + continueNegotiate(tid, myMeta); Util::sleep(500); } } @@ -504,16 +524,19 @@ namespace Mist { if (!curPageNum.count(tid) || nextPageNum != curPageNum[tid]) { if (curPageNum.count(tid)) { //Close the currently opened page when it exists - bufferFinalize(tid); + bufferFinalize(tid, myMeta); } //Open the new page - bufferStart(tid, nextPageNum); + bufferStart(tid, nextPageNum, myMeta); } //Buffer the packet - bufferNext(packet); + bufferNext(packet, myMeta); } void InOutBase::continueNegotiate(unsigned long tid) { + nProxy.continueNegotiate(tid, myMeta); + } + void negotiationProxy::continueNegotiate(unsigned long tid, DTSC::Meta & myMeta) { if (!tid) { return; } @@ -559,7 +582,7 @@ namespace Mist { if (!userClient.getData()){ char userPageName[100]; sprintf(userPageName, SHM_USERS, streamName.c_str()); - userClient = IPC::sharedClient(userPageName, 30, true); + userClient = IPC::sharedClient(userPageName, PLAY_EX_SIZE, true); } char * tmp = userClient.getData(); if (!tmp) { diff --git a/src/io.h b/src/io.h index 927c4dbd..c2103701 100644 --- a/src/io.h +++ b/src/io.h @@ -26,6 +26,42 @@ namespace Mist { unsigned long lastKeyTime;/// > pagesByTrack;/// trackOffset; ///< Offset of data on user page + std::map trackState; ///< State of the negotiation for tracks + std::map trackMap;/// metaPages;///< For each track, holds the page that describes which dataPages are mapped + std::map curPageNum;///< For each track, holds the number page that is currently being written. + std::map curPage;///< For each track, holds the page that is currently being written. + + IPC::sharedClient userClient;///< Shared memory used for connection to Mixer process. + + std::string streamName;///< Name of the stream to connect to + + bool encrypt; + Encryption::verimatrixData vmData; + std::map iVecs; + IPC::sharedPage encryptionPage; + + void continueNegotiate(unsigned long tid, DTSC::Meta & myMeta); + }; + ///\brief Class containing all basic input and output functions. class InOutBase { public: @@ -37,37 +73,23 @@ namespace Mist { void bufferRemove(unsigned long tid, unsigned long pageNumber); void bufferLivePacket(JSON::Value & packet); void bufferLivePacket(DTSC::Packet & packet); - bool isBuffered(unsigned long tid, unsigned long keyNum); - unsigned long bufferedOnPage(unsigned long tid, unsigned long keyNum); protected: + void continueNegotiate(unsigned long tid); + + + bool standAlone; static Util::Config * config; - void initiateEncryption();//LTS - void continueNegotiate(unsigned long tid); + negotiationProxy nProxy; DTSC::Packet thisPacket;//The current packet that is being parsed - std::string streamName;///< Name of the stream to connect to - IPC::sharedClient userClient;///< Shared memory used for connection to Mixer process. + std::string streamName; - DTSC::Meta myMeta;///< Stores either the input or output metadata std::set selectedTracks;///< Stores the track id's that are either selected for playback or input - std::map > pagesByTrack;/// trackOffset; ///< Offset of data on user page - std::map trackState; ///< State of the negotiation for tracks - std::map trackMap;/// metaPages;///< For each track, holds the page that describes which dataPages are mapped - std::map curPageNum;///< For each track, holds the number page that is currently being written. - std::map curPage;///< For each track, holds the page that is currently being written. - std::map > trackBuffer; ///< Buffer to be used during active track negotiation - - bool encrypt; - Encryption::verimatrixData vmData; - std::map iVecs; - IPC::sharedPage encryptionPage; + DTSC::Meta myMeta;///< Stores either the input or output metadata }; } diff --git a/src/output/mist_out.cpp b/src/output/mist_out.cpp index f2584aa9..9e217552 100644 --- a/src/output/mist_out.cpp +++ b/src/output/mist_out.cpp @@ -38,6 +38,7 @@ int main(int argc, char * argv[]) { std::cout << mistOut::capa.toString() << std::endl; return -1; } + conf.activate(); if (mistOut::listenMode()){ conf.serveForkedSocket(spawnForked); }else{ diff --git a/src/output/output.cpp b/src/output/output.cpp index 4cea2c6f..db2a468a 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -163,8 +163,8 @@ namespace Mist { if (lock){ liveMeta.wait(); } - if (metaPages[0].mapped){ - DTSC::Packet tmpMeta(metaPages[0].mapped, metaPages[0].len, true); + if (nProxy.metaPages[0].mapped){ + DTSC::Packet tmpMeta(nProxy.metaPages[0].mapped, nProxy.metaPages[0].len, true); if (tmpMeta.getVersion()){ myMeta.reinit(tmpMeta); } @@ -197,7 +197,7 @@ namespace Mist { if (isInitialized){ return; } - if (metaPages[0].mapped){ + if (nProxy.metaPages[0].mapped){ return; } if (streamName.size() < 1){ @@ -233,8 +233,8 @@ namespace Mist { /// Connects or reconnects to the stream. /// Assumes streamName class member has been set already. /// Will start input if not currently active, calls onFail() if this does not succeed. - /// After assuring stream is online, clears metaPages, then sets metaPages[0], statsPage and userClient to (hopefully) valid handles. - /// Finally, calls updateMeta() and stats() + /// After assuring stream is online, clears nProxy.metaPages, then sets nProxy.metaPages[0], statsPage and nProxy.userClient to (hopefully) valid handles. + /// Finally, calls updateMeta() void Output::reconnect(){ if (!Util::startInput(streamName)){ DEBUG_MSG(DLVL_FAIL, "Opening stream failed - aborting initalization"); @@ -243,9 +243,9 @@ namespace Mist { } char pageId[NAME_BUFFER_SIZE]; snprintf(pageId, NAME_BUFFER_SIZE, SHM_STREAM_INDEX, streamName.c_str()); - metaPages.clear(); - metaPages[0].init(pageId, DEFAULT_META_PAGE_SIZE); - if (!metaPages[0].mapped){ + nProxy.metaPages.clear(); + nProxy.metaPages[0].init(pageId, DEFAULT_META_PAGE_SIZE); + if (!nProxy.metaPages[0].mapped){ FAIL_MSG("Could not connect to server for %s", streamName.c_str()); onFail(); return; @@ -254,13 +254,12 @@ namespace Mist { statsPage.finish(); } statsPage = IPC::sharedClient(SHM_STATISTICS, STAT_EX_SIZE, true); - if (userClient.getData()){ - userClient.finish(); + if (nProxy.userClient.getData()){ + nProxy.userClient.finish(); } char userPageName[NAME_BUFFER_SIZE]; snprintf(userPageName, NAME_BUFFER_SIZE, SHM_USERS, streamName.c_str()); - userClient = IPC::sharedClient(userPageName, PLAY_EX_SIZE, true); - stats(); + nProxy.userClient = IPC::sharedClient(userPageName, PLAY_EX_SIZE, true); updateMeta(); } @@ -409,14 +408,14 @@ namespace Mist { } int Output::pageNumForKey(long unsigned int trackId, long long int keyNum){ - if (!metaPages.count(trackId)){ + if (!nProxy.metaPages.count(trackId)){ char id[NAME_BUFFER_SIZE]; snprintf(id, NAME_BUFFER_SIZE, SHM_TRACK_INDEX, streamName.c_str(), trackId); - metaPages[trackId].init(id, 8 * 1024); + nProxy.metaPages[trackId].init(id, 8 * 1024); } - int len = metaPages[trackId].len / 8; + int len = nProxy.metaPages[trackId].len / 8; for (int i = 0; i < len; i++){ - int * tmpOffset = (int *)(metaPages[trackId].mapped + (i * 8)); + int * tmpOffset = (int *)(nProxy.metaPages[trackId].mapped + (i * 8)); long amountKey = ntohl(tmpOffset[1]); if (amountKey == 0){continue;} long tmpKey = ntohl(tmpOffset[0]); @@ -429,7 +428,7 @@ namespace Mist { void Output::loadPageForKey(long unsigned int trackId, long long int keyNum){ if (myMeta.vod && keyNum > myMeta.tracks[trackId].keys.rbegin()->getNumber()){ - curPage.erase(trackId); + nProxy.curPage.erase(trackId); currKeyOpen.erase(trackId); return; } @@ -448,7 +447,7 @@ namespace Mist { } if (timeout > 100){ DEBUG_MSG(DLVL_FAIL, "Timeout while waiting for requested page %lld for track %lu. Aborting.", keyNum, trackId); - curPage.erase(trackId); + nProxy.curPage.erase(trackId); currKeyOpen.erase(trackId); return; } @@ -475,9 +474,9 @@ namespace Mist { } char id[NAME_BUFFER_SIZE]; snprintf(id, NAME_BUFFER_SIZE, SHM_TRACK_DATA, streamName.c_str(), trackId, pageNum); - curPage[trackId].init(id, DEFAULT_DATA_PAGE_SIZE); - if (!(curPage[trackId].mapped)){ - DEBUG_MSG(DLVL_FAIL, "Initializing page %s failed", curPage[trackId].name.c_str()); + nProxy.curPage[trackId].init(id, DEFAULT_DATA_PAGE_SIZE); + if (!(nProxy.curPage[trackId].mapped)){ + DEBUG_MSG(DLVL_FAIL, "Initializing page %s failed", nProxy.curPage[trackId].name.c_str()); return; } currKeyOpen[trackId] = pageNum; @@ -503,7 +502,7 @@ namespace Mist { bool Output::seek(unsigned int tid, unsigned long long pos, bool getNextKey){ loadPageForKey(tid, getKeyForTime(tid, pos) + (getNextKey?1:0)); - if (!curPage.count(tid) || !curPage[tid].mapped){ + if (!nProxy.curPage.count(tid) || !nProxy.curPage[tid].mapped){ INFO_MSG("Aborting seek to %llums in track %u, not available.", pos, tid); return false; } @@ -511,9 +510,9 @@ namespace Mist { tmp.tid = tid; tmp.offset = 0; DTSC::Packet tmpPack; - tmpPack.reInit(curPage[tid].mapped + tmp.offset, 0, true); + tmpPack.reInit(nProxy.curPage[tid].mapped + tmp.offset, 0, true); tmp.time = tmpPack.getTime(); - char * mpd = curPage[tid].mapped; + char * mpd = nProxy.curPage[tid].mapped; while ((long long)tmp.time < pos && tmpPack){ tmp.offset += tmpPack.getDataLen(); tmpPack.reInit(mpd + tmp.offset, 0, true); @@ -524,15 +523,15 @@ namespace Mist { return true; }else{ //don't print anything for empty packets - not sign of corruption, just unfinished stream. - if (curPage[tid].mapped[tmp.offset] != 0){ + if (nProxy.curPage[tid].mapped[tmp.offset] != 0){ DEBUG_MSG(DLVL_FAIL, "Noes! Couldn't find packet on track %d because of some kind of corruption error or somesuch.", tid); }else{ VERYHIGH_MSG("Track %d no data (key %u @ %u) - waiting...", tid, getKeyForTime(tid, pos) + (getNextKey?1:0), tmp.offset); unsigned int i = 0; - while (curPage[tid].mapped[tmp.offset] == 0 && ++i < 42){ + while (nProxy.curPage[tid].mapped[tmp.offset] == 0 && ++i < 42){ Util::wait(100); } - if (curPage[tid].mapped[tmp.offset] == 0){ + if (nProxy.curPage[tid].mapped[tmp.offset] == 0){ FAIL_MSG("Track %d no data (key %u) - timeout", tid, getKeyForTime(tid, pos) + (getNextKey?1:0)); }else{ return seek(tid, pos, getNextKey); @@ -879,7 +878,7 @@ namespace Mist { /*LTS-END*/ stats(); - userClient.finish(); + nProxy.userClient.finish(); statsPage.finish(); myConn.close(); return 0; @@ -941,15 +940,15 @@ namespace Mist { DEBUG_MSG(DLVL_DONTEVEN, "Loading track %u (next=%lu), %llu ms", nxt.tid, nxtKeyNum[nxt.tid], nxt.time); - if (nxt.offset >= curPage[nxt.tid].len){ + if (nxt.offset >= nProxy.curPage[nxt.tid].len){ nxtKeyNum[nxt.tid] = getKeyForTime(nxt.tid, thisPacket.getTime()); loadPageForKey(nxt.tid, ++nxtKeyNum[nxt.tid]); nxt.offset = 0; - if (curPage.count(nxt.tid) && curPage[nxt.tid].mapped){ - if (getDTSCTime(curPage[nxt.tid].mapped, nxt.offset) < nxt.time){ + if (nProxy.curPage.count(nxt.tid) && nProxy.curPage[nxt.tid].mapped){ + if (getDTSCTime(nProxy.curPage[nxt.tid].mapped, nxt.offset) < nxt.time){ ERROR_MSG("Time going backwards in track %u - dropping track.", nxt.tid); }else{ - nxt.time = getDTSCTime(curPage[nxt.tid].mapped, nxt.offset); + nxt.time = getDTSCTime(nProxy.curPage[nxt.tid].mapped, nxt.offset); buffer.insert(nxt); } prepareNext(); @@ -957,7 +956,7 @@ namespace Mist { } } - if (!curPage.count(nxt.tid) || !curPage[nxt.tid].mapped){ + if (!nProxy.curPage.count(nxt.tid) || !nProxy.curPage[nxt.tid].mapped){ //mapping failure? Drop this track and go to next. //not an error - usually means end of stream. DEBUG_MSG(DLVL_DEVEL, "Track %u no page - dropping track.", nxt.tid); @@ -966,7 +965,7 @@ namespace Mist { } //have we arrived at the end of the memory page? (4 zeroes mark the end) - if (!memcmp(curPage[nxt.tid].mapped + nxt.offset, "\000\000\000\000", 4)){ + if (!memcmp(nProxy.curPage[nxt.tid].mapped + nxt.offset, "\000\000\000\000", 4)){ //if we don't currently know where we are, we're lost. We should drop the track. if (!nxt.time){ DEBUG_MSG(DLVL_DEVEL, "Timeless empty packet on track %u - dropping track.", nxt.tid); @@ -994,8 +993,8 @@ namespace Mist { nxtKeyNum[nxt.tid] = getKeyForTime(nxt.tid, thisPacket.getTime()); loadPageForKey(nxt.tid, ++nxtKeyNum[nxt.tid]); nxt.offset = 0; - if (curPage.count(nxt.tid) && curPage[nxt.tid].mapped){ - unsigned long long nextTime = getDTSCTime(curPage[nxt.tid].mapped, nxt.offset); + if (nProxy.curPage.count(nxt.tid) && nProxy.curPage[nxt.tid].mapped){ + unsigned long long nextTime = getDTSCTime(nProxy.curPage[nxt.tid].mapped, nxt.offset); if (nextTime && nextTime < nxt.time){ DEBUG_MSG(DLVL_DEVEL, "Time going backwards in track %u - dropping track.", nxt.tid); }else{ @@ -1012,7 +1011,7 @@ namespace Mist { prepareNext(); return; } - thisPacket.reInit(curPage[nxt.tid].mapped + nxt.offset, 0, true); + thisPacket.reInit(nProxy.curPage[nxt.tid].mapped + nxt.offset, 0, true); if (thisPacket){ if (thisPacket.getTime() != nxt.time && nxt.time){ WARN_MSG("Loaded track %ld@%llu instead of %ld@%llu", thisPacket.getTrackId(), thisPacket.getTime(), nxt.tid, nxt.time); @@ -1073,9 +1072,9 @@ namespace Mist { completeKeyReadyTimeOut = false; } } - if (curPage[nxt.tid]){ - if (nxt.offset < curPage[nxt.tid].len){ - unsigned long long nextTime = getDTSCTime(curPage[nxt.tid].mapped, nxt.offset); + if (nProxy.curPage[nxt.tid]){ + if (nxt.offset < nProxy.curPage[nxt.tid].len){ + unsigned long long nextTime = getDTSCTime(nProxy.curPage[nxt.tid].mapped, nxt.offset); if (nextTime){ nxt.time = nextTime; }else{ @@ -1123,25 +1122,25 @@ namespace Mist { } } int tNum = 0; - if (!userClient.getData()){ + if (!nProxy.userClient.getData()){ char userPageName[NAME_BUFFER_SIZE]; snprintf(userPageName, NAME_BUFFER_SIZE, SHM_USERS, streamName.c_str()); - userClient = IPC::sharedClient(userPageName, PLAY_EX_SIZE, true); - if (!userClient.getData()){ + nProxy.userClient = IPC::sharedClient(userPageName, PLAY_EX_SIZE, true); + if (!nProxy.userClient.getData()){ DEBUG_MSG(DLVL_WARN, "Player connection failure - aborting output"); myConn.close(); return; } } - if (!trackMap.size()){ - IPC::userConnection userConn(userClient.getData()); + if (!nProxy.trackMap.size()){ + IPC::userConnection userConn(nProxy.userClient.getData()); for (std::set::iterator it = selectedTracks.begin(); it != selectedTracks.end() && tNum < SIMUL_TRACKS; it++){ userConn.setTrackId(tNum, *it); userConn.setKeynum(tNum, nxtKeyNum[*it]); tNum ++; } } - userClient.keepAlive(); + nProxy.userClient.keepAlive(); if (tNum > SIMUL_TRACKS){ DEBUG_MSG(DLVL_WARN, "Too many tracks selected, using only first %d", SIMUL_TRACKS); } diff --git a/src/output/output_hls.cpp b/src/output/output_hls.cpp index 4a945088..ee05c89e 100644 --- a/src/output/output_hls.cpp +++ b/src/output/output_hls.cpp @@ -5,38 +5,36 @@ namespace Mist { ///\brief Builds an index file for HTTP Live streaming. ///\return The index file for HTTP Live Streaming. - std::string OutHLS::liveIndex(){ + std::string OutHLS::liveIndex() { std::stringstream result; result << "#EXTM3U\r\n"; int audioId = -1; - std::string audioName; - for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++){ - if (it->second.codec == "AAC" || it->second.codec == "MP3" || it->second.codec == "AC3"){ + for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++) { + if (it->second.codec == "AAC" || it->second.codec == "MP3" || it->second.codec == "AC3") { audioId = it->first; - audioName = it->second.getIdentifier(); break; } } unsigned int vidTracks = 0; - for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++){ - if (it->second.codec == "H264" || it->second.codec == "HEVC"){ + for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++) { + if (it->second.codec == "H264" || it->second.codec == "HEVC") { vidTracks++; int bWidth = it->second.bps; - if (bWidth < 5){ + if (bWidth < 5) { bWidth = 5; } - if (audioId != -1){ + if (audioId != -1) { bWidth += myMeta.tracks[audioId].bps; } result << "#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=" << (bWidth * 8) << "\r\n"; result << it->first; - if (audioId != -1){ + if (audioId != -1) { result << "_" << audioId; } result << "/index.m3u8\r\n"; } } - if (!vidTracks && audioId){ + if (!vidTracks && audioId) { result << "#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=" << (myMeta.tracks[audioId].bps * 8) << "\r\n"; result << audioId << "/index.m3u8\r\n"; } @@ -44,83 +42,242 @@ namespace Mist { return result.str(); } - std::string OutHLS::liveIndex(int tid){ + std::string OutHLS::pushLiveIndex(){ + std::stringstream result; + result << "#EXTM3U\r\n"; + std::set audioTracks; + for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++) { + if (it->second.codec == "AAC" || it->second.codec == "MP3" || it->second.codec == "AC3") { + audioTracks.insert(it->first); + } + } + if (!audioTracks.size()){ + audioTracks.insert(-1); + } + unsigned int vidTracks = 0; + for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++) { + if (it->second.codec == "H264" || it->second.codec == "HEVC") { + for (std::set::iterator audIt = audioTracks.begin(); audIt != audioTracks.end(); audIt++){ + vidTracks++; + int bWidth = it->second.bps; + if (bWidth < 5) { + bWidth = 5; + } + if (*audIt != -1) { + bWidth += myMeta.tracks[*audIt].bps; + } + result << "#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=" << (bWidth * 8) << "\r\n"; + result << it->first; + if (*audIt != -1) { + result << "_" << *audIt; + } + result << "/index.m3u8\r\n"; + } + } + } + if (!vidTracks && audioTracks.size()) { + result << "#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=" << (myMeta.tracks[*audioTracks.begin()].bps * 8) << "\r\n"; + result << *audioTracks.begin() << "/index.m3u8\r\n"; + } + DEBUG_MSG(DLVL_HIGH, "Sending this index: %s", result.str().c_str()); + return result.str(); + } + + std::string OutHLS::pushLiveIndex(int tid, unsigned long bTime, unsigned long eTime){ + updateMeta(); + if (!myMeta.tracks[tid].fragments.size()) { + DEBUG_MSG(DLVL_FAIL, "liveIndex called with track %d, which has no fragments!", tid); + return ""; + } + std::stringstream result; + //parse single track + int longestFragment = 0; + for (std::deque::iterator it = myMeta.tracks[tid].fragments.begin(); (it + 1) != myMeta.tracks[tid].fragments.end(); it++) { + if (it->getDuration() > longestFragment) { + longestFragment = it->getDuration(); + } + } + if ((myMeta.tracks[tid].lastms - myMeta.tracks[tid].firstms) / myMeta.tracks[tid].fragments.size() > longestFragment) { + longestFragment = (myMeta.tracks[tid].lastms - myMeta.tracks[tid].firstms) / myMeta.tracks[tid].fragments.size(); + } + result << "#EXTM3U\r\n#EXT-X-TARGETDURATION:" << (longestFragment / 1000) + 1 << "\r\n"; + + std::deque lines; + unsigned int skippedLines = 0; + for (std::deque::iterator it = myMeta.tracks[tid].fragments.begin(); it != myMeta.tracks[tid].fragments.end(); it++) { + long long int starttime = myMeta.tracks[tid].getKey(it->getNumber()).getTime(); + long long duration = it->getDuration(); + if (duration <= 0) { + duration = myMeta.tracks[tid].lastms - starttime; + } + if (starttime < bTime){ + skippedLines++; + } + if (starttime >= bTime && (starttime + duration) <= eTime){ + char lineBuf[400]; + snprintf(lineBuf, 400, "#EXTINF:%lld, no desc\r\n%lld_%lld.ts\r\n", ((duration + 500) / 1000), starttime, starttime + duration); + lines.push_back(lineBuf); + } + } + + result << "#EXT-X-MEDIA-SEQUENCE:" << myMeta.tracks[tid].missedFrags + skippedLines << "\r\n"; + + while (lines.size()) { + result << lines.front(); + lines.pop_front(); + } + if (!myMeta.live && eTime >= myMeta.tracks[tid].lastms) { + result << "#EXT-X-ENDLIST\r\n"; + } + DEBUG_MSG(DLVL_HIGH, "Sending this index: %s", result.str().c_str()); + return result.str(); + } + + + std::string OutHLS::liveIndex(int tid) { updateMeta(); std::stringstream result; //parse single track int longestFragment = 0; - if (!myMeta.tracks[tid].fragments.size()){ + if (!myMeta.tracks[tid].fragments.size()) { DEBUG_MSG(DLVL_FAIL, "liveIndex called with track %d, which has no fragments!", tid); return ""; } - for (std::deque::iterator it = myMeta.tracks[tid].fragments.begin(); (it + 1) != myMeta.tracks[tid].fragments.end(); it++){ - if (it->getDuration() > longestFragment){ + for (std::deque::iterator it = myMeta.tracks[tid].fragments.begin(); (it + 1) != myMeta.tracks[tid].fragments.end(); it++) { + if (it->getDuration() > longestFragment) { longestFragment = it->getDuration(); } } - if ((myMeta.tracks[tid].lastms - myMeta.tracks[tid].firstms) / myMeta.tracks[tid].fragments.size() > longestFragment){ + if ((myMeta.tracks[tid].lastms - myMeta.tracks[tid].firstms) / myMeta.tracks[tid].fragments.size() > longestFragment) { longestFragment = (myMeta.tracks[tid].lastms - myMeta.tracks[tid].firstms) / myMeta.tracks[tid].fragments.size(); } result << "#EXTM3U\r\n#EXT-X-TARGETDURATION:" << (longestFragment / 1000) + 1 << "\r\n"; - + std::deque lines; - for (std::deque::iterator it = myMeta.tracks[tid].fragments.begin(); it != myMeta.tracks[tid].fragments.end(); it++){ + for (std::deque::iterator it = myMeta.tracks[tid].fragments.begin(); it != myMeta.tracks[tid].fragments.end(); it++) { long long int starttime = myMeta.tracks[tid].getKey(it->getNumber()).getTime(); - std::stringstream line; long long duration = it->getDuration(); - if (duration <= 0){ + if (duration <= 0) { duration = myMeta.tracks[tid].lastms - starttime; } - line << "#EXTINF:" << ((duration + 500) / 1000) << ", no desc\r\n" << starttime << "_" << duration + starttime << ".ts\r\n"; - lines.push_back(line.str()); + char lineBuf[400]; + snprintf(lineBuf, 400, "#EXTINF:%lld, no desc\r\n%lld_%lld,ts\r\n", ((duration + 500) / 1000), starttime, starttime + duration); + lines.push_back(lineBuf); } - + //skip the first fragment if live and there are more than 2 fragments. unsigned int skippedLines = 0; - if (myMeta.live){ - //only print the last segment when VoD - lines.pop_back(); - /*LTS-START*/ - unsigned int skip = (( myMeta.tracks[tid].fragments.size()-1) * config->getInteger("startpos")) / 1000u; - while (skippedLines < skip && lines.size()){ + if (myMeta.live) { + if (lines.size() > 2) { lines.pop_front(); skippedLines++; } - if (config->getInteger("listlimit")){ + //only print the last segment when VoD + lines.pop_back(); + /*LTS-START*/ + unsigned int skip = ((myMeta.tracks[tid].fragments.size() - 1) * config->getInteger("startpos")) / 1000u; + while (skippedLines < skip && lines.size()) { + lines.pop_front(); + skippedLines++; + } + if (config->getInteger("listlimit")) { unsigned long listlimit = config->getInteger("listlimit"); - while (lines.size() > listlimit){ + while (lines.size() > listlimit) { lines.pop_front(); skippedLines++; } } /*LTS-END*/ } - + result << "#EXT-X-MEDIA-SEQUENCE:" << myMeta.tracks[tid].missedFrags + skippedLines << "\r\n"; - - while (lines.size()){ + + while (lines.size()) { result << lines.front(); lines.pop_front(); } - if ( !myMeta.live){ + if (!myMeta.live) { result << "#EXT-X-ENDLIST\r\n"; } DEBUG_MSG(DLVL_HIGH, "Sending this index: %s", result.str().c_str()); return result.str(); } //liveIndex - - - OutHLS::OutHLS(Socket::Connection & conn) : TSOutput(conn){ + + std::string OutHLS::generatePushList() { + updateMeta(); + std::set videoTracks; + std::set audioTracks; + for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++) { + if (it->second.codec == "AAC" || it->second.codec == "MP3" || it->second.codec == "AC3") { + audioTracks.insert(it->first); + } + if (it->second.codec == "H264" || it->second.codec == "H265"){ + videoTracks.insert(it->first); + } + } + JSON::Value result; + for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++) { + std::stringstream tid; + tid << it->second.trackID; + result["tracks"][tid.str()] = it->second.toJSON(true); + } + for(std::set::iterator it = videoTracks.begin(); it != videoTracks.end(); it++){ + for(std::set::iterator it2 = audioTracks.begin(); it2 != audioTracks.end(); it2++){ + JSON::Value quality; + std::stringstream identifier; + identifier << "/" << *it << "_" << *it2; + quality["index"] = "/push" + identifier.str() + "/index_\%llu_\%llu.m3u8"; + quality["segment"] = identifier.str() + "/\%llu_\%llu.ts"; + quality["video"] = *it; + quality["audio"] = *it2; + quality["id"] = identifier.str(); + std::deque::iterator it3 = myMeta.tracks[*it].fragments.begin(); + for (int i = 0; i < 2; i++){ + if (it3 != myMeta.tracks[*it].fragments.end()){ + ++it3; + } + } + for (; it3 != myMeta.tracks[*it].fragments.end(); it3++) { + if (myMeta.live && it3 == (myMeta.tracks[*it].fragments.end() - 1)){ + //Skip the current last fragment if we are live + continue; + } + long long int starttime = myMeta.tracks[*it].getKey(it3->getNumber()).getTime(); + std::stringstream line; + long long duration = it3->getDuration(); + if (duration <= 0) { + duration = myMeta.tracks[*it].lastms - starttime; + } + std::stringstream segmenturl; + segmenturl << identifier.str() << "/" << starttime << "_" << duration + starttime << ".ts"; + JSON::Value segment; + //segment["url"] = segmenturl.str(); + segment["time"] = starttime; + segment["duration"] = duration; + segment["number"] = (unsigned int)it3->getNumber(); + quality["segments"].append(segment); + } + result["qualities"].append(quality); + } + } + return result.toString();; + } + + + + + OutHLS::OutHLS(Socket::Connection & conn) : TSOutput(conn) { realTime = 0; } - + OutHLS::~OutHLS() {} - - void OutHLS::init(Util::Config * cfg){ + + void OutHLS::init(Util::Config * cfg) { HTTPOutput::init(cfg); capa["name"] = "HLS"; capa["desc"] = "Enables HTTP protocol Apple-specific streaming (also known as HLS)."; capa["url_rel"] = "/hls/$/index.m3u8"; capa["url_prefix"] = "/hls/$/"; + capa["url_pushlist"] = "/hls/$/push/list"; capa["codecs"][0u][0u].append("HEVC"); capa["codecs"][0u][0u].append("H264"); capa["codecs"][0u][1u].append("AAC"); @@ -139,29 +296,29 @@ namespace Mist { /*LTS-END*/ } - int OutHLS::canSeekms(unsigned int ms){ + int OutHLS::canSeekms(unsigned int ms) { //no tracks? Frame too new by definition. - if ( !myMeta.tracks.size()){ + if (!myMeta.tracks.size()) { return 1; } - //check main track - DTSC::Track & mainTrack = myMeta.tracks[*selectedTracks.begin()]; - //return "too late" if one track is past this point - if (ms < mainTrack.firstms){ - return -1; - } - //return "too early" if one track is not yet at this point - if (ms > mainTrack.lastms){ - return 1; + //loop trough all the tracks + for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++) { + //return "too late" if one track is past this point + if (ms < it->second.firstms) { + return -1; + } + //return "too early" if one track is not yet at this point + if (ms > it->second.lastms) { + return 1; + } } return 0; } - void OutHLS::onHTTP(){ + void OutHLS::onHTTP() { std::string method = H.method; - - - if (H.url == "/crossdomain.xml"){ + + if (H.url == "/crossdomain.xml") { H.Clean(); H.SetHeader("Content-Type", "text/xml"); H.SetHeader("Server", "MistServer/" PACKAGE_VERSION); @@ -176,27 +333,81 @@ namespace Mist { H.Clean(); //clean for any possible next requests return; } //crossdomain.xml - - if (H.url.find("hls") == std::string::npos){ + + if (H.method == "OPTIONS") { + H.Clean(); + H.SetHeader("Content-Type", "application/octet-stream"); + H.SetHeader("Cache-Control", "no-cache"); + H.setCORSHeaders(); + H.SetBody(""); + H.SendResponse("200", "OK", myConn); + H.Clean(); + return; + } + + if (H.url.find("hls") == std::string::npos) { myConn.close(); return; } - - appleCompat = (H.GetHeader("User-Agent").find("iPad") != std::string::npos) || (H.GetHeader("User-Agent").find("iPod") != std::string::npos)|| (H.GetHeader("User-Agent").find("iPhone") != std::string::npos); + + + appleCompat = (H.GetHeader("User-Agent").find("iPad") != std::string::npos) || (H.GetHeader("User-Agent").find("iPod") != std::string::npos) || (H.GetHeader("User-Agent").find("iPhone") != std::string::npos); bool VLCworkaround = false; - if (H.GetHeader("User-Agent").substr(0, 3) == "VLC"){ + if (H.GetHeader("User-Agent").substr(0, 3) == "VLC") { std::string vlcver = H.GetHeader("User-Agent").substr(4); - if (vlcver[0] == '0' || vlcver[0] == '1' || (vlcver[0] == '2' && vlcver[2] < '2')){ + if (vlcver[0] == '0' || vlcver[0] == '1' || (vlcver[0] == '2' && vlcver[2] < '2')) { DEBUG_MSG(DLVL_INFO, "Enabling VLC version < 2.2.0 bug workaround."); VLCworkaround = true; } } initialize(); - if (H.url.find(".m3u") == std::string::npos){ - std::string tmpStr = H.getUrl().substr(5+streamName.size()); + if (H.url.substr(5 + streamName.size(), 5) == "/push"){ + std::string relPushUrl = H.url.substr(10 + streamName.size()); + H.Clean(); + if (relPushUrl == "/list"){ + H.SetBody(generatePushList()); + H.SendResponse("200", "OK", myConn); + H.Clean(); + return; + } + if (relPushUrl.find(".m3u8") != std::string::npos) { + H.SetHeader("Content-Type", "audio/x-mpegurl"); + } else { + H.SetHeader("Content-Type", "audio/mpegurl"); + } + if (relPushUrl == "/index.m3u8"){ + H.SetHeader("Cache-Control", "no-cache"); + H.setCORSHeaders(); + H.SetBody(pushLiveIndex()); + H.SendResponse("200", "OK", myConn); + H.Clean(); //clean for any possible next requests + return; + }else { + unsigned int vTrack; + unsigned int aTrack; + unsigned long long bTime; + unsigned long long eTime; + if (sscanf(relPushUrl.c_str(), "/%u_%u/index_%llu_%llu.m3u", &vTrack, &aTrack, &bTime, &eTime) == 4) { + if (eTime < bTime){ + eTime = bTime; + } + H.SetHeader("Cache-Control", "no-cache"); + H.setCORSHeaders(); + H.SetBody(pushLiveIndex(vTrack, bTime, eTime)); + H.SendResponse("200", "OK", myConn); + H.Clean(); //clean for any possible next requests + return; + } + } + H.SetBody("The HLS URL wasn't understood - what did you want, exactly?\n"); + myConn.SendNow(H.BuildResponse("404", "URL mismatch")); + H.Clean(); //clean for any possible next requests + return; + }else if (H.url.find(".m3u") == std::string::npos) { + std::string tmpStr = H.getUrl().substr(5 + streamName.size()); long long unsigned int from; - if (sscanf(tmpStr.c_str(), "/%u_%u/%llu_%llu.ts", &vidTrack, &audTrack, &from, &until) != 4){ - if (sscanf(tmpStr.c_str(), "/%u/%llu_%llu.ts", &vidTrack, &from, &until) != 3){ + if (sscanf(tmpStr.c_str(), "/%u_%u/%llu_%llu.ts", &vidTrack, &audTrack, &from, &until) != 4) { + if (sscanf(tmpStr.c_str(), "/%u/%llu_%llu.ts", &vidTrack, &from, &until) != 3) { DEBUG_MSG(DLVL_MEDIUM, "Could not parse URL: %s", H.getUrl().c_str()); H.Clean(); H.setCORSHeaders(); @@ -204,33 +415,38 @@ namespace Mist { myConn.SendNow(H.BuildResponse("404", "URL mismatch")); H.Clean(); //clean for any possible next requests return; - }else{ + } else { selectedTracks.clear(); selectedTracks.insert(vidTrack); } - }else{ + } else { selectedTracks.clear(); selectedTracks.insert(vidTrack); selectedTracks.insert(audTrack); } - - if (myMeta.live){ + for (std::map::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++){ + if (it->second.codec == "ID3"){ + selectedTracks.insert(it->first); + } + } + + if (myMeta.live) { unsigned int timeout = 0; int seekable; do { seekable = canSeekms(from); /// \todo Detection of out-of-range parts. - if (seekable > 0){ + if (seekable > 0) { //time out after 21 seconds - if (++timeout > 42){ + if (++timeout > 42) { myConn.close(); break; } Util::sleep(500); updateMeta(); } - }while (myConn && seekable > 0); - if (seekable < 0){ + } while (myConn && seekable > 0); + if (seekable < 0) { H.Clean(); H.setCORSHeaders(); H.SetBody("The requested fragment is no longer kept in memory on the server and cannot be served.\n"); @@ -240,7 +456,7 @@ namespace Mist { return; } } - + seek(from); ts_from = from; lastVid = from * 90; @@ -256,12 +472,12 @@ namespace Mist { H.StartResponse(H, myConn, VLCworkaround); unsigned int fragCounter = myMeta.tracks[vidTrack].missedFrags; - for (std::deque::iterator it = myMeta.tracks[vidTrack].fragments.begin(); it != myMeta.tracks[vidTrack].fragments.end(); it++){ - long long int starttime = myMeta.tracks[vidTrack].getKey(it->getNumber()).getTime(); - if (starttime <= from && starttime + it->getDuration() > from){ - EXTREME_MSG("setting continuity counter for PAT/PMT to %d",fragCounter); - contCounters[0]=fragCounter; //PAT continuity counter - contCounters[4096]=fragCounter; //PMT continuity counter + for (std::deque::iterator it = myMeta.tracks[vidTrack].fragments.begin(); it != myMeta.tracks[vidTrack].fragments.end(); it++) { + long long int starttime = myMeta.tracks[vidTrack].getKey(it->getNumber()).getTime(); + if (starttime <= from && starttime + it->getDuration() > from) { + EXTREME_MSG("setting continuity counter for PAT/PMT to %d", fragCounter); + contCounters[0] = fragCounter; //PAT continuity counter + contCounters[4096] = fragCounter; //PMT continuity counter break; } ++fragCounter; @@ -269,13 +485,13 @@ namespace Mist { packCounter = 0; parseData = true; wantRequest = false; - }else{ + } else { initialize(); std::string request = H.url.substr(H.url.find("/", 5) + 1); H.Clean(); - if (H.url.find(".m3u8") != std::string::npos){ + if (H.url.find(".m3u8") != std::string::npos) { H.SetHeader("Content-Type", "audio/x-mpegurl"); - }else{ + } else { H.SetHeader("Content-Type", "audio/mpegurl"); } H.SetHeader("Cache-Control", "no-cache"); @@ -286,10 +502,10 @@ namespace Mist { return; } std::string manifest; - if (request.find("/") == std::string::npos){ + if (request.find("/") == std::string::npos) { manifest = liveIndex(); - }else{ - int selectId = atoi(request.substr(0,request.find("/")).c_str()); + } else { + int selectId = atoi(request.substr(0, request.find("/")).c_str()); manifest = liveIndex(selectId); } H.SetBody(manifest); @@ -298,7 +514,7 @@ namespace Mist { } - void OutHLS::sendTS(const char * tsData, unsigned int len){ + void OutHLS::sendTS(const char * tsData, unsigned int len) { H.Chunkify(tsData, len, myConn); } } diff --git a/src/output/output_hls.h b/src/output/output_hls.h index 29afcc63..d90bcd1e 100644 --- a/src/output/output_hls.h +++ b/src/output/output_hls.h @@ -12,6 +12,13 @@ namespace Mist { protected: std::string liveIndex(); std::string liveIndex(int tid); + + + std::string pushLiveIndex(); + std::string pushLiveIndex(int tid, unsigned long bTime, unsigned long eTime); + + + std::string generatePushList(); int canSeekms(unsigned int ms); int keysToSend; unsigned int vidTrack; diff --git a/src/output/output_hss.cpp b/src/output/output_hss.cpp index 4b5dc62d..1a73678d 100644 --- a/src/output/output_hss.cpp +++ b/src/output/output_hss.cpp @@ -280,7 +280,7 @@ namespace Mist { moof_box.setContent(mfhd_box, 0); moof_box.setContent(traf_box, 1); /*LTS-START*/ - if (encrypt){ + if (nProxy.encrypt){ MP4::UUID_SampleEncryption sEnc; sEnc.setVersion(0); if (myMeta.tracks[tid].type == "audio") { @@ -340,10 +340,10 @@ namespace Mist { //Load the encryption data page char pageName[NAME_BUFFER_SIZE]; snprintf(pageName, NAME_BUFFER_SIZE, SHM_STREAM_ENCRYPT, streamName.c_str()); - encryptionPage.init(pageName, 8 * 1024 * 1024, false, false); - if (encryptionPage.mapped) { - vmData.read(encryptionPage.mapped); - encrypt = true; + nProxy.encryptionPage.init(pageName, 8 * 1024 * 1024, false, false); + if (nProxy.encryptionPage.mapped) { + nProxy.vmData.read(nProxy.encryptionPage.mapped); + nProxy.encrypt = true; } encryptionLoaded = true; } @@ -352,9 +352,9 @@ namespace Mist { std::string OutHSS::protectionHeader() { loadEncryption(); std::string xmlGen = "16AESCTR"; - xmlGen += vmData.keyid; + xmlGen += nProxy.vmData.keyid; xmlGen += ""; - xmlGen += vmData.laurl; + xmlGen += nProxy.vmData.laurl; xmlGen += ""; std::string tmp = toUTF16(xmlGen); tmp = tmp.substr(2); @@ -512,7 +512,7 @@ namespace Mist { Result << "\n"; } /*LTS-START*/ - if (encrypt) { + if (nProxy.encrypt) { Result << ""; Result << protectionHeader(); Result << ""; diff --git a/src/output/output_http.cpp b/src/output/output_http.cpp index d5ffdcf4..eb5ca745 100644 --- a/src/output/output_http.cpp +++ b/src/output/output_http.cpp @@ -173,7 +173,7 @@ namespace Mist { if (handler != capa["name"].asStringRef() || H.GetVar("stream") != streamName){ DEBUG_MSG(DLVL_MEDIUM, "Switching from %s (%s) to %s (%s)", capa["name"].asStringRef().c_str(), streamName.c_str(), handler.c_str(), H.GetVar("stream").c_str()); streamName = H.GetVar("stream"); - userClient.finish(); + nProxy.userClient.finish(); statsPage.finish(); reConnector(handler); H.Clean(); @@ -396,8 +396,6 @@ namespace Mist { return trustedProxies.count(ip); } /*LTS-END*/ - - /*begin-roxlu*/ void HTTPOutput::sendResponse(std::string message, std::string code) { diff --git a/src/output/output_push.cpp b/src/output/output_push.cpp new file mode 100644 index 00000000..e1a05226 --- /dev/null +++ b/src/output/output_push.cpp @@ -0,0 +1,346 @@ +#include "output_push.h" +#include +#include +#include +#include + + +#define PUSH_INDEX_SIZE 5 //Build index based on most recent X segments + +Util::Config * pConf; +std::string sName; + +std::string baseURL; + +long long srcPort; +std::string srcHost; + +std::string dstHost; +long long dstPort; +std::string dstUrl; + +//Used to keep track of all segments that can be pushed +std::map > pushableSegments; +//Used to keep track of the timestamp of each pushableSegment +std::map > pushableTimes; +//Used to keep track of the duration of each pushableSegment +std::map > pushableDurations; + + +//For each quality, store the latest number found in the push list +std::map latestNumber; + +//For each quality, store whether it is currently being pushed. +std::map parsing; + +//For each quality, store an fprint-style string of the relative url to the index__.m3u8 +std::map qualityIndex; +//For each quality, store an fprint-style string of the relative url to the segment. +std::map qualitySegment; + +//For each quality, store the last PUSH_INDEX_SIZE - 1 timestamps. Used to generate a time-constrained index.m3u8. +std::map > qualityBeginTimes; + + +//Parses a uri of the form 'http://[:]/, and split it into variables +void parseURI(const std::string & uri, std::string & host, long long & port, std::string & url){ + int loc = 0; + if (uri.find("http://") == 0){ + loc += 7; + } + host = uri.substr(loc, uri.find_first_of(":/", 7) - 7); + loc += host.size(); + if (uri[loc] == ':'){ + port = atoll(uri.c_str() + loc + 1); + loc = uri.find("/", loc); + } + url = uri.substr(loc); +} + +//Do an HTTP request, and route it into a post request on a different socket. +void proxyToPost(Socket::Connection & src, const std::string & srcUrl, Socket::Connection & dst, const std::string & dstUrl){ + INFO_MSG("Routing %s to %s", srcUrl.c_str(), dstUrl.c_str()); + + //Send the initial request + HTTP::Parser H; + H.url = srcUrl; + H.SendRequest(src); + H.Clean(); + + //Read only the headers of the reply + H.headerOnly = true; + while (src.connected()){ + if (src.Received().size() || src.spool()){ + if (H.Read(src)){ + break; + } + } + } + H.headerOnly = false; + + INFO_MSG("Reply from %s: %s %s", src.getHost().c_str(), H.url.c_str(), H.method.c_str()); + + //Change the headers of the reply to form a post request + H.method = "POST"; + H.url = dstUrl; + H.protocol = "HTTP/1.1"; + H.SetHeader("Host", dstHost); + //Start the post request + H.SendRequest(dst); + //Route the original payload. + H.Proxy(src, dst); + + H.Clean(); + while (dst.connected()){ + if (dst.Received().size() || dst.spool()){ + if (H.Read(dst)){ + break; + } + } + } + INFO_MSG("Reply from %s: %s %s", dst.getHost().c_str(), H.url.c_str(), H.method.c_str()); +} + + +///Push the first registered segment for this quality +void pushFirstElement(std::string qId) { + std::string semName = "MstPushLock" + sName; + IPC::semaphore pushLock(semName.c_str(), O_CREAT | O_RDWR, ACCESSPERMS, 1); + + std::string url; + int time; + int beginTime; + int duration; + //Wait for exclusive access to all globals + pushLock.wait(); + //Retrieve all globals for the segment to be pushed + if (pushableSegments[qId].size()){ + url = pushableSegments[qId].begin()->second; + time = pushableTimes[qId].begin()->second; + duration = pushableDurations[qId].begin()->second; + if (qualityBeginTimes[qId].size()){ + beginTime = qualityBeginTimes[qId].front(); + }else{ + beginTime = time; + } + } + //Give up exclusive access to all globals + pushLock.post(); + //Return if we do not have a segment to push + if (url == ""){ + return; + } + + //Create both source and destination connections + Socket::Connection srcConn(srcHost, srcPort, true); + Socket::Connection dstConn(dstHost, dstPort, true); + + //Set the locations to push to for this segment + std::string srcLocation = baseURL + url; + std::string dstLocation = dstUrl.substr(0, dstUrl.rfind("/")) + url; + + //Push the segment + proxyToPost(srcConn, srcLocation, dstConn, dstLocation); + + + srcConn = Socket::Connection(srcHost, srcPort, true); + + //Set the location to push to for the index containing this segment. + //The index will contain (at most) the last PUSH_INDEX_SIZE segments. + char srcIndex[200]; + snprintf(srcIndex, 200, qualityIndex[qId].c_str(), beginTime , time + duration); + srcLocation = baseURL + srcIndex; + dstLocation = dstLocation.substr(0, dstLocation.rfind("/")) + "/index.m3u8"; + + //Push the index + proxyToPost(srcConn, srcLocation, dstConn, dstLocation); + + + srcConn = Socket::Connection(srcHost, srcPort, true); + + //Set the location to push to for the global index containing all qualities. + srcLocation = baseURL + "/push/index.m3u8"; + dstLocation = dstLocation.substr(0, dstLocation.rfind("/")); + dstLocation = dstLocation.substr(0, dstLocation.rfind("/")) + "/index.m3u8"; + + //Push the global index + proxyToPost(srcConn, srcLocation, dstConn, dstLocation); + + //Close both connections + ///\todo Make the dstConn "persistent" for each thread? + srcConn.close(); + dstConn.close(); + + //Wait for exclusive access to all globals + pushLock.wait(); + //Update all globals to indicate the segment has been pushed correctly + pushableSegments[qId].erase(pushableSegments[qId].begin()); + pushableTimes[qId].erase(pushableTimes[qId].begin()); + pushableDurations[qId].erase(pushableDurations[qId].begin()); + qualityBeginTimes[qId].push_back(time); + //Remove the first elements fromt he beginTimes map to make sure we have PUSH_INDEX_SIZE elements in our index. + //We use -1 here, because we use the segment to currently push as well as everything stored in the map + while (qualityBeginTimes[qId].size() > PUSH_INDEX_SIZE - 1){ + qualityBeginTimes[qId].pop_front(); + } + //Give up exclusive access to all globals + pushLock.post(); +} + +///Thread used to push data. +void pushThread(void * nullPointer){ + std::string myThread; + + //Attempt to claim a non-claimed quality. + std::string semName = "MstPushClaim" + sName; + IPC::semaphore pushThreadLock(semName.c_str(), O_CREAT | O_RDWR, ACCESSPERMS, 1); + + pushThreadLock.wait(); + for (std::map >::iterator it = pushableSegments.begin(); it != pushableSegments.end(); it++){ + if (it->second.size()){//Make sure we dont try to "claim" pushing an empty track + if (!parsing.count(it->first) || !parsing[it->first]){ + INFO_MSG("Claiming thread %s", it->first.c_str()); + myThread = it->first; + parsing[it->first] = true; + break; + } + } + } + pushThreadLock.post(); + + //Return if we were unable to claim a quality + if (myThread == ""){ + INFO_MSG("No thread claimed"); + return; + } + + //While this output is active, push the first element in the list + while (pConf->is_active){ + pushFirstElement(myThread); + if (!pushableSegments[myThread].size()){ + Util::sleep(1000); + } + } + + parsing[myThread] = false; +} + + +namespace Mist { + + OutPush::OutPush(Socket::Connection & conn) : Output(conn){ + config->activate(); + } + OutPush::~OutPush(){} + + void OutPush::requestHandler() { + //Set aal basic data only the first time. + if (streamName == ""){ + srcPort = 80; + parseURI(config->getString("pushlist"), srcHost, srcPort, pushURL); + dstPort = 80; + parseURI(config->getString("destination"), dstHost, dstPort, dstUrl); + + + //Strip "/push/list" from the URL + baseURL = pushURL.substr(0, pushURL.rfind("/")); + baseURL = baseURL.substr(0, baseURL.rfind("/")); + + //Locate the streamname from the pushURL + int loc = baseURL.find("/", 1) + 1; + streamName = pushURL.substr(loc, pushURL.rfind("/") - loc); + sName = streamName; + + INFO_MSG("host: %s, port %lld, url %s, baseURL %s, streamName %s", srcHost.c_str(), srcPort, pushURL.c_str(), baseURL.c_str(), streamName.c_str()); + } + //Reconnect when disconnected + if (!listConn.connected()){ + listConn = Socket::Connection(srcHost, srcPort, true); + } + //Request the push list + if (listConn.connected()){ + HTTP::Parser hReq; + hReq.url = baseURL + "/push/list"; + hReq.SendRequest(listConn); + hReq.Clean(); + //Read the entire response, not just the headers! + while (!hReq.Read(listConn) && listConn.connected()){ + Util::sleep(100); + listConn.spool(); + } + + //Construct and parse the json list + JSON::Value reply = JSON::fromString(hReq.body); + int numQualities = reply["qualities"].size(); + for (int i = 0; i < numQualities; i++){ + JSON::Value & qRef = reply["qualities"][i]; + std::string qId = qRef["id"].asString(); + + //Set both the index and segment urls when not yet set. + if (!qualityIndex.count(qId)){ + qualityIndex[qId] = qRef["index"].asString(); + qualitySegment[qId] = qRef["segment"].asString(); + } + + //Save latest segment number before parsing + int curLatestNumber = latestNumber[qId]; + + //Loop over all segments + for (int j = 0; j < qRef["segments"].size(); j++){ + JSON::Value & segRef = qRef["segments"][j]; + int thisNumber = segRef["number"].asInt(); + + //Check if this segment is newer than the newest segment before parsing + if (thisNumber > curLatestNumber){ + //If it is the highest so far, store its number + if (thisNumber > latestNumber[qId]){ + latestNumber[qId] = thisNumber; + } + //If it is not yet added, add it. + if (!pushableSegments[qId].count(thisNumber)){ + char segmentUrl[200]; + //The qualitySegment map contains a printf-style string + snprintf(segmentUrl, 200, qualitySegment[qId].c_str(), segRef["time"].asInt(), segRef["time"].asInt() + segRef["duration"].asInt()); + pushableSegments[qId][segRef["number"].asInt()] = segmentUrl; + pushableTimes[qId][segRef["number"].asInt()] = segRef["time"].asInt(); + pushableDurations[qId][segRef["number"].asInt()] = segRef["duration"].asInt(); + } + } + } + } + } + //Calculate how many qualities are not yet being pushed + int threadsToSpawn = pushableSegments.size(); + for (std::map >::iterator it = pushableSegments.begin(); it != pushableSegments.end(); it++){ + if (parsing.count(it->first) && parsing[it->first]){ + threadsToSpawn --; + } + } + //And start a thread for each unpushed quality. + //Threads determine which quality to push for themselves. + for (int i = 0; i < threadsToSpawn; i++){ + tthread::thread thisThread(pushThread, 0); + thisThread.detach(); + } + Util::sleep(100); + } + + void OutPush::init(Util::Config * cfg){ + Output::init(cfg); + capa["name"] = "Push"; + capa["desc"] = "Enables HTTP Pushing."; + capa["required"]["pushlist"]["name"] = "URL location of the pushing list"; + capa["required"]["pushlist"]["help"] = "This is the location that will be checked for pushable data."; + capa["required"]["pushlist"]["option"] = "--pushlist"; + capa["required"]["pushlist"]["type"] = "str"; + cfg->addOption("pushlist", JSON::fromString("{\"arg\":\"string\",\"short\":\"p\",\"long\":\"pushlist\",\"help\":\"This is the location that will be checked for pushable data.\"}")); + capa["required"]["destination"]["name"] = "URL location of the destination"; + capa["required"]["destination"]["help"] = "This is the location that the date will be pushed to."; + capa["required"]["destination"]["option"] = "--destination"; + capa["required"]["destination"]["type"] = "str"; + cfg->addOption("destination", JSON::fromString("{\"arg\":\"string\",\"short\":\"D\",\"long\":\"destination\",\"help\":\"This is the location that the data will be checked for pushed to.\"}")); + cfg->addBasicConnectorOptions(capa); + pConf = cfg; + config = cfg; + } +} diff --git a/src/output/output_push.h b/src/output/output_push.h new file mode 100644 index 00000000..dfdb129e --- /dev/null +++ b/src/output/output_push.h @@ -0,0 +1,19 @@ +#include + +#include "output.h" + +namespace Mist { + class OutPush : public Output { + public: + OutPush(Socket::Connection & conn); + ~OutPush(); + static bool listenMode(){return false;} + virtual void requestHandler(); + static void init(Util::Config * cfg); + protected: + Socket::Connection listConn; + std::string pushURL; + }; +} + +typedef Mist::OutPush mistOut; diff --git a/src/output/output_rtmp.cpp b/src/output/output_rtmp.cpp index 62465fa8..cc223245 100644 --- a/src/output/output_rtmp.cpp +++ b/src/output/output_rtmp.cpp @@ -903,12 +903,13 @@ namespace Mist { } JSON::Value pack_out = F.toJSON(myMeta, *amf_storage, next.cs_id*3 + (F.data[0] == 0x09 ? 0 : (F.data[0] == 0x08 ? 1 : 2) )); if ( !pack_out.isNull()){ - if (!userClient.getData()){ + if (!nProxy.userClient.getData()){ char userPageName[NAME_BUFFER_SIZE]; snprintf(userPageName, NAME_BUFFER_SIZE, SHM_USERS, streamName.c_str()); - userClient = IPC::sharedClient(userPageName, 30, true); + nProxy.userClient = IPC::sharedClient(userPageName, PLAY_EX_SIZE, true); } continueNegotiate(pack_out["trackid"].asInt()); + nProxy.streamName = streamName; bufferLivePacket(pack_out); } break; diff --git a/src/output/output_ts_base.cpp b/src/output/output_ts_base.cpp index ce4c94ae..56b03458 100644 --- a/src/output/output_ts_base.cpp +++ b/src/output/output_ts_base.cpp @@ -33,7 +33,7 @@ namespace Mist { if (packData.getBytesFree() == 184){ packData.clear(); - packData.setPID(0x100 - 1 + thisPacket.getTrackId()); + packData.setPID(thisPacket.getTrackId()); packData.setContinuityCounter(++contCounters[packData.getPID()]); if (first[thisPacket.getTrackId()]){ packData.setUnitStart(1);