From 0af992d4057cef8ebef53eb67be809c713262af2 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Wed, 21 Apr 2021 18:11:46 +0200 Subject: [PATCH] Various fixes, among which: - Fixed segfault when attempting to initialseek on disconnected streams - Fix 100% CPU bug in controller's stats code - WebRTC UDP bind socket improvements - Several segfault fixes - Increased packet reordering buffer size from 30 to 150 packets - Tweaks to default output/buffer behaviour for incoming pushes - Added message for load balancer checks - Fixed HLS content type - Stats fixes - Exit reason fixes - Fixed socket IP address detection - Fixed non-string arguments for stream settings - Added caching for getConnectedBinHost() - Added WebRTC playback rate control - Added/completed VP8/VP9 support to WebRTC/RTSP - Added live seek option to WebRTC - Fixed seek to exactly newest timestamp - Fixed HLS input # Conflicts: # lib/defines.h # src/input/input.cpp --- CMakeLists.txt | 2 + lib/comms.cpp | 13 +- lib/comms.h | 1 + lib/config.cpp | 22 +- lib/config.h | 6 +- lib/defines.h | 6 +- lib/dtsc.cpp | 83 +++++- lib/dtsc.h | 3 + lib/mp4_generic.cpp | 25 +- lib/mp4_generic.h | 2 +- lib/procs.cpp | 70 +++-- lib/rtmpchunks.cpp | 25 +- lib/rtp.cpp | 156 +++++----- lib/rtp.h | 4 + lib/sdp.cpp | 17 +- lib/sdp_media.cpp | 6 +- lib/srtp.cpp | 26 +- lib/stream.cpp | 197 ++++++------- lib/stream.h | 12 +- lib/ts_packet.cpp | 16 +- lib/ts_packet.h | 2 + lib/url.h | 1 + lib/util.cpp | 24 +- lib/util.h | 5 +- src/analysers/analyser_dtsc.cpp | 16 +- src/analysers/analyser_dtsc.h | 1 + src/analysers/analyser_rtmp.cpp | 10 +- src/analysers/analyser_rtmp.h | 1 + src/controller/controller.cpp | 2 +- src/controller/controller_api.cpp | 6 + src/controller/controller_capabilities.cpp | 6 + src/controller/controller_connectors.cpp | 4 +- src/controller/controller_push.cpp | 53 +++- src/controller/controller_push.h | 3 + src/controller/controller_statistics.cpp | 93 +++--- src/controller/controller_storage.cpp | 4 +- src/controller/controller_storage.h | 2 +- src/input/input.cpp | 253 +++++++++------- src/input/input.h | 6 +- src/input/input_balancer.cpp | 67 +++++ src/input/input_buffer.cpp | 89 +++--- src/input/input_buffer.h | 3 +- src/input/input_dtsc.cpp | 1 - src/input/input_dtsc.h | 5 + src/input/input_ebml.cpp | 17 +- src/input/input_hls.cpp | 84 +++--- src/input/input_hls.h | 6 +- src/input/input_playlist.cpp | 16 +- src/input/input_playlist.h | 5 +- src/input/input_rtsp.cpp | 52 ++-- src/input/input_rtsp.h | 7 +- src/input/input_ts.cpp | 18 +- src/input/input_ts.h | 7 +- src/io.cpp | 16 +- src/output/mist_out.cpp | 4 +- src/output/output.cpp | 325 +++++++++++++-------- src/output/output.h | 4 +- src/output/output_cmaf.cpp | 11 +- src/output/output_dtsc.cpp | 2 +- src/output/output_hls.cpp | 23 +- src/output/output_http_internal.cpp | 7 +- src/output/output_https.cpp | 5 + src/output/output_rtmp.cpp | 2 +- src/output/output_rtsp.cpp | 2 + src/output/output_ts.cpp | 2 +- src/output/output_ts_base.cpp | 7 +- src/output/output_webrtc.cpp | 200 ++++++++++--- src/output/output_webrtc.h | 5 + src/process/process_exec.cpp | 23 +- src/process/process_exec.h | 14 +- src/process/process_ffmpeg.cpp | 37 ++- src/process/process_ffmpeg.h | 1 + src/sourcery.cpp | 1 + src/utils/util_load.cpp | 4 +- test/status.cpp | 46 +++ 75 files changed, 1512 insertions(+), 790 deletions(-) create mode 100644 test/status.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b25d30e..10767a23 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -802,4 +802,6 @@ target_link_libraries(resolvetest mist) add_executable(bitwritertest test/bitwriter.cpp ${BINARY_DIR}/mist/.headers) target_link_libraries(bitwritertest mist) add_test(BitWriterTest COMMAND bitwritertest) +add_executable(streamstatustest test/status.cpp ${BINARY_DIR}/mist/.headers) +target_link_libraries(streamstatustest mist) diff --git a/lib/comms.cpp b/lib/comms.cpp index bc7fc0e3..5bd08276 100644 --- a/lib/comms.cpp +++ b/lib/comms.cpp @@ -115,7 +115,7 @@ namespace Comms{ do{ for (size_t i = firstValid(); i < endValid(); i++){ if (getStatus(i) == COMM_STATUS_INVALID){continue;} - setStatus(COMM_STATUS_DISCONNECT, i); + setStatus(COMM_STATUS_REQDISCONNECT, i); } while (getStatus(firstValid()) == COMM_STATUS_INVALID){deleteFirst();} }while (firstValid() < endValid() && ++c < 10); @@ -174,7 +174,7 @@ namespace Comms{ dataAccX.addField("lastsecond", RAX_64UINT); dataAccX.addField("down", RAX_64UINT); dataAccX.addField("up", RAX_64UINT); - dataAccX.addField("host", RAX_STRING, 16); + dataAccX.addField("host", RAX_RAW, 16); dataAccX.addField("stream", RAX_STRING, 100); dataAccX.addField("connector", RAX_STRING, 20); dataAccX.addField("crc", RAX_32UINT); @@ -277,8 +277,13 @@ namespace Comms{ up.set(_up, idx); } - std::string Statistics::getHost() const{return host.string(index);} - std::string Statistics::getHost(size_t idx) const{return (master ? host.string(idx) : "");} + std::string Statistics::getHost() const{ + return std::string(host.ptr(index), 16); + } + std::string Statistics::getHost(size_t idx) const{ + if (!master){return std::string((size_t)16, (char)'\000');} + return std::string(host.ptr(idx), 16); + } void Statistics::setHost(std::string _host){host.set(_host, index);} void Statistics::setHost(std::string _host, size_t idx){ if (!master){return;} diff --git a/lib/comms.h b/lib/comms.h index af096f70..1e603f3d 100644 --- a/lib/comms.h +++ b/lib/comms.h @@ -5,6 +5,7 @@ #define COMM_STATUS_DONOTTRACK 0x40 #define COMM_STATUS_SOURCE 0x80 +#define COMM_STATUS_REQDISCONNECT 0xFD #define COMM_STATUS_DISCONNECT 0xFE #define COMM_STATUS_INVALID 0xFF diff --git a/lib/config.cpp b/lib/config.cpp index 1eb43778..28461b5e 100644 --- a/lib/config.cpp +++ b/lib/config.cpp @@ -33,13 +33,22 @@ #include #include #include +#include // for va_list bool Util::Config::is_active = false; bool Util::Config::is_restarting = false; static Socket::Server *serv_sock_pointer = 0; uint32_t Util::Config::printDebugLevel = DEBUG; // std::string Util::Config::streamName; -std::string Util::Config::exitReason; +char Util::exitReason[256] = {0}; + +void Util::logExitReason(const char *format, ...){ + if (exitReason[0]){return;} + va_list args; + va_start(args, format); + vsnprintf(exitReason, 255, format, args); + va_end(args); +} std::string Util::listenInterface; uint32_t Util::listenPort = 0; @@ -450,7 +459,16 @@ void Util::Config::signal_handler(int signum, siginfo_t *sigInfo, void *ignore){ static int ctr = 0; if (!is_active && ++ctr > 4){BACKTRACE;} #endif - logExitReason("Setting is_active to false due to received signal interrupt"); + switch (sigInfo->si_code){ + case SI_USER: + case SI_QUEUE: + case SI_TIMER: + case SI_ASYNCIO: + case SI_MESGQ: + logExitReason("signal %s (%d) from process %d", strsignal(signum), signum, sigInfo->si_pid); + break; + default: logExitReason("signal %s (%d)", strsignal(signum), signum); + } is_active = false; default: switch (sigInfo->si_code){ diff --git a/lib/config.h b/lib/config.h index e3d34a55..573519b5 100644 --- a/lib/config.h +++ b/lib/config.h @@ -13,6 +13,8 @@ /// Contains utility code, not directly related to streaming media namespace Util{ + extern char exitReason[256]; + void logExitReason(const char * format, ...); /// Deals with parsing configuration from commandline options. class Config{ @@ -27,10 +29,6 @@ namespace Util{ static bool is_restarting; ///< Set to true when restarting, set to false on boot. static uint32_t printDebugLevel; static std::string streamName; ///< Used by debug messages to identify the stream name - static std::string exitReason; - static void logExitReason(const std::string &reason){ - if (!exitReason.size()){exitReason = reason;} - } // functions Config(); Config(std::string cmd); diff --git a/lib/defines.h b/lib/defines.h index 8d5bbbc3..4096cbea 100644 --- a/lib/defines.h +++ b/lib/defines.h @@ -163,7 +163,7 @@ static inline void show_stackframe(){} // assumed #define DEFAULT_PAGE_COUNT DEFAULT_KEY_COUNT // Assume every page is a key to ensure enough space -#define DEFAULT_FRAGMENT_DURATION 5000 +#define DEFAULT_FRAGMENT_DURATION 1900 #define META_META_OFFSET 104 #define META_META_RECORDSIZE 576 @@ -214,6 +214,7 @@ static inline void show_stackframe(){} #define SHM_TRIGGER "MstTRGR%s" //%s trigger name #define SEM_LIVE "/MstLIVE%s" //%s stream name #define SEM_INPUT "/MstInpt%s" //%s stream name +#define SEM_TRACKLIST "/MstTRKS%s" //%s stream name #define SEM_SESSCACHE "/MstSessCacheLock" #define SHM_CAPA "MstCapa" #define SHM_PROTO "MstProt" @@ -243,6 +244,9 @@ static inline void show_stackframe(){} // Setting this value to lower than 2 seconds **WILL** cause stuttering in playback due to buffer negotiation. #define SIMULATED_LIVE_BUFFER 7000 +/// The time between virtual audio "keyframes" +#define AUDIO_KEY_INTERVAL 2047 + #define STAT_EX_SIZE 177 #define PLAY_EX_SIZE 2 + 6 * SIMUL_TRACKS diff --git a/lib/dtsc.cpp b/lib/dtsc.cpp index 1d734c6c..63f36474 100644 --- a/lib/dtsc.cpp +++ b/lib/dtsc.cpp @@ -11,10 +11,6 @@ #include #include -#define AUDIO_KEY_INTERVAL \ - 5000 ///< This define controls the keyframe interval for non-video tracks, such as audio and - ///< metadata tracks. - namespace DTSC{ char Magic_Header[] = "DTSC"; char Magic_Packet[] = "DTPD"; @@ -448,6 +444,14 @@ namespace DTSC{ Bit::htobll(data + 12, _time); } + void Packet::nullMember(const std::string & memb){ + if (!master){ + INFO_MSG("Can't null '%s' for this packet, as it is not master.", memb.c_str()); + return; + } + getScan().nullMember(memb); + } + ///\brief Returns the track id of the packet. ///\return The track id of this packet. size_t Packet::getTrackId() const{ @@ -544,6 +548,32 @@ namespace DTSC{ return Scan(); } + /// If this is an object type and contains the given indice/len, sets the indice name to all zeroes. + void Scan::nullMember(const std::string & indice){ + nullMember(indice.data(), indice.size()); + } + + /// If this is an object type and contains the given indice/len, sets the indice name to all zeroes. + void Scan::nullMember(const char * indice, const size_t ind_len){ + if (getType() != DTSC_OBJ && getType() != DTSC_CON){return;} + char * i = p + 1; + //object, scan contents + while (i[0] + i[1] != 0 && i < p + len) { //while not encountering 0x0000 (we assume 0x0000EE) + if (i + 2 >= p + len) { + return;//out of packet! + } + uint16_t strlen = Bit::btohs(i); + i += 2; + if (ind_len == strlen && strncmp(indice, i, strlen) == 0) { + memset(i, 0, strlen); + return; + } + i = skipDTSC(i + strlen, p + len); + if (!i) {return;} + } + return; + } + /// Returns an object representing the named indice of this object. /// Returns an invalid object if this indice doesn't exist or this isn't an object type. bool Scan::hasMember(const std::string &indice) const{ @@ -1153,7 +1183,7 @@ namespace DTSC{ /// Does not clear "tracks" beforehand, so it may contain stale information afterwards if it was /// already populated. void Meta::refresh(){ - if (!stream.getPointer("tracks")){ + if (!stream.isReady() || !stream.getPointer("tracks")){ INFO_MSG("No track pointer, not refreshing."); return; } @@ -1480,6 +1510,15 @@ namespace DTSC{ /// Adds a track to the metadata structure. /// To be called from the various inputs/outputs whenever they want to add a track. size_t Meta::addTrack(size_t fragCount, size_t keyCount, size_t partCount, size_t pageCount, bool setValid){ + char pageName[NAME_BUFFER_SIZE]; + + snprintf(pageName, NAME_BUFFER_SIZE, SEM_TRACKLIST, streamName.c_str()); + IPC::semaphore trackLock(pageName, O_CREAT | O_RDWR, ACCESSPERMS, 1); + if (!trackLock){ + FAIL_MSG("Could not open semaphore to add track!"); + return -1; + } + trackLock.wait(); size_t pageSize = TRACK_TRACK_OFFSET + TRACK_TRACK_RECORDSIZE + (TRACK_FRAGMENT_OFFSET + (TRACK_FRAGMENT_RECORDSIZE * fragCount)) + (TRACK_KEY_OFFSET + (TRACK_KEY_RECORDSIZE * keyCount)) + @@ -1488,7 +1527,6 @@ namespace DTSC{ size_t tNumber = trackList.getPresent(); - char pageName[NAME_BUFFER_SIZE]; snprintf(pageName, NAME_BUFFER_SIZE, SHM_STREAM_TM, streamName.c_str(), getpid(), tNumber); Track &t = tracks[tNumber]; @@ -1511,7 +1549,7 @@ namespace DTSC{ trackList.setInt(trackPidField, getpid(), tNumber); trackList.setInt(trackSourceTidField, INVALID_TRACK_ID, tNumber); if (setValid){validateTrack(tNumber);} - + trackLock.post(); return tNumber; } @@ -1740,6 +1778,7 @@ namespace DTSC{ } std::string Meta::getLang(size_t trackIdx) const{ const DTSC::Track &t = tracks.at(trackIdx); + if (!t.track.isReady()){return "";} return t.track.getPointer(t.trackLangField); } @@ -1864,8 +1903,14 @@ namespace DTSC{ if (getType(*it) != "video"){continue;} DTSC::Parts p(parts(*it)); size_t ctr = 0; + int64_t prevOffset = 0; + bool firstOffset = true; for (size_t i = p.getFirstValid(); i < p.getEndValid(); ++i){ - if (p.getOffset(i)){return true;} + if (firstOffset){ + firstOffset = false; + prevOffset = p.getOffset(i); + } + if (p.getOffset(i) != prevOffset){return true;} if (++ctr >= 100){break;} } } @@ -1903,7 +1948,7 @@ namespace DTSC{ std::string(trackList.getPointer(trackEncryptionField, i)) != ""){ res.erase(trackList.getInt(trackSourceTidField, i)); } - if (!tracks.count(i)){res.erase(i);} + if (!tracks.count(i) || !tracks.at(i).track.isReady()){res.erase(i);} if (skipEmpty){ if (res.count(i) && !tracks.at(i).parts.getPresent()){res.erase(i);} } @@ -2032,10 +2077,12 @@ namespace DTSC{ curJitter = 0; } if (t > lastTime + 2500){ - if ((x % 4) == 0 && maxJitter > 50 && curJitter < maxJitter - 50){ - HIGH_MSG("Jitter lowered from %" PRIu64 " to %" PRIu64 " ms", maxJitter, curJitter); - maxJitter = curJitter; - curJitter = 0; + if ((x % 4) == 0){ + if (maxJitter > 50 && curJitter < maxJitter - 50){ + MEDIUM_MSG("Jitter lowered from %" PRIu64 " to %" PRIu64 " ms", maxJitter, curJitter); + maxJitter = curJitter; + } + curJitter = maxJitter*0.75; } ++x; trueTime[x % 8] = curMs; @@ -2055,7 +2102,11 @@ namespace DTSC{ // Postive jitter = packets arriving too late. // We need to delay playback at least by this amount to account for it. if ((uint64_t)jitter > maxJitter){ - HIGH_MSG("Jitter increased from %" PRIu64 " to %" PRId64 " ms", maxJitter, jitter); + if (jitter - maxJitter > 420){ + INFO_MSG("Jitter increased from %" PRIu64 " to %" PRId64 " ms", maxJitter, jitter); + }else{ + HIGH_MSG("Jitter increased from %" PRIu64 " to %" PRId64 " ms", maxJitter, jitter); + } maxJitter = (uint64_t)jitter; } if (curJitter < (uint64_t)jitter){curJitter = (uint64_t)jitter;} @@ -2291,6 +2342,8 @@ namespace DTSC{ if (streamPage.mapped && stream.isReady()){stream.setExit();} streamPage.master = true; } + stream = Util::RelAccX(); + trackList = Util::RelAccX(); streamPage.close(); tM.clear(); tracks.clear(); @@ -2875,6 +2928,7 @@ namespace DTSC{ const Util::RelAccX &pages = tracks.at(idx).pages; size_t res = pages.getStartPos(); for (size_t i = pages.getStartPos(); i < pages.getEndPos(); ++i){ + if (pages.getInt("avail", i) == 0){continue;} if (pages.getInt("firsttime", i) > time){break;} res = i; } @@ -2887,6 +2941,7 @@ namespace DTSC{ const Util::RelAccX &pages = tracks.at(idx).pages; size_t res = pages.getStartPos(); for (size_t i = pages.getStartPos(); i < pages.getEndPos(); ++i){ + if (pages.getInt("avail", i) == 0){continue;} if (pages.getInt("firstkey", i) > keyNum){break;} res = i; } diff --git a/lib/dtsc.h b/lib/dtsc.h index 2538cefa..78ae5528 100644 --- a/lib/dtsc.h +++ b/lib/dtsc.h @@ -63,6 +63,8 @@ namespace DTSC{ Scan getMember(const std::string &indice) const; Scan getMember(const char *indice) const; Scan getMember(const char *indice, size_t ind_len) const; + void nullMember(const std::string & indice); + void nullMember(const char * indice, size_t ind_len); Scan getIndice(size_t num) const; std::string getIndiceName(size_t num) const; size_t getSize() const; @@ -111,6 +113,7 @@ namespace DTSC{ void setKeyFrame(bool kf); virtual uint64_t getTime() const; void setTime(uint64_t _time); + void nullMember(const std::string & memb); size_t getTrackId() const; char *getData() const; size_t getDataLen() const; diff --git a/lib/mp4_generic.cpp b/lib/mp4_generic.cpp index b4580949..fb692aca 100644 --- a/lib/mp4_generic.cpp +++ b/lib/mp4_generic.cpp @@ -1063,8 +1063,8 @@ namespace MP4{ ESDS::ESDS(){memcpy(data + 4, "esds", 4);} - ESDS::ESDS(std::string init){ - ///\todo Do this better, in a non-hardcoded way. + ESDS::ESDS(const DTSC::Meta & M, size_t idx){ + std::string init = M.getInit(idx); memcpy(data + 4, "esds", 4); reserve(payloadOffset, 0, init.size() ? init.size() + 28 : 26); unsigned int i = 12; @@ -1084,14 +1084,10 @@ namespace MP4{ data[i++] = 0; // buffer size data[i++] = 0; // buffer size data[i++] = 0; // buffer size - data[i++] = 0; // maxbps - data[i++] = 0; // maxbps - data[i++] = 0; // maxbps - data[i++] = 0; // maxbps - data[i++] = 0; // avgbps - data[i++] = 0; // avgbps - data[i++] = 0; // avgbps - data[i++] = 0; // avgbps + Bit::htobl(data+i, M.getMaxBps(idx));//maxbps + i += 4; + Bit::htobl(data+i, M.getBps(idx));//avgbps + i += 4; if (init.size()){ data[i++] = 0x5; // DecSpecificInfoTag data[i++] = init.size(); @@ -2825,17 +2821,20 @@ namespace MP4{ AudioSampleEntry::AudioSampleEntry(const DTSC::Meta &M, size_t idx){ std::string tCodec = M.getCodec(idx); initialize(); - if (tCodec == "AAC" || tCodec == "MP3"){setCodec("mp4a");} - if (tCodec == "AC3"){setCodec("ac-3");} setDataReferenceIndex(1); setSampleRate(M.getRate(idx)); setChannelCount(M.getChannels(idx)); setSampleSize(M.getSize(idx)); + if (tCodec == "AAC" || tCodec == "MP3"){ + setCodec("mp4a"); + setSampleSize(16); + } + if (tCodec == "AC3"){setCodec("ac-3");} if (tCodec == "AC3"){ MP4::DAC3 dac3Box(M.getRate(idx), M.getChannels(idx)); setCodecBox(dac3Box); }else{// other codecs use the ESDS box - MP4::ESDS esdsBox(M.getInit(idx)); + MP4::ESDS esdsBox(M, idx); setCodecBox(esdsBox); } } diff --git a/lib/mp4_generic.h b/lib/mp4_generic.h index 8eb66000..b16adbb3 100644 --- a/lib/mp4_generic.h +++ b/lib/mp4_generic.h @@ -241,7 +241,7 @@ namespace MP4{ class ESDS : public fullBox{ public: ESDS(); - ESDS(std::string init); + ESDS(const DTSC::Meta & M, size_t idx); ESDescriptor getESDescriptor(); bool isAAC(); std::string getCodec(); diff --git a/lib/procs.cpp b/lib/procs.cpp index 256b01d1..08557f82 100644 --- a/lib/procs.cpp +++ b/lib/procs.cpp @@ -325,42 +325,50 @@ pid_t Util::Procs::StartPiped(const char *const *argv, int *fdin, int *fdout, in pid = fork(); if (pid == 0){// child handler_set = false; + if (!fdin){ + dup2(devnull, 100); + }else if (*fdin == -1){ + close(pipein[1]); // close unused write end + dup2(pipein[0], 100); + close(pipein[0]); + }else{ + dup2(*fdin, 100); + } + if (!fdout){ + dup2(devnull, 101); + }else if (*fdout == -1){ + close(pipeout[0]); // close unused read end + dup2(pipeout[1], 101); + close(pipeout[1]); + }else{ + dup2(*fdout, 101); + } + if (!fderr){ + dup2(devnull, 102); + }else if (*fderr == -1){ + close(pipeerr[0]); // close unused read end + dup2(pipeerr[1], 102); + close(pipeerr[1]); + }else{ + dup2(*fderr, 102); + } + if (fdin && *fdin != -1){close(*fdin);} + if (fdout && *fdout != -1){close(*fdout);} + if (fderr && *fderr != -1){close(*fderr);} + if (devnull != -1){close(devnull);} // Close all sockets in the socketList for (std::set::iterator it = Util::Procs::socketList.begin(); it != Util::Procs::socketList.end(); ++it){ close(*it); } - if (!fdin){ - dup2(devnull, STDIN_FILENO); - }else if (*fdin == -1){ - close(pipein[1]); // close unused write end - dup2(pipein[0], STDIN_FILENO); - close(pipein[0]); - }else if (*fdin != STDIN_FILENO){ - dup2(*fdin, STDIN_FILENO); - } - if (!fdout){ - dup2(devnull, STDOUT_FILENO); - }else if (*fdout == -1){ - close(pipeout[0]); // close unused read end - dup2(pipeout[1], STDOUT_FILENO); - close(pipeout[1]); - }else if (*fdout != STDOUT_FILENO){ - dup2(*fdout, STDOUT_FILENO); - } - if (!fderr){ - dup2(devnull, STDERR_FILENO); - }else if (*fderr == -1){ - close(pipeerr[0]); // close unused read end - dup2(pipeerr[1], STDERR_FILENO); - close(pipeerr[1]); - }else if (*fderr != STDERR_FILENO){ - dup2(*fderr, STDERR_FILENO); - } - if (fdin && *fdin != -1 && *fdin != STDIN_FILENO){close(*fdin);} - if (fdout && *fdout != -1 && *fdout != STDOUT_FILENO){close(*fdout);} - if (fderr && *fderr != -1 && *fderr != STDERR_FILENO){close(*fderr);} - if (devnull != -1){close(devnull);} + //Black magic to make sure if 0/1/2 are not what we think they are, we end up with them not mixed up and weird. + dup2(100, 0); + dup2(101, 1); + dup2(102, 2); + close(100); + close(101); + close(102); + //There! Now we normalized our stdio // Because execvp requires a char* const* and we have a const char* const* execvp(argv[0], (char *const *)argv); /*LTS-START*/ diff --git a/lib/rtmpchunks.cpp b/lib/rtmpchunks.cpp index 3183821a..c4370b46 100644 --- a/lib/rtmpchunks.cpp +++ b/lib/rtmpchunks.cpp @@ -354,7 +354,10 @@ bool RTMPStream::Chunk::Parse(Socket::Buffer &buffer){ switch (headertype){ case 0x00: - if (!buffer.available(i + 11)){return false;}// can't read whole header + if (!buffer.available(i + 11)){ + DONTEVEN_MSG("Cannot read whole header"); + return false; + }// can't read whole header indata = buffer.copy(i + 11); timestamp = indata[i++] * 256 * 256; timestamp += indata[i++] * 256; @@ -372,7 +375,10 @@ bool RTMPStream::Chunk::Parse(Socket::Buffer &buffer){ msg_stream_id += indata[i++] * 256 * 256 * 256; break; case 0x40: - if (!buffer.available(i + 7)){return false;}// can't read whole header + if (!buffer.available(i + 7)){ + DONTEVEN_MSG("Cannot read whole header"); + return false; + }// can't read whole header indata = buffer.copy(i + 7); if (!allow_short){WARN_MSG("Warning: Header type 0x40 with no valid previous chunk!");} timestamp = indata[i++] * 256 * 256; @@ -391,7 +397,10 @@ bool RTMPStream::Chunk::Parse(Socket::Buffer &buffer){ msg_stream_id = prev.msg_stream_id; break; case 0x80: - if (!buffer.available(i + 3)){return false;}// can't read whole header + if (!buffer.available(i + 3)){ + DONTEVEN_MSG("Cannot read whole header"); + return false; + }// can't read whole header indata = buffer.copy(i + 3); if (!allow_short){WARN_MSG("Warning: Header type 0x80 with no valid previous chunk!");} timestamp = indata[i++] * 256 * 256; @@ -435,7 +444,10 @@ bool RTMPStream::Chunk::Parse(Socket::Buffer &buffer){ // read extended timestamp, if necessary if (ts_header == 0x00ffffff){ - if (!buffer.available(i + 4)){return false;}// can't read timestamp + if (!buffer.available(i + 4)){ + DONTEVEN_MSG("Cannot read timestamp"); + return false; + }// can't read timestamp indata = buffer.copy(i + 4); timestamp += indata[i++] * 256 * 256 * 256; timestamp += indata[i++] * 256 * 256; @@ -447,7 +459,10 @@ bool RTMPStream::Chunk::Parse(Socket::Buffer &buffer){ // read data if length > 0, and allocate it if (real_len > 0){ - if (!buffer.available(i + real_len)){return false;}// can't read all data (yet) + if (!buffer.available(i + real_len)){ + DONTEVEN_MSG("Cannot read all data yet"); + return false; + }// can't read all data (yet) buffer.remove(i); // remove the header if (prev.len_left > 0){ data = prev.data + buffer.remove(real_len); // append the data and remove from buffer diff --git a/lib/rtp.cpp b/lib/rtp.cpp index b0d452ba..0f885ad3 100644 --- a/lib/rtp.cpp +++ b/lib/rtp.cpp @@ -58,12 +58,13 @@ namespace RTP{ if ((payload[0] & 0x1F) == 12){return;} /// \todo This function probably belongs in DMS somewhere. if (payloadlen + getHsize() + 2 <= maxDataLen){ + data[1] &= 0x7F; // setting the RTP marker bit to 0 if (lastOfAccesUnit){ data[1] |= 0x80; // setting the RTP marker bit to 1 } uint8_t nal_type = (payload[0] & 0x1F); if (nal_type < 1 || nal_type > 5){ - data[1] &= ~0x80; // but not for non-vlc types + data[1] &= 0x7F; // but not for non-vlc types } memcpy(data + getHsize(), payload, payloadlen); callBack(socket, data, getHsize() + payloadlen, channel); @@ -239,6 +240,10 @@ namespace RTP{ sendVP8(socket, callBack, payload, payloadlen, channel); return; } + if (codec == "VP9"){ + sendVP8(socket, callBack, payload, payloadlen, channel); + return; + } if (codec == "HEVC"){ unsigned long sent = 0; while (sent < payloadlen){ @@ -414,6 +419,18 @@ namespace RTP{ data = (char *)dat; } + /// Describes a packet in human-readable terms + std::string Packet::toString() const{ + std::stringstream ret; + ret << maxDataLen << "b RTP packet "; + if (getMarker()){ret << "(marked) ";} + ret << "payload type " << getPayloadType() << ", #" << getSequence() << ", @" << getTimeStamp(); + ret << " (" << getHsize() << "b header, " << getPayloadSize() << "b payload, " << getPadding() << "b padding)"; + return ret.str(); + } + + + MPEGVideoHeader::MPEGVideoHeader(char *d){data = d;} uint16_t MPEGVideoHeader::getTotalLen() const{ @@ -481,8 +498,8 @@ namespace RTP{ /// Calls the callback with packets in sorted order, whenever it becomes possible to do so. void Sorter::addPacket(const Packet &pack){ if (!rtpSeq){rtpSeq = pack.getSequence();} - // packet is very early - assume dropped after 30 packets - while ((int16_t)(rtpSeq - ((uint16_t)pack.getSequence())) < -30){ + // packet is very early - assume dropped after 150 packets + while ((int16_t)(rtpSeq - ((uint16_t)pack.getSequence())) < -150){ WARN_MSG("Giving up on packet %u", rtpSeq); ++rtpSeq; ++lostTotal; @@ -574,7 +591,10 @@ namespace RTP{ if (M.getType(tid) == "video" || M.getCodec(tid) == "MP2" || M.getCodec(tid) == "MP3"){ m = 90.0; } - setProperties(tid, M.getCodec(tid), M.getType(tid), M.getInit(tid), m); + if (M.getCodec(tid) == "opus"){ + m = 48.0; + } + setProperties(M.getID(tid), M.getCodec(tid), M.getType(tid), M.getInit(tid), m); } void toDTSC::setCallbacks(void (*cbP)(const DTSC::Packet &pkt), @@ -627,6 +647,9 @@ namespace RTP{ if (codec == "VP8"){ return handleVP8(msTime, pl, plSize, missed, (pkt.getPadding() == 1) ? true : false); } + if (codec == "VP9"){ + return handleVP8(msTime, pl, plSize, missed, (pkt.getPadding() == 1) ? true : false); + } // Trivial codecs just fill a packet with raw data and continue. Easy peasy, lemon squeezy. if (codec == "ALAW" || codec == "opus" || codec == "PCM" || codec == "ULAW"){ DTSC::Packet nextPack; @@ -902,6 +925,51 @@ namespace RTP{ // Header data? Compare to init, set if needed, and throw away uint8_t nalType = (buffer[4] & 0x1F); if (nalType == 9 && len < 20){return;}// ignore delimiter-only packets + if (!h264OutBuffer.size()){ + currH264Time = ts; + h264BufferWasKey = isKey; + } + + //Send an outPacket every time the timestamp updates + if (currH264Time != ts){ + //calculate the "packet" (which might be more than one actual packet) timestamp + uint32_t offset = 0; + uint64_t newTs = currH264Time; + + if (fps > 1){ + // Assume a steady frame rate, clip the timestamp based on frame number. + uint64_t frameNo = (currH264Time / (1000.0 / fps)) + 0.5; + while (frameNo < packCount){packCount--;} + // More than 32 frames behind? We probably skipped something, somewhere... + if ((frameNo - packCount) > 32){packCount = frameNo;} + // After some experimentation, we found that the time offset is the difference between the + // frame number and the packet counter, times the frame rate in ms + offset = (frameNo - packCount) * (1000.0 / fps); + //... and the timestamp is the packet counter times the frame rate in ms. + newTs = packCount * (1000.0 / fps); + VERYHIGH_MSG("Packing time %" PRIu64 " = %sframe %" PRIu64 " (%.2f FPS). Expected %" PRIu64 + " -> +%" PRIu64 "/%" PRIu32, + ts, isKey ? "key" : "i", frameNo, fps, packCount, (frameNo - packCount), offset); + }else{ + // For non-steady frame rate, assume no offsets are used and the timestamp is already + // correct + VERYHIGH_MSG("Packing time %" PRIu64 " = %sframe %" PRIu64 " (variable rate)", currH264Time, + isKey ? "key" : "i", packCount); + } + // Fill the new DTSC packet, buffer it. + DTSC::Packet nextPack; + nextPack.genericFill(newTs, offset, trackId, h264OutBuffer, h264OutBuffer.size(), 0, h264BufferWasKey); + packCount++; + outPacket(nextPack); + + //Clear the buffers, reset the time to current + h264OutBuffer.assign(0, 0); + currH264Time = ts; + h264BufferWasKey = isKey; + } + h264BufferWasKey |= isKey; + + switch (nalType){ case 6: // SEI return; @@ -950,80 +1018,26 @@ namespace RTP{ } return; case 5:{ - // @todo add check if ppsData and spsData are not empty? - static Util::ResizeablePointer tmp; - tmp.assign(0, 0); + //If this is a keyframe and we have no buffer yet, prepend the SPS/PPS + if (!h264OutBuffer.size()){ + char sizeBuffer[4]; + Bit::htobl(sizeBuffer, spsData.size()); + h264OutBuffer.append(sizeBuffer, 4); + h264OutBuffer.append(spsData.data(), spsData.size()); - char sizeBuffer[4]; - Bit::htobl(sizeBuffer, spsData.size()); - tmp.append(sizeBuffer, 4); - tmp.append(spsData.data(), spsData.size()); - - Bit::htobl(sizeBuffer, ppsData.size()); - tmp.append(sizeBuffer, 4); - tmp.append(ppsData.data(), ppsData.size()); - tmp.append(buffer, len); - - uint32_t offset = 0; - uint64_t newTs = ts; - - if (fps > 1){ - // Assume a steady frame rate, clip the timestamp based on frame number. - uint64_t frameNo = (ts / (1000.0 / fps)) + 0.5; - while (frameNo < packCount){packCount--;} - // More than 32 frames behind? We probably skipped something, somewhere... - if ((frameNo - packCount) > 32){packCount = frameNo;} - // After some experimentation, we found that the time offset is the difference between the - // frame number and the packet counter, times the frame rate in ms - offset = (frameNo - packCount) * (1000.0 / fps); - //... and the timestamp is the packet counter times the frame rate in ms. - newTs = packCount * (1000.0 / fps); - VERYHIGH_MSG("Packing time %" PRIu64 " = %sframe %" PRIu64 " (%.2f FPS). Expected %" PRIu64 - " -> +%" PRIu64 "/%" PRIu32, - ts, isKey ? "key" : "i", frameNo, fps, packCount, (frameNo - packCount), offset); - }else{ - // For non-steady frame rate, assume no offsets are used and the timestamp is already - // correct - VERYHIGH_MSG("Packing time %" PRIu64 " = %sframe %" PRIu64 " (variable rate)", ts, - isKey ? "key" : "i", packCount); + Bit::htobl(sizeBuffer, ppsData.size()); + h264OutBuffer.append(sizeBuffer, 4); + h264OutBuffer.append(ppsData.data(), ppsData.size()); } - // Fill the new DTSC packet, buffer it. - DTSC::Packet nextPack; - nextPack.genericFill(newTs, offset, trackId, tmp, tmp.size(), 0, isKey); - packCount++; - outPacket(nextPack); - return; + //Note: no return, we still want to buffer the packet itself, below! } default: // others, continue parsing break; } - uint32_t offset = 0; - uint64_t newTs = ts; - if (fps > 1){ - // Assume a steady frame rate, clip the timestamp based on frame number. - uint64_t frameNo = (ts / (1000.0 / fps)) + 0.5; - while (frameNo < packCount){packCount--;} - // More than 32 frames behind? We probably skipped something, somewhere... - if ((frameNo - packCount) > 32){packCount = frameNo;} - // After some experimentation, we found that the time offset is the difference between the - // frame number and the packet counter, times the frame rate in ms - offset = (frameNo - packCount) * (1000.0 / fps); - //... and the timestamp is the packet counter times the frame rate in ms. - newTs = packCount * (1000.0 / fps); - VERYHIGH_MSG("Packing time %" PRIu64 " = %sframe %" PRIu64 " (%.2f FPS). Expected %" PRIu64 - " -> +%" PRIu64 "/%" PRIu32, - ts, isKey ? "key" : "i", frameNo, fps, packCount, (frameNo - packCount), offset); - }else{ - // For non-steady frame rate, assume no offsets are used and the timestamp is already correct - VERYHIGH_MSG("Packing time %" PRIu64 " = %sframe %" PRIu64 " (variable rate)", ts, - isKey ? "key" : "i", packCount); - } - // Fill the new DTSC packet, buffer it. - DTSC::Packet nextPack; - nextPack.genericFill(newTs, offset, trackId, buffer, len, 0, isKey); - packCount++; - outPacket(nextPack); + //Buffer the packet + h264OutBuffer.append(buffer, len); + } /// Handles a single H264 packet, checking if others are appended at the end in Annex B format. diff --git a/lib/rtp.h b/lib/rtp.h index 393ae387..04b120cf 100644 --- a/lib/rtp.h +++ b/lib/rtp.h @@ -77,6 +77,7 @@ namespace RTP{ Packet(const char *dat, uint64_t len); const char *getData(); char *ptr() const{return data;} + std::string toString() const; }; /// Sorts RTP packets, outputting them through a callback in correct order. @@ -163,6 +164,9 @@ namespace RTP{ h265::initData hevcInfo; ///< For HEVC init parsing Util::ResizeablePointer fuaBuffer; ///< For H264/HEVC FU-A packets Util::ResizeablePointer packBuffer; ///< For H264/HEVC regular and STAP packets + uint64_t currH264Time;//Time of the DTSC packet currently being built (pre-conversion) + Util::ResizeablePointer h264OutBuffer; ///< For collecting multiple timestamps into one packet + bool h264BufferWasKey; void handleH264(uint64_t msTime, char *pl, uint32_t plSize, bool missed, bool hasPadding); void handleH264Single(uint64_t ts, const char *buffer, const uint32_t len, bool isKey); void handleH264Multi(uint64_t ts, char *buffer, const uint32_t len); diff --git a/lib/sdp.cpp b/lib/sdp.cpp index b8b97d39..d035784d 100644 --- a/lib/sdp.cpp +++ b/lib/sdp.cpp @@ -462,9 +462,18 @@ namespace SDP{ myMeta->setCodec(tid, "HEVC"); myMeta->setRate(tid, 90000); } + if (trCodec == "VP8"){ + myMeta->setCodec(tid, "VP8"); + myMeta->setRate(tid, 90000); + } + if (trCodec == "VP9"){ + myMeta->setCodec(tid, "VP9"); + myMeta->setRate(tid, 90000); + } if (trCodec == "OPUS"){ myMeta->setCodec(tid, "opus"); myMeta->setInit(tid, "OpusHead\001\002\170\000\200\273\000\000\000\000\000", 19); + myMeta->setRate(tid, 48000); } if (trCodec == "PCMA"){myMeta->setCodec(tid, "ALAW");} if (trCodec == "PCMU"){myMeta->setCodec(tid, "ULAW");} @@ -484,7 +493,10 @@ namespace SDP{ myMeta->setCodec(tid, "PCM"); myMeta->setSize(tid, 24); } - if (trCodec == "MPEG4-GENERIC"){myMeta->setCodec(tid, "AAC");} + if (trCodec == "MPEG4-GENERIC"){ + myMeta->setCodec(tid, "AAC"); + myMeta->setSize(tid, 16); + } if (!myMeta->getCodec(tid).size()){ ERROR_MSG("Unsupported RTP mapping: %s", mediaType.c_str()); }else{ @@ -669,6 +681,9 @@ namespace SDP{ if (M->getType(tid) == "video" || M->getCodec(tid) == "MP2" || M->getCodec(tid) == "MP3"){ return 90.0; } + if (M->getCodec(tid) == "opus"){ + return 48.0; + } return ((double)M->getRate(tid) / 1000.0); } diff --git a/lib/sdp_media.cpp b/lib/sdp_media.cpp index a603433c..3da14a2d 100644 --- a/lib/sdp_media.cpp +++ b/lib/sdp_media.cpp @@ -12,6 +12,8 @@ namespace SDP{ return "H264"; }else if (codec == "VP8"){ return "VP8"; + }else if (codec == "VP9"){ + return "VP9"; }else if (codec == "AC3"){ return "AC3"; }else if (codec == "PCMA"){ @@ -49,6 +51,8 @@ namespace SDP{ return "H264"; }else if (codec == "VP8"){ return "VP8"; + }else if (codec == "VP9"){ + return "VP9"; }else if (codec == "AC3"){ return "AC3"; }else if (codec == "ALAW"){ @@ -184,7 +188,7 @@ namespace SDP{ return 90000; }else if (encodingName == "VP8"){ return 90000; - }else if (encodingName == "vp9"){ + }else if (encodingName == "VP9"){ return 90000; } diff --git a/lib/srtp.cpp b/lib/srtp.cpp index 8c5c9d07..7f2c5fe6 100644 --- a/lib/srtp.cpp +++ b/lib/srtp.cpp @@ -110,11 +110,13 @@ int SRTPReader::shutdown(){ int r = 0; - srtp_err_status_t status = srtp_dealloc(session); - if (srtp_err_status_ok != status){ - ERROR_MSG("Failed to cleanly shutdown the SRTP session. Status: %s", - srtp_status_to_string(status).c_str()); - r -= 5; + if (session){ + srtp_err_status_t status = srtp_dealloc(session); + if (srtp_err_status_ok != status){ + ERROR_MSG("Failed to cleanly shutdown the SRTP session. Status: %s", + srtp_status_to_string(status).c_str()); + r -= 5; + } } memset((void *)&policy, 0x00, sizeof(policy)); @@ -292,12 +294,14 @@ error: int SRTPWriter::shutdown(){ int r = 0; - - srtp_err_status_t status = srtp_dealloc(session); - if (srtp_err_status_ok != status){ - ERROR_MSG("Failed to cleanly shutdown the SRTP session. Status: %s", - srtp_status_to_string(status).c_str()); - r -= 5; + + if (session){ + srtp_err_status_t status = srtp_dealloc(session); + if (srtp_err_status_ok != status){ + ERROR_MSG("Failed to cleanly shutdown the SRTP session. Status: %s", + srtp_status_to_string(status).c_str()); + r -= 5; + } } memset((char *)&policy, 0x00, sizeof(policy)); diff --git a/lib/stream.cpp b/lib/stream.cpp index 2105d8a2..3428977c 100644 --- a/lib/stream.cpp +++ b/lib/stream.cpp @@ -462,7 +462,7 @@ bool Util::startInput(std::string streamname, std::string filename, bool forkFir while (!streamAlive(streamname) && ++waiting < 240){ Util::wait(250); if (!Util::Procs::isRunning(pid)){ - FAIL_MSG("Input process shut down before stream coming online, aborting."); + FAIL_MSG("Input process (PID %d) shut down before stream coming online, aborting.", pid); break; } } @@ -548,7 +548,7 @@ JSON::Value Util::getInputBySource(const std::string &filename, bool isProvider) /// streamname MUST be pre-sanitized /// target gets variables replaced and may be altered by the PUSH_OUT_START trigger response. /// Attempts to match the altered target to an output that can push to it. -pid_t Util::startPush(const std::string &streamname, std::string &target){ +pid_t Util::startPush(const std::string &streamname, std::string &target, int debugLvl){ if (Triggers::shouldTrigger("PUSH_OUT_START", streamname)){ std::string payload = streamname + "\n" + target; std::string filepath_response = target; @@ -562,6 +562,8 @@ pid_t Util::startPush(const std::string &streamname, std::string &target){ // Set original target string in environment setenv("MST_ORIG_TARGET", target.c_str(), 1); + //If no debug level set, default to level of starting process + if (debugLvl < 0){debugLvl = Util::Config::printDebugLevel;} // The target can hold variables like current time etc streamVariables(target, streamname); @@ -604,9 +606,13 @@ pid_t Util::startPush(const std::string &streamname, std::string &target){ } INFO_MSG("Pushing %s to %s through %s", streamname.c_str(), target.c_str(), output_bin.c_str()); // Start output. + std::string dLvl = JSON::Value(debugLvl).asString(); char *argv[] ={(char *)output_bin.c_str(), (char *)"--stream", (char *)streamname.c_str(), - (char *)target.c_str(), (char *)NULL}; - + (char *)target.c_str(), 0, 0, 0}; + if (debugLvl != DEBUG){ + argv[4] = (char*)"-g"; + argv[5] = (char*)dLvl.c_str(); + } int stdErr = 2; // Cache return value so we can do some cleaning before we return pid_t ret = Util::Procs::StartPiped(argv, 0, 0, &stdErr); @@ -688,10 +694,10 @@ DTSC::Scan Util::DTSCShmReader::getScan(){ /// Does not do any checks if the protocol supports these tracks, just selects blindly. /// It is necessary to follow up with a selectDefaultTracks() call to strip unsupported /// codecs/combinations. -std::set Util::findTracks(const DTSC::Meta &M, const std::string &trackType, const std::string &trackVal){ +std::set Util::findTracks(const DTSC::Meta &M, const JSON::Value &capa, const std::string &trackType, const std::string &trackVal, const std::string &UA){ std::set result; if (!trackVal.size()){return result;} - if (trackVal == "-1" | trackVal == "none"){return result;}// don't select anything in particular + if (trackVal == "-1" || trackVal == "none"){return result;}// don't select anything in particular if (trackVal.find(',') != std::string::npos){ // Comma-separated list, recurse. std::stringstream ss(trackVal); @@ -708,7 +714,7 @@ std::set Util::findTracks(const DTSC::Meta &M, const std::string &trackT WARN_MSG("Track %zu does not exist in stream, cannot select", idx); return result; } - if (M.getType(idx) != trackType && M.getCodec(idx) != trackType){ + if (trackType.size() && M.getType(idx) != trackType && M.getCodec(idx) != trackType){ WARN_MSG("Track %zu is not %s (%s/%s), cannot select", idx, trackType.c_str(), M.getType(idx).c_str(), M.getCodec(idx).c_str()); return result; @@ -720,23 +726,22 @@ std::set Util::findTracks(const DTSC::Meta &M, const std::string &trackT Util::stringToLower(trackLow); if (trackLow == "all" || trackLow == "*"){ // select all tracks of this type - std::set validTracks = M.getValidTracks(); + std::set validTracks = capa?getSupportedTracks(M, capa):M.getValidTracks(); for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); it++){ - if (M.getType(*it) == trackType || M.getCodec(*it) == trackType){result.insert(*it);} + if (!trackType.size() || M.getType(*it) == trackType || M.getCodec(*it) == trackType){result.insert(*it);} } return result; } if (trackLow == "highbps" || trackLow == "bestbps" || trackLow == "maxbps"){ // select highest bit rate track of this type + std::set validTracks = capa?getSupportedTracks(M, capa):M.getValidTracks(); size_t currVal = INVALID_TRACK_ID; uint32_t currRate = 0; - std::set validTracks = getSupportedTracks(M, capa); for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); it++){ - const DTSC::Track &Trk = M.tracks.at(*it); - if (!trackType.size() || Trk.type == trackType || Trk.codec == trackType){ - if (currRate < Trk.bps){ + if (!trackType.size() || M.getType(*it) == trackType || M.getCodec(*it) == trackType){ + if (currRate < M.getBps(*it)){ currVal = *it; - currRate = Trk.bps; + currRate = M.getBps(*it); } } } @@ -745,66 +750,50 @@ std::set Util::findTracks(const DTSC::Meta &M, const std::string &trackT } if (trackLow == "lowbps" || trackLow == "worstbps" || trackLow == "minbps"){ // select lowest bit rate track of this type + std::set validTracks = capa?getSupportedTracks(M, capa):M.getValidTracks(); size_t currVal = INVALID_TRACK_ID; uint32_t currRate = 0xFFFFFFFFul; - std::set validTracks = getSupportedTracks(M, capa); for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); it++){ - const DTSC::Track &Trk = M.tracks.at(*it); - if (!trackType.size() || Trk.type == trackType || Trk.codec == trackType){ - if (currRate > Trk.bps){ + if (!trackType.size() || M.getType(*it) == trackType || M.getCodec(*it) == trackType){ + if (currRate > M.getBps(*it)){ currVal = *it; - currRate = Trk.bps; + currRate = M.getBps(*it); } } } if (currVal != INVALID_TRACK_ID){result.insert(currVal);} return result; } - // less-than or greater-than track matching on bit rate or resolution + //less-than or greater-than track matching on bit rate or resolution if (trackLow[0] == '<' || trackLow[0] == '>'){ unsigned int bpsVal; uint64_t targetBps = 0; - if (trackLow.find("bps") != std::string::npos && sscanf(trackLow.c_str(), "<%ubps", &bpsVal) == 1){ - targetBps = bpsVal; - } - if (trackLow.find("kbps") != std::string::npos && sscanf(trackLow.c_str(), "<%ukbps", &bpsVal) == 1){ - targetBps = bpsVal * 1024; - } - if (trackLow.find("mbps") != std::string::npos && sscanf(trackLow.c_str(), "<%umbps", &bpsVal) == 1){ - targetBps = bpsVal * 1024 * 1024; - } - if (trackLow.find("bps") != std::string::npos && sscanf(trackLow.c_str(), ">%ubps", &bpsVal) == 1){ - targetBps = bpsVal; - } - if (trackLow.find("kbps") != std::string::npos && sscanf(trackLow.c_str(), ">%ukbps", &bpsVal) == 1){ - targetBps = bpsVal * 1024; - } - if (trackLow.find("mbps") != std::string::npos && sscanf(trackLow.c_str(), ">%umbps", &bpsVal) == 1){ - targetBps = bpsVal * 1024 * 1024; - } + if (trackLow.find("bps") != std::string::npos && sscanf(trackLow.c_str(), "<%ubps", &bpsVal) == 1){targetBps = bpsVal;} + if (trackLow.find("kbps") != std::string::npos && sscanf(trackLow.c_str(), "<%ukbps", &bpsVal) == 1){targetBps = bpsVal*1024;} + if (trackLow.find("mbps") != std::string::npos && sscanf(trackLow.c_str(), "<%umbps", &bpsVal) == 1){targetBps = bpsVal*1024*1024;} + if (trackLow.find("bps") != std::string::npos && sscanf(trackLow.c_str(), ">%ubps", &bpsVal) == 1){targetBps = bpsVal;} + if (trackLow.find("kbps") != std::string::npos && sscanf(trackLow.c_str(), ">%ukbps", &bpsVal) == 1){targetBps = bpsVal*1024;} + if (trackLow.find("mbps") != std::string::npos && sscanf(trackLow.c_str(), ">%umbps", &bpsVal) == 1){targetBps = bpsVal*1024*1024;} if (targetBps){ - targetBps >>= 3; // select all tracks of this type that match the requirements - std::set validTracks = getSupportedTracks(M, capa); + std::set validTracks = capa?getSupportedTracks(M, capa):M.getValidTracks(); for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); it++){ - const DTSC::Track &Trk = M.tracks.at(*it); - if (!trackType.size() || Trk.type == trackType || Trk.codec == trackType){ - if (trackLow[0] == '>' && Trk.bps > targetBps){result.insert(*it);} - if (trackLow[0] == '<' && Trk.bps < targetBps){result.insert(*it);} + if (!trackType.size() || M.getType(*it) == trackType || M.getCodec(*it) == trackType){ + if (trackLow[0] == '>' && M.getBps(*it) > targetBps){result.insert(*it);} + if (trackLow[0] == '<' && M.getBps(*it) < targetBps){result.insert(*it);} } } return result; } unsigned int resX, resY; uint64_t targetArea = 0; - if (sscanf(trackLow.c_str(), "<%ux%u", &resX, &resY) == 2){targetArea = resX * resY;} - if (sscanf(trackLow.c_str(), ">%ux%u", &resX, &resY) == 2){targetArea = resX * resY;} + if (sscanf(trackLow.c_str(), "<%ux%u", &resX, &resY) == 2){targetArea = resX*resY;} + if (sscanf(trackLow.c_str(), ">%ux%u", &resX, &resY) == 2){targetArea = resX*resY;} if (targetArea){ - std::set validTracks = getSupportedTracks(M, capa); + std::set validTracks = capa?getSupportedTracks(M, capa):M.getValidTracks(); for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); it++){ - const DTSC::Track &Trk = M.tracks.at(*it); - if (!trackType.size() || Trk.type == trackType || Trk.codec == trackType){ - uint64_t trackArea = Trk.width * Trk.height; + if (!trackType.size() || M.getType(*it) == trackType || M.getCodec(*it) == trackType){ + uint64_t trackArea = M.getWidth(*it)*M.getHeight(*it); if (trackLow[0] == '>' && trackArea > targetArea){result.insert(*it);} if (trackLow[0] == '<' && trackArea < targetArea){result.insert(*it);} } @@ -812,32 +801,23 @@ std::set Util::findTracks(const DTSC::Meta &M, const std::string &trackT return result; } } - // approx bitrate matching + //approx bitrate matching { unsigned int bpsVal; uint64_t targetBps = 0; - if (trackLow.find("bps") != std::string::npos && sscanf(trackLow.c_str(), "%ubps", &bpsVal) == 1){ - targetBps = bpsVal; - } - if (trackLow.find("kbps") != std::string::npos && sscanf(trackLow.c_str(), "%ukbps", &bpsVal) == 1){ - targetBps = bpsVal * 1024; - } - if (trackLow.find("mbps") != std::string::npos && sscanf(trackLow.c_str(), "%umbps", &bpsVal) == 1){ - targetBps = bpsVal * 1024 * 1024; - } + if (trackLow.find("bps") != std::string::npos && sscanf(trackLow.c_str(), "%ubps", &bpsVal) == 1){targetBps = bpsVal;} + if (trackLow.find("kbps") != std::string::npos && sscanf(trackLow.c_str(), "%ukbps", &bpsVal) == 1){targetBps = bpsVal*1024;} + if (trackLow.find("mbps") != std::string::npos && sscanf(trackLow.c_str(), "%umbps", &bpsVal) == 1){targetBps = bpsVal*1024*1024;} if (targetBps){ - targetBps >>= 3; // select nearest bit rate track of this type + std::set validTracks = capa?getSupportedTracks(M, capa):M.getValidTracks(); size_t currVal = INVALID_TRACK_ID; uint32_t currDist = 0; - std::set validTracks = getSupportedTracks(M, capa); for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); it++){ - const DTSC::Track &Trk = M.tracks.at(*it); - if (!trackType.size() || Trk.type == trackType || Trk.codec == trackType){ - if (currVal == INVALID_TRACK_ID || (Trk.bps >= targetBps && currDist > (Trk.bps - targetBps)) || - (Trk.bps < targetBps && currDist > (targetBps - Trk.bps))){ + if (!trackType.size() || M.getType(*it) == trackType || M.getCodec(*it) == trackType){ + if (currVal == INVALID_TRACK_ID || (M.getBps(*it) >= targetBps && currDist > (M.getBps(*it)-targetBps)) || (M.getBps(*it) < targetBps && currDist > (targetBps-M.getBps(*it)))){ currVal = *it; - currDist = (Trk.bps >= targetBps) ? (Trk.bps - targetBps) : (targetBps - Trk.bps); + currDist = (M.getBps(*it) >= targetBps)?(M.getBps(*it)-targetBps):(targetBps-M.getBps(*it)); } } } @@ -880,13 +860,12 @@ std::set Util::findTracks(const DTSC::Meta &M, const std::string &trackT if (!trackType.size() || trackType == "video"){ if (trackLow == "highres" || trackLow == "bestres" || trackLow == "maxres"){ // select highest resolution track of this type + std::set validTracks = capa?getSupportedTracks(M, capa):M.getValidTracks(); size_t currVal = INVALID_TRACK_ID; uint64_t currRes = 0; - std::set validTracks = getSupportedTracks(M, capa); for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); it++){ - const DTSC::Track &Trk = M.tracks.at(*it); - if (!trackType.size() || Trk.type == trackType || Trk.codec == trackType){ - uint64_t trackRes = Trk.width * Trk.height; + if (!trackType.size() || M.getType(*it) == trackType || M.getCodec(*it) == trackType){ + uint64_t trackRes = M.getWidth(*it)*M.getHeight(*it); if (currRes < trackRes){ currVal = *it; currRes = trackRes; @@ -898,13 +877,12 @@ std::set Util::findTracks(const DTSC::Meta &M, const std::string &trackT } if (trackLow == "lowres" || trackLow == "worstres" || trackLow == "minres"){ // select lowest resolution track of this type + std::set validTracks = capa?getSupportedTracks(M, capa):M.getValidTracks(); size_t currVal = INVALID_TRACK_ID; uint64_t currRes = 0xFFFFFFFFFFFFFFFFull; - std::set validTracks = getSupportedTracks(M, capa); for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); it++){ - const DTSC::Track &Trk = M.tracks.at(*it); - if (!trackType.size() || Trk.type == trackType || Trk.codec == trackType){ - uint64_t trackRes = Trk.width * Trk.height; + if (!trackType.size() || M.getType(*it) == trackType || M.getCodec(*it) == trackType){ + uint64_t trackRes = M.getWidth(*it)*M.getHeight(*it); if (currRes > trackRes){ currVal = *it; currRes = trackRes; @@ -918,18 +896,16 @@ std::set Util::findTracks(const DTSC::Meta &M, const std::string &trackT unsigned int resX, resY; if (sscanf(trackLow.c_str(), "~%ux%u", &resX, &resY) == 2){ // select nearest resolution track of this type + std::set validTracks = capa?getSupportedTracks(M, capa):M.getValidTracks(); size_t currVal = INVALID_TRACK_ID; uint64_t currDist = 0; - uint64_t targetArea = resX * resY; - std::set validTracks = getSupportedTracks(M, capa); + uint64_t targetArea = resX*resY; for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); it++){ - const DTSC::Track &Trk = M.tracks.at(*it); - if (!trackType.size() || Trk.type == trackType || Trk.codec == trackType){ - uint64_t trackArea = Trk.width * Trk.height; - if (currVal == INVALID_TRACK_ID || (trackArea >= targetArea && currDist > (trackArea - targetArea)) || - (trackArea < targetArea && currDist > (targetArea - trackArea))){ + if (!trackType.size() || M.getType(*it) == trackType || M.getCodec(*it) == trackType){ + uint64_t trackArea = M.getWidth(*it)*M.getHeight(*it); + if (currVal == INVALID_TRACK_ID || (trackArea >= targetArea && currDist > (trackArea-targetArea)) || (trackArea < targetArea && currDist > (targetArea-trackArea))){ currVal = *it; - currDist = (trackArea >= targetArea) ? (trackArea - targetArea) : (targetArea - trackArea); + currDist = (trackArea >= targetArea)?(trackArea-targetArea):(targetArea-trackArea); } } } @@ -941,12 +917,26 @@ std::set Util::findTracks(const DTSC::Meta &M, const std::string &trackT // attempt to do language/codec matching // convert 2-character language codes into 3-character language codes if (trackLow.size() == 2){trackLow = Encodings::ISO639::twoToThree(trackLow);} - std::set validTracks = M.getValidTracks(); + std::set validTracks = capa?getSupportedTracks(M, capa):M.getValidTracks(); for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); it++){ - if (M.getType(*it) == trackType || M.getCodec(*it) == trackType){ + if (!trackType.size() || M.getType(*it) == trackType || M.getCodec(*it) == trackType){ std::string codecLow = M.getCodec(*it); Util::stringToLower(codecLow); if (M.getLang(*it) == trackLow || trackLow == codecLow){result.insert(*it);} + if (!trackType.size() || trackType == "video"){ + unsigned int resX, resY; + if (trackLow == "720p" && M.getWidth(*it) == 1280 && M.getHeight(*it) == 720){result.insert(*it);} + if (trackLow == "1080p" && M.getWidth(*it) == 1920 && M.getHeight(*it) == 1080){result.insert(*it);} + if (trackLow == "1440p" && M.getWidth(*it) == 2560 && M.getHeight(*it) == 1440){result.insert(*it);} + if (trackLow == "2k" && M.getWidth(*it) == 2048 && M.getHeight(*it) == 1080){result.insert(*it);} + if (trackLow == "4k" && M.getWidth(*it) == 3840 && M.getHeight(*it) == 2160){result.insert(*it);} + if (trackLow == "5k" && M.getWidth(*it) == 5120 && M.getHeight(*it) == 2880){result.insert(*it);} + if (trackLow == "8k" && M.getWidth(*it) == 7680 && M.getHeight(*it) == 4320){result.insert(*it);} + //match "XxY" format + if (sscanf(trackLow.c_str(), "%ux%u", &resX, &resY) == 2){ + if (M.getWidth(*it) == resX && M.getHeight(*it) == resY){result.insert(*it);} + } + } } } return result; @@ -1021,7 +1011,7 @@ std::set Util::getSupportedTracks(const DTSC::Meta &M, const JSON::Value std::string encryptionType = M.getEncryption(*it); encryptionType = encryptionType.substr(0, encryptionType.find('/')); bool found = false; - jsonForEach(capa["encryption"], itb){ + jsonForEachConst(capa["encryption"], itb){ if (itb->asStringRef() == encryptionType){ found = true; break; @@ -1067,7 +1057,8 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map validTracks = getSupportedTracks(M, capa); + std::set validTracks = M.getValidTracks(); + if (capa){validTracks = getSupportedTracks(M, capa);} // check which tracks don't actually exist std::set toRemove; @@ -1076,10 +1067,6 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map::iterator it = toRemove.begin(); it != toRemove.end(); it++){ @@ -1103,22 +1090,22 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map::iterator trit = validTracks.begin(); trit != validTracks.end(); trit++){ - const DTSC::Track &Trk = M.tracks.at(*trit); - bool problems = false; - if (capa.isMember("exceptions") && capa["exceptions"].isObject() && capa["exceptions"].size()){ - jsonForEachConst(capa["exceptions"], ex){ - if (ex.key() == "codec:" + Trk.codec){ - problems = !Util::checkException(*ex, UA); - break; + bool problems = false; + if (capa.isMember("exceptions") && capa["exceptions"].isObject() && + capa["exceptions"].size()){ + jsonForEachConst(capa["exceptions"], ex){ + if (ex.key() == "codec:" + M.getCodec(*trit)){ + problems = !Util::checkException(*ex, UA); + break; + } } } - } - // if (!allowBFrames && M.hasBFrames(*trit)){problems = true;} - if (problems){continue;} - if (noSelAudio && Trk.type == "audio"){continue;} - if (noSelVideo && Trk.type == "video"){continue;} - if (noSelSub && (Trk.type == "subtitle" || Trk.codec == "subtitle")){continue;} - result.insert(*trit); + if (!allowBFrames && M.hasBFrames(*trit)){problems = true;} + if (problems){continue;} + if (noSelAudio && M.getType(*trit) == "audio"){continue;} + if (noSelVideo && M.getType(*trit) == "video"){continue;} + if (noSelSub && (M.getType(*trit) == "subtitle" || M.getCodec(*trit) == "subtitle")){continue;} + result.insert(*trit); } return result; } diff --git a/lib/stream.h b/lib/stream.h index c9d35311..f8ef94a3 100644 --- a/lib/stream.h +++ b/lib/stream.h @@ -9,6 +9,8 @@ #include "util.h" #include +const JSON::Value empty; + namespace Util{ void streamVariables(std::string &str, const std::string &streamname, const std::string &source = ""); std::string getTmpFolder(); @@ -18,7 +20,7 @@ namespace Util{ bool isProvider = false, const std::map &overrides = std::map(), pid_t *spawn_pid = NULL); - int startPush(const std::string &streamname, std::string &target); + int startPush(const std::string &streamname, std::string &target, int debugLvl = -1); JSON::Value getStreamConfig(const std::string &streamname); JSON::Value getGlobalConfig(const std::string &optionName); JSON::Value getInputBySource(const std::string &filename, bool isProvider = false); @@ -26,13 +28,13 @@ namespace Util{ bool checkException(const JSON::Value &ex, const std::string &useragent); std::string codecString(const std::string &codec, const std::string &initData = ""); - std::set getSupportedTracks(const DTSC::Meta &M, JSON::Value &capa, + std::set getSupportedTracks(const DTSC::Meta &M, const JSON::Value &capa, const std::string &type = "", const std::string &UA = ""); - std::set findTracks(const DTSC::Meta &M, const std::string &trackType, const std::string &trackVal); + std::set findTracks(const DTSC::Meta &M, const JSON::Value &capa, const std::string &trackType, const std::string &trackVal, const std::string &UA = ""); std::set wouldSelect(const DTSC::Meta &M, const std::string &trackSelector = "", - JSON::Value capa = JSON::Value(), const std::string &UA = ""); + const JSON::Value &capa = empty, const std::string &UA = ""); std::set wouldSelect(const DTSC::Meta &M, const std::map &targetParams, - JSON::Value capa = JSON::Value(), const std::string &UA = ""); + const JSON::Value &capa = empty, const std::string &UA = "", uint64_t seekTarget = 0); class DTSCShmReader{ public: diff --git a/lib/ts_packet.cpp b/lib/ts_packet.cpp index 365a9ebc..608a03de 100644 --- a/lib/ts_packet.cpp +++ b/lib/ts_packet.cpp @@ -1162,6 +1162,14 @@ namespace TS{ return output.str(); } + + size_t getUniqTrackID(const DTSC::Meta &M, size_t idx){ + return idx+255; + //size_t ret = M.getID(idx); + //if (ret < 255){ret += 255;} + //return ret; + } + /// Construct a PMT (special 188B ts packet) from a set of selected tracks and metadata. /// This function is not part of the packet class, but it is in the TS namespace. /// It uses an internal static TS packet for PMT storage. @@ -1201,16 +1209,12 @@ namespace TS{ } } if (vidTrack == -1){vidTrack = *(selectedTracks.begin());} - size_t pcrPid = M.getID(vidTrack); - if (pcrPid < 255){pcrPid += 255;} - PMT.setPCRPID(pcrPid); + PMT.setPCRPID(getUniqTrackID(M, vidTrack)); PMT.setProgramInfoLength(0); ProgramMappingEntry entry = PMT.getEntry(0); for (std::set::iterator it = selectedTracks.begin(); it != selectedTracks.end(); it++){ std::string codec = M.getCodec(*it); - size_t pkgId = M.getID(*it); - if (pkgId < 255){pkgId += 255;} - entry.setElementaryPid(pkgId); + entry.setElementaryPid(getUniqTrackID(M, *it)); entry.setESInfo(""); if (codec == "H264"){ entry.setStreamType(0x1B); diff --git a/lib/ts_packet.h b/lib/ts_packet.h index 730c19d3..e3ab320d 100644 --- a/lib/ts_packet.h +++ b/lib/ts_packet.h @@ -245,6 +245,8 @@ namespace TS{ extern char PAT[188]; + size_t getUniqTrackID(const DTSC::Meta &M, size_t idx); + const char *createPMT(std::set &selectedTracks, const DTSC::Meta &M, int contCounter = 0); const char *createSDT(const std::string &streamName, int contCounter = 0); diff --git a/lib/url.h b/lib/url.h index d912f5e6..98df3ece 100644 --- a/lib/url.h +++ b/lib/url.h @@ -4,6 +4,7 @@ #pragma once #include #include +#include /// Holds all HTTP processing related code. namespace HTTP{ diff --git a/lib/util.cpp b/lib/util.cpp index 0f8796dd..0f70811f 100644 --- a/lib/util.cpp +++ b/lib/util.cpp @@ -276,7 +276,7 @@ namespace Util{ /// Parses log messages from the given file descriptor in, printing them to out, optionally /// calling the given callback for each valid message. Closes the file descriptor on read error void logParser(int in, int out, bool colored, - void callback(const std::string &, const std::string &, const std::string &, bool)){ + void callback(const std::string &, const std::string &, const std::string &, uint64_t, bool)){ char buf[1024]; FILE *output = fdopen(in, "r"); @@ -347,7 +347,7 @@ namespace Util{ while (j < 1023 && buf[j] != '\n' && buf[j] != 0){++j;} buf[j] = 0; // print message - if (callback){callback(kind, message, strmNm, true);} + if (callback){callback(kind, message, strmNm, JSON::Value(progpid).asInt(), true);} color_msg = color_end; if (colored){ if (!strcmp(kind, "CONF")){color_msg = CONF_msg;} @@ -386,9 +386,11 @@ namespace Util{ uint64_t FieldAccX::uint(size_t recordNo) const{return src->getInt(field, recordNo);} std::string FieldAccX::string(size_t recordNo) const{ - std::string res(src->getPointer(field, recordNo)); - if (res.size() > field.size){res.resize(field.size);} - return res; + return std::string(src->getPointer(field, recordNo)); + } + + const char * FieldAccX::ptr(size_t recordNo) const{ + return src->getPointer(field, recordNo); } void FieldAccX::set(uint64_t val, size_t recordNo){src->setInt(field, val, recordNo);} @@ -396,7 +398,9 @@ namespace Util{ void FieldAccX::set(const std::string &val, size_t recordNo){ char *place = src->getPointer(field, recordNo); memcpy(place, val.data(), std::min((size_t)field.size, val.size())); - place[std::min((size_t)field.size - 1, val.size())] = 0; + if ((field.type & 0xF0) == RAX_STRING){ + place[std::min((size_t)field.size - 1, val.size())] = 0; + } } /// If waitReady is true (default), waits for isReady() to return true in 50ms sleep increments. @@ -597,7 +601,7 @@ namespace Util{ char *ptr = getPointer(it->first, i); size_t sz = getSize(it->first, i); size_t zeroCount = 0; - for (size_t j = 0; j < sz && j < 100 && zeroCount < 10; ++j){ + for (size_t j = 0; j < sz && j < 100 && zeroCount < 16; ++j){ r << "0x" << std::hex << std::setw(2) << std::setfill('0') << (int)ptr[j] << std::dec << " "; if (ptr[j] == 0x00){ zeroCount++; @@ -770,13 +774,13 @@ namespace Util{ } void RelAccX::setString(const RelAccXFieldData &fd, const std::string &val, uint64_t recordNo){ - if ((fd.type & 0xF0) != RAX_STRING){ - WARN_MSG("Setting non-string"); + if ((fd.type & 0xF0) != RAX_STRING && (fd.type & 0xF0) != RAX_RAW){ + WARN_MSG("Setting non-string data type to a string value"); return; } char *ptr = RECORD_POINTER; memcpy(ptr, val.data(), std::min((uint32_t)val.size(), fd.size)); - ptr[std::min((uint32_t)val.size(), fd.size - 1)] = 0; + if ((fd.type & 0xF0) == RAX_STRING){ptr[std::min((uint32_t)val.size(), fd.size - 1)] = 0;} } /// Writes the given int to the given field in the given record. diff --git a/lib/util.h b/lib/util.h index 2d244792..4b4a9f7f 100644 --- a/lib/util.h +++ b/lib/util.h @@ -20,7 +20,7 @@ namespace Util{ class DataCallback{ public: virtual void dataCallback(const char *ptr, size_t size){ - INFO_MSG("default callback, size: %llu", size); + INFO_MSG("default callback, size: %zu", size); } }; @@ -54,7 +54,7 @@ namespace Util{ }; void logParser(int in, int out, bool colored, - void callback(const std::string &, const std::string &, const std::string &, bool) = 0); + void callback(const std::string &, const std::string &, const std::string &, uint64_t, bool) = 0); void redirectLogsIfNeeded(); /// Holds type, size and offset for RelAccX class internal data fields. @@ -194,6 +194,7 @@ namespace Util{ FieldAccX(RelAccX *_src = NULL, RelAccXFieldData _field = RelAccXFieldData()); uint64_t uint(size_t recordNo) const; std::string string(size_t recordNo) const; + const char * ptr(size_t recordNo) const; void set(uint64_t val, size_t recordNo = 0); void set(const std::string &val, size_t recordNo = 0); diff --git a/src/analysers/analyser_dtsc.cpp b/src/analysers/analyser_dtsc.cpp index 4dddc6b2..bb10997b 100644 --- a/src/analysers/analyser_dtsc.cpp +++ b/src/analysers/analyser_dtsc.cpp @@ -9,6 +9,10 @@ void AnalyserDTSC::init(Util::Config &conf){ opt["short"] = "H"; opt["help"] = "Parse entire file or streams as a single headless DTSC packet"; conf.addOption("headless", opt); + opt["long"] = "sizeprepended"; + opt["short"] = "s"; + opt["help"] = "If set, data of packets is considered to be size-prepended"; + conf.addOption("sizeprepended", opt); opt.null(); } @@ -21,6 +25,7 @@ bool AnalyserDTSC::open(const std::string &filename){ AnalyserDTSC::AnalyserDTSC(Util::Config &conf) : Analyser(conf){ headLess = conf.getBool("headless"); + sizePrepended = conf.getBool("sizeprepended"); } bool AnalyserDTSC::parsePacket(){ @@ -60,10 +65,19 @@ bool AnalyserDTSC::parsePacket(){ char *payDat; size_t payLen; P.getString("data", payDat, payLen); + uint32_t currLen = 0; + uint64_t byteCounter = 0; for (uint64_t i = 0; i < payLen; ++i){ - if ((i % 32) == 0){std::cout << std::endl;} + if (sizePrepended && !currLen){ + currLen = 4+Bit::btohl(payDat+i); + byteCounter = 0; + } + if ((byteCounter % 32) == 0){std::cout << std::endl;} std::cout << std::hex << std::setw(2) << std::setfill('0') << (unsigned int)payDat[i]; + ++byteCounter; + --currLen; } + std::cout << std::dec << std::endl; } break; } diff --git a/src/analysers/analyser_dtsc.h b/src/analysers/analyser_dtsc.h index b8c06396..9b6ac820 100644 --- a/src/analysers/analyser_dtsc.h +++ b/src/analysers/analyser_dtsc.h @@ -10,6 +10,7 @@ public: private: bool headLess; + bool sizePrepended; DTSC::Packet P; Socket::Connection conn; uint64_t totalBytes; diff --git a/src/analysers/analyser_rtmp.cpp b/src/analysers/analyser_rtmp.cpp index d2af848f..38c1a342 100644 --- a/src/analysers/analyser_rtmp.cpp +++ b/src/analysers/analyser_rtmp.cpp @@ -16,6 +16,11 @@ void AnalyserRTMP::init(Util::Config &conf){ opt.null(); } +/// Checks if standard input is still valid. +bool AnalyserRTMP::isOpen(){ + return (*isActive) && (std::cin.good() || strbuf.size()); +} + AnalyserRTMP::AnalyserRTMP(Util::Config &conf) : Analyser(conf){ if (conf.getString("reconstruct") != ""){ reconstruct.open(conf.getString("reconstruct").c_str()); @@ -43,7 +48,10 @@ bool AnalyserRTMP::parsePacket(){ // While we can't parse a packet, while (!next.Parse(strbuf)){ // fill our internal buffer "strbuf" in (up to) 1024 byte chunks - if (!std::cin.good()){return false;} + if (!std::cin.good()){ + strbuf.clear(); + return false; + } size_t charCount = 0; std::string tmpbuffer; tmpbuffer.reserve(1024); diff --git a/src/analysers/analyser_rtmp.h b/src/analysers/analyser_rtmp.h index 2627027d..7ced6f8f 100644 --- a/src/analysers/analyser_rtmp.h +++ b/src/analysers/analyser_rtmp.h @@ -18,4 +18,5 @@ public: static void init(Util::Config &conf); bool parsePacket(); virtual bool open(const std::string &filename); + virtual bool isOpen(); }; diff --git a/src/controller/controller.cpp b/src/controller/controller.cpp index e2350faf..9ec6fd3d 100644 --- a/src/controller/controller.cpp +++ b/src/controller/controller.cpp @@ -343,7 +343,7 @@ int main_loop(int argc, char **argv){ WARN_MSG("You have very little free RAM available (%" PRIu64 " MiB). While Mist will run just fine with this amount, do note that random crashes " "may occur should you ever run out of free RAM. Please be pro-active and keep an " - "eye on the RAM usage!"); + "eye on the RAM usage!", (mem_free + mem_bufcache)/1024); } if (shm_free < 1024 * 1024 && mem_total > 1024 * 1024 * 1.12){ WARN_MSG("You have very little shared memory available (%" PRIu64 diff --git a/src/controller/controller_api.cpp b/src/controller/controller_api.cpp index 17780b35..827d94c7 100644 --- a/src/controller/controller_api.cpp +++ b/src/controller/controller_api.cpp @@ -502,6 +502,12 @@ void Controller::handleAPICommands(JSON::Value &Request, JSON::Value &Response){ Controller::triggerStats[Request["trigger_fail"].asStringRef()].failCount++; return; } + if (Request.isMember("push_status_update")){ + JSON::Value &statUp = Request["push_status_update"]; + if (statUp.isMember("id") && statUp.isMember("status")){ + setPushStatus(statUp["id"].asInt(), statUp["status"]); + } + } /*LTS-END*/ // Parse config and streams from the request. if (Request.isMember("config") && Request["config"].isObject()){ diff --git a/src/controller/controller_capabilities.cpp b/src/controller/controller_capabilities.cpp index 0643b9ec..a58dba6f 100644 --- a/src/controller/controller_capabilities.cpp +++ b/src/controller/controller_capabilities.cpp @@ -200,6 +200,12 @@ namespace Controller{ trgs["DEFAULT_STREAM"]["response_action"] = "Overrides the default stream setting (for this view) to the response value. If empty, " "fails loading the stream and returns an error to the viewer/user."; + + trgs["PUSH_END"]["when"] = "Every time a push stops, for any reason"; + trgs["PUSH_END"]["stream_specific"] = true; + trgs["PUSH_END"]["payload"] = "push ID (integer)\nstream name (string)\ntarget URI, before variables/triggers affected it (string)\ntarget URI, afterwards, as actually used (string)\nlast 10 log messages (JSON array string)\nmost recent push status (JSON object string)"; + trgs["PUSH_END"]["response"] = "ignored"; + trgs["PUSH_END"]["response_action"] = "None."; } /// Aquire list of available protocols, storing in global 'capabilities' JSON::Value. diff --git a/src/controller/controller_connectors.cpp b/src/controller/controller_connectors.cpp index a7fcd56e..e5db39de 100644 --- a/src/controller/controller_connectors.cpp +++ b/src/controller/controller_connectors.cpp @@ -164,8 +164,6 @@ namespace Controller{ std::set runningConns; // used for building args - int zero = 0; - int out = fileno(stdout); int err = fileno(stderr); char *argarr[15]; // approx max # of args (with a wide margin) int i; @@ -259,7 +257,7 @@ namespace Controller{ JSON::Value p = JSON::fromString(*runningConns.begin()); buildPipedArguments(p, (char **)&argarr, capabilities); // start piped w/ generated args - currentConnectors[*runningConns.begin()] = Util::Procs::StartPiped(argarr, &zero, &out, &err); + currentConnectors[*runningConns.begin()] = Util::Procs::StartPiped(argarr, 0, 0, &err); Triggers::doTrigger("OUTPUT_START", *runningConns.begin()); // LTS } runningConns.erase(runningConns.begin()); diff --git a/src/controller/controller_push.cpp b/src/controller/controller_push.cpp index 5017ad9a..f12c9221 100644 --- a/src/controller/controller_push.cpp +++ b/src/controller/controller_push.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace Controller{ @@ -38,6 +39,38 @@ namespace Controller{ } } + void setPushStatus(uint64_t id, const JSON::Value & status){ + if (!activePushes.count(id)){return;} + activePushes[id][5].extend(status); + } + + void pushLogMessage(uint64_t id, const JSON::Value & msg){ + JSON::Value &log = activePushes[id][4]; + log.append(msg); + log.shrink(10); + } + + bool isPushActive(uint64_t id){ + while (Controller::conf.is_active && !pushListRead){Util::sleep(100);} + return activePushes.count(id); + } + + /// Only used internally, to remove pushes + static void removeActivePush(pid_t id){ + //ignore if the push does not exist + if (!activePushes.count(id)){return;} + + JSON::Value p = activePushes[id]; + if (Triggers::shouldTrigger("PUSH_END", p[1].asStringRef())){ + std::string payload = p[0u].asString() + "\n" + p[1u].asString() + "\n" + p[2u].asString() + "\n" + p[3u].asString() + "\n" + p[4u].toString() + "\n" + p[5u].toString(); + Triggers::doTrigger("PUSH_END", payload, p[1].asStringRef()); + } + + //actually remove, make sure next pass the new list is written out too + activePushes.erase(id); + mustWritePushList = true; + } + /// Returns true if the push is currently active, false otherwise. bool isPushActive(const std::string &streamname, const std::string &target){ while (Controller::conf.is_active && !pushListRead){Util::sleep(100);} @@ -52,8 +85,7 @@ namespace Controller{ } } while (toWipe.size()){ - activePushes.erase(*toWipe.begin()); - mustWritePushList = true; + removeActivePush(*toWipe.begin()); toWipe.erase(toWipe.begin()); } return false; @@ -75,8 +107,7 @@ namespace Controller{ } } while (toWipe.size()){ - activePushes.erase(*toWipe.begin()); - mustWritePushList = true; + removeActivePush(*toWipe.begin()); toWipe.erase(toWipe.begin()); } } @@ -198,6 +229,17 @@ namespace Controller{ break; } } + //Check if any pushes have ended, clean them up + std::set toWipe; + for (std::map::iterator it = activePushes.begin(); it != activePushes.end(); ++it){ + if (!Util::Procs::isActive(it->first)){toWipe.insert(it->first);} + } + while (toWipe.size()){ + removeActivePush(*toWipe.begin()); + toWipe.erase(toWipe.begin()); + mustWritePushList = true; + } + //write push list to shared memory, for restarting/crash recovery/etc if (mustWritePushList && pushPage.mapped){ writePushList(pushPage.mapped); mustWritePushList = false; @@ -227,8 +269,7 @@ namespace Controller{ } } while (toWipe.size()){ - activePushes.erase(*toWipe.begin()); - mustWritePushList = true; + removeActivePush(*toWipe.begin()); toWipe.erase(toWipe.begin()); } } diff --git a/src/controller/controller_push.h b/src/controller/controller_push.h index 36bf3d71..070e3e64 100644 --- a/src/controller/controller_push.h +++ b/src/controller/controller_push.h @@ -8,6 +8,9 @@ namespace Controller{ void startPush(const std::string &streamname, std::string &target); void stopPush(unsigned int ID); void listPush(JSON::Value &output); + void pushLogMessage(uint64_t id, const JSON::Value & msg); + void setPushStatus(uint64_t id, const JSON::Value & status); + bool isPushActive(uint64_t id); // Functions for automated pushes, add/remove void addPush(JSON::Value &request); diff --git a/src/controller/controller_statistics.cpp b/src/controller/controller_statistics.cpp index c4c86090..25a0714a 100644 --- a/src/controller/controller_statistics.cpp +++ b/src/controller/controller_statistics.cpp @@ -50,6 +50,7 @@ bool Controller::killOnExit = KILL_ON_EXIT; tthread::mutex Controller::statsMutex; unsigned int Controller::maxConnsPerIP = 0; uint64_t Controller::statDropoff = 0; +static uint64_t cpu_use = 0; char noBWCountMatches[1717]; uint64_t bwLimit = 128 * 1024 * 1024; // gigabit default limit @@ -76,6 +77,21 @@ void Controller::updateBandwidthConfig(){ } } } + //Localhost is always excepted from counts + { + std::string newbins = Socket::getBinForms("::1"); + if (offset + newbins.size() < 1700){ + memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); + offset += newbins.size(); + } + } + { + std::string newbins = Socket::getBinForms("127.0.0.1/8"); + if (offset + newbins.size() < 1700){ + memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); + offset += newbins.size(); + } + } } // For server-wide totals. Local to this file only. @@ -107,7 +123,7 @@ Controller::sessIndex::sessIndex(){ /// into strings. This extracts the host, stream name, connector and crc field, ignoring everything /// else. Controller::sessIndex::sessIndex(const Comms::Statistics &statComm, size_t id){ - host = statComm.getHost(id); + Socket::hostBytesToStr(statComm.getHost(id).data(), 16, host); streamName = statComm.getStream(id); connector = statComm.getConnector(id); crc = statComm.getCRC(id); @@ -336,6 +352,28 @@ void Controller::SharedMemStats(void *config){ bool firstRun = true; while (((Util::Config *)config)->is_active){ { + std::ifstream cpustat("/proc/stat"); + if (cpustat){ + char line[300]; + while (cpustat.getline(line, 300)){ + static unsigned long long cl_total = 0, cl_idle = 0; + unsigned long long c_user, c_nice, c_syst, c_idle, c_total; + if (sscanf(line, "cpu %Lu %Lu %Lu %Lu", &c_user, &c_nice, &c_syst, &c_idle) == 4){ + c_total = c_user + c_nice + c_syst + c_idle; + if (c_total > cl_total){ + cpu_use = (long long)(1000 - ((c_idle - cl_idle) * 1000) / (c_total - cl_total)); + }else{ + cpu_use = 0; + } + cl_total = c_total; + cl_idle = c_idle; + break; + } + } + } + } + { + tthread::lock_guard guard(Controller::configMutex); tthread::lock_guard guard2(statsMutex); cacheLock->wait(); /*LTS*/ @@ -522,7 +560,8 @@ uint32_t Controller::statSession::kill(){ /// Updates the given active connection with new stats data. void Controller::statSession::update(uint64_t index, Comms::Statistics &statComm){ - std::string myHost = statComm.getHost(index); + std::string myHost; + Socket::hostBytesToStr(statComm.getHost(index).data(), 16, myHost); std::string myStream = statComm.getStream(index); std::string myConnector = statComm.getConnector(index); // update the sync byte: 0 = requesting fill, 2 = requesting refill, 1 = needs checking, > 2 = @@ -613,12 +652,12 @@ void Controller::statSession::update(uint64_t index, Comms::Statistics &statComm } if (currDown + currUp >= COUNTABLE_BYTES){ if (sessionType == SESS_UNSET){ - if (myConnector == "INPUT"){ + if (myConnector.size() >= 5 && myConnector.substr(0, 5) == "INPUT"){ ++servInputs; streamStats[myStream].inputs++; streamStats[myStream].currIns++; sessionType = SESS_INPUT; - }else if (myConnector == "OUTPUT"){ + }else if (myConnector.size() >= 6 && myConnector.substr(0, 6) == "OUTPUT"){ ++servOutputs; streamStats[myStream].outputs++; streamStats[myStream].currOuts++; @@ -632,19 +671,21 @@ void Controller::statSession::update(uint64_t index, Comms::Statistics &statComm } // If previous < COUNTABLE_BYTES, we haven't counted any data so far. // We need to count all the data in that case, otherwise we only count the difference. - if (prevUp + prevDown < COUNTABLE_BYTES){ - if (!myStream.size() || myStream[0] == 0){ - if (streamStats.count(myStream)){streamStats.erase(myStream);} + if (noBWCount != 2){ //only count connections that are countable + if (prevUp + prevDown < COUNTABLE_BYTES){ + if (!myStream.size() || myStream[0] == 0){ + if (streamStats.count(myStream)){streamStats.erase(myStream);} + }else{ + streamStats[myStream].upBytes += currUp; + streamStats[myStream].downBytes += currDown; + } }else{ - streamStats[myStream].upBytes += currUp; - streamStats[myStream].downBytes += currDown; - } - }else{ - if (!myStream.size() || myStream[0] == 0){ - if (streamStats.count(myStream)){streamStats.erase(myStream);} - }else{ - streamStats[myStream].upBytes += currUp - prevUp; - streamStats[myStream].downBytes += currDown - prevDown; + if (!myStream.size() || myStream[0] == 0){ + if (streamStats.count(myStream)){streamStats.erase(myStream);} + }else{ + streamStats[myStream].upBytes += currUp - prevUp; + streamStats[myStream].downBytes += currDown - prevDown; + } } } } @@ -1437,29 +1478,9 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int H.StartResponse("200", "OK", H, conn, true); // Collect core server stats - uint64_t cpu_use = 0; uint64_t mem_total = 0, mem_free = 0, mem_bufcache = 0; uint64_t bw_up_total = 0, bw_down_total = 0; { - std::ifstream cpustat("/proc/stat"); - if (cpustat){ - char line[300]; - while (cpustat.getline(line, 300)){ - static unsigned long long cl_total = 0, cl_idle = 0; - unsigned long long c_user, c_nice, c_syst, c_idle, c_total; - if (sscanf(line, "cpu %Lu %Lu %Lu %Lu", &c_user, &c_nice, &c_syst, &c_idle) == 4){ - c_total = c_user + c_nice + c_syst + c_idle; - if (c_total > cl_total){ - cpu_use = (long long int)(1000 - ((c_idle - cl_idle) * 1000) / (c_total - cl_total)); - }else{ - cpu_use = 0; - } - cl_total = c_total; - cl_idle = c_idle; - break; - } - } - } std::ifstream meminfo("/proc/meminfo"); if (meminfo){ char line[300]; diff --git a/src/controller/controller_storage.cpp b/src/controller/controller_storage.cpp index e16459a9..1171a393 100644 --- a/src/controller/controller_storage.cpp +++ b/src/controller/controller_storage.cpp @@ -1,5 +1,6 @@ #include "controller_capabilities.h" #include "controller_storage.h" +#include "controller_push.h" //LTS #include #include #include @@ -41,7 +42,7 @@ namespace Controller{ ///\brief Store and print a log message. ///\param kind The type of message. ///\param message The message to be logged. - void Log(const std::string &kind, const std::string &message, const std::string &stream, bool noWriteToLog){ + void Log(const std::string &kind, const std::string &message, const std::string &stream, uint64_t progPid, bool noWriteToLog){ if (noWriteToLog){ tthread::lock_guard guard(logMutex); JSON::Value m; @@ -52,6 +53,7 @@ namespace Controller{ m.append(stream); Storage["log"].append(m); Storage["log"].shrink(100); // limit to 100 log messages + if (isPushActive(progPid)){pushLogMessage(progPid, m);} //LTS logCounter++; if (rlxLogs && rlxLogs->isReady()){ if (!firstLog){firstLog = logCounter;} diff --git a/src/controller/controller_storage.h b/src/controller/controller_storage.h index 0fe7e85f..6339cfd0 100644 --- a/src/controller/controller_storage.h +++ b/src/controller/controller_storage.h @@ -22,7 +22,7 @@ namespace Controller{ Util::RelAccX *streamsAccessor(); /// Store and print a log message. - void Log(const std::string &kind, const std::string &message, const std::string &stream = "", + void Log(const std::string &kind, const std::string &message, const std::string &stream = "", uint64_t progPid = 0, bool noWriteToLog = false); void logAccess(const std::string &sessId, const std::string &strm, const std::string &conn, const std::string &host, uint64_t duration, uint64_t up, uint64_t down, diff --git a/src/input/input.cpp b/src/input/input.cpp index a32a9866..28839fbe 100644 --- a/src/input/input.cpp +++ b/src/input/input.cpp @@ -77,7 +77,8 @@ namespace Mist{ option["help"] = "Generate .dtsh, then exit"; config->addOption("headeronly", option); - /*LTS-START + /*LTS-START*/ + /* //Encryption option.null(); option["arg"] = "string"; @@ -86,31 +87,6 @@ namespace Mist{ option["help"] = "a KID:KEY combo for auto-encrypting tracks"; config->addOption("encryption", option); - - option.null(); - option["long"] = "realtime"; - option["short"] = "r"; - option["help"] = "Feed the results of this input in realtime to the buffer"; - config->addOption("realtime", option); - capa["optional"]["realtime"]["name"] = "Simulated Live"; - capa["optional"]["realtime"]["help"] = "Make this input run as a simulated live stream"; - capa["optional"]["realtime"]["option"] = "--realtime"; - - option.null(); - option["long"] = "simulated-starttime"; - option["arg"] = "integer"; - option["short"] = "S"; - option["help"] = "Unix timestamp on which the simulated start of the stream is based."; - option["value"].append(0); - config->addOption("simulated-starttime", option); - capa["optional"]["simulated-starttime"]["name"] = "Simulated start time"; - capa["optional"]["simulated-starttime"]["help"] = - "The unix timestamp on which this stream is assumed to have started playback, or 0 for " - "automatic"; - capa["optional"]["simulated-starttime"]["option"] = "--simulated-starttime"; - capa["optional"]["simulated-starttime"]["type"] = "uint"; - capa["optional"]["simulated-starttime"]["default"] = 0; - option.null(); option["arg"] = "string"; option["short"] = "B"; @@ -175,8 +151,33 @@ namespace Mist{ capa["optional"]["playready"]["help"] = "The header to use for PlayReady encryption."; capa["optional"]["playready"]["option"] = "--playready"; capa["optional"]["playready"]["type"] = "string"; - LTS-END*/ + */ + option.null(); + option["long"] = "realtime"; + option["short"] = "r"; + option["help"] = "Feed the results of this input in realtime to the buffer"; + config->addOption("realtime", option); + capa["optional"]["realtime"]["name"] = "Simulated Live"; + capa["optional"]["realtime"]["help"] = "Make this input run as a simulated live stream"; + capa["optional"]["realtime"]["option"] = "--realtime"; + + option.null(); + option["long"] = "simulated-starttime"; + option["arg"] = "integer"; + option["short"] = "S"; + option["help"] = "Unix timestamp on which the simulated start of the stream is based."; + option["value"].append(0); + config->addOption("simulated-starttime", option); + capa["optional"]["simulated-starttime"]["name"] = "Simulated start time"; + capa["optional"]["simulated-starttime"]["help"] = + "The unix timestamp on which this stream is assumed to have started playback, or 0 for " + "automatic"; + capa["optional"]["simulated-starttime"]["option"] = "--simulated-starttime"; + capa["optional"]["simulated-starttime"]["type"] = "uint"; + capa["optional"]["simulated-starttime"]["default"] = 0; + + /*LTS-END*/ capa["optional"]["debug"]["name"] = "debug"; capa["optional"]["debug"]["help"] = "The debug level at which messages need to be printed."; capa["optional"]["debug"]["option"] = "--debug"; @@ -637,7 +638,7 @@ namespace Mist{ if (streamStatus){streamStatus.mapped[0] = STRMSTAT_SHUTDOWN;} config->is_active = false; finish(); - INFO_MSG("Input for stream %s closing clean", streamName.c_str()); + INFO_MSG("Input closing clean, reason: %s", Util::exitReason); userSelect.clear(); if (streamStatus){streamStatus.mapped[0] = STRMSTAT_OFF;} } @@ -669,6 +670,9 @@ namespace Mist{ } } /*LTS-END*/ + if (!ret && ((Util::bootSecs() - activityCounter) >= INPUT_TIMEOUT)){ + Util::logExitReason("no activity for %u seconds", Util::bootSecs() - activityCounter); + } return ret; } @@ -682,28 +686,29 @@ namespace Mist{ /// - call getNext() in a loop, buffering packets void Input::stream(){ - if (!config->getBool("realtime") && Util::streamAlive(streamName)){ - WARN_MSG("Stream already online, cancelling"); - return; - } - std::map overrides; overrides["throughboot"] = ""; - if (isSingular()){ - if (Util::streamAlive(streamName)){ - WARN_MSG("Stream already online, cancelling"); - return; - } - overrides["singular"] = ""; - } if (config->getBool("realtime") || (capa.isMember("hardcoded") && capa["hardcoded"].isMember("resume") && capa["hardcoded"]["resume"])){ overrides["resume"] = "1"; } - if (!Util::startInput(streamName, "push://INTERNAL_ONLY:" + config->getString("input"), true, - true, overrides)){// manually override stream url to start the buffer - WARN_MSG("Could not start buffer, cancelling"); - return; + if (isSingular()){ + if (!config->getBool("realtime") && Util::streamAlive(streamName)){ + WARN_MSG("Stream already online, cancelling"); + return; + } + overrides["singular"] = ""; + if (!Util::startInput(streamName, "push://INTERNAL_ONLY:" + config->getString("input"), true, + true, overrides)){// manually override stream url to start the buffer + WARN_MSG("Could not start buffer, cancelling"); + return; + } + }else{ + if (!Util::startInput(streamName, "push://INTERNAL_PUSH:" + capa["name"].asStringRef(), true, + true, overrides)){// manually override stream url to start the buffer + WARN_MSG("Could not start buffer, cancelling"); + return; + } } INFO_MSG("Input started"); @@ -715,19 +720,23 @@ namespace Mist{ } parseStreamHeader(); - std::set validTracks = M.getMySourceTracks(getpid()); - if (!validTracks.size()){ - userSelect.clear(); - finish(); - INFO_MSG("No tracks found, cancelling"); - return; + std::set validTracks; + + if (publishesTracks()){ + validTracks = M.getMySourceTracks(getpid()); + if (!validTracks.size()){ + userSelect.clear(); + finish(); + INFO_MSG("No tracks found, cancelling"); + return; + } } timeOffset = 0; uint64_t minFirstMs = 0; // If resume mode is on, find matching tracks and set timeOffset values to make sure we append to the tracks. - if (config->getBool("realtime")){ + if (publishesTracks() && config->getBool("realtime")){ seek(0); minFirstMs = 0xFFFFFFFFFFFFFFFFull; @@ -736,12 +745,11 @@ namespace Mist{ uint64_t maxLastMs = 0; // track lowest firstms value - for (std::map::iterator it = myMeta.tracks.begin(); - it != myMeta.tracks.end(); ++it){ - if (it->second.firstms < minFirstMs){minFirstMs = it->second.firstms;} - if (it->second.firstms > maxFirstMs){maxFirstMs = it->second.firstms;} - if (it->second.lastms < minLastMs){minLastMs = it->second.lastms;} - if (it->second.lastms > maxLastMs){maxLastMs = it->second.lastms;} + for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); ++it){ + if (meta.getFirstms(*it) < minFirstMs){minFirstMs = meta.getFirstms(*it);} + if (meta.getFirstms(*it) > maxFirstMs){maxFirstMs = meta.getFirstms(*it);} + if (meta.getLastms(*it) < minLastMs){minLastMs = meta.getLastms(*it);} + if (meta.getLastms(*it) > maxLastMs){maxLastMs = meta.getLastms(*it);} } if (maxFirstMs - minFirstMs > 500){ WARN_MSG("Begin timings of tracks for this file are %" PRIu64 @@ -756,10 +764,8 @@ namespace Mist{ maxLastMs - minLastMs, minLastMs, maxLastMs); } // find highest current time - for (std::map::iterator secondIt = tmpM.tracks.begin(); - secondIt != tmpM.tracks.end(); ++secondIt){ - VERYHIGH_MSG("Track %u starts at %" PRIu64, secondIt->first, secondIt->second.lastms); - timeOffset = std::max(timeOffset, (int64_t)secondIt->second.lastms); + for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); ++it){ + timeOffset = std::max(timeOffset, (int64_t)meta.getLastms(*it)); } if (timeOffset){ @@ -771,13 +777,11 @@ namespace Mist{ timeOffset -= minFirstMs; // we don't need to add the lowest firstms value to the offset, as it's already there } } - - for (std::map::iterator it = myMeta.tracks.begin(); - it != myMeta.tracks.end(); it++){ - it->second.firstms += timeOffset; - it->second.lastms = 0; - selectedTracks.insert(it->first); - it->second.minKeepAway = SIMULATED_LIVE_BUFFER; + if (publishesTracks()){ + for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); ++it){ + meta.setFirstms(*it, meta.getFirstms(*it)+timeOffset); + meta.setLastms(*it, 0); + } } simStartTime = config->getInteger("simulated-starttime"); @@ -785,9 +789,9 @@ namespace Mist{ std::string reason; if (config->getBool("realtime")){ - reason = realtimeMainLoop(); + realtimeMainLoop(); }else{ - reason = streamMainLoop(); + streamMainLoop(); } closeStreamSource(); @@ -795,11 +799,68 @@ namespace Mist{ userSelect.clear(); finish(); - INFO_MSG("Input closing clean; reason: %s", reason.c_str()); + INFO_MSG("Input closing clean; reason: %s", Util::exitReason); return; } - std::string Input::streamMainLoop(){ + void Input::streamMainLoop(){ + uint64_t statTimer = 0; + uint64_t startTime = Util::bootSecs(); + size_t tid; + size_t idx; + Comms::Statistics statComm; + getNext(); + tid = thisPacket.getTrackId(); + idx = M.trackIDToIndex(tid, getpid()); + if (thisPacket && !userSelect.count(idx)){ + userSelect[idx].reload(streamName, idx, COMM_STATUS_SOURCE | COMM_STATUS_DONOTTRACK); + } + while (thisPacket && config->is_active && userSelect[idx].isAlive()){ + if (userSelect[idx].getStatus() == COMM_STATUS_REQDISCONNECT){ + Util::logExitReason("buffer requested shutdown"); + break; + } + bufferLivePacket(thisPacket); + userSelect[idx].keepAlive(); + getNext(); + if (!thisPacket){ + Util::logExitReason("invalid packet from getNext"); + break; + } + tid = thisPacket.getTrackId(); + idx = M.trackIDToIndex(tid, getpid()); + if (thisPacket && !userSelect.count(idx)){ + userSelect[idx].reload(streamName, idx, COMM_STATUS_SOURCE | COMM_STATUS_DONOTTRACK); + } + + if (Util::bootSecs() - statTimer > 1){ + // Connect to stats for INPUT detection + if (!statComm){statComm.reload();} + if (statComm){ + if (!statComm.isAlive()){ + config->is_active = false; + Util::logExitReason("received shutdown request from controller"); + return; + } + uint64_t now = Util::bootSecs(); + statComm.setNow(now); + statComm.setCRC(getpid()); + statComm.setStream(streamName); + statComm.setConnector("INPUT:" + capa["name"].asStringRef()); + statComm.setUp(0); + statComm.setDown(streamByteCount()); + statComm.setTime(now - startTime); + statComm.setLastSecond(0); + statComm.setHost(getConnectedBinHost()); + statComm.keepAlive(); + } + + statTimer = Util::bootSecs(); + } + } + } + + void Input::realtimeMainLoop(){ uint64_t statTimer = 0; uint64_t startTime = Util::bootSecs(); Comms::Statistics statComm; @@ -809,7 +870,18 @@ namespace Mist{ userSelect[tid].reload(streamName, tid, COMM_STATUS_SOURCE | COMM_STATUS_DONOTTRACK); } while (thisPacket && config->is_active && userSelect[thisPacket.getTrackId()].isAlive()){ + thisPacket.nullMember("bpos"); + while (config->is_active && userSelect[thisPacket.getTrackId()].isAlive() && + Util::bootMS() + SIMULATED_LIVE_BUFFER < (thisPacket.getTime() + timeOffset) + simStartTime){ + Util::sleep(std::min(((thisPacket.getTime() + timeOffset) + simStartTime) - (Util::getMS() + SIMULATED_LIVE_BUFFER), + (uint64_t)1000)); + userSelect[thisPacket.getTrackId()].keepAlive(); + } + uint64_t originalTime = thisPacket.getTime(); + thisPacket.setTime(originalTime + timeOffset); bufferLivePacket(thisPacket); + thisPacket.setTime(originalTime); + userSelect[thisPacket.getTrackId()].keepAlive(); getNext(); if (thisPacket && !userSelect.count(thisPacket.getTrackId())){ @@ -823,50 +895,31 @@ namespace Mist{ if (statComm){ if (!statComm.isAlive()){ config->is_active = false; - return "received shutdown request from controller"; + Util::logExitReason("received shutdown request from controller"); + return; } uint64_t now = Util::bootSecs(); statComm.setNow(now); statComm.setCRC(getpid()); statComm.setStream(streamName); - statComm.setConnector("INPUT"); + statComm.setConnector("INPUT:" + capa["name"].asStringRef()); statComm.setUp(0); statComm.setDown(streamByteCount()); statComm.setTime(now - startTime); statComm.setLastSecond(0); + statComm.setHost(getConnectedBinHost()); statComm.keepAlive(); } statTimer = Util::bootSecs(); } } - if (!nProxy.userClient.isAlive()){return "buffer shutdown";} - if (!config->is_active){return "received deactivate signal";} - if (!thisPacket){return "Invalid packet";} - return "Unknown"; - } - - std::string Input::realtimeMainLoop(){ - getNext(); - while (thisPacket && config->is_active && nProxy.userClient.isAlive()){ - thisPacket.nullMember("bpos"); - while (config->is_active && nProxy.userClient.isAlive() && - Util::bootMS() + SIMULATED_LIVE_BUFFER < (thisPacket.getTime() + timeOffset) + simStartTime){ - Util::sleep(std::min(((thisPacket.getTime() + timeOffset) + simStartTime) - (Util::getMS() + SIMULATED_LIVE_BUFFER), - (uint64_t)1000)); - nProxy.userClient.keepAlive(); - } - uint64_t originalTime = thisPacket.getTime(); - thisPacket.setTime(originalTime + timeOffset); - nProxy.bufferLivePacket(thisPacket, myMeta); - thisPacket.setTime(originalTime); - getNext(); - nProxy.userClient.keepAlive(); + if (!thisPacket){ + Util::logExitReason("invalid packet from getNext"); + } + if (thisPacket && !userSelect[thisPacket.getTrackId()].isAlive()){ + Util::logExitReason("buffer shutdown"); } - if (!thisPacket){return "end of file";} - if (!config->is_active){return "received deactivate signal";} - if (!userSelect[thisPacket.getTrackId()].isAlive()){return "buffer shutdown";} - return "Unknown"; } void Input::finish(){ diff --git a/src/input/input.h b/src/input/input.h index 41bfb1e4..500eab6b 100644 --- a/src/input/input.h +++ b/src/input/input.h @@ -33,6 +33,7 @@ namespace Mist{ bool hasMeta() const; static Util::Config *config; virtual bool needsLock(){return !config->getBool("realtime");} + virtual bool publishesTracks(){return true;} protected: virtual bool checkArguments() = 0; @@ -54,11 +55,12 @@ namespace Mist{ virtual void convert(); virtual void serve(); virtual void stream(); + virtual std::string getConnectedBinHost(){return std::string("\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001", 16);} virtual size_t streamByteCount(){ return 0; }; // For live streams: to update the stats with correct values. - virtual std::string streamMainLoop(); - virtual std::string realtimeMainLoop(); + virtual void streamMainLoop(); + virtual void realtimeMainLoop(); bool isAlwaysOn(); virtual void userLeadIn(); diff --git a/src/input/input_balancer.cpp b/src/input/input_balancer.cpp index 007caae6..448e6eff 100644 --- a/src/input/input_balancer.cpp +++ b/src/input/input_balancer.cpp @@ -16,6 +16,73 @@ namespace Mist{ capa["source_match"] = "balance:*"; capa["priority"] = 9; capa["morphic"] = 1; + + JSON::Value option; + option["arg"] = "integer"; + option["long"] = "buffer"; + option["short"] = "b"; + option["help"] = "DVR buffer time in ms"; + option["value"].append(50000); + config->addOption("bufferTime", option); + capa["optional"]["DVR"]["name"] = "Buffer time (ms)"; + capa["optional"]["DVR"]["help"] = + "The target available buffer time for this live stream, in milliseconds. This is the time " + "available to seek around in, and will automatically be extended to fit whole keyframes as " + "well as the minimum duration needed for stable playback."; + capa["optional"]["DVR"]["option"] = "--buffer"; + capa["optional"]["DVR"]["type"] = "uint"; + capa["optional"]["DVR"]["default"] = 50000; + + option.null(); + option["arg"] = "integer"; + option["long"] = "cut"; + option["short"] = "c"; + option["help"] = "Any timestamps before this will be cut from the live buffer"; + option["value"].append(0); + config->addOption("cut", option); + capa["optional"]["cut"]["name"] = "Cut time (ms)"; + capa["optional"]["cut"]["help"] = + "Any timestamps before this will be cut from the live buffer."; + capa["optional"]["cut"]["option"] = "--cut"; + capa["optional"]["cut"]["type"] = "uint"; + capa["optional"]["cut"]["default"] = 0; + + option.null(); + + option["arg"] = "integer"; + option["long"] = "resume"; + option["short"] = "R"; + option["help"] = "Enable resuming support (1) or disable resuming support (0, default)"; + option["value"].append(0); + config->addOption("resume", option); + capa["optional"]["resume"]["name"] = "Resume support"; + capa["optional"]["resume"]["help"] = + "If enabled, the buffer will linger after source disconnect to allow resuming the stream " + "later. If disabled, the buffer will instantly close on source disconnect."; + capa["optional"]["resume"]["option"] = "--resume"; + capa["optional"]["resume"]["type"] = "select"; + capa["optional"]["resume"]["select"][0u][0u] = "0"; + capa["optional"]["resume"]["select"][0u][1u] = "Disabled"; + capa["optional"]["resume"]["select"][1u][0u] = "1"; + capa["optional"]["resume"]["select"][1u][1u] = "Enabled"; + capa["optional"]["resume"]["default"] = 0; + + option.null(); + + option["arg"] = "integer"; + option["long"] = "segment-size"; + option["short"] = "S"; + option["help"] = "Target time duration in milliseconds for segments"; + option["value"].append(5000); + config->addOption("segmentsize", option); + capa["optional"]["segmentsize"]["name"] = "Segment size (ms)"; + capa["optional"]["segmentsize"]["help"] = "Target time duration in milliseconds for segments."; + capa["optional"]["segmentsize"]["option"] = "--segment-size"; + capa["optional"]["segmentsize"]["type"] = "uint"; + capa["optional"]["segmentsize"]["default"] = 5000; + capa["codecs"][0u][0u].append("*"); + capa["codecs"][0u][1u].append("*"); + capa["codecs"][0u][2u].append("*"); } int inputBalancer::boot(int argc, char *argv[]){ diff --git a/src/input/input_buffer.cpp b/src/input/input_buffer.cpp index 6a7d6ab4..04a4335a 100644 --- a/src/input/input_buffer.cpp +++ b/src/input/input_buffer.cpp @@ -30,6 +30,7 @@ namespace Mist{ capa["optional"].removeMember("realtime"); + lastReTime = 0; /*LTS*/ finalMillis = 0; liveMeta = 0; capa["name"] = "Buffer"; @@ -118,6 +119,7 @@ namespace Mist{ cutTime = 0; segmentSize = 1900; hasPush = false; + everHadPush = false; resumeMode = false; } @@ -360,7 +362,7 @@ namespace Mist{ if (!(users.getStatus(i) & COMM_STATUS_SOURCE)){continue;} if (users.getTrack(i) != tid){continue;} // We have found the right track here (pid matches, and COMM_STATUS_SOURCE set) - users.setStatus(COMM_STATUS_DISCONNECT, i); + users.setStatus(COMM_STATUS_REQDISCONNECT, i); break; } @@ -450,7 +452,7 @@ namespace Mist{ // firstVideo = 1 happens when there are no tracks, in which case we don't care any more uint32_t firstKey = keys.getFirstValid(); uint32_t endKey = keys.getEndValid(); - if (type != "video"){ + if (type != "video" && videoFirstms != 0xFFFFFFFFFFFFFFFFull){ if ((endKey - firstKey) < 2 || keys.getTime(firstKey + 1) > videoFirstms){continue;} } // Buffer cutting @@ -464,19 +466,6 @@ namespace Mist{ } } updateMeta(); - if (config->is_active){ - if (streamStatus){streamStatus.mapped[0] = hasPush ? STRMSTAT_READY : STRMSTAT_WAIT;} - } - static bool everHadPush = false; - if (hasPush){ - hasPush = false; - everHadPush = true; - }else if (everHadPush && !resumeMode && config->is_active){ - INFO_MSG("Shutting down buffer because resume mode is disabled and the source disconnected"); - if (streamStatus){streamStatus.mapped[0] = STRMSTAT_SHUTDOWN;} - config->is_active = false; - userSelect.clear(); - } } void inputBuffer::userLeadIn(){ @@ -487,22 +476,21 @@ namespace Mist{ /*LTS-END*/ connectedUsers = 0; + //Store child process PIDs in generatePids. + //These are controlled by the buffer (usually processes) and should not count towards incoming pushes generatePids.clear(); for (std::map::iterator it = runningProcs.begin(); it != runningProcs.end(); it++){ generatePids.insert(it->second); } + hasPush = false; } void inputBuffer::userOnActive(size_t id){ ///\todo Add tracing of earliest watched keys, to prevent data going out of memory for /// still-watching viewers if (users.getStatus(id) != COMM_STATUS_DISCONNECT && users.getStatus(id) & COMM_STATUS_SOURCE){ sourcePids[users.getPid(id)].insert(users.getTrack(id)); - if (!M.trackValid(users.getTrack(id))){ - users.setStatus(COMM_STATUS_DISCONNECT, id); - return; - } // GeneratePids holds the pids of the process that generate data, so ignore those for determining if a push is ingested. - if (!generatePids.count(users.getPid(id))){hasPush = true;} + if (M.trackValid(users.getTrack(id)) && !generatePids.count(users.getPid(id))){hasPush = true;} } if (!(users.getStatus(id) & COMM_STATUS_DONOTTRACK)){++connectedUsers;} @@ -516,11 +504,20 @@ namespace Mist{ } } void inputBuffer::userLeadOut(){ + if (config->is_active && streamStatus){streamStatus.mapped[0] = hasPush ? STRMSTAT_READY : STRMSTAT_WAIT;} + if (hasPush){everHadPush = true;} + if (!hasPush && everHadPush && !resumeMode && config->is_active){ + Util::logExitReason("source disconnected for non-resumable stream"); + if (streamStatus){streamStatus.mapped[0] = STRMSTAT_SHUTDOWN;} + config->is_active = false; + userSelect.clear(); + } /*LTS-START*/ static std::set prevValidTracks; std::set validTracks = M.getValidTracks(); if (validTracks != prevValidTracks){ + MEDIUM_MSG("Valid tracks count changed from %lu to %lu", prevValidTracks.size(), validTracks.size()); prevValidTracks = validTracks; if (Triggers::shouldTrigger("LIVE_TRACK_LIST")){ JSON::Value triggerPayload; @@ -545,7 +542,6 @@ namespace Mist{ bool inputBuffer::preRun(){ // This function gets run periodically to make sure runtime updates of the config get parsed. - lastReTime = Util::epoch(); /*LTS*/ std::string strName = config->getString("streamname"); Util::sanitizeName(strName); strName = strName.substr(0, (strName.find_first_of("+ "))); @@ -553,16 +549,21 @@ namespace Mist{ snprintf(tmpBuf, NAME_BUFFER_SIZE, SHM_STREAM_CONF, strName.c_str()); Util::DTSCShmReader rStrmConf(tmpBuf); DTSC::Scan streamCfg = rStrmConf.getScan(); + if (streamCfg){ + JSON::Value configuredProcesses = streamCfg.getMember("processes").asJSON(); + checkProcesses(configuredProcesses); + } + //Check if bufferTime setting is correct uint64_t tmpNum = retrieveSetting(streamCfg, "DVR", "bufferTime"); if (tmpNum < 1000){tmpNum = 1000;} - // if the new value is different, print a message and apply it if (bufferTime != tmpNum){ DEVEL_MSG("Setting bufferTime from %" PRIu64 " to new value of %" PRIu64, bufferTime, tmpNum); bufferTime = tmpNum; } /*LTS-START*/ + //Check if cutTime setting is correct tmpNum = retrieveSetting(streamCfg, "cut"); // if the new value is different, print a message and apply it if (cutTime != tmpNum){ @@ -570,28 +571,27 @@ namespace Mist{ cutTime = tmpNum; } + //Check if resume setting is correct tmpNum = retrieveSetting(streamCfg, "resume"); - // if the new value is different, print a message and apply it if (resumeMode != (bool)tmpNum){ INFO_MSG("Setting resume mode from %s to new value of %s", resumeMode ? "enabled" : "disabled", tmpNum ? "enabled" : "disabled"); resumeMode = tmpNum; } + if (!meta){return true;}//abort the rest if we can't write metadata + lastReTime = Util::epoch(); /*LTS*/ + + //Check if segmentsize setting is correct tmpNum = retrieveSetting(streamCfg, "segmentsize"); - if (M && tmpNum < M.biggestFragment() / 2){tmpNum = M.biggestFragment() / 2;} - // if the new value is different, print a message and apply it + if (tmpNum < meta.biggestFragment() / 2){tmpNum = meta.biggestFragment() / 2;} + segmentSize = meta.getMinimumFragmentDuration(); if (segmentSize != tmpNum){ INFO_MSG("Setting segmentSize from %" PRIu64 " to new value of %" PRIu64, segmentSize, tmpNum); segmentSize = tmpNum; - if (M && M.getMinimumFragmentDuration() == 0){ - meta.setMinimumFragmentDuration(segmentSize); - } - } - if (streamCfg){ - JSON::Value configuredProcesses = streamCfg.getMember("processes").asJSON(); - checkProcesses(configuredProcesses); + meta.setMinimumFragmentDuration(segmentSize); } + /*LTS-END*/ return true; } @@ -643,11 +643,8 @@ namespace Mist{ void inputBuffer::checkProcesses(const JSON::Value &procs){ if (!M.getValidTracks().size()){return;} std::set newProcs; - std::map wouldSelect; // used for building args - int zero = 0; - int out = fileno(stdout); int err = fileno(stderr); char *argarr[3]; @@ -660,22 +657,14 @@ namespace Mist{ continue; } if (tmp.isMember("source_track")){ - std::string sourceTrack = tmp["source_track"].asString(); - if (sourceTrack != "null" && findTrack(sourceTrack) == INVALID_TRACK_ID){ - // No match - skip this process - continue; - } + std::set wouldSelect = Util::findTracks(M, JSON::Value(), "", tmp["source_track"].asStringRef()); + // No match - skip this process + if (!wouldSelect.size()){continue;} } - std::stringstream s; if (tmp.isMember("track_select")){ std::set wouldSelect = Util::wouldSelect(M, tmp["track_select"].asStringRef()); - if (!wouldSelect.size()){ - // No match - skip this process - continue; - } - for (std::set::iterator it = wouldSelect.begin(); it != wouldSelect.end(); it++){ - s << *it << " "; - } + // No match - skip this process + if (!wouldSelect.size()){continue;} } if (tmp.isMember("track_inhibit")){ std::set wouldSelect = Util::wouldSelect( @@ -693,7 +682,6 @@ namespace Mist{ } } newProcs.insert(tmp.toString()); - wouldSelect[tmp.toString()] = s.str(); } // shut down deleted/changed processes @@ -722,8 +710,7 @@ namespace Mist{ argarr[1] = (char *)config.c_str(); argarr[2] = 0; INFO_MSG("Starting process: %s %s", argarr[0], argarr[1]); - INFO_MSG(" WouldSelect is %s", wouldSelect.at(*newProcs.begin()).c_str()); - runningProcs[*newProcs.begin()] = Util::Procs::StartPiped(argarr, &zero, &out, &err); + runningProcs[*newProcs.begin()] = Util::Procs::StartPiped(argarr, 0, 0, &err); } newProcs.erase(newProcs.begin()); } diff --git a/src/input/input_buffer.h b/src/input/input_buffer.h index 6811465b..69ef877e 100644 --- a/src/input/input_buffer.h +++ b/src/input/input_buffer.h @@ -17,7 +17,8 @@ namespace Mist{ size_t segmentSize; /*LTS*/ uint64_t lastReTime; /*LTS*/ uint64_t finalMillis; - bool hasPush; + bool hasPush;//Is a push currently being received? + bool everHadPush;//Was there ever a push received? bool resumeMode; IPC::semaphore *liveMeta; diff --git a/src/input/input_dtsc.cpp b/src/input/input_dtsc.cpp index e13581f8..2f35d74a 100644 --- a/src/input/input_dtsc.cpp +++ b/src/input/input_dtsc.cpp @@ -361,7 +361,6 @@ namespace Mist{ thisPacket.reInit(srcConn); // read the next packet before continuing continue; // parse the next packet before returning } - thisPacket = DTSC::Packet(thisPacket, M.trackIDToIndex(thisPacket.getTrackId(), getpid())); return; // we have a packet } } diff --git a/src/input/input_dtsc.h b/src/input/input_dtsc.h index aa7d4afb..6075ccb2 100644 --- a/src/input/input_dtsc.h +++ b/src/input/input_dtsc.h @@ -27,6 +27,11 @@ namespace Mist{ inputDTSC(Util::Config *cfg); bool needsLock(); + virtual std::string getConnectedBinHost(){ + if (srcConn){return srcConn.getBinHost();} + return Input::getConnectedBinHost(); + } + protected: // Private Functions bool openStreamSource(); diff --git a/src/input/input_ebml.cpp b/src/input/input_ebml.cpp index ebb07a59..9f077762 100644 --- a/src/input/input_ebml.cpp +++ b/src/input/input_ebml.cpp @@ -114,12 +114,17 @@ namespace Mist{ while (ptr.size() < needed){ if (!ptr.allocate(needed)){return false;} int64_t toRead = needed - ptr.size(); - if (!fread(ptr + ptr.size(), toRead, 1, inFile)){ - // We assume if there is no current data buffered, that we are at EOF and don't print a warning - if (ptr.size()){ - FAIL_MSG("Could not read more data! (have %zu, need %" PRIu32 ")", ptr.size(), needed); + int readResult = 0; + while (!readResult){ + readResult = fread(ptr + ptr.size(), toRead, 1, inFile); + if (!readResult){ + if (errno == EINTR){continue;} + // At EOF we don't print a warning + if (!feof(inFile)){ + FAIL_MSG("Could not read more data! (have %zu, need %" PRIu32 ")", ptr.size(), needed); + } + return false; } - return false; } totalBytes += toRead; ptr.size() = needed; @@ -463,7 +468,7 @@ namespace Mist{ }break; } } - thisPacket.genericFill(C.time, C.offset, M.trackIDToIndex(C.track, getpid()), C.ptr, C.dsize, + thisPacket.genericFill(C.time, C.offset, C.track, C.ptr, C.dsize, C.bpos, C.key); } diff --git a/src/input/input_hls.cpp b/src/input/input_hls.cpp index ae8a0f43..c79bd774 100644 --- a/src/input/input_hls.cpp +++ b/src/input/input_hls.cpp @@ -525,8 +525,7 @@ namespace Mist{ std::string test = root.link(entry.filename).getFilePath(); fileSource.open(test.c_str(), std::ios::ate | std::ios::binary); if (!fileSource.good()){WARN_MSG("file: %s, error: %s", test.c_str(), strerror(errno));} - entry.byteEnd = fileSource.tellg(); - totalBytes += entry.byteEnd; + totalBytes += fileSource.tellg(); } entry.timestamp = lastTimestamp + startTime; @@ -592,12 +591,9 @@ namespace Mist{ void inputHLS::parseStreamHeader(){ if (!initPlaylist(config->getString("input"))){ FAIL_MSG("Failed to load HLS playlist, aborting"); - myMeta = DTSC::Meta(); return; } - myMeta = DTSC::Meta(); - myMeta.live = true; - myMeta.vod = false; + meta.reInit(config->getString("streamname"), false); INFO_MSG("Parsing live stream to create header..."); TS::Packet packet; // to analyse and extract data int counter = 1; @@ -612,7 +608,7 @@ namespace Mist{ for (std::deque::iterator entryIt = pListIt->second.begin(); entryIt != pListIt->second.end(); ++entryIt){ - nProxy.userClient.keepAlive(); + keepAlive(); if (!segDowner.loadSegment(*entryIt)){ WARN_MSG("Skipping segment that could not be loaded in an attempt to recover"); tsStream.clear(); @@ -633,21 +629,24 @@ namespace Mist{ tsStream.getEarliestPacket(headerPack); int tmpTrackId = headerPack.getTrackId(); uint64_t packetId = pidMapping[(((uint64_t)pListIt->first) << 32) + tmpTrackId]; - + if (packetId == 0){ pidMapping[(((uint64_t)pListIt->first) << 32) + headerPack.getTrackId()] = counter; pidMappingR[counter] = (((uint64_t)pListIt->first) << 32) + headerPack.getTrackId(); packetId = counter; - VERYHIGH_MSG("Added file %s, trackid: %zu, mapped to: %d", - entryIt->filename.c_str(), headerPack.getTrackId(), counter); + VERYHIGH_MSG("Added file %s, trackid: %zu, mapped to: %d", entryIt->filename.c_str(), + headerPack.getTrackId(), counter); counter++; } - if ((!myMeta.tracks.count(packetId) || !myMeta.tracks[packetId].codec.size())){ - tsStream.initializeMetadata(myMeta, tmpTrackId, packetId); - myMeta.tracks[packetId].minKeepAway = globalWaitTime * 2000; - VERYHIGH_MSG("setting minKeepAway = %d for track: %" PRIu64, - myMeta.tracks[packetId].minKeepAway, packetId); + size_t idx = M.trackIDToIndex(packetId, getpid()); + if ((idx == INVALID_TRACK_ID || !M.getCodec(idx).size())){ + tsStream.initializeMetadata(meta, tmpTrackId, packetId); + idx = M.trackIDToIndex(packetId, getpid()); + if (idx != INVALID_TRACK_ID){ + meta.setMinKeepAway(idx, globalWaitTime * 2000); + VERYHIGH_MSG("setting minKeepAway = %" PRIu32 " for track: %zu", globalWaitTime * 2000, idx); + } } } break; // we have all tracks discovered, next playlist! @@ -655,8 +654,6 @@ namespace Mist{ }while (!segDowner.atEnd()); if (preCounter < counter){break;}// We're done reading this playlist! } - - in.close(); } tsStream.clear(); currentPlaylist = 0; @@ -673,8 +670,6 @@ namespace Mist{ meta.reInit(config->getString("streamname"), config->getString("input") + ".dtsh"); hasHeader = (bool)M; - if (M){return true;} - if (!hasHeader){meta.reInit(config->getString("streamname"), true);} TS::Packet packet; // to analyse and extract data @@ -704,7 +699,7 @@ namespace Mist{ DTSC::Packet headerPack; tsStream.getEarliestPacket(headerPack); - int tmpTrackId = headerPack.getTrackId(); + size_t tmpTrackId = headerPack.getTrackId(); uint64_t packetId = pidMapping[(((uint64_t)pListIt->first) << 32) + tmpTrackId]; if (packetId == 0){ @@ -717,10 +712,8 @@ namespace Mist{ } size_t idx = M.trackIDToIndex(packetId, getpid()); - INFO_MSG("PacketID: %" PRIu64 ", pid: %d, mapped to %zu", packetId, getpid(), idx); if (!hasHeader && (idx == INVALID_TRACK_ID || !M.getCodec(idx).size())){ tsStream.initializeMetadata(meta, tmpTrackId, packetId); - INFO_MSG("InitializingMeta for track %zu -> %zu", tmpTrackId, packetId); idx = M.trackIDToIndex(packetId, getpid()); } @@ -757,6 +750,7 @@ namespace Mist{ counter++; } + size_t idx = M.trackIDToIndex(packetId, getpid()); if (!hasHeader && (idx == INVALID_TRACK_ID || !M.getCodec(idx).size())){ tsStream.initializeMetadata(meta, tmpTrackId, packetId); idx = M.trackIDToIndex(packetId, getpid()); @@ -781,7 +775,6 @@ namespace Mist{ INFO_MSG("write header file..."); M.toFile((config->getString("input") + ".dtsh").c_str()); - in.close(); return true; } @@ -794,26 +787,29 @@ namespace Mist{ INSANE_MSG("Getting next"); uint32_t tid = 0; bool finished = false; - if (userSelect.size()){tid = userSelect.begin()->first;} thisPacket.null(); while (config->is_active && (needsLock() || keepAlive())){ // Check if we have a packet bool hasPacket = false; - if (streamIsLive){ + if (idx == INVALID_TRACK_ID){ hasPacket = tsStream.hasPacketOnEachTrack() || (segDowner.atEnd() && tsStream.hasPacket()); }else{ - hasPacket = tsStream.hasPacket(M.getID(idx) & 0xFFFF); + hasPacket = tsStream.hasPacket(getMappedTrackId(M.getID(idx))); } // Yes? Excellent! Read and return it. if (hasPacket){ // Read - if (M.getLive()){ + if (idx == INVALID_TRACK_ID){ tsStream.getEarliestPacket(thisPacket); - tid = M.trackIDToIndex((((uint64_t)currentPlaylist) << 16) + thisPacket.getTrackId(), getpid()); + tid = getOriginalTrackId(currentPlaylist, thisPacket.getTrackId()); + if (!tid){ + INFO_MSG("Track %" PRIu64 " on PLS %" PRIu64 " -> %" PRIu32, thisPacket.getTrackId(), currentPlaylist, tid); + continue; + } }else{ - tsStream.getPacket(M.getID(idx) & 0xFFFF, thisPacket); + tsStream.getPacket(getMappedTrackId(M.getID(idx)), thisPacket); } if (!thisPacket){ FAIL_MSG("Could not getNext TS packet!"); @@ -850,8 +846,8 @@ namespace Mist{ plsTimeOffset[currentPlaylist] += (int64_t)(plsLastTime[currentPlaylist] + plsInterval[currentPlaylist]) - (int64_t)newTime; newTime = thisPacket.getTime() + plsTimeOffset[currentPlaylist]; - INFO_MSG("[Guess; New offset: %" PRId64 " -> %" PRId64 "] Packet %" PRIu32 - "@%" PRIu64 "ms -> %" PRIu64 "ms", + INFO_MSG("[Guess; New offset: %" PRId64 " -> %" PRId64 "] Packet %" PRIu32 "@%" PRIu64 + "ms -> %" PRIu64 "ms", prevOffset, plsTimeOffset[currentPlaylist], tid, thisPacket.getTime(), newTime); } } @@ -891,24 +887,28 @@ namespace Mist{ // No? Then we want to try reading the next file. // No segments? Wait until next playlist reloading time. - currentPlaylist = firstSegment(); + if (idx != INVALID_TRACK_ID){ + currentPlaylist = getMappedTrackPlaylist(M.getID(idx)); + }else{ + currentPlaylist = firstSegment(); + } if (currentPlaylist < 0){ VERYHIGH_MSG("Waiting for segments..."); - if (nProxy.userClient.isAlive()){nProxy.userClient.keepAlive();} + keepAlive(); Util::wait(500); continue; } // Now that we know our playlist is up-to-date, actually try to read the file. - VERYHIGH_MSG("Moving on to next TS segment (variant %" PRIu32 ")", currentPlaylist); + VERYHIGH_MSG("Moving on to next TS segment (variant %" PRIu64 ")", currentPlaylist); if (readNextFile()){ MEDIUM_MSG("Next segment read successfully"); finished = false; continue; // Success! Continue regular parsing. }else{ - if (selectedTracks.size() > 1){ + if (userSelect.size() > 1){ // failed to read segment for playlist, dropping it - WARN_MSG("Dropping variant %" PRIu32 " because we couldn't read anything from it", currentPlaylist); + WARN_MSG("Dropping variant %" PRIu64 " because we couldn't read anything from it", currentPlaylist); tthread::lock_guard guard(entryMutex); listEntries.erase(currentPlaylist); if (listEntries.size()){continue;} @@ -946,17 +946,17 @@ namespace Mist{ currentIndex = plistEntry - 1; currentPlaylist = getMappedTrackPlaylist(trackId); - INFO_MSG("Seeking to index %d on playlist %d", currentIndex, currentPlaylist); + INFO_MSG("Seeking to index %zu on playlist %" PRIu64, currentIndex, currentPlaylist); {// Lock mutex for listEntries tthread::lock_guard guard(entryMutex); if (!listEntries.count(currentPlaylist)){ - WARN_MSG("Playlist %d not loaded, aborting seek", currentPlaylist); + WARN_MSG("Playlist %" PRIu64 " not loaded, aborting seek", currentPlaylist); return; } std::deque &curPlaylist = listEntries[currentPlaylist]; if (curPlaylist.size() <= currentIndex){ - WARN_MSG("Playlist %d has %zu <= %d entries, aborting seek", currentPlaylist, + WARN_MSG("Playlist %" PRIu64 " has %zu <= %zu entries, aborting seek", currentPlaylist, curPlaylist.size(), currentIndex); return; } @@ -1179,7 +1179,7 @@ namespace Mist{ tthread::lock_guard guard(entryMutex); std::deque &curList = listEntries[currentPlaylist]; if (!curList.size()){ - WARN_MSG("no entries found in playlist: %d!", currentPlaylist); + WARN_MSG("no entries found in playlist: %" PRIu64 "!", currentPlaylist); return false; } if (!streamIsLive){ @@ -1204,7 +1204,7 @@ namespace Mist{ if (Util::bootSecs() < ntry.timestamp){ VERYHIGH_MSG("Slowing down to realtime..."); while (Util::bootSecs() < ntry.timestamp){ - if (nProxy.userClient.isAlive()){nProxy.userClient.keepAlive();} + keepAlive(); Util::wait(250); } } @@ -1228,7 +1228,7 @@ namespace Mist{ /// this will keep the playlists in sync while reading segments. size_t inputHLS::firstSegment(){ // Only one selected? Immediately return the right playlist. - if (userSelect.size() == 1){return ((M.getID(userSelect.begin()->first) >> 16) & 0xFFFF);} + if (userSelect.size() == 1){return getMappedTrackPlaylist(M.getID(userSelect.begin()->first));} uint64_t firstTimeStamp = 0; int tmpId = -1; int segCount = 0; diff --git a/src/input/input_hls.h b/src/input/input_hls.h index 9d328132..59d763a2 100644 --- a/src/input/input_hls.h +++ b/src/input/input_hls.h @@ -121,7 +121,7 @@ namespace Mist{ Socket::Connection conn; TS::Packet tsBuf; - int firstSegment(); + size_t firstSegment(); void waitForNextSegment(); void readPMT(); bool checkArguments(); @@ -130,7 +130,6 @@ namespace Mist{ bool needHeader(){return true;} void getNext(size_t idx = INVALID_TRACK_ID); void seek(uint64_t seekTime, size_t idx = INVALID_TRACK_ID); - FILE *inFile; FILE *tsFile; @@ -141,6 +140,9 @@ namespace Mist{ void parseStreamHeader(); + uint32_t getMappedTrackId(uint64_t id); + uint32_t getMappedTrackPlaylist(uint64_t id); + uint64_t getOriginalTrackId(uint32_t playlistId, uint32_t id); size_t getEntryId(uint32_t playlistId, uint64_t bytePos); }; }// namespace Mist diff --git a/src/input/input_playlist.cpp b/src/input/input_playlist.cpp index b61146aa..2d6e8928 100644 --- a/src/input/input_playlist.cpp +++ b/src/input/input_playlist.cpp @@ -39,17 +39,19 @@ namespace Mist{ return true; } - std::string inputPlaylist::streamMainLoop(){ + void inputPlaylist::streamMainLoop(){ bool seenValidEntry = true; uint64_t startTime = Util::bootMS(); - while (config->is_active && nProxy.userClient.isAlive()){ + while (config->is_active){ struct tm *wTime; time_t nowTime = time(0); wTime = localtime(&nowTime); wallTime = wTime->tm_hour * 60 + wTime->tm_min; - nProxy.userClient.keepAlive(); reloadPlaylist(); - if (!playlist.size()){return "No entries in playlist";} + if (!playlist.size()){ + Util::logExitReason("No entries in playlist"); + return; + } ++playlistIndex; if (playlistIndex >= playlist.size()){ if (!seenValidEntry){ @@ -103,7 +105,7 @@ namespace Mist{ continue; } seenValidEntry = true; - while (Util::Procs::isRunning(spawn_pid) && nProxy.userClient.isAlive() && config->is_active){ + while (Util::Procs::isRunning(spawn_pid) && config->is_active){ Util::sleep(1000); if (reloadOn != 0xFFFF){ time_t nowTime = time(0); @@ -117,13 +119,9 @@ namespace Mist{ Util::Procs::Stop(spawn_pid); } } - nProxy.userClient.keepAlive(); } if (!config->is_active && Util::Procs::isRunning(spawn_pid)){Util::Procs::Stop(spawn_pid);} } - if (!config->is_active){return "received deactivate signal";} - if (!nProxy.userClient.isAlive()){return "buffer shutdown";} - return "Unknown"; } void inputPlaylist::reloadPlaylist(){ diff --git a/src/input/input_playlist.h b/src/input/input_playlist.h index 5ffc33fd..c9bbf766 100644 --- a/src/input/input_playlist.h +++ b/src/input/input_playlist.h @@ -11,9 +11,10 @@ namespace Mist{ protected: bool checkArguments(); bool readHeader(){return true;} - virtual void parseStreamHeader(){myMeta.tracks[1].codec = "PLACEHOLDER";} - std::string streamMainLoop(); + virtual void parseStreamHeader(){} + void streamMainLoop(); virtual bool needHeader(){return false;} + virtual bool publishesTracks(){return false;} private: void reloadPlaylist(); diff --git a/src/input/input_rtsp.cpp b/src/input/input_rtsp.cpp index b555eb12..0412d52e 100644 --- a/src/input/input_rtsp.cpp +++ b/src/input/input_rtsp.cpp @@ -45,6 +45,8 @@ namespace Mist{ capa["codecs"][0u][0u].append("H264"); capa["codecs"][0u][0u].append("HEVC"); capa["codecs"][0u][0u].append("MPEG2"); + capa["codecs"][0u][0u].append("VP8"); + capa["codecs"][0u][0u].append("VP9"); capa["codecs"][0u][1u].append("AAC"); capa["codecs"][0u][1u].append("MP3"); capa["codecs"][0u][1u].append("AC3"); @@ -194,45 +196,45 @@ namespace Mist{ tcpCon.close(); } - std::string InputRTSP::streamMainLoop(){ - IPC::sharedClient statsPage = IPC::sharedClient(SHM_STATISTICS, STAT_EX_SIZE, true); + void InputRTSP::streamMainLoop(){ + Comms::Statistics statComm; uint64_t startTime = Util::epoch(); uint64_t lastPing = Util::bootSecs(); + uint64_t lastSecs = 0; while (keepAlive() && parsePacket()){ + uint64_t currSecs = Util::bootSecs(); handleUDP(); if (Util::bootSecs() - lastPing > 30){ sendCommand("GET_PARAMETER", url.getUrl(), ""); lastPing = Util::bootSecs(); } if (lastSecs != currSecs){ - if (!statsPage.getData()){ - statsPage = IPC::sharedClient(SHM_STATISTICS, STAT_EX_SIZE, true); - } - if (statsPage.getData()){ - if (!statsPage.isAlive()){ + lastSecs = currSecs; + // Connect to stats for INPUT detection + statComm.reload(); + if (statComm){ + if (!statComm.isAlive()){ config->is_active = false; - statsPage.finish(); - return "received shutdown request from controller"; + Util::logExitReason("received shutdown request from controller"); + return; } - uint64_t now = Util::epoch(); - IPC::statExchange tmpEx(statsPage.getData()); - tmpEx.now(now); - tmpEx.crc(getpid()); - tmpEx.streamName(streamName); - tmpEx.connector("INPUT"); - tmpEx.up(tcpCon.dataUp()); - tmpEx.down(tcpCon.dataDown()); - tmpEx.time(now - startTime); - tmpEx.lastSecond(0); - statsPage.keepAlive(); + uint64_t now = Util::bootSecs(); + statComm.setNow(now); + statComm.setCRC(getpid()); + statComm.setStream(streamName); + statComm.setConnector("INPUT:" + capa["name"].asStringRef()); + statComm.setUp(tcpCon.dataUp()); + statComm.setDown(tcpCon.dataDown()); + statComm.setTime(now - startTime); + statComm.setLastSecond(0); + statComm.setHost(getConnectedBinHost()); + statComm.keepAlive(); } } } - statsPage.finish(); - if (!tcpCon){return "TCP connection closed";} - if (!config->is_active){return "received deactivate signal";} - if (!keepAlive()){return "buffer shutdown";} - return "Unknown"; + if (!tcpCon){ + Util::logExitReason("TCP connection closed"); + } } bool InputRTSP::parsePacket(bool mustHave){ diff --git a/src/input/input_rtsp.h b/src/input/input_rtsp.h index e228c981..a254c21c 100644 --- a/src/input/input_rtsp.h +++ b/src/input/input_rtsp.h @@ -17,6 +17,11 @@ namespace Mist{ void incoming(const DTSC::Packet &pkt); void incomingRTP(const uint64_t track, const RTP::Packet &p); + virtual std::string getConnectedBinHost(){ + if (tcpCon){return tcpCon.getBinHost();} + return Input::getConnectedBinHost(); + } + protected: // Private Functions bool checkArguments(); @@ -29,7 +34,7 @@ namespace Mist{ const std::map *extraHeaders = 0, bool reAuth = true); bool parsePacket(bool mustHave = false); bool handleUDP(); - std::string streamMainLoop(); + void streamMainLoop(); Socket::Connection tcpCon; HTTP::Parser sndH, recH; HTTP::URL url; diff --git a/src/input/input_ts.cpp b/src/input/input_ts.cpp index e2ff6134..04679b3c 100644 --- a/src/input/input_ts.cpp +++ b/src/input/input_ts.cpp @@ -463,7 +463,7 @@ namespace Mist{ tmpIdx = meta.addTrack(0, 0, 0, 0); } - std::string inputTS::streamMainLoop(){ + void inputTS::streamMainLoop(){ meta.removeTrack(tmpIdx); INFO_MSG("Removed temptrack %zu", tmpIdx); Comms::Statistics statComm; @@ -495,7 +495,8 @@ namespace Mist{ } if (!tcpCon){ config->is_active = false; - return "end of streamed input"; + Util::logExitReason("end of streamed input"); + return; } }else{ std::string leftData; @@ -557,17 +558,19 @@ namespace Mist{ if (statComm){ if (!statComm.isAlive()){ config->is_active = false; - return "received shutdown request from controller"; + Util::logExitReason("received shutdown request from controller"); + return; } uint64_t now = Util::bootSecs(); statComm.setNow(now); statComm.setCRC(getpid()); statComm.setStream(streamName); - statComm.setConnector("INPUT"); + statComm.setConnector("INPUT:" + capa["name"].asStringRef()); statComm.setUp(0); statComm.setDown(downCounter + tcpCon.dataDown()); statComm.setTime(now - startTime); statComm.setLastSecond(0); + statComm.setHost(getConnectedBinHost()); statComm.keepAlive(); } @@ -577,7 +580,8 @@ namespace Mist{ if (hasStarted && !threadTimer.size()){ if (!isAlwaysOn()){ config->is_active = false; - return "no active threads and we had input in the past"; + Util::logExitReason("no active threads and we had input in the past"); + return; }else{ hasStarted = false; } @@ -607,13 +611,13 @@ namespace Mist{ if (Util::bootSecs() - noDataSince > 20){ if (!isAlwaysOn()){ config->is_active = false; - return "No packets received for 20 seconds - terminating"; + Util::logExitReason("no packets received for 20 seconds"); + return; }else{ noDataSince = Util::bootSecs(); } } } - return "received shutdown request"; } void inputTS::finish(){ diff --git a/src/input/input_ts.h b/src/input/input_ts.h index bb307fc1..994c8c9c 100644 --- a/src/input/input_ts.h +++ b/src/input/input_ts.h @@ -14,6 +14,11 @@ namespace Mist{ ~inputTS(); bool needsLock(); + virtual std::string getConnectedBinHost(){ + if (tcpCon){return tcpCon.getBinHost();} + /// \TODO Handle UDP + return Input::getConnectedBinHost(); + } protected: // Private Functions bool checkArguments(); @@ -25,7 +30,7 @@ namespace Mist{ void readPMT(); bool openStreamSource(); void parseStreamHeader(); - std::string streamMainLoop(); + void streamMainLoop(); void finish(); FILE *inFile; ///< The input file with ts data TS::Stream tsStream; ///< Used for parsing the incoming ts stream diff --git a/src/io.cpp b/src/io.cpp index 0143392d..67a84933 100644 --- a/src/io.cpp +++ b/src/io.cpp @@ -303,10 +303,15 @@ namespace Mist{ /// Initiates/continues negotiation with the buffer as well ///\param packet The packet to buffer void InOutBase::bufferLivePacket(const DTSC::Packet &packet){ + size_t idx = M.trackIDToIndex(packet.getTrackId(), getpid()); + if (idx == INVALID_TRACK_ID){ + INFO_MSG("Packet for track %zu has no valid index!", packet.getTrackId()); + return; + } char *data; size_t dataLen; packet.getString("data", data, dataLen); - bufferLivePacket(packet.getTime(), packet.getInt("offset"), packet.getTrackId(), data, dataLen, + bufferLivePacket(packet.getTime(), packet.getInt("offset"), idx, data, dataLen, packet.getInt("bpos"), packet.getFlag("keyframe")); /// \TODO META Build something that should actually be able to deal with "extra" values } @@ -329,7 +334,7 @@ namespace Mist{ // Assume this is the first packet on the track isKeyframe = true; }else{ - if (packTime - tPages.getInt("lastkeytime", tPages.getEndPos() - 1) >= 5000){ + if (packTime - tPages.getInt("lastkeytime", tPages.getEndPos() - 1) >= AUDIO_KEY_INTERVAL){ isKeyframe = true; } } @@ -343,9 +348,8 @@ namespace Mist{ packTime, M.getLastms(packTrack)); return; } - if (packet.getTime() > myMeta.tracks[tid].lastms + 30000 && myMeta.tracks[tid].lastms){ - WARN_MSG("Sudden jump in timestamp from %" PRIu64 " to %" PRIu64, myMeta.tracks[tid].lastms, - packet.getTime()); + if (packTime > M.getLastms(packTrack) + 30000 && M.getLastms(packTrack)){ + WARN_MSG("Sudden jump in timestamp from %" PRIu64 " to %" PRIu64, M.getLastms(packTrack), packTime); } } @@ -361,7 +365,7 @@ namespace Mist{ tPages.setInt("firstkey", 0, 0); tPages.setInt("firsttime", packTime, 0); tPages.setInt("size", DEFAULT_DATA_PAGE_SIZE, 0); - tPages.setInt("keycount", 0, endPage); + tPages.setInt("keycount", 0, 0); tPages.setInt("avail", 0, 0); ++endPage; } diff --git a/src/output/mist_out.cpp b/src/output/mist_out.cpp index 78f7927a..03c65fbc 100644 --- a/src/output/mist_out.cpp +++ b/src/output/mist_out.cpp @@ -12,7 +12,7 @@ int spawnForked(Socket::Connection &S){ void handleUSR1(int signum, siginfo_t *sigInfo, void *ignore){ HIGH_MSG("USR1 received - triggering rolling restart"); Util::Config::is_restarting = true; - Util::Config::logExitReason("setting is_active to false because of received USR1"); + Util::logExitReason("signal USR1"); Util::Config::is_active = false; } @@ -47,5 +47,7 @@ int main(int argc, char *argv[]){ return tmp.run(); } } + INFO_MSG("Exit reason: %s", Util::exitReason); return 0; } + diff --git a/src/output/output.cpp b/src/output/output.cpp index 638c7dcd..7cb37b68 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -48,7 +48,6 @@ namespace Mist{ Output::Output(Socket::Connection &conn) : myConn(conn){ pushing = false; - pushIsOngoing = false; firstTime = 0; firstPacketTime = 0xFFFFFFFFFFFFFFFFull; lastPacketTime = 0; @@ -67,6 +66,10 @@ namespace Mist{ lastRecv = Util::bootSecs(); if (myConn){ setBlocking(true); + //Make sure that if the socket is a non-stdio socket, we close it when forking + if (myConn.getSocket() > 2){ + Util::Procs::socketList.insert(myConn.getSocket()); + } }else{ WARN_MSG("Warning: MistOut created with closed socket!"); } @@ -129,6 +132,7 @@ namespace Mist{ }else{ MEDIUM_MSG("onFail '%s': %s", streamName.c_str(), msg.c_str()); } + Util::logExitReason(msg.c_str()); isInitialized = false; wantRequest = true; parseData = false; @@ -143,7 +147,7 @@ namespace Mist{ } reconnect(); // if the connection failed, fail - if (streamName.size() < 1){ + if (!meta || streamName.size() < 1){ onFail("Could not connect to stream", true); return; } @@ -181,19 +185,8 @@ namespace Mist{ if (shmSessions.mapped){ char shmEmpty[SHM_SESSIONS_ITEM]; memset(shmEmpty, 0, SHM_SESSIONS_ITEM); - std::string host = statComm.getHost(); - if (host.substr(0, 12) == - std::string("\000\000\000\000\000\000\000\000\000\000\377\377", 12)){ - char tmpstr[16]; - snprintf(tmpstr, 16, "%hhu.%hhu.%hhu.%hhu", host[12], host[13], host[14], host[15]); - host = tmpstr; - }else{ - char tmpstr[40]; - snprintf(tmpstr, 40, "%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x:%.2x%.2x", - host[0], host[1], host[2], host[3], host[4], host[5], host[6], host[7], - host[8], host[9], host[10], host[11], host[12], host[13], host[14], host[15]); - host = tmpstr; - } + std::string host; + Socket::hostBytesToStr(statComm.getHost().data(), 16, host); uint32_t shmOffset = 0; const std::string &cName = capa["name"].asStringRef(); while (shmOffset + SHM_SESSIONS_ITEM < SHM_SESSIONS_SIZE){ @@ -259,12 +252,19 @@ namespace Mist{ std::string Output::getConnectedHost(){return myConn.getHost();} - std::string Output::getConnectedBinHost(){return myConn.getBinHost();} + std::string Output::getConnectedBinHost(){ + if (!prevHost.size()){ + if (myConn && myConn.getPureSocket() != -1){ + prevHost = myConn.getBinHost(); + } + if (!prevHost.size()){prevHost.assign("\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001", 16);} + } + return prevHost; + } bool Output::isReadyForPlay(){ // If a protocol does not support any codecs, we assume you know what you're doing if (!capa.isMember("codecs")){return true;} - if (isPushing()){return true;} if (!isInitialized){initialize();} meta.refresh(); if (getSupportedTracks().size()){ @@ -363,27 +363,25 @@ namespace Mist{ while (!meta && ++attempts < 20 && Util::streamAlive(streamName)){ meta.reInit(streamName, false); } - if (!meta){ - onFail("Could not connect to stream data", true); - return; - } + if (!meta){return;} meta.refresh(); isInitialized = true; statComm.reload(); stats(true); - if (!pushing){selectDefaultTracks();} - if (!M.getVod() && !isReadyForPlay()){ - uint64_t waitUntil = Util::epoch() + 30; + if (isPushing()){return;} + if (!isRecording() && !M.getVod() && !isReadyForPlay()){ + uint64_t waitUntil = Util::bootSecs() + 45; while (!M.getVod() && !isReadyForPlay()){ - if (Util::epoch() > waitUntil + 45 || (!userSelect.size() && Util::epoch() > waitUntil)){ - INFO_MSG("Giving up waiting for playable tracks. Stream: %s, IP: %s", streamName.c_str(), - getConnectedHost().c_str()); + if (Util::bootSecs() > waitUntil || (!userSelect.size() && Util::bootSecs() > waitUntil)){ + INFO_MSG("Giving up waiting for playable tracks. IP: %s", getConnectedHost().c_str()); break; } Util::wait(500); + meta.refresh(); stats(); } } + selectDefaultTracks(); } std::set Output::getSupportedTracks(const std::string &type) const{ @@ -403,22 +401,11 @@ namespace Mist{ bool autoSeek = buffer.size(); uint64_t seekTarget = currentTime(); std::set newSelects = - Util::wouldSelect(myMeta, targetParams, capa, UA, autoSeek ? seekTarget : 0); - - std::set oldSel; - for (std::set::iterator selIt = selectedTracks.begin(); - selIt != selectedTracks.end(); ++selIt){ - oldSel.insert(*selIt); - } - - if (oldSel == newSelects){ - // No new selections? Do nothing, return no change. - return false; - } + Util::wouldSelect(M, targetParams, capa, UA, autoSeek ? seekTarget : 0); if (autoSeek){ std::set toRemove; - for (std::set::iterator it = wouldSelect.begin(); it != wouldSelect.end(); it++){ + for (std::set::iterator it = newSelects.begin(); it != newSelects.end(); it++){ // autoSeeking and target not in bounds? Drop it too. if (M.getLastms(*it) < std::max(seekTarget, (uint64_t)6000lu) - 6000){ toRemove.insert(*it); @@ -426,7 +413,7 @@ namespace Mist{ } // remove those from selectedtracks for (std::set::iterator it = toRemove.begin(); it != toRemove.end(); it++){ - wouldSelect.erase(*it); + newSelects.erase(*it); } } @@ -435,7 +422,7 @@ namespace Mist{ userSelect.clear(); // Select tracks here! - for (std::set::iterator it = wouldSelect.begin(); it != wouldSelect.end(); it++){ + for (std::set::iterator it = newSelects.begin(); it != newSelects.end(); it++){ userSelect[*it].reload(streamName, *it); } @@ -453,6 +440,14 @@ namespace Mist{ parseData = false; } + ///Returns the timestamp of the next upcoming keyframe after thisPacket, or 0 if that cannot be determined (yet). + uint64_t Output::nextKeyTime(){ + DTSC::Keys keys(M.keys(getMainSelectedTrack())); + if (!keys.getValidCount()){return 0;} + size_t keyNum = keys.getNumForTime(lastPacketTime); + return keys.getTime(keyNum+1); + } + uint64_t Output::pageNumForKey(size_t trackId, size_t keyNum){ const Util::RelAccX &tPages = M.pages(trackId); for (uint64_t i = tPages.getDeleted(); i < tPages.getEndPos(); i++){ @@ -671,7 +666,7 @@ namespace Mist{ for (std::set::iterator it = seekTracks.begin(); it != seekTracks.end(); it++){ seek(*it, pos, false); } - firstTime = Util::bootMS() - currentTime(); + firstTime = Util::bootMS() - (currentTime() * realTime / 1000); } bool Output::seek(size_t tid, uint64_t pos, bool getNextKey){ @@ -690,7 +685,7 @@ namespace Mist{ stats(); } } - if (meta.getLastms(tid) <= pos){ + if (meta.getLastms(tid) < pos){ WARN_MSG("Aborting seek to %" PRIu64 "ms in track %zu: past end of track (= %" PRIu64 "ms).", pos, tid, meta.getLastms(tid)); userSelect.erase(tid); @@ -751,7 +746,7 @@ namespace Mist{ if (curPage[tid].mapped[tmp.offset]){return seek(tid, pos, getNextKey);} FAIL_MSG("Track %zu no data (key %zu@%" PRIu64 ") - timeout", tid, keyNum + (getNextKey ? 1 : 0), tmp.offset); userSelect.erase(tid); - firstTime = Util::bootMS() - buffer.begin()->time; + firstTime = Util::bootMS() - (buffer.begin()->time * realTime / 1000); return false; } @@ -761,6 +756,7 @@ namespace Mist{ /// needsLookAhead+minKeepAway ms from the end. Unless lastms < 5000, then it seeks to the first /// keyframe of the main track. Aborts if there is no main track or it has no keyframes. void Output::initialSeek(){ + if (!meta){return;} uint64_t seekPos = 0; if (meta.getLive()){ size_t mainTrack = getMainSelectedTrack(); @@ -980,6 +976,7 @@ namespace Mist{ /// It seeks to the last sync'ed keyframe of the main track, no closer than needsLookAhead+minKeepAway ms from the end. /// Aborts if not live, there is no main track or it has no keyframes. bool Output::liveSeek(){ + if (!realTime){return false;}//Makes no sense when playing in turbo mode static uint32_t seekCount = 2; uint64_t seekPos = 0; if (!meta.getLive()){return false;} @@ -989,11 +986,15 @@ namespace Mist{ uint64_t cTime = thisPacket.getTime(); uint64_t mKa = getMinKeepAway(); if (!maxSkipAhead){ + bool noReturn = false; uint64_t newSpeed = 1000; if (lMs - mKa - needsLookAhead - extraKeepAway > cTime + 50){ // We need to speed up! uint64_t diff = (lMs - mKa - needsLookAhead - extraKeepAway) - cTime; - if (diff > 1000){ + if (diff > 3000){ + noReturn = true; + newSpeed = 1000; + }else if (diff > 1000){ newSpeed = 750; }else if (diff > 500){ newSpeed = 900; @@ -1002,13 +1003,11 @@ namespace Mist{ } } if (realTime != newSpeed){ - HIGH_MSG("Changing playback speed from %" PRIu64 " to %" PRIu64 "(%" PRIu64 - " ms LA, %" PRIu64 " ms mKA, %lu eKA)", - realTime, newSpeed, needsLookAhead, mKa, extraKeepAway); - firstTime = Util::bootMS() - cTime; + HIGH_MSG("Changing playback speed from %" PRIu64 " to %" PRIu64 "(%" PRIu64 " ms LA, %" PRIu64 " ms mKA, %lu eKA)", realTime, newSpeed, needsLookAhead, mKa, extraKeepAway); + firstTime = Util::bootMS() - (cTime * newSpeed / 1000); realTime = newSpeed; } - return false; + if (!noReturn){return false;} } // cancel if there are no keys in the main track if (mainTrack == INVALID_TRACK_ID){return false;} @@ -1167,8 +1166,14 @@ namespace Mist{ while (keepGoing() && (wantRequest || parseData)){ if (wantRequest){requestHandler();} if (parseData){ - if (!isInitialized){initialize();} - if (!sentHeader){ + if (!isInitialized){ + initialize(); + if (!isInitialized){ + onFail("Stream initialization failed"); + break; + } + } + if (!sentHeader && keepGoing()){ DONTEVEN_MSG("sendHeader"); sendHeader(); } @@ -1186,6 +1191,8 @@ namespace Mist{ Util::sleep(std::min(thisPacket.getTime() - ((((Util::bootMS() - firstTime) * 1000) + maxSkipAhead) / realTime), 1000ul)); + //Make sure we stay responsive to requests and stats while waiting + if (wantRequest){requestHandler();} stats(); } } @@ -1217,6 +1224,8 @@ namespace Mist{ }else{ playbackSleep(sleepTime); } + //Make sure we stay responsive to requests and stats while waiting + if (wantRequest){requestHandler();} stats(); } if (!timeoutTries){ @@ -1237,6 +1246,7 @@ namespace Mist{ INFO_MSG("Switching to next push target filename: %s", newTarget.c_str()); if (!connectToFile(newTarget)){ FAIL_MSG("Failed to open file, aborting: %s", newTarget.c_str()); + Util::logExitReason("failed to open file, aborting: %s", newTarget.c_str()); onFinish(); break; } @@ -1247,13 +1257,14 @@ namespace Mist{ }else{ if (!onFinish()){ INFO_MSG("Shutting down because planned stopping point reached"); + Util::logExitReason("planned stopping point reached"); break; } } } sendNext(); }else{ - INFO_MSG("Shutting down because of stream end"); + Util::logExitReason("end of stream"); /*LTS-START*/ if (Triggers::shouldTrigger("CONN_STOP", streamName)){ std::string payload = @@ -1265,18 +1276,14 @@ namespace Mist{ } } if (!meta){ - Util::Config::logExitReason("No connection to the metadata"); + Util::logExitReason("lost internal connection to stream data"); break; } } stats(); } - MEDIUM_MSG("MistOut client handler shutting down: %s, %s, %s", - myConn.connected() ? "conn_active" : "conn_closed", wantRequest ? "want_request" : "no_want_request", - parseData ? "parsing_data" : "not_parsing_data"); - if (Util::Config::exitReason.size()){ - INFO_MSG("Logged exit reason: %s", Util::Config::exitReason.c_str()); - } + if (!myConn){Util::logExitReason("remote connection closed");} + INFO_MSG("Client handler shutting down, exit reason: %s", Util::exitReason); onFinish(); /*LTS-START*/ @@ -1460,42 +1467,66 @@ namespace Mist{ DTSC::Keys keys(M.keys(nxt.tid)); size_t thisKey = keys.getNumForTime(nxt.time); - while (!nextTime && keepGoing()){ - // The next packet is either not available yet, or on another page - // Check if we have a next valid packet - if (memcmp(curPage[nxt.tid].mapped + nxt.offset + preLoad.getDataLen(), "\000\000\000\000", 4)){ - nextTime = getDTSCTime(curPage[nxt.tid].mapped, nxt.offset + preLoad.getDataLen()); - // After 500ms, we assume the time will not update anymore - if (++emptyCount >= 20){break;} + + // Check if we have a next valid packet + if (memcmp(curPage[nxt.tid].mapped + nxt.offset + preLoad.getDataLen(), "\000\000\000\000", 4)){ + nextTime = getDTSCTime(curPage[nxt.tid].mapped, nxt.offset + preLoad.getDataLen()); + if (!nextTime){ + WARN_MSG("Next packet is available, but has no time. Please warn the developers if you see this message!"); + dropTrack(nxt.tid, "EOP: invalid next packet"); + return false; + } + }else{ + //no next packet yet! + //Check if this is the last packet of a VoD stream. Return success and drop the track. + if (M.getVod() && nxt.time >= M.getLastms(nxt.tid)){ + thisPacket.reInit(curPage[nxt.tid].mapped + nxt.offset, 0, true); + thisIdx = nxt.tid; + dropTrack(nxt.tid, "end of VoD track reached", false); + return true; + } + //Check if there exists a different page for the next key + size_t nextKeyPage = M.getPageNumberForKey(nxt.tid, thisKey + 1); + if (nextKeyPage != INVALID_KEY_NUM && nextKeyPage != currentPage[nxt.tid]){ + // If so, the next key is our next packet + nextTime = keys.getTime(thisKey + 1); }else{ - if (M.getVod() && nxt.time >= M.getLastms(nxt.tid)){break;} - if (M.getPageNumberForKey(nxt.tid, thisKey + 1) != currentPage[nxt.tid]){ - // Check if its on a different page - nextTime = keys.getTime(thisKey + 1); - } + //Okay, there's no next page yet, and no next packet on this page either. + //That means we're waiting for data to show up, somewhere. + // after ~25 seconds, give up and drop the track. if (++emptyCount >= 1000){ - // after ~25 seconds, give up and drop the track. dropTrack(nxt.tid, "EOP: data wait timeout"); return false; } - Util::sleep(25); - // we're waiting for new data to show up - if (emptyCount % 640 == 0){ - reconnect(); // reconnect every 16 seconds - // if we don't have a connection to the metadata here, this means the stream has gone offline in the meanwhile. - if (!meta){return false;} + //every ~1 second, check if the stream is not offline + if (emptyCount % 40 == 0 && M.getLive() && Util::getStreamStatus(streamName) == STRMSTAT_OFF){ + Util::logExitReason("Stream source shut down"); + thisPacket.null(); + return true; } + //every ~16 seconds, reconnect to metadata + if (emptyCount % 640 == 0){ + reconnect(); + if (!meta){ + onFail("Could not connect to stream data", true); + thisPacket.null(); + return true; + } + // if we don't have a connection to the metadata here, this means the stream has gone offline in the meanwhile. + if (!meta){ + Util::logExitReason("Attempted reconnect to source failed"); + thisPacket.null(); + return true; + } + return false;//no sleep after reconnect + } + //Fine! We didn't want a packet, anyway. Let's try again later. + Util::sleep(25); + return false; } } - if (M.getVod() && nxt.time >= M.getLastms(nxt.tid)){ - thisPacket.reInit(curPage[nxt.tid].mapped + nxt.offset, 0, true); - thisIdx = nxt.tid; - dropTrack(nxt.tid, "end of VoD track reached", false); - return true; - } - // If we don't have a timestamp at all, this is due to a different cause. - if (!nextTime && (emptyCount < 20)){return false;} + //If the next packet should've been before the current packet, something is wrong. Abort, abort! if (nextTime < nxt.time){ dropTrack(nxt.tid, "time going backwards"); return false; @@ -1513,7 +1544,7 @@ namespace Mist{ emptyCount = 0; // valid packet - reset empty counter if (!userSelect[nxt.tid].isAlive()){ - INFO_MSG("Track %zu is not alive!", nxt.tid); + INFO_MSG("Track %zu is not alive!", nxt.tid); return false; } @@ -1526,12 +1557,6 @@ namespace Mist{ // exchange the current packet in the buffer for the next one buffer.erase(buffer.begin()); - - if (M.getVod() && nxt.time > M.getLastms(nxt.tid)){ - dropTrack(nxt.tid, "detected last VoD packet"); - return true; - } - buffer.insert(nxt); return true; @@ -1541,8 +1566,10 @@ namespace Mist{ /// Outputs used as an input should return INPUT, outputs used for automation should return OUTPUT, others should return their proper name. /// The default implementation is usually good enough for all the non-INPUT types. std::string Output::getStatsName(){ - if (isPushing()){return "INPUT";} - if (config->hasOption("target") && config->getString("target").size()){return "OUTPUT";} + if (isPushing()){return "INPUT:" + capa["name"].asStringRef();} + if (config->hasOption("target") && config->getString("target").size()){ + return "OUTPUT:" + capa["name"].asStringRef(); + } return capa["name"].asStringRef(); } @@ -1554,6 +1581,25 @@ namespace Mist{ uint64_t now = Util::bootSecs(); if (now == lastStats && !force){return;} + if (isRecording()){ + static uint64_t lastPushUpdate = now; + if (lastPushUpdate + 5 <= now){ + JSON::Value pStat; + pStat["push_status_update"]["id"] = getpid(); + JSON::Value & pData = pStat["push_status_update"]["status"]; + pData["mediatime"] = currentTime(); + for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ + pData["tracks"].append(it->first); + } + pData["bytes"] = myConn.dataUp(); + pData["active_seconds"] = (now - myConn.connTime()); + Socket::UDPConnection uSock; + uSock.SetDestination("localhost", 4242); + uSock.SendNow(pStat.toString()); + lastPushUpdate = now; + } + } + if (!statComm){statComm.reload();} if (!statComm){return;} @@ -1562,16 +1608,13 @@ namespace Mist{ HIGH_MSG("Writing stats: %s, %s, %u, %lu, %lu", getConnectedHost().c_str(), streamName.c_str(), crc & 0xFFFFFFFFu, myConn.dataUp(), myConn.dataDown()); /*LTS-START*/ - if (statComm.getStatus() == COMM_STATUS_DISCONNECT){ + if (statComm.getStatus() == COMM_STATUS_REQDISCONNECT){ onFail("Shutting down on controller request"); return; } /*LTS-END*/ statComm.setNow(now); - if (statComm.getHost() == - std::string("\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", 16)){ - statComm.setHost(getConnectedBinHost()); - } + statComm.setHost(getConnectedBinHost()); statComm.setCRC(crc); statComm.setStream(streamName); statComm.setConnector(getStatsName()); @@ -1597,16 +1640,38 @@ namespace Mist{ doSync(); - for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ - it->second.keepAlive(); - if (it->second.isAlive() && M.getLive() && it->second.getStatus() & COMM_STATUS_SOURCE){ - if (Util::bootSecs() - M.getLastUpdated(it->first) > 3){ - INFO_MSG("Not updating data for track %zu?", it->first); + if (isPushing()){ + for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ + it->second.keepAlive(); + if (it->second.getStatus() == COMM_STATUS_REQDISCONNECT){ + if (dropPushTrack(it->second.getTrack(), "disconnect request from buffer")){break;} } + if (!it->second.isAlive()){ + if (dropPushTrack(it->second.getTrack(), "track mapping no longer valid")){break;} + } + //if (Util::bootSecs() - M.getLastUpdated(it->first) > 5){ + // if (dropPushTrack(it->second.getTrack(), "track updates being ignored by buffer")){break;} + //} + } + }else{ + for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ + it->second.keepAlive(); } } } + bool Output::dropPushTrack(uint32_t trackId, const std::string & dropReason){ + for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ + if (it->second.getTrack() == trackId){ + WARN_MSG("Dropping input track %" PRIu32 ": %s", trackId, dropReason.c_str()); + userSelect.erase(it); + return true; + break; + } + } + return false; + } + void Output::onRequest(){ // simply clear the buffer, we don't support any kind of input by default myConn.Received().clear(); @@ -1651,7 +1716,7 @@ namespace Mist{ // Initialize the stream source if needed, connect to it waitForStreamPushReady(); // pull the source setting from metadata - strmSource = meta.getSource(); + if (meta){strmSource = meta.getSource();} if (!strmSource.size()){ FAIL_MSG("Push rejected - stream %s not configured or unavailable", streamName.c_str()); @@ -1683,13 +1748,10 @@ namespace Mist{ } } - std::string smp = streamName.substr(0, streamName.find_first_of("+ ")); - if (Triggers::shouldTrigger("STREAM_PUSH", smp)){ - std::string payload = - streamName + "\n" + getConnectedHost() + "\n" + capa["name"].asStringRef() + "\n" + reqUrl; - if (!Triggers::doTrigger("STREAM_PUSH", payload, smp)){ - FAIL_MSG("Push from %s to %s rejected - STREAM_PUSH trigger denied the push", - getConnectedHost().c_str(), streamName.c_str()); + if (Triggers::shouldTrigger("STREAM_PUSH", streamName)){ + std::string payload = streamName + "\n" + getConnectedHost() + "\n" + capa["name"].asStringRef() + "\n" + reqUrl; + if (!Triggers::doTrigger("STREAM_PUSH", payload, streamName)){ + WARN_MSG("Push from %s rejected by STREAM_PUSH trigger", getConnectedHost().c_str()); pushing = false; return false; } @@ -1698,8 +1760,7 @@ namespace Mist{ if (IP != ""){ if (!myConn.isAddress(IP)){ - FAIL_MSG("Push from %s to %s rejected - source host not whitelisted", - getConnectedHost().c_str(), streamName.c_str()); + WARN_MSG("Push from %s rejected; not whitelisted", getConnectedHost().c_str()); pushing = false; return false; } @@ -1712,7 +1773,31 @@ namespace Mist{ void Output::waitForStreamPushReady(){ uint8_t streamStatus = Util::getStreamStatus(streamName); MEDIUM_MSG("Current status for %s buffer is %u", streamName.c_str(), streamStatus); - while (streamStatus != STRMSTAT_WAIT && streamStatus != STRMSTAT_READY && keepGoing()){ + if (streamStatus == STRMSTAT_READY){ + reconnect(); + std::set vTracks = M.getValidTracks(true); + INFO_MSG("Stream already active (%zu valid tracks) - check if it's not shutting down...", vTracks.size()); + uint64_t oneTime = 0; + uint64_t twoTime = 0; + for (std::set::iterator it = vTracks.begin(); it != vTracks.end(); ++it){ + if (M.getLastms(*it) > oneTime){oneTime = M.getLastms(*it);} + } + Util::wait(2000); + for (std::set::iterator it = vTracks.begin(); it != vTracks.end(); ++it){ + if (M.getLastms(*it) > twoTime){twoTime = M.getLastms(*it);} + } + if (twoTime <= oneTime+500){ + disconnect(); + INFO_MSG("Waiting for stream reset before attempting push input accept (%" PRIu64 " <= %" PRIu64 "+500)", twoTime, oneTime); + while (streamStatus != STRMSTAT_OFF && keepGoing()){ + userSelect.clear(); + Util::wait(1000); + streamStatus = Util::getStreamStatus(streamName); + } + reconnect(); + } + } + while (((streamStatus != STRMSTAT_WAIT && streamStatus != STRMSTAT_READY) || !meta) && keepGoing()){ INFO_MSG("Waiting for %s buffer to be ready... (%u)", streamName.c_str(), streamStatus); disconnect(); userSelect.clear(); @@ -1720,11 +1805,15 @@ namespace Mist{ streamStatus = Util::getStreamStatus(streamName); if (streamStatus == STRMSTAT_OFF || streamStatus == STRMSTAT_WAIT || streamStatus == STRMSTAT_READY){ INFO_MSG("Reconnecting to %s buffer... (%u)", streamName.c_str(), streamStatus); + Util::wait(500); reconnect(); streamStatus = Util::getStreamStatus(streamName); } } if (streamStatus == STRMSTAT_READY || streamStatus == STRMSTAT_WAIT){reconnect();} + if (!meta){ + onFail("Could not connect to stream data", true); + } } void Output::selectAllTracks(){ diff --git a/src/output/output.h b/src/output/output.h index 51713d5d..f4b2af66 100644 --- a/src/output/output.h +++ b/src/output/output.h @@ -65,6 +65,7 @@ namespace Mist{ /// This function is called whenever a packet is ready for sending. /// Inside it, thisPacket is guaranteed to contain a valid packet. virtual void sendNext(){}// REQUIRED! Others are optional. + virtual bool dropPushTrack(uint32_t trackId, const std::string & dropReason); bool getKeyFrame(); bool prepareNext(); virtual void dropTrack(size_t trackId, const std::string &reason, bool probablyBad = true); @@ -105,6 +106,7 @@ namespace Mist{ std::set buffer; ///< A sorted list of next-to-be-loaded packets. bool sought; ///< If a seek has been done, this is set to true. Used for seeking on ///< prepareNext(). + std::string prevHost; ///< Old value for getConnectedBinHost, for caching protected: // these are to be messed with by child classes virtual bool inlineRestartCapable() const{ return false; @@ -128,6 +130,7 @@ namespace Mist{ Comms::Statistics statComm; bool isBlocking; ///< If true, indicates that myConn is blocking. uint32_t crc; ///< Checksum, if any, for usage in the stats. + uint64_t nextKeyTime(); // stream delaying variables uint64_t maxSkipAhead; ///< Maximum ms that we will go ahead of the intended timestamps. @@ -148,7 +151,6 @@ namespace Mist{ virtual bool isPushing(){return pushing;}; bool allowPush(const std::string &passwd); void waitForStreamPushReady(); - bool pushIsOngoing; uint64_t firstPacketTime; uint64_t lastPacketTime; diff --git a/src/output/output_cmaf.cpp b/src/output/output_cmaf.cpp index bbcaccb0..5c03c680 100644 --- a/src/output/output_cmaf.cpp +++ b/src/output/output_cmaf.cpp @@ -13,6 +13,8 @@ #include #include +uint64_t bootMsOffset; + namespace Mist{ OutCMAF::OutCMAF(Socket::Connection &conn) : HTTPOutput(conn){ @@ -68,6 +70,8 @@ namespace Mist{ void OutCMAF::onHTTP(){ initialize(); + bootMsOffset = 0; + if (M.getLive()){bootMsOffset = M.getBootMsOffset();} if (H.url.size() < streamName.length() + 7){ H.Clean(); @@ -440,6 +444,12 @@ namespace Mist{ } void hlsSegment(uint64_t start, uint64_t duration, std::stringstream &s, bool first){ + if (bootMsOffset){ + uint64_t unixMs = start + bootMsOffset + (Util::unixMS() - Util::bootMS()); + time_t uSecs = unixMs/1000; + struct tm * tVal = gmtime(&uSecs); + s << "#EXT-X-PROGRAM-DATE-TIME: " << (tVal->tm_year+1900) << "-" << std::setw(2) << std::setfill('0') << (tVal->tm_mon+1) << "-" << std::setw(2) << std::setfill('0') << tVal->tm_mday << "T" << std::setw(2) << std::setfill('0') << tVal->tm_hour << ":" << std::setw(2) << std::setfill('0') << tVal->tm_min << ":" << std::setw(2) << std::setfill('0') << tVal->tm_sec << "." << std::setw(3) << std::setfill('0') << (unixMs%1000) << "Z" << std::endl; + } s << "#EXTINF:" << (((double)duration) / 1000) << ",\r\nchunk_" << start << ".m4s" << std::endl; } @@ -537,7 +547,6 @@ namespace Mist{ result << "#EXTM3U\r\n" "#EXT-X-VERSION:7\r\n" - "#EXT-X-DISCONTINUITY\r\n" "#EXT-X-TARGETDURATION:" << targetDuration << "\r\n"; if (M.getLive()){result << "#EXT-X-MEDIA-SEQUENCE:" << firstFragment << "\r\n";} diff --git a/src/output/output_dtsc.cpp b/src/output/output_dtsc.cpp index 7a926ff8..0eaff95c 100644 --- a/src/output/output_dtsc.cpp +++ b/src/output/output_dtsc.cpp @@ -63,7 +63,7 @@ namespace Mist{ config = cfg; } - std::string OutDTSC::getStatsName(){return (pushing ? "INPUT" : "OUTPUT");} + std::string OutDTSC::getStatsName(){return (pushing ? "INPUT:DTSC" : "OUTPUT:DTSC");} /// Seeks to the first sync'ed keyframe of the main track. /// Aborts if there is no main track or it has no keyframes. diff --git a/src/output/output_hls.cpp b/src/output/output_hls.cpp index d7cfd856..466f074b 100644 --- a/src/output/output_hls.cpp +++ b/src/output/output_hls.cpp @@ -6,6 +6,8 @@ namespace Mist{ bool OutHLS::isReadyForPlay(){ + if (!isInitialized){initialize();} + meta.refresh(); if (!M.getValidTracks().size()){return false;} uint32_t mainTrack = M.mainTrack(); if (mainTrack == INVALID_TRACK_ID){return false;} @@ -244,7 +246,11 @@ namespace Mist{ bool isTS = (HTTP::URL(H.url).getExt().substr(0, 3) != "m3u"); H.Clean(); H.setCORSHeaders(); - H.SetHeader("Content-Type", "application/octet-stream"); + if (isTS){ + H.SetHeader("Content-Type", "video/mp2t"); + }else{ + H.SetHeader("Content-Type", "application/vnd.apple.mpegurl"); + } if (isTS && !hasSessionIDs()){ H.SetHeader("Cache-Control", "public, max-age=600, immutable"); H.SetHeader("Pragma", ""); @@ -343,6 +349,7 @@ namespace Mist{ std::string request = H.url.substr(H.url.find("/", 5) + 1); H.Clean(); H.setCORSHeaders(); + H.SetHeader("Content-Type", "application/vnd.apple.mpegurl"); if (!M.getValidTracks().size()){ H.SendResponse("404", "Not online or found", myConn); H.Clean(); @@ -378,16 +385,14 @@ namespace Mist{ wantRequest = true; parseData = false; - // Ensure alignment of contCounters for selected tracks, to prevent discontinuities. - for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ - uint32_t pkgPid = 255 + it->first; - uint16_t &contPkg = contCounters[pkgPid]; - if (contPkg % 16 != 0){ + // Ensure alignment of contCounters, to prevent discontinuities. + for (std::map::iterator it = contCounters.begin(); it != contCounters.end(); it++){ + if (it->second % 16 != 0){ packData.clear(); - packData.setPID(pkgPid); + packData.setPID(it->first); packData.addStuffing(); - while (contPkg % 16 != 0){ - packData.setContinuityCounter(++contPkg); + while (it->second % 16 != 0){ + packData.setContinuityCounter(++it->second); sendTS(packData.checkAndGetBuffer()); } packData.clear(); diff --git a/src/output/output_http_internal.cpp b/src/output/output_http_internal.cpp index ab581345..df178534 100644 --- a/src/output/output_http_internal.cpp +++ b/src/output/output_http_internal.cpp @@ -111,6 +111,7 @@ namespace Mist{ capa["provides"] = "HTTP"; capa["protocol"] = "http://"; capa["url_rel"] = "/$.html"; + capa["codecs"][0u][0u].append("+*"); capa["url_match"].append("/crossdomain.xml"); capa["url_match"].append("/clientaccesspolicy.xml"); capa["url_match"].append("/$.html"); @@ -1028,7 +1029,7 @@ namespace Mist{ snprintf(pageName, NAME_BUFFER_SIZE, SHM_STREAM_STATE, streamName.c_str()); IPC::sharedPage streamStatus(pageName, 1, false, false); uint8_t prevState, newState, pingCounter = 0; - uint64_t prevTracks; + std::set prevTracks; prevState = newState = STRMSTAT_INVALID; while (keepGoing()){ if (!streamStatus || !streamStatus.exists()){streamStatus.init(pageName, 1, false, false);} @@ -1038,10 +1039,10 @@ namespace Mist{ newState = streamStatus.mapped[0]; } - if (newState != prevState || (newState == STRMSTAT_READY && M.getValidTracks().size() != prevTracks)){ + if (newState != prevState || (newState == STRMSTAT_READY && M.getValidTracks() != prevTracks)){ if (newState == STRMSTAT_READY){ reconnect(); - prevTracks = M.getValidTracks().size(); + prevTracks = M.getValidTracks(); }else{ disconnect(); } diff --git a/src/output/output_https.cpp b/src/output/output_https.cpp index 9e67ac9e..f06b5454 100644 --- a/src/output/output_https.cpp +++ b/src/output/output_https.cpp @@ -92,6 +92,7 @@ namespace Mist{ char error_buf[200]; mbedtls_strerror(ret, error_buf, 200); MEDIUM_MSG("Could not handshake, SSL error: %s (%d)", error_buf, ret); + Util::logExitReason("Could not handshake, SSL error: %s (%d)", error_buf, ret); C.close(); return; }else{ @@ -110,6 +111,7 @@ namespace Mist{ int fd[2]; if (socketpair(PF_LOCAL, SOCK_STREAM, 0, fd) != 0){ FAIL_MSG("Could not open anonymous socket for SSL<->HTTP connection!"); + Util::logExitReason("Could not open anonymous socket for SSL<->HTTP connection!"); return 1; } std::deque args; @@ -135,6 +137,7 @@ namespace Mist{ close(fd[1]); if (http_proc < 2){ FAIL_MSG("Could not spawn MistOutHTTP process for SSL connection!"); + Util::logExitReason("Could not spawn MistOutHTTP process for SSL connection!"); return 1; } Socket::Connection http(fd[0]); @@ -150,6 +153,7 @@ namespace Mist{ if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE){ if (ret <= 0){ HIGH_MSG("SSL disconnect!"); + Util::logExitReason("SSL client disconnected"); break; } // we received ret bytes of data to pass on. Do so. @@ -168,6 +172,7 @@ namespace Mist{ ret = mbedtls_ssl_write(&ssl, (const unsigned char *)http_buf.get().data() + done, toSend - done); if (ret == MBEDTLS_ERR_NET_CONN_RESET || ret == MBEDTLS_ERR_SSL_CLIENT_RECONNECT){ HIGH_MSG("SSL disconnect!"); + Util::logExitReason("SSL client disconnected"); http.close(); break; } diff --git a/src/output/output_rtmp.cpp b/src/output/output_rtmp.cpp index 9c5f99a6..24f0983a 100644 --- a/src/output/output_rtmp.cpp +++ b/src/output/output_rtmp.cpp @@ -1229,7 +1229,7 @@ namespace Mist{ static std::map pushMeta; static std::map lastTagTime; static std::map reTrackToID; - if (!isInitialized){ + if (!isInitialized || !meta){ MEDIUM_MSG("Received useless media data"); onFinish(); break; diff --git a/src/output/output_rtsp.cpp b/src/output/output_rtsp.cpp index 9f5ac225..83697557 100644 --- a/src/output/output_rtsp.cpp +++ b/src/output/output_rtsp.cpp @@ -91,6 +91,8 @@ namespace Mist{ capa["codecs"][0u][0u].append("H264"); capa["codecs"][0u][0u].append("HEVC"); capa["codecs"][0u][0u].append("MPEG2"); + capa["codecs"][0u][0u].append("VP8"); + capa["codecs"][0u][0u].append("VP9"); capa["codecs"][0u][1u].append("AAC"); capa["codecs"][0u][1u].append("MP3"); capa["codecs"][0u][1u].append("AC3"); diff --git a/src/output/output_ts.cpp b/src/output/output_ts.cpp index d2a0e04e..6c097b67 100644 --- a/src/output/output_ts.cpp +++ b/src/output/output_ts.cpp @@ -189,7 +189,7 @@ namespace Mist{ std::string OutTS::getStatsName(){ if (!parseData){ - return "INPUT"; + return "INPUT:" + capa["name"].asStringRef(); }else{ return Output::getStatsName(); } diff --git a/src/output/output_ts_base.cpp b/src/output/output_ts_base.cpp index dcfe693e..b8239ed4 100644 --- a/src/output/output_ts_base.cpp +++ b/src/output/output_ts_base.cpp @@ -65,17 +65,12 @@ namespace Mist{ std::string type = M.getType(thisIdx); std::string codec = M.getCodec(thisIdx); bool video = (type == "video"); - - size_t pkgPid = M.getID(thisIdx); - if (pkgPid < 255){pkgPid += 255;} - + size_t pkgPid = TS::getUniqTrackID(M, thisIdx); bool &firstPack = first[thisIdx]; uint16_t &contPkg = contCounters[pkgPid]; - uint64_t packTime = thisPacket.getTime(); bool keyframe = thisPacket.getInt("keyframe"); firstPack = true; - char *dataPointer = 0; size_t dataLen = 0; thisPacket.getString("data", dataPointer, dataLen); // data diff --git a/src/output/output_webrtc.cpp b/src/output/output_webrtc.cpp index fda13a68..b3ba80bb 100644 --- a/src/output/output_webrtc.cpp +++ b/src/output/output_webrtc.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include // ifaddr, listing ip addresses. namespace Mist{ @@ -31,10 +32,12 @@ namespace Mist{ /* ------------------------------------------------ */ OutWebRTC::OutWebRTC(Socket::Connection &myConn) : HTTPOutput(myConn){ + lastPackMs = 0; vidTrack = INVALID_TRACK_ID; prevVidTrack = INVALID_TRACK_ID; audTrack = INVALID_TRACK_ID; stayLive = true; + target_rate = 0.0; firstKey = true; repeatInit = true; @@ -95,6 +98,7 @@ namespace Mist{ capa["url_match"] = "/webrtc/$"; capa["codecs"][0u][0u].append("H264"); capa["codecs"][0u][0u].append("VP8"); + capa["codecs"][0u][0u].append("VP9"); capa["codecs"][0u][1u].append("opus"); capa["codecs"][0u][1u].append("ALAW"); capa["codecs"][0u][1u].append("ULAW"); @@ -107,7 +111,7 @@ namespace Mist{ capa["optional"]["preferredvideocodec"]["help"] = "Comma separated list of video codecs you want to support in preferred order. e.g. " "H264,VP8"; - capa["optional"]["preferredvideocodec"]["default"] = "H264,VP8"; + capa["optional"]["preferredvideocodec"]["default"] = "H264,VP9,VP8"; capa["optional"]["preferredvideocodec"]["type"] = "string"; capa["optional"]["preferredvideocodec"]["option"] = "--webrtc-video-codecs"; capa["optional"]["preferredvideocodec"]["short"] = "V"; @@ -129,14 +133,26 @@ namespace Mist{ capa["optional"]["bindhost"]["option"] = "--bindhost"; capa["optional"]["bindhost"]["short"] = "B"; - capa["optional"]["mergesessions"]["name"] = "Merge sessions"; + capa["optional"]["mergesessions"]["name"] = "merge sessions"; capa["optional"]["mergesessions"]["help"] = - "If enabled, merges together all views from a single user into a single combined session. " - "If disabled, each view (reconnection of the signalling websocket) is a separate session."; + "if enabled, merges together all views from a single user into a single combined session. " + "if disabled, each view (reconnection of the signalling websocket) is a separate session."; capa["optional"]["mergesessions"]["option"] = "--mergesessions"; capa["optional"]["mergesessions"]["short"] = "m"; capa["optional"]["mergesessions"]["default"] = 0; + capa["optional"]["nackdisable"]["name"] = "Disallow NACKs for viewers"; + capa["optional"]["nackdisable"]["help"] = "Disallows viewers to send NACKs for lost packets"; + capa["optional"]["nackdisable"]["option"] = "--nackdisable"; + capa["optional"]["nackdisable"]["short"] = "n"; + capa["optional"]["nackdisable"]["default"] = 0; + + capa["optional"]["jitterlog"]["name"] = "Write jitter log"; + capa["optional"]["jitterlog"]["help"] = "Writes log of frame transmit jitter to /tmp/ for each outgoing connection"; + capa["optional"]["jitterlog"]["option"] = "--jitterlog"; + capa["optional"]["jitterlog"]["short"] = "J"; + capa["optional"]["jitterlog"]["default"] = 0; + config->addOptionsFromCapabilities(capa); } @@ -280,13 +296,61 @@ namespace Mist{ parseData = true; selectDefaultTracks(); } - stayLive = (endTime() < seek_time + 5000); + stayLive = (target_rate == 0.0) && (endTime() < seek_time + 5000); + if (command["seek_time"].asStringRef() == "live"){stayLive = true;} if (stayLive){seek_time = endTime();} seek(seek_time, true); JSON::Value commandResult; commandResult["type"] = "on_seek"; commandResult["result"] = true; if (M.getLive()){commandResult["live_point"] = stayLive;} + if (target_rate == 0.0){ + commandResult["play_rate_curr"] = "auto"; + }else{ + commandResult["play_rate_curr"] = target_rate; + } + webSock->sendFrame(commandResult.toString()); + onIdle(); + return; + } + + if (command["type"] == "set_speed"){ + if (!command.isMember("play_rate")){ + sendSignalingError("on_speed", "Received a playback speed setting request but no `play_rate` property."); + return; + } + double set_rate = command["play_rate"].asDouble(); + if (!parseData){ + parseData = true; + selectDefaultTracks(); + } + JSON::Value commandResult; + commandResult["type"] = "on_speed"; + if (target_rate == 0.0){ + commandResult["play_rate_prev"] = "auto"; + }else{ + commandResult["play_rate_prev"] = target_rate; + } + if (set_rate == 0.0){ + commandResult["play_rate_curr"] = "auto"; + }else{ + commandResult["play_rate_curr"] = set_rate; + } + if (target_rate != set_rate){ + target_rate = set_rate; + if (target_rate == 0.0){ + realTime = 1000;//set playback speed to default + firstTime = Util::bootMS() - currentTime(); + maxSkipAhead = 0;//enabled automatic rate control + }else{ + stayLive = false; + //Set new realTime speed + realTime = 1000 / target_rate; + firstTime = Util::bootMS() - (currentTime() / target_rate); + maxSkipAhead = 1;//disable automatic rate control + } + } + if (M.getLive()){commandResult["live_point"] = stayLive;} webSock->sendFrame(commandResult.toString()); onIdle(); return; @@ -337,6 +401,15 @@ namespace Mist{ sendSignalingError(command["type"].asString(), "Unhandled command type: " + command["type"].asString()); } + bool OutWebRTC::dropPushTrack(uint32_t trackId, const std::string & dropReason){ + JSON::Value commandResult; + commandResult["type"] = "on_track_drop"; + commandResult["track"] = trackId; + commandResult["mediatype"] = M.getType(trackId); + webSock->sendFrame(commandResult.toString()); + return Output::dropPushTrack(trackId, dropReason); + } + void OutWebRTC::sendSignalingError(const std::string &commandType, const std::string &errorMessage){ JSON::Value commandResult; commandResult["type"] = "on_error"; @@ -357,6 +430,12 @@ namespace Mist{ initialize(); selectDefaultTracks(); + if (config && config->hasOption("jitterlog") && config->getBool("jitterlog")){ + std::string fileName = "/tmp/jitter_"+JSON::Value(getpid()).asString(); + jitterLog.open(fileName.c_str()); + lastPackMs = Util::bootMS(); + } + if (0 == udpPort){bindUDPSocketOnLocalCandidateAddress(0);} std::string videoCodec; @@ -389,8 +468,10 @@ namespace Mist{ return false; } videoTrack.rtpPacketizer = RTP::Packet(videoTrack.payloadType, rand(), 0, videoTrack.SSRC, 0); - // Enabled NACKs - sdpAnswer.videoLossPrevention = SDP_LOSS_PREVENTION_NACK; + if (!config || !config->hasOption("nackdisable") || !config->getBool("nackdisable")){ + // Enable NACKs + sdpAnswer.videoLossPrevention = SDP_LOSS_PREVENTION_NACK; + } videoTrack.sorter.tmpVideoLossPrevention = sdpAnswer.videoLossPrevention; } } @@ -500,7 +581,7 @@ namespace Mist{ capa["codecs"].null(); - const char *videoCodecPreference[] ={"H264", "VP8", NULL}; + const char *videoCodecPreference[] ={"H264", "VP9", "VP8", NULL}; const char **videoCodec = videoCodecPreference; SDP::Media *videoMediaOffer = sdpSession.getMediaForType("video"); if (videoMediaOffer){ @@ -535,12 +616,12 @@ namespace Mist{ if (0 == udpPort){bindUDPSocketOnLocalCandidateAddress(0);} - std::string prefVideoCodec = "VP8,H264"; + std::string prefVideoCodec = "VP9,VP8,H264"; if (config && config->hasOption("preferredvideocodec")){ prefVideoCodec = config->getString("preferredvideocodec"); if (prefVideoCodec.empty()){ WARN_MSG("No preferred video codec value set; resetting to default."); - prefVideoCodec = "VP8,H264"; + prefVideoCodec = "VP9,VP8,H264"; } } @@ -579,10 +660,11 @@ namespace Mist{ SDP::MediaFormat *fmtRED = sdpSession.getMediaFormatByEncodingName("video", "RED"); SDP::MediaFormat *fmtULPFEC = sdpSession.getMediaFormatByEncodingName("video", "ULPFEC"); - if (fmtRED && fmtULPFEC){ + if (fmtRED || fmtULPFEC){ videoTrack.ULPFECPayloadType = fmtULPFEC->payloadType; videoTrack.REDPayloadType = fmtRED->payloadType; payloadTypeToWebRTCTrack[fmtRED->payloadType] = videoTrack.payloadType; + payloadTypeToWebRTCTrack[fmtULPFEC->payloadType] = videoTrack.payloadType; } sdpAnswer.videoLossPrevention = SDP_LOSS_PREVENTION_NACK; videoTrack.sorter.tmpVideoLossPrevention = sdpAnswer.videoLossPrevention; @@ -594,7 +676,7 @@ namespace Mist{ videoTrack.rtpToDTSC.setProperties(meta, vIdx); videoTrack.rtpToDTSC.setCallbacks(onDTSCConverterHasPacketCallback, onDTSCConverterHasInitDataCallback); - videoTrack.sorter.setCallback(vIdx, onRTPSorterHasPacketCallback); + videoTrack.sorter.setCallback(M.getID(vIdx), onRTPSorterHasPacketCallback); userSelect[vIdx].reload(streamName, vIdx, COMM_STATUS_SOURCE); INFO_MSG("Video push received on track %zu", vIdx); @@ -616,7 +698,7 @@ namespace Mist{ audioTrack.rtpToDTSC.setProperties(meta, aIdx); audioTrack.rtpToDTSC.setCallbacks(onDTSCConverterHasPacketCallback, onDTSCConverterHasInitDataCallback); - audioTrack.sorter.setCallback(aIdx, onRTPSorterHasPacketCallback); + audioTrack.sorter.setCallback(M.getID(aIdx), onRTPSorterHasPacketCallback); userSelect[aIdx].reload(streamName, aIdx, COMM_STATUS_SOURCE); INFO_MSG("Audio push received on track %zu", aIdx); @@ -639,13 +721,47 @@ namespace Mist{ return false; } - udpPort = - udp.bind(port, (config && config->hasOption("bindhost") && config->getString("bindhost").size()) - ? config->getString("bindhost") - : myConn.getBoundAddress()); - Util::Procs::socketList.insert(udp.getSock()); - sdpAnswer.setCandidate(externalAddr, udpPort); + std::string bindAddr; + //If a bind host has been put in as override, use it + if (config && config->hasOption("bindhost") && config->getString("bindhost").size()){ + bindAddr = config->getString("bindhost"); + udpPort = udp.bind(port, bindAddr); + if (!udpPort){ + WARN_MSG("UDP bind address not valid - ignoring setting and using best guess instead"); + bindAddr.clear(); + }else{ + INFO_MSG("Bound to pre-configured UDP bind address"); + } + } + //use the best IPv4 guess we have + if (!bindAddr.size()){ + bindAddr = Socket::resolveHostToBestExternalAddrGuess(externalAddr, AF_INET, myConn.getBoundAddress()); + if (!bindAddr.size()){ + WARN_MSG("UDP bind to best guess failed - using same address as incoming connection as a last resort"); + bindAddr.clear(); + }else{ + udpPort = udp.bind(port, bindAddr); + if (!udpPort){ + WARN_MSG("UDP bind to best guess failed - using same address as incoming connection as a last resort"); + bindAddr.clear(); + }else{ + INFO_MSG("Bound to public UDP bind address derived from hostname"); + } + } + } + if (!bindAddr.size()){ + bindAddr = myConn.getBoundAddress(); + udpPort = udp.bind(port, bindAddr); + if (!udpPort){ + FAIL_MSG("UDP bind to connected address failed - we're out of options here, I'm afraid..."); + bindAddr.clear(); + }else{ + INFO_MSG("Bound to same UDP address as TCP address - this is potentially wrong, but used as a last resort"); + } + } + Util::Procs::socketList.insert(udp.getSock()); + sdpAnswer.setCandidate(bindAddr, udpPort); return true; } @@ -813,8 +929,8 @@ namespace Mist{ if (idx == INVALID_TRACK_ID || !webrtcTracks.count(idx)){ FAIL_MSG("Received an RTP packet for a track that we didn't prepare for. PayloadType is " - "%" PRIu32, - rtp_pkt.getPayloadType()); + "%" PRIu32 ", idx %zu", + rtp_pkt.getPayloadType(), idx); return; } @@ -830,6 +946,8 @@ namespace Mist{ FAIL_MSG("Failed to unprotect a RTP packet."); return; } + RTP::Packet unprotPack(udp.data, len); + DONTEVEN_MSG("%s", unprotPack.toString().c_str()); // Here follows a very rudimentary algo for requesting lost // packets; I guess after some experimentation a better @@ -843,11 +961,11 @@ namespace Mist{ rtcTrack.prevReceivedSequenceNumber = currSeqNum; - if (rtp_pkt.getPayloadType() == rtcTrack.REDPayloadType){ + if (rtp_pkt.getPayloadType() == rtcTrack.REDPayloadType || rtp_pkt.getPayloadType() == rtcTrack.ULPFECPayloadType){ rtcTrack.sorter.addREDPacket(udp.data, len, rtcTrack.payloadType, rtcTrack.REDPayloadType, rtcTrack.ULPFECPayloadType); }else{ - rtcTrack.sorter.addPacket(RTP::Packet(udp.data, len)); + rtcTrack.sorter.addPacket(unprotPack); } }else if ((pt >= 64) && (pt < 96)){ @@ -909,7 +1027,7 @@ namespace Mist{ void OutWebRTC::onDTSCConverterHasPacket(const DTSC::Packet &pkt){ // extract meta data (init data, width/height, etc); - size_t idx = pkt.getTrackId(); + size_t idx = M.trackIDToIndex(pkt.getTrackId(), getpid()); std::string codec = M.getCodec(idx); if (codec == "H264"){ if (M.getInit(idx).empty()){ @@ -919,9 +1037,10 @@ namespace Mist{ } if (codec == "VP8" && pkt.getFlag("keyframe")){extractFrameSizeFromVP8KeyFrame(pkt);} + if (codec == "VP9" && pkt.getFlag("keyframe")){extractFrameSizeFromVP8KeyFrame(pkt);} // create rtcp packet (set bitrate and request keyframe). - if (codec == "H264" || codec == "VP8"){ + if (codec == "H264" || codec == "VP8" || codec == "VP9"){ uint64_t now = Util::bootMS(); if (now >= rtcpTimeoutInMillis){ @@ -942,13 +1061,15 @@ namespace Mist{ INFO_MSG("Validated track %zu in meta", idx); meta.validateTrack(idx); } + DONTEVEN_MSG("DTSC: %s", pkt.toSummary().c_str()); bufferLivePacket(pkt); } - void OutWebRTC::onDTSCConverterHasInitData(size_t idx, const std::string &initData){ + void OutWebRTC::onDTSCConverterHasInitData(size_t trackId, const std::string &initData){ + size_t idx = M.trackIDToIndex(trackId, getpid()); if (idx == INVALID_TRACK_ID || !webrtcTracks.count(idx)){ ERROR_MSG( - "Recieved init data for a track that we don't manager. TrackID %zu /PayloadType: %zu", + "Recieved init data for a track that we don't manage. TrackID %zu /PayloadType: %zu", idx, M.getID(idx)); return; } @@ -972,9 +1093,10 @@ namespace Mist{ meta.setInit(idx, avccbox.payload(), avccbox.payloadSize()); } - void OutWebRTC::onRTPSorterHasPacket(size_t idx, const RTP::Packet &pkt){ + void OutWebRTC::onRTPSorterHasPacket(size_t trackId, const RTP::Packet &pkt){ + size_t idx = M.trackIDToIndex(trackId, getpid()); if (idx == INVALID_TRACK_ID || !webrtcTracks.count(idx)){ - ERROR_MSG("Received a sorted RTP packet for track %zu but we don't manage this track.", idx); + ERROR_MSG("Received a sorted RTP packet for payload %zu (idx %zu) but we don't manage this track.", trackId, idx); return; } webrtcTracks[idx].rtpToDTSC.addRTP(pkt); @@ -989,7 +1111,7 @@ namespace Mist{ int protectedSize = nbytes; if (srtpWriter.protectRtp((uint8_t *)(void *)rtpOutBuffer, &protectedSize) != 0){ - ERROR_MSG("Failed to protect the RTCP message."); + ERROR_MSG("Failed to protect the RTP message."); return; } @@ -1076,6 +1198,14 @@ namespace Mist{ // If we see this is audio or video, use the webrtc track we negotiated if (M.getType(tid) == "video" && webrtcTracks.count(vidTrack)){ trackPointer = &webrtcTracks[vidTrack]; + + if (lastPackMs){ + uint64_t newMs = Util::bootMS(); + jitterLog << (newMs - lastPackMs) << std::endl; + lastPackMs = newMs; + } + + } if (M.getType(tid) == "audio" && webrtcTracks.count(audTrack)){ trackPointer = &webrtcTracks[audTrack]; @@ -1093,19 +1223,17 @@ namespace Mist{ WebRTCTrack &rtcTrack = *trackPointer; uint64_t timestamp = thisPacket.getTime(); - rtcTrack.rtpPacketizer.setTimestamp(timestamp * SDP::getMultiplier(&M, thisIdx)); + uint64_t newTime = timestamp * SDP::getMultiplier(&M, thisIdx); + rtcTrack.rtpPacketizer.setTimestamp(newTime); bool isKeyFrame = thisPacket.getFlag("keyframe"); didReceiveKeyFrame = isKeyFrame; if (M.getCodec(thisIdx) == "H264"){ if (isKeyFrame && firstKey){ - char *data; - size_t dataLen; - thisPacket.getString("data", data, dataLen); size_t offset = 0; while (offset + 4 < dataLen){ - size_t nalLen = Bit::btohl(data + offset); - uint8_t nalType = data[offset + 4] & 0x1F; + size_t nalLen = Bit::btohl(dataPointer + offset); + uint8_t nalType = dataPointer[offset + 4] & 0x1F; if (nalType == 7 || nalType == 8){// Init data already provided in-band, skip repeating // it. repeatInit = false; diff --git a/src/output/output_webrtc.h b/src/output/output_webrtc.h index a60957a1..73cb8790 100644 --- a/src/output/output_webrtc.h +++ b/src/output/output_webrtc.h @@ -67,6 +67,7 @@ #include #include #include +#include #define NACK_BUFFER_SIZE 1024 @@ -130,6 +131,7 @@ namespace Mist{ virtual void sendNext(); virtual void onWebsocketFrame(); virtual void preWebsocketConnect(); + virtual bool dropPushTrack(uint32_t trackId, const std::string & dropReason); void onIdle(); bool onFinish(); bool doesWebsockets(){return true;} @@ -142,6 +144,8 @@ namespace Mist{ void onRTPPacketizerHasRTCPPacket(const char *data, uint32_t nbytes); private: + uint64_t lastPackMs; + std::ofstream jitterLog; std::string externalAddr; void ackNACK(uint32_t SSRC, uint16_t seq); bool handleWebRTCInputOutput(); ///< Reads data from the UDP socket. Returns true when we read @@ -198,6 +202,7 @@ namespace Mist{ ///< the signaling channel. Defaults to 6mbit. size_t audTrack, vidTrack, prevVidTrack; + double target_rate; ///< Target playback speed rate (1.0 = normal, 0 = auto) bool didReceiveKeyFrame; /* TODO burst delay */ int64_t packetOffset; ///< For timestamp rewrite with BMO diff --git a/src/process/process_exec.cpp b/src/process/process_exec.cpp index 7b42efce..b8400ad2 100644 --- a/src/process/process_exec.cpp +++ b/src/process/process_exec.cpp @@ -9,7 +9,7 @@ #include //for stat #include //for stat -int pipein[2], pipeout[2], pipeerr[2]; +int pipein[2], pipeout[2]; Util::Config co; Util::Config conf; @@ -134,8 +134,14 @@ int main(int argc, char *argv[]){ } // create pipe pair before thread - pipe(pipein); - pipe(pipeout); + if (pipe(pipein) || pipe(pipeout)){ + FAIL_MSG("Could not create pipes for process!"); + return 1; + } + Util::Procs::socketList.insert(pipeout[0]); + Util::Procs::socketList.insert(pipeout[1]); + Util::Procs::socketList.insert(pipein[0]); + Util::Procs::socketList.insert(pipein[1]); // stream which connects to input tthread::thread source(sourceThread, 0); @@ -188,9 +194,18 @@ namespace Mist{ int ffer = 2; pid_t execd_proc = -1; + + std::string streamName = opt["sink"].asString(); + if (!streamName.size()){streamName = opt["source"].asStringRef();} + Util::streamVariables(streamName, opt["source"].asStringRef()); + + //Do variable substitution on command + std::string tmpCmd = opt["exec"].asStringRef(); + Util::streamVariables(tmpCmd, streamName, opt["source"].asStringRef()); + // exec command char exec_cmd[10240]; - strncpy(exec_cmd, opt["exec"].asString().c_str(), 10240); + strncpy(exec_cmd, tmpCmd.c_str(), 10240); INFO_MSG("Executing command: %s", exec_cmd); uint8_t argCnt = 0; char *startCh = 0; diff --git a/src/process/process_exec.h b/src/process/process_exec.h index 39631872..45b30eea 100644 --- a/src/process/process_exec.h +++ b/src/process/process_exec.h @@ -21,7 +21,9 @@ namespace Mist{ class ProcessSink : public InputEBML{ public: - ProcessSink(Util::Config *cfg) : InputEBML(cfg){}; + ProcessSink(Util::Config *cfg) : InputEBML(cfg){ + capa["name"] = "MKVExec"; + }; void getNext(size_t idx = INVALID_TRACK_ID){ static bool recurse = false; if (recurse){return InputEBML::getNext(idx);} @@ -52,7 +54,15 @@ namespace Mist{ class ProcessSource : public OutEBML{ public: - ProcessSource(Socket::Connection &c) : OutEBML(c){realTime = 1000;}; + bool isRecording(){return false;} + ProcessSource(Socket::Connection &c) : OutEBML(c){ + capa["name"] = "MKVExec"; + realTime = 0; + }; + void sendHeader(){ + realTime = 0; + OutEBML::sendHeader(); + }; void sendNext(){ extraKeepAway = 0; needsLookAhead = 0; diff --git a/src/process/process_ffmpeg.cpp b/src/process/process_ffmpeg.cpp index 0b5026f2..93ffeeee 100644 --- a/src/process/process_ffmpeg.cpp +++ b/src/process/process_ffmpeg.cpp @@ -13,7 +13,7 @@ int ofin = -1, ofout = 1, oferr = 2; int ifin = -1, ifout = -1, iferr = 2; -int pipein[2], pipeout[2], pipeerr[2]; +int pipein[2], pipeout[2]; Util::Config co; Util::Config conf; @@ -307,8 +307,12 @@ int main(int argc, char *argv[]){ } // create pipe pair before thread - pipe(pipein); - pipe(pipeout); + if (pipe(pipein) || pipe(pipeout)){ + FAIL_MSG("Could not create pipes for process!"); + return 1; + } + Util::Procs::socketList.insert(pipeout[0]); + Util::Procs::socketList.insert(pipein[1]); // stream which connects to input tthread::thread source(sourceThread, 0); @@ -384,14 +388,14 @@ namespace Mist{ std::string EncodeOutputEBML::getTrackType(int tid){return M.getType(tid);} void EncodeOutputEBML::setVideoTrack(std::string tid){ - std::set tracks = Util::findTracks(M, "video", tid); + std::set tracks = Util::findTracks(M, capa, "video", tid); for (std::set::iterator it = tracks.begin(); it != tracks.end(); it++){ userSelect[*it].reload(streamName, *it); } } void EncodeOutputEBML::setAudioTrack(std::string tid){ - std::set tracks = Util::findTracks(M, "audio", tid); + std::set tracks = Util::findTracks(M, capa, "audio", tid); for (std::set::iterator it = tracks.begin(); it != tracks.end(); it++){ userSelect[*it].reload(streamName, *it); } @@ -571,7 +575,7 @@ namespace Mist{ if (!preset.empty()){options.append(" -preset " + preset);} - snprintf(ffcmd, 10240, "ffmpeg -hide_banner -loglevel warning -f lavfi -i color=c=black:s=%dx%d %s %s -c:v %s %s %s %s -an -f matroska - ", + snprintf(ffcmd, 10240, "ffmpeg -hide_banner -loglevel warning -f lavfi -i color=c=black:s=%dx%d %s %s -c:v %s %s %s %s -an -force_key_frames source -f matroska - ", res_x, res_y, s_input.c_str(), s_overlay.c_str(), codec.c_str(), options.c_str(), getBitrateSetting().c_str(), flags.c_str()); @@ -681,16 +685,26 @@ namespace Mist{ }else{ // sources array missing, create empty object in array opt["sources"][0u]["src"] = "-"; - - WARN_MSG("No stdin input set in config, adding input stream with default settings"); + if (opt.isMember("resolution")){ + opt["sources"][0u]["width"] = -1; + opt["sources"][0u]["height"] = res_y; + opt["sources"][0u]["anchor"] = "center"; + } + INFO_MSG("Default source: input stream at preserved-aspect same height"); stdinSource = true; } if (!stdinSource){ // no stdin source item found in sources configuration, add source object at the beginning - opt["sources"].prepend(JSON::fromString("{\"src\':\"-\"}")); - WARN_MSG("No stdin input stream found in 'inputs' config, adding stdin input stream at the " - "beginning of the array"); + JSON::Value nOpt; + nOpt["src"] = "-"; + if (opt.isMember("resolution")){ + nOpt["width"] = -1; + nOpt["height"] = res_y; + nOpt["anchor"] = "center"; + } + opt["sources"].prepend(nOpt); + WARN_MSG("Source is not used: adding source stream at preserved-aspect same height"); } return true; @@ -799,7 +813,6 @@ namespace Mist{ } prepareCommand(); - MEDIUM_MSG("Starting ffmpeg process..."); ffout = p.StartPiped(args, &pipein[0], &pipeout[1], &ffer); while (conf.is_active && p.isRunning(ffout)){Util::sleep(200);} diff --git a/src/process/process_ffmpeg.h b/src/process/process_ffmpeg.h index a0550fd2..eac7c4c8 100644 --- a/src/process/process_ffmpeg.h +++ b/src/process/process_ffmpeg.h @@ -52,6 +52,7 @@ namespace Mist{ class EncodeOutputEBML : public OutEBML{ public: EncodeOutputEBML(Socket::Connection &c) : OutEBML(c){}; // realTime = 0;}; + bool isRecording(){return false;} void setVideoTrack(std::string tid); void setAudioTrack(std::string tid); void sendNext(); diff --git a/src/sourcery.cpp b/src/sourcery.cpp index a867f8db..39a3166e 100644 --- a/src/sourcery.cpp +++ b/src/sourcery.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include std::string getContents(const char *fileName){ diff --git a/src/utils/util_load.cpp b/src/utils/util_load.cpp index 28bea4aa..3df34713 100644 --- a/src/utils/util_load.cpp +++ b/src/utils/util_load.cpp @@ -664,7 +664,9 @@ void handleServer(void *hostEntryPointer){ entry->state = STATE_ERROR; }else{ if (down){ - WARN_MSG("Connection established with %s", url.host.c_str()); + std::string ipStr; + Socket::hostBytesToStr(DL.getSocket().getBinHost().data(), 16, ipStr); + WARN_MSG("Connection established with %s (%s)", url.host.c_str(), ipStr.c_str()); memcpy(entry->details->binHost, DL.getSocket().getBinHost().data(), 16); entry->state = STATE_ONLINE; down = false; diff --git a/test/status.cpp b/test/status.cpp new file mode 100644 index 00000000..617b6964 --- /dev/null +++ b/test/status.cpp @@ -0,0 +1,46 @@ +#include +#include +#include +#include + +int main(int argc, char **argv){ + Util::Config cfg; + cfg.activate(); + uint8_t prevStat = 255; + while (cfg.is_active){ + uint8_t currStat = Util::getStreamStatus(argv[1]); + if (currStat != prevStat){ + std::cout << "Stream status: "; + switch (currStat){ + case STRMSTAT_OFF: + std::cout << "Off"; + break; + case STRMSTAT_INIT: + std::cout << "Init"; + break; + case STRMSTAT_BOOT: + std::cout << "Boot"; + break; + case STRMSTAT_WAIT: + std::cout << "Wait"; + break; + case STRMSTAT_READY: + std::cout << "Ready"; + break; + case STRMSTAT_SHUTDOWN: + std::cout << "Shutdown"; + break; + case STRMSTAT_INVALID: + std::cout << "Invalid"; + break; + default: + std::cout << "??? (" << currStat << ")"; + } + std::cout << std::endl; + prevStat = currStat; + } + Util::sleep(200); + } + return 0; +} +