From 871bee3436efb201a8f9801312530090b303a5ae Mon Sep 17 00:00:00 2001 From: Thulinma Date: Tue, 25 Jul 2017 11:50:04 +0200 Subject: [PATCH 1/3] Improvements to Downloader class to allow for more flexibility in usage --- lib/downloader.cpp | 70 +++++++++++++++++++++++++++------------------- lib/downloader.h | 3 ++ 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/lib/downloader.cpp b/lib/downloader.cpp index 2a07f654..2bebdeea 100644 --- a/lib/downloader.cpp +++ b/lib/downloader.cpp @@ -35,6 +35,47 @@ namespace HTTP{ /// Clears all extra/override headers for outgoing requests. void Downloader::clearHeaders(){extraHeaders.clear();} + /// Returns a reference to the internal HTTP class instance. + Parser &Downloader::getHTTP(){return H;} + + /// Returns a reference to the internal Socket::Connection class instance. + Socket::Connection &Downloader::getSocket(){return S;} + + /// Sends a request for the given URL, does no waiting. + void Downloader::doRequest(const HTTP::URL &link){ + if (link.protocol != "http"){ + FAIL_MSG("Protocol not supported: %s", link.protocol.c_str()); + return; + } + INFO_MSG("Retrieving %s", link.getUrl().c_str()); + H.Clean(); + // Reconnect if needed + if (!S || link.host != connectedHost || link.getPort() != connectedPort){ + S.close(); + connectedHost = link.host; + connectedPort = link.getPort(); + S = Socket::Connection(connectedHost, connectedPort, true); + } + H.url = "/" + link.path; + if (link.args.size()){H.url += "?" + link.args;} + if (link.port.size()){ + H.SetHeader("Host", link.host + ":" + link.port); + }else{ + H.SetHeader("Host", link.host); + } + H.SetHeader("User-Agent", "MistServer " PACKAGE_VERSION); + H.SetHeader("X-Version", PACKAGE_VERSION); + H.SetHeader("Accept", "*/*"); + if (extraHeaders.size()){ + for (std::map::iterator it = extraHeaders.begin(); + it != extraHeaders.end(); ++it){ + H.SetHeader(it->first, it->second); + } + } + H.SendRequest(S); + H.Clean(); + } + /// Downloads the given URL into 'H', returns true on success. /// Makes at most 5 attempts, and will wait no longer than 5 seconds without receiving data. bool Downloader::get(const HTTP::URL &link, uint8_t maxRecursiveDepth){ @@ -43,36 +84,9 @@ namespace HTTP{ FAIL_MSG("Protocol not supported: %s", link.protocol.c_str()); return false; } - INFO_MSG("Retrieving %s", link.getUrl().c_str()); unsigned int loop = 6; // max 5 attempts - while (--loop){// loop while we are unsuccessful - H.Clean(); - // Reconnect if needed - if (!S || link.host != connectedHost || link.getPort() != connectedPort){ - S.close(); - connectedHost = link.host; - connectedPort = link.getPort(); - S = Socket::Connection(connectedHost, connectedPort, true); - } - H.url = "/" + link.path; - if (link.args.size()){H.url += "?" + link.args;} - if (link.port.size()){ - H.SetHeader("Host", link.host + ":" + link.port); - }else{ - H.SetHeader("Host", link.host); - } - H.SetHeader("User-Agent", "MistServer " PACKAGE_VERSION); - H.SetHeader("X-Version", PACKAGE_VERSION); - H.SetHeader("Accept", "*/*"); - if (extraHeaders.size()){ - for (std::map::iterator it = extraHeaders.begin(); - it != extraHeaders.end(); ++it){ - H.SetHeader(it->first, it->second); - } - } - H.SendRequest(S); - H.Clean(); + doRequest(link); uint64_t reqTime = Util::bootSecs(); while (S && Util::bootSecs() < reqTime + 5){ // No data? Wait for a second or so. diff --git a/lib/downloader.h b/lib/downloader.h index 0c40b707..a209e170 100644 --- a/lib/downloader.h +++ b/lib/downloader.h @@ -6,6 +6,7 @@ namespace HTTP{ public: Downloader(){progressCallback = 0;} std::string &data(); + void doRequest(const HTTP::URL &link); bool get(const std::string &link); bool get(const HTTP::URL &link, uint8_t maxRecursiveDepth = 6); std::string getHeader(const std::string &headerName); @@ -15,6 +16,8 @@ namespace HTTP{ bool (*progressCallback)(); ///< Called every time the socket stalls, up to 4X per second. void setHeader(const std::string &name, const std::string &val); void clearHeaders(); + Parser &getHTTP(); + Socket::Connection &getSocket(); private: std::map extraHeaders; ///< Holds extra headers to sent with request From c534a2d3cae0c029f1369ecd34e2c484a53e5bb6 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Wed, 26 Jul 2017 15:51:36 +0200 Subject: [PATCH 2/3] Increased SIMUL_TRACKS from 10 to 20 --- lib/defines.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/defines.h b/lib/defines.h index 33870c95..2a2fa24f 100644 --- a/lib/defines.h +++ b/lib/defines.h @@ -136,5 +136,5 @@ static inline void show_stackframe(){} #define SHM_CONF "MstConf" #define NAME_BUFFER_SIZE 200 //char buffer size for snprintf'ing shm filenames -#define SIMUL_TRACKS 10 +#define SIMUL_TRACKS 20 From 3446d022e82fa7c8773a461d9aefb5c39d766885 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Wed, 26 Jul 2017 17:00:36 +0200 Subject: [PATCH 3/3] Added header read speed timers, simplified header logic for VoD. --- src/input/input.cpp | 21 +++++++++++++-------- src/input/input.h | 2 +- src/input/input_buffer.cpp | 11 ----------- src/input/input_buffer.h | 9 +++++---- src/input/input_dtsc.cpp | 14 +++++--------- src/input/input_dtsc.h | 1 + src/input/input_flv.cpp | 2 -- src/input/input_mp3.cpp | 2 -- 8 files changed, 25 insertions(+), 37 deletions(-) diff --git a/src/input/input.cpp b/src/input/input.cpp index f6833e40..2a2cedc7 100644 --- a/src/input/input.cpp +++ b/src/input/input.cpp @@ -168,13 +168,20 @@ namespace Mist { } int Input::run() { + myMeta.sourceURI = config->getString("input"); if (streamStatus){streamStatus.mapped[0] = STRMSTAT_BOOT;} checkHeaderTimes(config->getString("input")); - if (!readHeader()){ - std::cerr << "Reading header for " << config->getString("input") << " failed." << std::endl; - return 0; + if (needHeader()){ + uint64_t timer = Util::bootMS(); + bool headerSuccess = readHeader(); + if (!headerSuccess) { + std::cerr << "Reading header for " << config->getString("input") << " failed." << std::endl; + return 0; + }else{ + timer = Util::bootMS() - timer; + DEBUG_MSG(DLVL_DEVEL, "Read header for '%s' in %llums", streamName.c_str(), timer); + } } - myMeta.sourceURI = config->getString("input"); if (myMeta.vod){ parseHeader(); MEDIUM_MSG("Header parsed, %lu tracks", myMeta.tracks.size()); @@ -585,7 +592,7 @@ namespace Mist { return false; } //Update keynum to point to the corresponding page - uint64_t bufferTimer = Util::getMS(); + uint64_t bufferTimer = Util::bootMS(); INFO_MSG("Loading key %u from page %lu", keyNum, (--(nProxy.pagesByTrack[track].upper_bound(keyNum)))->first); keyNum = (--(nProxy.pagesByTrack[track].upper_bound(keyNum)))->first; if (!bufferStart(track, keyNum)){ @@ -620,7 +627,7 @@ namespace Mist { getNext(); } bufferFinalize(track); - bufferTimer = Util::getMS() - bufferTimer; + bufferTimer = Util::bootMS() - bufferTimer; DEBUG_MSG(DLVL_DEVEL, "Done buffering page %d (%llu packets, %llu bytes) for track %d in %llums", keyNum, packCounter, byteCounter, track, bufferTimer); pageCounter[track][keyNum] = 15; return true; @@ -646,7 +653,6 @@ namespace Mist { playing = -1; playUntil = until; initialTime = 0; - benchMark = Util::getMS(); } void Input::playOnce(){ @@ -654,7 +660,6 @@ namespace Mist { playing = 1; } ++playing; - benchMark = Util::getMS(); } void Input::quitPlay(){ diff --git a/src/input/input.h b/src/input/input.h index e274e086..ae2c68e8 100644 --- a/src/input/input.h +++ b/src/input/input.h @@ -29,6 +29,7 @@ namespace Mist { static void callbackWrapper(char * data, size_t len, unsigned int id); virtual bool checkArguments() = 0; virtual bool readHeader() = 0; + virtual bool needHeader(){return !readExistingHeader();} virtual bool preRun(){return true;} virtual bool readExistingHeader(); virtual bool atKeyFrame(); @@ -59,7 +60,6 @@ namespace Mist { int initialTime; int playing; unsigned int playUntil; - unsigned int benchMark; bool isBuffer; uint64_t activityCounter; diff --git a/src/input/input_buffer.cpp b/src/input/input_buffer.cpp index ff6c6272..eb173e55 100644 --- a/src/input/input_buffer.cpp +++ b/src/input/input_buffer.cpp @@ -774,16 +774,5 @@ namespace Mist { return true; } - bool inputBuffer::readHeader() { - return true; - } - - void inputBuffer::getNext(bool smart) {} - - void inputBuffer::seek(int seekTime) {} - - void inputBuffer::trackSelect(std::string trackSpec) {} } - - diff --git a/src/input/input_buffer.h b/src/input/input_buffer.h index 96cc8898..26ed8496 100644 --- a/src/input/input_buffer.h +++ b/src/input/input_buffer.h @@ -19,12 +19,13 @@ namespace Mist { bool preRun(); bool checkArguments(){return true;} void updateMeta(); - bool readHeader(); - void getNext(bool smart = true); + bool readHeader(){return false;} + bool needHeader(){return false;} + void getNext(bool smart = true){} void updateTrackMeta(unsigned long tNum); void updateMetaFromPage(unsigned long tNum, unsigned long pageNum); - void seek(int seekTime); - void trackSelect(std::string trackSpec); + void seek(int seekTime){} + void trackSelect(std::string trackSpec){} bool removeKey(unsigned int tid); void removeUnused(); void eraseTrackDataPages(unsigned long tid); diff --git a/src/input/input_dtsc.cpp b/src/input/input_dtsc.cpp index 7836487e..5b96a854 100644 --- a/src/input/input_dtsc.cpp +++ b/src/input/input_dtsc.cpp @@ -208,19 +208,15 @@ namespace Mist { return true; } + bool inputDTSC::needHeader(){ + if (!needsLock()){return false;} + return Input::needHeader(); + } + bool inputDTSC::readHeader() { - if (!needsLock()) { - return true; - } if (!inFile) { return false; } - DTSC::File tmp(config->getString("input") + ".dtsh"); - if (tmp) { - myMeta = tmp.getMeta(); - DEBUG_MSG(DLVL_HIGH, "Meta read in with %lu tracks", myMeta.tracks.size()); - return true; - } if (inFile.getMeta().moreheader < 0 || inFile.getMeta().tracks.size() == 0) { DEBUG_MSG(DLVL_FAIL, "Missing external header file"); return false; diff --git a/src/input/input_dtsc.h b/src/input/input_dtsc.h index 496d6b5a..0a0704e5 100644 --- a/src/input/input_dtsc.h +++ b/src/input/input_dtsc.h @@ -13,6 +13,7 @@ namespace Mist { void parseStreamHeader(); bool checkArguments(); bool readHeader(); + bool needHeader(); void getNext(bool smart = true); void seek(int seekTime); void trackSelect(std::string trackSpec); diff --git a/src/input/input_flv.cpp b/src/input/input_flv.cpp index a60f3b60..bebb880f 100644 --- a/src/input/input_flv.cpp +++ b/src/input/input_flv.cpp @@ -78,8 +78,6 @@ namespace Mist { bool inputFLV::readHeader() { if (!inFile){return false;} - //See whether a separate header file exists. - if (readExistingHeader()){return true;} //Create header file from FLV data Util::fseek(inFile, 13, SEEK_SET); AMF::Object amf_storage; diff --git a/src/input/input_mp3.cpp b/src/input/input_mp3.cpp index ebeb89ef..c640fda5 100644 --- a/src/input/input_mp3.cpp +++ b/src/input/input_mp3.cpp @@ -51,8 +51,6 @@ namespace Mist { bool inputMP3::readHeader() { if (!inFile){return false;} - //See whether a separate header file exists. - if (readExistingHeader()){return true;} myMeta = DTSC::Meta(); myMeta.tracks[1].trackID = 1; myMeta.tracks[1].type = "audio";