More backports from Pro edition, among which HTTPS/TLS support

src/analysers/analyser_dtsc.cpp

@@ -14,7 +14,7 @@ void AnalyserDTSC::init(Util::Config &conf){
 
 bool AnalyserDTSC::open(const std::string &filename){
   if (!Analyser::open(filename)){return false;}
-  conn = Socket::Connection(1, 0);
+  conn.open(1, 0);
   totalBytes = 0;
   return true;
 }

src/controller/controller_api.cpp

@@ -492,6 +492,7 @@ void Controller::handleAPICommands(JSON::Value & Request, JSON::Value & Response
     Response["capabilities"] = capabilities;
   }
 
+
   if(Request.isMember("browse")){                    
     if(Request["browse"] == ""){
       Request["browse"] = ".";

src/controller/controller_connectors.cpp

@@ -37,9 +37,11 @@ namespace Controller {
     }
     memset(f.mapped, 0, 32);
     Util::RelAccX A(f.mapped, false);
+    if (!A.isReady()){
     A.addField("cmd", RAX_128STRING);
     A.addField("pid", RAX_64UINT);
     A.setReady();
+    }
     uint32_t count = 0;
     std::map<std::string, pid_t>::iterator it;
     for (it = currentConnectors.begin(); it != currentConnectors.end(); ++it){

src/input/input_buffer.cpp

@@ -581,16 +581,34 @@ namespace Mist {
 
         std::string trackIdentifier = trackMeta.tracks.find(value)->second.getIdentifier();
         DEBUG_MSG(DLVL_HIGH, "Attempting colision detection for track %s", trackIdentifier.c_str());
-
+        //Get the identifier for the track, and attempt colission detection.
+        int collidesWith = -1;
+        for (std::map<unsigned int, DTSC::Track>::iterator it = myMeta.tracks.begin(); it != myMeta.tracks.end(); it++) {
+          //If the identifier of an existing track and the current track match, assume the are the same track and reject the negotiated one.
+          ///\todo Maybe switch to a new form of detecting collisions, especially with regards to multiple audio languages and camera angles.
+          if (it->second.getIdentifier() == trackIdentifier) {
+            collidesWith = it->first;
+            break;
+          }
+        }
         //Remove the "negotiate" status in either case
         negotiatingTracks.erase(value);
         //Set master to true before erasing the page, because we are responsible for cleaning up unused pages
         nProxy.metaPages[value].master = true;
         nProxy.metaPages.erase(value);
 
-        int finalMap = 3;
-        if (trackMeta.tracks.find(value)->second.type == "video"){finalMap = 1;}
-        if (trackMeta.tracks.find(value)->second.type == "audio"){finalMap = 2;}
+        //Check if the track collides, and whether the track it collides with is active.
+        if (collidesWith != -1 && activeTracks.count(collidesWith)) { /*LTS*/
+          //Print a warning message and set the state of the track to rejected.
+          WARN_MSG("Collision of temporary track %lu with existing track %d detected. Handling as a new valid track.", value, collidesWith);
+          collidesWith = -1;
+        }
+        uint64_t finalMap = collidesWith;
+        if (finalMap == -1) {
+          //No collision has been detected, assign a new final number
+          finalMap = (myMeta.tracks.size() ? myMeta.tracks.rbegin()->first : 0) + 1;
+          DEBUG_MSG(DLVL_DEVEL, "No colision detected for temporary track %lu from user %u, assigning final track number %lu", value, id, finalMap);
+        }
         //Resume either if we have more than 1 keyframe on the replacement track (assume it was already pushing before the track "dissapeared")
         //or if the firstms of the replacement track is later than the lastms on the existing track
         if (!myMeta.tracks.count(finalMap) || trackMeta.tracks.find(value)->second.keys.size() > 1 || trackMeta.tracks.find(value)->second.firstms >= myMeta.tracks[finalMap].lastms) {

src/input/input_dtsc.cpp

@@ -144,7 +144,7 @@ namespace Mist {
   bool inputDTSC::openStreamSource() {
     std::string source = config->getString("input");
     if (source == "-"){
-      srcConn = Socket::Connection(fileno(stdout),fileno(stdin));
+      srcConn.open(fileno(stdout),fileno(stdin));
       return true;
     }
     if (source.find("dtsc://") == 0) {
@@ -159,7 +159,7 @@ namespace Mist {
     if (streamName == "") {
       streamName = givenStream;
     }
-    srcConn = Socket::Connection(host, port, true);
+    srcConn.open(host, port, true);
     if (!srcConn.connected()){
       return false;
     }

src/input/input_h264.cpp

@@ -44,9 +44,9 @@ namespace Mist{
 
       int fin = -1, fout = -1;
       inputProcess = Util::Procs::StartPiped(args, &fin, &fout, 0);
-      myConn = Socket::Connection(-1, fout);
+      myConn.open(-1, fout);
     }else{
-      myConn = Socket::Connection(fileno(stdout), fileno(stdin));
+      myConn.open(fileno(stdout), fileno(stdin));
     }
     myConn.Received().splitter.assign("\000\000\001", 3);
     myMeta.vod = false;

src/io.cpp

@@ -57,16 +57,17 @@ namespace Mist {
   bool InOutBase::bufferStart(unsigned long tid, unsigned long pageNumber) {
     VERYHIGH_MSG("bufferStart for stream %s, track %lu, page %lu", streamName.c_str(), tid, pageNumber);
     //Initialize the stream metadata if it does not yet exist
+#ifndef TSLIVE_INPUT
     if (!nProxy.metaPages.count(0)) {
       initiateMeta();
     }
+#endif
     //If we are a stand-alone player skip track negotiation, as there will be nothing to negotiate with.
     if (standAlone) {
       if (!nProxy.trackMap.count(tid)) {
         nProxy.trackMap[tid] = tid;
       }
     }
-    //Negotiate the requested track if needed.
     return nProxy.bufferStart(tid, pageNumber, myMeta);
   }
 
@@ -667,6 +668,10 @@ namespace Mist {
           tmpMeta.tracks[newTid] = myMeta.tracks[tid];
           tmpMeta.tracks[newTid].trackID = newTid;
           JSON::Value tmpVal = tmpMeta.toJSON();
+          if (!myMeta.tracks[tid].type.size() || !myMeta.tracks[tid].codec.size()){
+            FAIL_MSG("Negotiating a track without metadata. This is a serious issue, please report this to the developers.");
+            BACKTRACE;
+          }
           std::string tmpStr = tmpVal.toNetPacked();
           memcpy(metaPages[tid].mapped, tmpStr.data(), tmpStr.size());
           HIGH_MSG("Temporary metadata written for incoming track %lu, handling as track %lu", tid, newTid);

src/output/output.cpp

@@ -1053,7 +1053,7 @@ namespace Mist{
       //actually drop what we found.
       //if both of the above cases occur, the next prepareNext iteration will take care of that
       for (std::set<uint32_t>::iterator it = dropTracks.begin(); it != dropTracks.end(); ++it){
-        dropTrack(*it, "seek/select mismatch", true);
+        dropTrack(*it, "seek/select mismatch");
       }
       return false;
     }
@@ -1061,7 +1061,7 @@ namespace Mist{
     sortedPageInfo nxt = *(buffer.begin());
 
     if (!myMeta.tracks.count(nxt.tid)){
-      dropTrack(nxt.tid, "disappeared from metadata", true);
+      dropTrack(nxt.tid, "disappeared from metadata");
       return false;
     }
 
@@ -1105,7 +1105,7 @@ namespace Mist{
       }
       //for VoD, check if we've reached the end of the track, if so, drop it
       if (myMeta.vod && nxt.time > myMeta.tracks[nxt.tid].lastms){
-        dropTrack(nxt.tid, "Reached end of track");
+        dropTrack(nxt.tid, "Reached end of track", false);
       }
       //if this is a live stream, we might have just reached the live point.
       //check where the next key is
@@ -1137,9 +1137,9 @@ namespace Mist{
       loadPageForKey(nxt.tid, ++nxtKeyNum[nxt.tid]);
       nxt.offset = 0;
       if (nProxy.curPage.count(nxt.tid) && nProxy.curPage[nxt.tid].mapped){
-        unsigned long long nextTime = getDTSCTime(nProxy.curPage[nxt.tid].mapped, nxt.offset);
+        uint64_t nextTime = getDTSCTime(nProxy.curPage[nxt.tid].mapped, nxt.offset);
         if (nextTime && nextTime < nxt.time){
-          dropTrack(nxt.tid, "time going backwards");
+          dropTrack(nxt.tid, "EOP: time going backwards ("+JSON::Value(nextTime).asString()+" < "+JSON::Value(nxt.time).asString()+")");
         }else{
           if (nextTime){
             nxt.time = nextTime;

src/output/output.h

@@ -22,9 +22,9 @@ namespace Mist {
       }
       return (time == rhs.time && tid < rhs.tid);
     }
-    unsigned int tid;
-    long long unsigned int time;
-    unsigned int offset;
+    uint64_t tid;
+    uint64_t time;
+    uint32_t offset;
   };
 
   /// The output class is intended to be inherited by MistOut process classes.

src/output/output_http_internal.cpp

@@ -41,8 +41,7 @@ namespace Mist {
       std::string host = getConnectedHost();
       dup2(myConn.getSocket(), STDIN_FILENO);
       dup2(myConn.getSocket(), STDOUT_FILENO);
-      myConn.drop();
-      myConn = Socket::Connection(STDOUT_FILENO, STDIN_FILENO);
+      myConn.open(STDOUT_FILENO, STDIN_FILENO);
       myConn.setHost(host);
     }
     if (config->getOption("wrappers",true).size() == 0 || config->getString("wrappers") == ""){
@@ -279,6 +278,10 @@ namespace Mist {
     
     std::string devSkin = "";
     if (H.GetVar("dev").size()) { devSkin = ",skin:\"dev\""; }
+    H.SetVar("stream", "");
+    H.SetVar("dev", "");
+    devSkin += ",urlappend:\"" + H.allVars() + "\"";
+    H.SetVar("stream", streamName);
     
     H.Clean();
     H.SetHeader("Content-Type", "text/html");

src/output/output_https.cpp

@@ -0,0 +1,241 @@
+#include "output_https.h"
+#include <mist/procs.h>
+
+namespace Mist{
+  mbedtls_entropy_context OutHTTPS::entropy;
+  mbedtls_ctr_drbg_context OutHTTPS::ctr_drbg;
+  mbedtls_ssl_config OutHTTPS::sslConf;
+  mbedtls_x509_crt OutHTTPS::srvcert;
+  mbedtls_pk_context OutHTTPS::pkey;
+
+  void OutHTTPS::init(Util::Config *cfg){
+    Output::init(cfg);
+    capa["name"] = "HTTPS";
+    capa["friendly"] = "HTTPS (HTTP+TLS)";
+    capa["desc"] = "HTTPS connection handler, provides all enabled HTTP-based outputs";
+    capa["provides"] = "HTTP";
+    capa["protocol"] = "https://";
+    capa["required"]["cert"]["name"] = "Certificate";
+    capa["required"]["cert"]["help"] = "(Root) certificate(s) file(s) to append to chain";
+    capa["required"]["cert"]["option"] = "--cert";
+    capa["required"]["cert"]["short"] = "C";
+    capa["required"]["cert"]["default"] = "";
+    capa["required"]["cert"]["type"] = "str";
+    capa["required"]["key"]["name"] = "Key";
+    capa["required"]["key"]["help"] = "Private key for SSL";
+    capa["required"]["key"]["option"] = "--key";
+    capa["required"]["key"]["short"] = "K";
+    capa["required"]["key"]["default"] = "";
+    capa["required"]["key"]["type"] = "str";
+
+    capa["optional"]["wrappers"]["name"] = "Active players";
+    capa["optional"]["wrappers"]["help"] = "Which players are attempted and in what order.";
+    capa["optional"]["wrappers"]["default"] = "";
+    capa["optional"]["wrappers"]["type"] = "ord_multi_sel";
+    capa["optional"]["wrappers"]["allowed"].append("html5");
+    capa["optional"]["wrappers"]["allowed"].append("videojs");
+    capa["optional"]["wrappers"]["allowed"].append("dashjs");
+    capa["optional"]["wrappers"]["allowed"].append("flash_strobe");
+    capa["optional"]["wrappers"]["allowed"].append("silverlight");
+    capa["optional"]["wrappers"]["allowed"].append("img");
+    capa["optional"]["wrappers"]["option"] = "--wrappers";
+    capa["optional"]["wrappers"]["short"] = "w";
+    cfg->addConnectorOptions(4433, capa);
+    config = cfg;
+  }
+
+  OutHTTPS::OutHTTPS(Socket::Connection &C) : Output(C){
+    int ret;
+    mbedtls_net_init(&client_fd);
+    client_fd.fd = C.getSocket();
+    mbedtls_ssl_init(&ssl);
+    if ((ret = mbedtls_ctr_drbg_reseed(&ctr_drbg, (const unsigned char *)"child", 5)) != 0){
+      FAIL_MSG("Could not reseed");
+      C.close();
+      return;
+    }
+
+    // Set up the SSL connection
+    if ((ret = mbedtls_ssl_setup(&ssl, &sslConf)) != 0){
+      FAIL_MSG("Could not set up SSL connection");
+      C.close();
+      return;
+    }
+
+    // Inform mbedtls how we'd like to use the connection (uses default bio handlers)
+    // We tell it to use non-blocking IO here
+    mbedtls_net_set_nonblock(&client_fd);
+    mbedtls_ssl_set_bio(&ssl, &client_fd, mbedtls_net_send, mbedtls_net_recv, NULL);
+    // do the SSL handshake
+    while ((ret = mbedtls_ssl_handshake(&ssl)) != 0){
+      if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE){
+        char error_buf[200];
+        mbedtls_strerror(ret, error_buf, 200);
+        MEDIUM_MSG("Could not handshake, SSL error: %s (%d)", error_buf, ret);
+        C.close();
+        return;
+      }else{
+        Util::sleep(100);
+      }
+    }
+    HIGH_MSG("Started SSL connection handler");
+  }
+
+  int OutHTTPS::run(){
+    unsigned char buf[1024 * 4]; // 4k internal buffer
+    int ret;
+
+    // Start a MistOutHTTP process, connected to this SSL connection
+    int fderr = 2;
+    int fd[2];
+    if (socketpair(PF_LOCAL, SOCK_STREAM, 0, fd) != 0){
+      FAIL_MSG("Could not open anonymous socket for SSL<->HTTP connection!");
+      return 1;
+    }
+    std::deque<std::string> args;
+    args.push_back(Util::getMyPath() + "MistOutHTTP");
+    args.push_back("--ip");
+    args.push_back(myConn.getHost());
+    if (config->getString("nostreamtext").size()){
+      args.push_back("--nostreamtext");
+      args.push_back(config->getString("nostreamtext"));
+    }
+    if (config->getString("pubaddr").size()){
+      args.push_back("--public-address");
+      args.push_back(config->getString("pubaddr"));
+    }
+    args.push_back("");
+    Util::Procs::socketList.insert(fd[0]);
+    pid_t http_proc = Util::Procs::StartPiped(args, &(fd[1]), &(fd[1]), &fderr);
+    close(fd[1]);
+    if (http_proc < 2){
+      FAIL_MSG("Could not spawn MistOutHTTP process for SSL connection!");
+      return 1;
+    }
+    Socket::Connection http(fd[0]);
+    http.setBlocking(false);
+    Socket::Buffer &http_buf = http.Received();
+
+    // pass data back and forth between the SSL connection and HTTP process while connected
+    while (config->is_active && http){
+      bool activity = false;
+      // attempt to read SSL data, pass to HTTP
+      ret = mbedtls_ssl_read(&ssl, buf, sizeof(buf));
+      if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE){
+        if (ret <= 0){
+          HIGH_MSG("SSL disconnect!");
+          break;
+        }
+        // we received ret bytes of data to pass on. Do so.
+        activity = true;
+        http.SendNow((const char *)buf, ret);
+      }
+
+      // attempt to read HTTP data, pass to SSL
+      if (http.spool() || http_buf.size()){
+        // We have data - pass it on
+        activity = true;
+        while (http_buf.size() && http){
+          int todo = http_buf.get().size();
+          int done = 0;
+          while (done < todo){
+            ret = mbedtls_ssl_write(&ssl, (const unsigned char*)http_buf.get().data() + done, todo - done);
+            if (ret == MBEDTLS_ERR_NET_CONN_RESET || ret == MBEDTLS_ERR_SSL_CLIENT_RECONNECT){
+              HIGH_MSG("SSL disconnect!");
+              http.close();
+              break;
+            }
+            if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE){
+              done += ret;
+            }else{
+              Util::sleep(50);
+            }
+          }
+          http_buf.get().clear();
+        }
+      }
+      if (!activity){
+        Util::sleep(50);
+      }
+    }
+    // close the HTTP process (close stdio, kill its PID)
+    http.close();
+    Util::Procs::Stop(http_proc);
+    uint16_t waiting = 0;
+    while (++waiting < 100){
+      if (!Util::Procs::isRunning(http_proc)){break;}
+      Util::sleep(100);
+    }
+    return 0;
+  }
+
+
+  OutHTTPS::~OutHTTPS(){
+    HIGH_MSG("Ending SSL connection handler");
+    // close when we're done
+    mbedtls_ssl_close_notify(&ssl);
+    mbedtls_ssl_free(&ssl);
+    mbedtls_net_free(&client_fd);
+    myConn.close();
+  }
+
+  /// Listens for HTTPS requests, accepting them and connecting them to a HTTP socket
+  void OutHTTPS::listener(Util::Config &conf, int (*callback)(Socket::Connection &S)){
+    if (config->getOption("cert", true).size() < 2 || config->getOption("key", true).size() < 2){
+      FAIL_MSG("The cert/key required options were not passed!");
+      return;
+    }
+
+    //Declare and set up all required mbedtls structures
+    int ret;
+    mbedtls_ssl_config_init(&sslConf);
+    mbedtls_entropy_init(&entropy);
+    mbedtls_pk_init(&pkey);
+    mbedtls_x509_crt_init(&srvcert);
+    mbedtls_ctr_drbg_init(&ctr_drbg);
+
+    // seed the rng
+    if ((ret = mbedtls_ctr_drbg_seed(&ctr_drbg, mbedtls_entropy_func, &entropy, (const unsigned char *)"MistServer", 10)) != 0){
+      FAIL_MSG("Could not seed the random number generator!");
+    }
+
+    //Read certificate chain(s) from cmdline option(s)
+    JSON::Value certs = config->getOption("cert", true);
+    jsonForEach(certs, it){
+      if (it->asStringRef().size()){//Ignore empty entries (default is empty)
+        ret = mbedtls_x509_crt_parse_file(&srvcert, it->asStringRef().c_str());
+        if (ret != 0){
+          WARN_MSG("Could not load any certificates from file: %s", it->asStringRef().c_str());
+        }
+      }
+    }
+
+    //Read key from cmdline option
+    ret = mbedtls_pk_parse_keyfile(&pkey, config->getString("key").c_str(), 0);
+    if (ret != 0){
+      FAIL_MSG("Could not load any keys from file: %s", config->getString("key").c_str());
+      return;
+    }
+
+    if ((ret = mbedtls_ssl_config_defaults(&sslConf, MBEDTLS_SSL_IS_SERVER, MBEDTLS_SSL_TRANSPORT_STREAM, MBEDTLS_SSL_PRESET_DEFAULT)) != 0){
+      FAIL_MSG("SSL config defaults failed");
+      return;
+    }
+    mbedtls_ssl_conf_rng(&sslConf, mbedtls_ctr_drbg_random, &ctr_drbg);
+    mbedtls_ssl_conf_ca_chain(&sslConf, srvcert.next, NULL);
+    if ((ret = mbedtls_ssl_conf_own_cert(&sslConf, &srvcert, &pkey)) != 0){
+      FAIL_MSG("SSL config own certificate failed");
+      return;
+    }
+
+    Output::listener(conf, callback);
+
+    //Free all the mbedtls structures
+    mbedtls_x509_crt_free(&srvcert);
+    mbedtls_pk_free(&pkey);
+    mbedtls_ssl_config_free(&sslConf);
+    mbedtls_ctr_drbg_free(&ctr_drbg);
+    mbedtls_entropy_free(&entropy);
+  }
+}
+

src/output/output_https.h

@@ -0,0 +1,34 @@
+#pragma once
+#include <mist/defines.h>
+#include "output.h"
+#include <mbedtls/certs.h>
+#include <mbedtls/ctr_drbg.h>
+#include <mbedtls/entropy.h>
+#include <mbedtls/net.h>
+#include <mbedtls/ssl.h>
+#include <mbedtls/timing.h>
+#include <mbedtls/x509.h>
+
+namespace Mist {
+
+  class OutHTTPS : public Output {
+    public:
+      OutHTTPS(Socket::Connection & C);
+      virtual ~OutHTTPS();
+      void onRequest(){};
+      int run();
+      static bool listenMode(){return true;}
+      static void init(Util::Config * cfg);
+      static void listener(Util::Config & conf, int (*callback)(Socket::Connection & S));
+    private:
+      mbedtls_net_context client_fd;
+      mbedtls_ssl_context ssl;
+      static mbedtls_entropy_context entropy;
+      static mbedtls_ctr_drbg_context ctr_drbg;
+      static mbedtls_ssl_config sslConf;
+      static mbedtls_x509_crt srvcert;
+      static mbedtls_pk_context pkey;
+  };
+}
+
+typedef Mist::OutHTTPS mistOut;

src/output/output_ts.cpp

@@ -38,7 +38,7 @@ namespace Mist {
     capa["friendly"] = "TS over TCP";
     capa["desc"] = "Real time streaming in MPEG2/TS format over raw TCP";
     capa["deps"] = "";
-    capa["required"]["streamname"]["name"] = "Stream";
+    capa["required"]["streamname"]["name"] = "Source stream";
     capa["required"]["streamname"]["help"] = "What streamname to serve. For multiple streams, add this protocol multiple times using different ports.";
     capa["required"]["streamname"]["type"] = "str";
     capa["required"]["streamname"]["option"] = "--stream";