Added support for external writers

src/controller/controller.cpp

@@ -2,6 +2,7 @@
 /// Contains all code for the controller executable.
 
 #include "controller_api.h"
+#include "controller_external_writers.h"
 #include "controller_capabilities.h"
 #include "controller_connectors.h"
 #include "controller_push.h"
@@ -586,6 +587,9 @@ int main_loop(int argc, char **argv){
 #endif
   /*LTS-END*/
 
+  // Init external writer config
+  Controller::externalWritersToShm();
+
   // start main loop
   while (Controller::conf.is_active){
     Controller::conf.serveThreadedSocket(Controller::handleAPIConnection);

src/controller/controller_api.cpp

@@ -4,6 +4,7 @@
 #include "controller_statistics.h"
 #include "controller_storage.h"
 #include "controller_streams.h"
+#include "controller_external_writers.h"
 #include <dirent.h> //for browse API call
 #include <fstream>
 #include <mist/auth.h>
@@ -1203,6 +1204,13 @@ void Controller::handleAPICommands(JSON::Value &Request, JSON::Value &Response){
   if (Request.isMember("variable_add")){Controller::addVariable(Request["variable_add"], Response["variable_list"]);}
   if (Request.isMember("variable_remove")){Controller::removeVariable(Request["variable_remove"], Response["variable_list"]);}
 
+  if (Request.isMember("external_writer_remove")){Controller::removeExternalWriter(Request["external_writer_remove"]);}
+  if (Request.isMember("external_writer_add")){Controller::addExternalWriter(Request["external_writer_add"]);}
+  if (Request.isMember("external_writer_remove") || Request.isMember("external_writer_add") || 
+      Request.isMember("external_writer_list")){
+    Controller::listExternalWriters(Response["external_writer_list"]);
+  }
+
   Controller::writeConfig();
   Controller::configChanged = false;
 

src/controller/controller_external_writers.cpp

@@ -0,0 +1,173 @@
+#include "controller_external_writers.h"
+#include "controller_statistics.h"
+#include "controller_storage.h"
+#include <mist/downloader.h>
+#include <mist/bitfields.h>
+#include <mist/config.h>
+#include <mist/json.h>
+#include <mist/stream.h>
+#include <mist/triggers.h>
+#include <string>
+
+namespace Controller{
+  // Size of the shared memory page
+  static uint64_t pageSize = EXTWRITERS_INITSIZE;
+
+  /// \brief Writes external writers from the server config to shared memory
+  void externalWritersToShm(){
+    uint64_t writerCount = Controller::Storage["extwriters"].size();
+    IPC::sharedPage writersPage(EXTWRITERS, pageSize, false, false);
+    // If we have an existing page, set the reload flag
+    if (writersPage.mapped){
+      writersPage.master = true;
+      Util::RelAccX binAccx = Util::RelAccX(writersPage.mapped, false);
+      // Check if we need a bigger page
+      uint64_t sizeRequired = binAccx.getOffset() + binAccx.getRSize() * writerCount;
+      if (pageSize < sizeRequired){pageSize = sizeRequired;}
+      binAccx.setReload();
+    }
+    // Close & unlink any existing page and create a new one
+    writersPage.close();
+    writersPage.init(EXTWRITERS, pageSize, true, false);
+    Util::RelAccX exwriAccx = Util::RelAccX(writersPage.mapped, false);
+    exwriAccx = Util::RelAccX(writersPage.mapped, false);
+    exwriAccx.addField("name", RAX_32STRING);
+    exwriAccx.addField("cmdline", RAX_256STRING);
+    exwriAccx.addField("protocols", RAX_NESTED, RAX_64STRING * writerCount * 8);
+    // Set amount of records that can fit and how many will be used
+    uint64_t reqCount = (pageSize - exwriAccx.getOffset()) / exwriAccx.getRSize();
+    exwriAccx.setRCount(reqCount);
+    exwriAccx.setPresent(reqCount);
+    exwriAccx.setEndPos(writerCount);
+    // Do the same for the nested protocol field
+    uint64_t index = 0;
+    jsonForEach(Controller::Storage["extwriters"], it){
+      std::string name = (*it)[0u].asString();
+      std::string cmdline = (*it)[1u].asString();
+      exwriAccx.setString("name", name, index);
+      exwriAccx.setString("cmdline", cmdline, index);
+      // Create nested field for source match
+      uint8_t protocolCount = (*it)[2u].size();
+      Util::RelAccX protocolAccx = Util::RelAccX(exwriAccx.getPointer("protocols", index), false);
+      protocolAccx.addField("protocol", RAX_64STRING);
+      protocolAccx.setRCount(protocolCount);
+      protocolAccx.setPresent(protocolCount);
+      protocolAccx.setEndPos(protocolCount);
+      uint8_t binIt = 0;
+      jsonForEach((*it)[2u], protIt){
+        std::string thisProtocol = (*protIt).asString();
+        protocolAccx.setString("protocol", thisProtocol, binIt);
+        binIt++;
+      }
+      index++;
+      protocolAccx.setReady();
+    }
+    exwriAccx.setReady();
+    // Leave the page in memory after returning
+    writersPage.master = false;
+  }
+
+  /// \brief Adds a new generic writer binary to the server config
+  /// The request should contain:
+  /// - name: name given to this binary in order to edit/remove it's entry in Mists config
+  /// - cmdline: command line including arguments
+  /// - supported URL protocols: used to identify for what targets we need to run the executable
+  void addExternalWriter(JSON::Value &request){
+    std::string name;
+    std::string cmdline;
+    JSON::Value protocols;
+    bool isNew = true;
+    if (request.isArray()){
+      if(request.size() == 4){
+        name = request[0u].asString();
+        cmdline = request[1u].asString();
+        protocols = request[2u];
+      }else{
+        ERROR_MSG("Cannot add external writer, as the request contained %u variables. Required variables are: name, cmdline and protocols", request.size());
+        return;
+      }
+    }else{
+      name = request["name"].asString();
+      cmdline = request["cmdline"].asString();
+      protocols = request["protocols"];
+    }
+    //convert protocols from string to array if needed
+    if (protocols.isString()){protocols.append(protocols.asString());}
+    if (!name.size()){
+      ERROR_MSG("Blank or missing name in request");
+      return;
+    }
+    if (!cmdline.size()){
+      ERROR_MSG("Blank or missing cmdline in request");
+      return;
+    }
+    if (!protocols.size()){
+      ERROR_MSG("Missing protocols in request");
+      return;
+    }
+    if (name.size() > 31){
+      name = name.substr(0, 31);
+      WARN_MSG("Maximum name length is 31 characters, truncating name to '%s'", name.c_str());
+    }
+    if (cmdline.size() > 255){
+      cmdline.erase(255);
+      WARN_MSG("Maximum cmdline length is 255 characters, truncating cmdline to '%s'", cmdline.c_str());
+    }
+    jsonForEach(protocols, protIt){
+      if ((*protIt).size() > 63){
+        (*protIt) = (*protIt).asString().substr(0, 63);
+        WARN_MSG("Maximum protocol length is 63 characters, truncating protocol to '%s'", (*protIt).asStringRef().c_str());
+      }
+    }
+
+    // Check if we have an existing variable with the same name to modify
+    jsonForEach(Controller::Storage["extwriters"], it){
+      if ((*it)[0u].asString() == name){
+        INFO_MSG("Modifying existing external writer '%s'", name.c_str());
+        (*it)[1u] = cmdline;
+        (*it)[2u] = protocols;
+        isNew = false;
+        break;
+      }
+    }
+    // Else push a new custom variable to the list
+    if (isNew){
+      INFO_MSG("Adding new external writer '%s'", name.c_str());
+      JSON::Value thisVar;
+      thisVar.append(name);
+      thisVar.append(cmdline);
+      thisVar.append(protocols);
+      Controller::Storage["extwriters"].append(thisVar);
+    }
+    // Modify shm
+    externalWritersToShm();
+  }
+
+  /// \brief Fills output with all defined external writers
+  void listExternalWriters(JSON::Value &output){
+    output = Controller::Storage["extwriters"];
+  }
+
+  /// \brief Removes the external writer name contained in the request from shm and the sever config
+  void removeExternalWriter(const JSON::Value &request){
+    std::string name;
+    if (request.isString()){
+      name = request.asStringRef();
+    }else if (request.isArray()){
+      name = request[0u].asStringRef();
+    }else if (request.isMember("name")){
+      name = request["name"].asStringRef();
+    }
+    if (!name.size()){
+      WARN_MSG("Aborting request to remove an external writer, as no name was given");
+      return;
+    }
+    // Modify config
+    jsonForEach(Controller::Storage["extwriters"], it){
+      if ((*it)[0u].asString() == name){it.remove();}
+    }
+    // Modify shm
+    externalWritersToShm();
+  }
+}// namespace Controller
+

src/controller/controller_external_writers.h

@@ -0,0 +1,14 @@
+#include <mist/config.h>
+#include <mist/json.h>
+#include <mist/tinythread.h>
+#include <string>
+
+namespace Controller{
+  // API calls to manage external writers
+  void addExternalWriter(JSON::Value &request);
+  void listExternalWriters(JSON::Value &output);
+  void removeExternalWriter(const JSON::Value &request);
+
+  // internal use only
+  void externalWritersToShm();
+}// namespace Controller

src/controller/meson.build

@@ -3,6 +3,7 @@ executables += {
   'name': 'MistController', 
   'sources' : [
     files( 'controller.cpp',
+           'controller_external_writers.cpp',
            'controller_updater.cpp',
            'controller_streams.cpp',
            'controller_storage.cpp',

src/input/input.cpp

@@ -8,11 +8,11 @@
 #include <iterator>
 #include <mist/auth.h>
 #include <mist/defines.h>
-#include <mist/downloader.h>
 #include <mist/encode.h>
 #include <mist/procs.h>
 #include <mist/stream.h>
 #include <mist/triggers.h>
+#include <mist/urireader.h>
 #include <sstream>
 #include <sys/wait.h>
 
@@ -587,6 +587,7 @@ namespace Mist{
     }
     // close file
     file.close();
+    // Export DTSH to a file or remote target
     outMeta.toFile(fileName + ".dtsh");
   }
 
@@ -1496,7 +1497,20 @@ namespace Mist{
         return true;
       }
     }
-    meta.reInit(config->getString("streamname"), config->getString("input") + ".dtsh");
+    // Try to read any existing DTSH file
+    std::string fileName = config->getString("input") + ".dtsh";
+    HIGH_MSG("Loading metadata for stream '%s' from file '%s'", streamName.c_str(), fileName.c_str());
+    char *scanBuf;
+    uint64_t fileSize;
+    HTTP::URIReader inFile(fileName);
+    if (!inFile){return false;}
+    inFile.readAll(scanBuf, fileSize);
+    inFile.close();
+    if (!fileSize){return false;}
+    DTSC::Packet pkt(scanBuf, fileSize, true);
+    HIGH_MSG("Retrieved header of %lu bytes", fileSize);
+    meta.reInit(streamName, pkt.getScan());
+
     if (meta.version != DTSH_VERSION){
       INFO_MSG("Updating wrong version header file from version %u to %u", meta.version, DTSH_VERSION);
       return false;

src/output/output.cpp

@@ -2287,24 +2287,20 @@ namespace Mist{
     sentHeader = true;
   }
 
+  /// \brief Makes the generic writer available to output classes
+  /// \param file target URL or filepath
+  /// \param append whether to open this connection in truncate or append mode
+  /// \param conn connection which will be used to send data. Will use Output's internal myConn if not initialised
   bool Output::connectToFile(std::string file, bool append, Socket::Connection *conn){
-    if (!conn){conn = &myConn;}
-    int flags = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
-    int mode = O_RDWR | O_CREAT | (append ? O_APPEND : O_TRUNC);
-    if (!Util::createPathFor(file)){
-      ERROR_MSG("Cannot not create file %s: could not create parent folder", file.c_str());
-      return false;
-    }
-    int outFile = open(file.c_str(), mode, flags);
-    if (outFile < 0){
-      ERROR_MSG("Failed to open file %s, error: %s", file.c_str(), strerror(errno));
-      return false;
-    }
-    if (*conn){
+    int outFile = -1;
+    if (!conn) {conn = &myConn;}
+    bool isFileTarget = HTTP::URL(file).isLocalPath();
+    if (!Util::externalWriter(file, outFile, append)){return false;}
+    if (*conn && isFileTarget) {
       flock(conn->getSocket(), LOCK_UN | LOCK_NB);
     }
     // Lock the file in exclusive mode to ensure no other processes write to it
-    if(flock(outFile, LOCK_EX | LOCK_NB)){
+    if(isFileTarget && flock(outFile, LOCK_EX | LOCK_NB)){
       ERROR_MSG("Failed to lock file %s, error: %s", file.c_str(), strerror(errno));
       return false;
     }
@@ -2319,7 +2315,7 @@ namespace Mist{
 
     int r = dup2(outFile, conn->getSocket());
     if (r == -1){
-      ERROR_MSG("Failed to create an alias for the socket using dup2: %s.", strerror(errno));
+      ERROR_MSG("Failed to create an alias for the socket %d -> %d using dup2: %s.", outFile, conn->getSocket(), strerror(errno));
       return false;
     }
     close(outFile);