Custom variables support

Change-Id: Ia847a868d1803dc80fdb8a627d57bba571cdee64

lib/defines.h

@@ -204,6 +204,8 @@ static inline void show_stackframe(){}
 #define COMMS_SESSIONS "MstSession%s"
 #define COMMS_SESSIONS_INITSIZE 8 * 1024 * 1024
 
+#define CUSTOM_VARIABLES_INITSIZE 64 * 1024
+
 #define SEM_STATISTICS "/MstStat"
 #define SEM_USERS "/MstUser%s" //%s stream name
 
@@ -238,6 +240,7 @@ static inline void show_stackframe(){}
 #define SHM_STATE_LOGS "MstStateLogs"
 #define SHM_STATE_ACCS "MstStateAccs"
 #define SHM_STATE_STREAMS "MstStateStreams"
+#define SHM_CUSTOM_VARIABLES "MstVars"
 #define NAME_BUFFER_SIZE 200 // char buffer size for snprintf'ing shm filenames
 #define SHM_SESSIONS "/MstSess"
 #define SHM_SESSIONS_ITEM 165     // 4 byte crc, 100b streamname, 20b connector, 40b host, 1b sync

lib/procs.cpp

@@ -234,19 +234,96 @@ void Util::Procs::childsig_handler(int signum){
 }
 
 /// Runs the given command and returns the stdout output as a string.
-std::string Util::Procs::getOutputOf(char *const *argv){
+/// \param maxWait amount of milliseconds to wait for new output to come in over stdout before aborting
+std::string Util::Procs::getOutputOf(char *const *argv, uint64_t maxWait){
   int fin = 0, fout = -1, ferr = 0;
+  uint64_t waitedFor = 0;
+  uint8_t tries = 0;
   pid_t myProc = StartPiped(argv, &fin, &fout, &ferr);
   Socket::Connection O(-1, fout);
+  O.setBlocking(false);
   Util::ResizeablePointer ret;
   while (childRunning(myProc) || O){
     if (O.spool() || O.Received().size()){
+      waitedFor = 0;
+      tries = 0;
       while (O.Received().size()){
         std::string & t = O.Received().get();
         ret.append(t);
         t.clear();
       }
     }else{
+      if (maxWait && waitedFor > maxWait){
+        WARN_MSG("Timeout while getting output of '%s', returning %luB of data",  (char *)argv, ret.size());
+        break;
+      }
+      else if(maxWait){
+        uint64_t waitTime = Util::expBackoffMs(tries++, 10, maxWait);
+        Util::sleep(waitTime);
+        waitedFor += waitTime;
+      }
+      else{
+        Util::sleep(50);
+      }
+    }
+  }
+  return std::string(ret, ret.size());
+}
+
+/// Runs the given command and returns the stdout output as a string.
+/// \param maxWait amount of milliseconds to wait before shutting down the spawned process
+/// \param maxValBytes amount of Bytes allowed in the output before shutting down the spawned process
+std::string Util::Procs::getLimitedOutputOf(char *const *argv, uint64_t maxWait, uint32_t maxValBytes){
+  int fin = 0, fout = -1, ferr = 0;
+  uint64_t waitedFor = 0;
+  uint8_t tries = 0;
+  pid_t myProc = StartPiped(argv, &fin, &fout, &ferr);
+  Socket::Connection O(-1, fout);
+  O.setBlocking(false);
+  Util::ResizeablePointer ret;
+  std::string fullCmd;
+  uint8_t idx = 0;
+  while (argv[idx]){
+    fullCmd += argv[idx++];
+    fullCmd += " ";
+  }
+  while (childRunning(myProc) || O){
+    if (O.spool() || O.Received().size()){
+      tries = 0;
+      while (O.Received().size()){
+        std::string & t = O.Received().get();
+        ret.append(t);
+        t.clear();
+      }
+    }else{
+      if (waitedFor > maxWait){
+        WARN_MSG("Reached timeout of %lu ms. Killing process with command %s...", maxWait, fullCmd.c_str());
+        break;
+      }
+      else {
+        uint64_t waitTime = Util::expBackoffMs(tries++, 10, maxWait);
+        Util::sleep(waitTime);
+        waitedFor += waitTime;
+      }
+    }
+    if (ret.size() > maxValBytes){
+      WARN_MSG("Have a limit of %uB, but received %luB of data. Killing process with command %s...",  maxValBytes, ret.size(), fullCmd.c_str());
+      break;
+    }
+  }
+  // Stop the process if it is still running
+  if (childRunning(myProc)){
+    close(fout);
+    Stop(myProc);
+    waitedFor = 0;
+  }
+  // Give it a few seconds, but then forcefully stop it
+  while (childRunning(myProc)){
+    if (waitedFor > 2000){
+      Murder(myProc);
+      break;
+    }else{
+      waitedFor += 50;
       Util::sleep(50);
     }
   }
@@ -261,10 +338,10 @@ char *const *Util::Procs::dequeToArgv(std::deque<std::string> &argDeq){
   return ret;
 }
 
-std::string Util::Procs::getOutputOf(std::deque<std::string> &argDeq){
+std::string Util::Procs::getOutputOf(std::deque<std::string> &argDeq, uint64_t maxWait){
   std::string ret;
   char *const *argv = dequeToArgv(argDeq); // Note: Do not edit deque before executing command
-  ret = getOutputOf(argv);
+  ret = getOutputOf(argv, maxWait);
   return ret;
 }
 

lib/procs.h

@@ -8,6 +8,7 @@
 #include <string>
 #include <unistd.h>
 #include <vector>
+#include <stdint.h> 
 
 /// Contains utility code, not directly related to streaming media
 namespace Util{
@@ -29,8 +30,9 @@ namespace Util{
     static void fork_prepare();
     static void fork_complete();
     static void setHandler();
-    static std::string getOutputOf(char *const *argv);
-    static std::string getOutputOf(std::deque<std::string> &argDeq);
+    static std::string getOutputOf(char *const *argv, uint64_t maxWait = 0);
+    static std::string getOutputOf(std::deque<std::string> &argDeq, uint64_t maxWait = 0);
+    static std::string getLimitedOutputOf(char *const *argv, uint64_t maxWait, uint32_t maxValBytes);
     static pid_t StartPiped(const char *const *argv, int *fdin, int *fdout, int *fderr);
     static pid_t StartPiped(std::deque<std::string> &argDeq, int *fdin, int *fdout, int *fderr);
     static void Stop(pid_t name);

lib/stream.cpp

@@ -107,35 +107,87 @@ std::string Util::codecString(const std::string &codec, const std::string &initD
   return "";
 }
 
+/// Local-only helper function that replaces a variable and returns the amount of replacements done
+size_t replaceVar(std::string & input, const std::string & var, const std::string & rep){
+  size_t count = 0;
+  const std::string withBraces = "${"+var+"}";
+  const std::string noBraces = "$"+var;
+  count += Util::replace(input, withBraces, rep);
+  count += Util::replace(input, noBraces, rep);
+  return count;
+}
+
+size_t Util::streamCustomVariables(std::string &str){
+  size_t count = 0;
+  // Read shared memory page containing custom variables
+  static IPC::sharedPage variablePage(SHM_CUSTOM_VARIABLES, 0, false, false);
+  // Check if the page needs to be reopened
+  if (variablePage.mapped){
+    Util::RelAccX varAccX(variablePage.mapped, false);
+    if (varAccX.isReload()){variablePage.close();}
+  }
+  // Reopen memory page if it has been closed
+  if (!variablePage.mapped){
+    variablePage.init(SHM_CUSTOM_VARIABLES, 0, false, false);
+    if(!variablePage.mapped){
+      ERROR_MSG("Unable to substitute custom variables, as memory page %s failed to open", SHM_CUSTOM_VARIABLES);
+      return 0;
+    }
+  }
+  // Extract variables
+  Util::RelAccX varAccX(variablePage.mapped, false);
+  for (size_t i = 0; i < varAccX.getEndPos(); i++){
+    // Replace $thisName with $thisVal
+    if (varAccX.getPointer("name", i)){
+      std::string thisName = "$" + std::string(varAccX.getPointer("name", i));
+      std::string thisVal = std::string(varAccX.getPointer("lastVal", i));
+      count += replaceVar(str, thisName, thisVal);
+    }
+  }
+  return count;
+}
+
 /// Replaces all stream-related variables in the given 'str' with their values.
-void Util::streamVariables(std::string &str, const std::string &streamname, const std::string &source){
-  Util::replace(str, "$source", source);
-  Util::replace(str, "$datetime", "$year.$month.$day.$hour.$minute.$second");
-  Util::replace(str, "$day", strftime_now("%d"));
-  Util::replace(str, "$month", strftime_now("%m"));
-  Util::replace(str, "$year", strftime_now("%Y"));
-  Util::replace(str, "$hour", strftime_now("%H"));
-  Util::replace(str, "$minute", strftime_now("%M"));
-  Util::replace(str, "$second", strftime_now("%S"));
-  Util::replace(str, "$wday", strftime_now("%u")); // weekday, 1-7, monday=1
-  Util::replace(str, "$yday", strftime_now("%j")); // yearday, 001-366
-  Util::replace(str, "$week", strftime_now("%V")); // week number, 01-53
-  Util::replace(str, "$stream", streamname);
+size_t Util::streamVariables(std::string &str, const std::string &streamname, const std::string &source, uint8_t depth){
+  size_t replaced = 0;
+  if (depth > 9){
+    WARN_MSG("Reached a depth of %u when replacing stream variables", depth);
+    return 0;
+  }
+  // If there are no variables, abort
+  if (str.find('$') == std::string::npos){return 0;}
+  // Find and replace any custom variables
+  replaced += streamCustomVariables(str);
+  replaced += replaceVar(str, "source", source);
+  replaced += replaceVar(str, "datetime", "$year.$month.$day.$hour.$minute.$second");
+  replaced += replaceVar(str, "day", strftime_now("%d"));
+  replaced += replaceVar(str, "month", strftime_now("%m"));
+  replaced += replaceVar(str, "year", strftime_now("%Y"));
+  replaced += replaceVar(str, "hour", strftime_now("%H"));
+  replaced += replaceVar(str, "minute", strftime_now("%M"));
+  replaced += replaceVar(str, "second", strftime_now("%S"));
+  replaced += replaceVar(str, "wday", strftime_now("%u")); // weekday, 1-7, monday=1
+  replaced += replaceVar(str, "yday", strftime_now("%j")); // yearday, 001-366
+  replaced += replaceVar(str, "week", strftime_now("%V")); // week number, 01-53
+  replaced += replaceVar(str, "stream", streamname);
   if (streamname.find('+') != std::string::npos){
     std::string strbase = streamname.substr(0, streamname.find('+'));
     std::string strext = streamname.substr(streamname.find('+') + 1);
-    Util::replace(str, "$basename", strbase);
-    Util::replace(str, "$wildcard", strext);
+    replaced += Util::replace(str, "basename", strbase);
+    replaced += Util::replace(str, "wildcard", strext);
     if (strext.size()){
-      Util::replace(str, "$pluswildcard", "+" + strext);
+      replaced += Util::replace(str, "pluswildcard", "+" + strext);
     }else{
-      Util::replace(str, "$pluswildcard", "");
+      replaced += Util::replace(str, "pluswildcard", "");
     }
   }else{
-    Util::replace(str, "$basename", streamname);
-    Util::replace(str, "$wildcard", "");
-    Util::replace(str, "$pluswildcard", "");
+    replaced += Util::replace(str, "basename", streamname);
+    replaced += Util::replace(str, "wildcard", "");
+    replaced += Util::replace(str, "pluswildcard", "");
   }
+  // Continue recursively if we've replaced a variable which exposed another variable to be replaced
+  if (replaced && str.find('$') != std::string::npos){replaced += streamVariables(str, streamName, source, ++depth);}
+  return replaced;
 }
 
 std::string Util::getTmpFolder(){

lib/stream.h

@@ -13,7 +13,8 @@
 const JSON::Value empty;
 
 namespace Util{
-  void streamVariables(std::string &str, const std::string &streamname, const std::string &source = "");
+  size_t streamCustomVariables(std::string &str);
+  size_t streamVariables(std::string &str, const std::string &streamname, const std::string &source = "", uint8_t depth = 0);
   std::string getTmpFolder();
   void sanitizeName(std::string &streamname);
   bool streamAlive(std::string &streamname);

lib/triggers.cpp

@@ -94,9 +94,9 @@ namespace Triggers{
         while (Util::Procs::isActive(myProc) && counter < 150){
           Util::sleep(100);
           ++counter;
-          if (counter >= 150){
-            if (counter == 150){FAIL_MSG("Trigger taking too long - killing process");}
-            if (counter >= 250){
+          if (counter >= 100){
+            if (counter == 100){FAIL_MSG("Trigger taking too long - killing process");}
+            if (counter >= 140){
               Util::Procs::Stop(myProc);
             }else{
               Util::Procs::Murder(myProc);

lib/util.cpp

@@ -150,14 +150,41 @@ namespace Util{
     }
   }
 
-  /// Replaces any occurrences of 'from' with 'to' in 'str'.
-  void replace(std::string &str, const std::string &from, const std::string &to){
-    if (from.empty()){return;}
+  /// Replaces any occurrences of 'from' with 'to' in 'str', returns how many replacements were made
+  size_t replace(std::string &str, const std::string &from, const std::string &to){
+    if (from.empty()){return 0;}
+    size_t counter = 0;
     size_t start_pos = 0;
     while ((start_pos = str.find(from, start_pos)) != std::string::npos){
       str.replace(start_pos, from.length(), to);
+      ++counter;
       start_pos += to.length();
     }
+    return counter;
+  }
+
+  /// \brief Removes whitespace from the beginning and end of a given string
+  void stringTrim(std::string &val){
+    if (!val.size()){ return; }
+    uint64_t startPos = 0;
+    uint64_t length = 0;
+    // Set startPos to the first character which does not have value 09-13
+    for (uint64_t i = 0; i < val.size(); i++){
+      if (val[i] == 32){continue;}
+      if (val[i] < 9 || val[i] > 13){
+        startPos = i;
+        break;
+      }
+    }
+    // Same thing in reverse for endPos
+    for (uint64_t i = val.size() - 1; i > 0 ; i--){
+      if (val[i] == 32){continue;}
+      if (val[i] < 9 || val[i] > 13){
+        length = i + 1 - startPos;
+        break;
+      }
+    }
+    val = val.substr(startPos, length);
   }
 
   //Returns the time to wait in milliseconds for exponential back-off waiting.

lib/util.h

@@ -13,7 +13,8 @@ namespace Util{
   bool createPath(const std::string &path);
   bool stringScan(const std::string &src, const std::string &pattern, std::deque<std::string> &result);
   void stringToLower(std::string &val);
-  void replace(std::string &str, const std::string &from, const std::string &to);
+  size_t replace(std::string &str, const std::string &from, const std::string &to);
+  void stringTrim(std::string &val);
 
   int64_t expBackoffMs(const size_t currIter, const size_t maxIter, const int64_t maxWait);