Added H264 analyser (most code by Erik Zandvliet)

CMakeLists.txt

@@ -273,6 +273,7 @@ makeAnalyser(RAX rax)
 #makeAnalyser(RTP rtp) #LTS
 makeAnalyser(RTSP rtsp_rtp) #LTS
 makeAnalyser(TS ts) #LTS
+makeAnalyser(H264 h264) #LTS
 makeAnalyser(TSStream tsstream) #LTS
 makeAnalyser(Stats stats) #LTS
 

lib/bitstream.cpp

@@ -151,6 +151,91 @@ namespace Utils {
     return golombPeeker() - 1;
   }
 
+
+  bitWriter::bitWriter(){
+    dataBuffer = NULL;
+    bufferSize = 0;
+    reallocate(0);
+    dataSize = 0;
+  }
+
+  bitWriter::~bitWriter(){
+    if (dataBuffer != NULL){
+      free(dataBuffer);
+    }
+  }
+
+  void bitWriter::reallocate(size_t newSize){
+    size_t sizeBefore = bufferSize / 8;
+    char * tmp;
+    if (dataBuffer != NULL){
+      tmp = (char*)realloc(dataBuffer, (newSize / 8) + 1);
+    }else{
+      tmp = (char*)malloc((newSize / 8) + 1);
+    }
+    if (tmp){
+      dataBuffer = tmp;
+      bufferSize = ((newSize / 8) + 1) * 8;
+      memset(dataBuffer + sizeBefore, 0x00, (bufferSize / 8) - sizeBefore);
+    }else{
+      FAIL_MSG("Could not reallocate!!");
+    }
+  }
+
+  size_t bitWriter::size() {
+    return dataSize;
+  }
+
+  void bitWriter::append(uint64_t value, size_t bitLength){
+    if (dataSize + bitLength > bufferSize){
+      reallocate(dataSize + bitLength);
+    }
+
+    int64_t fullShift = (bitLength / 8) * 8;
+    uint64_t firstMask = ((0x01ull << (bitLength % 8)) - 1) << fullShift;
+
+    appendData( ((value & firstMask) >> fullShift), bitLength - fullShift);
+    while (fullShift > 0) {
+      fullShift -= 8;
+      uint64_t mask = (0xFFull) << fullShift;
+      appendData((value & mask) >> fullShift, 8);
+    }
+  }
+
+  void bitWriter::appendData(uint8_t data, size_t len){
+    size_t byteOffset = dataSize / 8;
+    size_t bitOffset = dataSize % 8;
+    if (len <= 8 - bitOffset){
+      dataBuffer[byteOffset] |= (data << (8 - bitOffset - len));
+      dataSize += len;
+    }else{
+      size_t shift = (len - (8 - bitOffset));
+      dataBuffer[byteOffset] |= (data >> shift);
+      dataSize += (len - shift);
+      appendData(data, shift);
+    }
+  }
+
+  size_t bitWriter::UExpGolombEncodedSize(uint64_t value){
+    value ++;
+    size_t res = 1;
+    size_t maxVal = 1;
+    while (value > maxVal){
+      maxVal = (maxVal << 1 | 0x01);
+      res += 1;
+    }
+    return 2 * res - 1;
+  }
+
+  void bitWriter::appendExpGolomb(uint64_t value){
+    append(value + 1, UExpGolombEncodedSize(value));
+  }
+
+  void bitWriter::appendUExpGolomb(uint64_t value){
+    append(value + 1, UExpGolombEncodedSize(value));
+  }
+
+
 //Note: other bitstream here
   bitstreamLSBF::bitstreamLSBF() {
     readBufferOffset = 0;

lib/bitstream.h

@@ -1,5 +1,6 @@
 #pragma once
 #include<string>
+#include "defines.h"
 
 namespace Utils {
   class bitstream {
@@ -26,6 +27,17 @@ namespace Utils {
       long long unsigned int getUExpGolomb();
       long long int peekExpGolomb();
       long long unsigned int peekUExpGolomb();
+
+      static size_t bitSizeUExpGolomb(size_t value){
+        size_t i = 1;
+        size_t power = 2;
+        while (power - 2 < value){
+          i+= 2;
+          power *= 2;
+        }
+        return i;
+      }
+
     private:
       bool checkBufferSize(unsigned int size);
       long long unsigned int golombGetter();
@@ -36,6 +48,27 @@ namespace Utils {
       size_t bufferSize;
   };
 
+  class bitWriter {
+    public:
+      bitWriter();
+      ~bitWriter();
+      size_t size();
+      void append(uint64_t value, size_t bitLength);
+      void appendExpGolomb(uint64_t value);
+      void appendUExpGolomb(uint64_t value);
+      static size_t UExpGolombEncodedSize(uint64_t value);
+      std::string str() { return std::string(dataBuffer, (dataSize / 8) + (dataSize % 8 ? 1 : 0)); }
+    protected:
+      void reallocate(size_t newSize);
+      void appendData(uint8_t data, size_t len);
+
+      char * dataBuffer;
+      //NOTE: ALL SIZES IN BITS!
+      size_t bufferSize;
+
+      size_t dataSize;
+  };
+
   class bitstreamLSBF {
     public:
       bitstreamLSBF();

lib/h264.h

@@ -1,7 +1,12 @@
+#pragma once
 #include <deque>
 #include <string>
+#include <cstdlib>
+#include <cstdio>
 
 #include "nal.h"
+#include "bitfields.h"
+#include "bitstream.h"
 
 namespace h264 {
 
@@ -67,4 +72,229 @@ namespace h264 {
       unsigned long dataLen;
   };
 
+  class nalUnit {
+    public:
+      nalUnit(const char * data, size_t len) : payload(data, len) {}
+      uint8_t getType() { return payload[0] & 0x1F; }
+      virtual void toPrettyString(std::ostream & out){
+        out << "Nal unit of type " << (((uint8_t)payload[0]) & 0x1F) << ", " << payload.size() << " bytes long" << std::endl;
+      }
+      void write(std::ostream & out){
+        //always writes in annex_b style
+        out.write("\000\000\000\001", 4);
+        out.write(payload.data(), payload.size());
+      }
+      virtual std::string generate() { return ""; }
+      virtual void setSPSNumber(size_t newNumber) {}
+      virtual void setPPSNumber(size_t newNumber) {}
+    protected:
+      std::string payload;
+  };
+
+  class hrd_parameters {
+    public:
+      hrd_parameters() {}
+      hrd_parameters(Utils::bitstream & bs);
+      void toPrettyString(std::ostream & out, size_t indent = 0);
+      
+      uint64_t cpbCntMinus1;
+      uint8_t bitRateScale;
+      uint8_t cpbSizeScale;
+  };
+
+  class vui_parameters {
+    public:
+      vui_parameters() {};
+      vui_parameters(Utils::bitstream & bs);
+      void generate(Utils::bitWriter & bw);
+      void toPrettyString(std::ostream & out, size_t indent = 0);
+      
+      bool aspectRatioInfoPresentFlag;
+      uint8_t aspectRatioIdc;
+      uint16_t sarWidth;
+      uint16_t sarHeight;
+      bool overscanInfoPresentFlag;
+      bool overscanAppropriateFlag;
+      bool videoSignalTypePresentFlag;
+      uint8_t videoFormat;
+      bool videoFullRangeFlag;
+      bool colourDescriptionPresentFlag;
+      uint8_t colourPrimaries;
+      uint8_t transferCharacteristics;
+      uint8_t matrixCoefficients;
+      bool chromaLocInfoPresentFlag;
+      uint64_t chromaSampleLocTypeTopField;
+      uint64_t chromaSampleLocTypeBottomField;
+      bool timingInfoPresentFlag;
+      uint32_t numUnitsInTick;
+      uint32_t timeScale;
+      bool fixedFrameRateFlag;
+      bool nalHrdParametersPresentFlag;
+
+      bool vclHrdParametersPresentFlag;
+
+      bool lowDelayHrdFlag;
+      bool picStructPresentFlag;
+      bool bitstreamRestrictionFlag;
+      bool motionVectorsOverPicBoundariesFlag;
+      uint64_t maxBytesPerPicDenom;
+      uint64_t maxBitsPerMbDenom;
+      uint64_t log2MaxMvLengthHorizontal;
+      uint64_t log2MaxMvLengthVertical;
+      uint64_t numReorderFrames;
+      uint64_t maxDecFrameBuffering;
+  };
+
+  class spsUnit : public nalUnit {
+    public:
+      spsUnit(const char * data, size_t len);
+      ~spsUnit(){
+        if (scalingListPresentFlags != NULL){
+          free(scalingListPresentFlags);
+        }
+      }
+      std::string generate();
+      void toPrettyString(std::ostream & out);
+      void scalingList(uint64_t * scalingList, size_t sizeOfScalingList, bool & useDefaultScalingMatrixFlag, Utils::bitstream & bs);
+      void setSPSNumber(size_t newNumber);
+      uint8_t profileIdc;
+      bool constraintSet0Flag;
+      bool constraintSet1Flag;
+      bool constraintSet2Flag;
+      bool constraintSet3Flag;
+      bool constraintSet4Flag;
+      bool constraintSet5Flag;
+      uint8_t levelIdc;
+      uint64_t seqParameterSetId;
+      
+      uint64_t chromaFormatIdc;
+      bool separateColourPlaneFlag;
+      uint64_t bitDepthLumaMinus8;
+      uint64_t bitDepthChromaMinus8;
+      bool qpprimeYZeroTransformBypassFlag;
+      bool seqScalingMatrixPresentFlag;
+      //Here go scaling lists
+      uint8_t * scalingListPresentFlags;
+
+      uint64_t ** scalingList4x4;
+      bool * useDefaultScalingMatrix4x4Flag;
+      uint64_t ** scalingList8x8;
+      bool * useDefaultScalingMatrix8x8Flag;
+
+
+      uint64_t log2MaxFrameNumMinus4;
+      uint64_t picOrderCntType;
+      uint64_t log2MaxPicOrderCntLsbMinus4;
+      
+      //Here go values for pic_order_cnt_type == 1
+      
+      uint64_t maxNumRefFrames;
+      bool gapsInFrameNumValueAllowedFlag;
+      uint64_t picWidthInMbsMinus1; 
+      uint64_t picHeightInMapUnitsMinus1; 
+      bool frameMbsOnlyFlag;
+      bool mbAdaptiveFrameFieldFlag;
+      bool direct8x8InferenceFlag;
+      bool frameCroppingFlag;
+      uint64_t frameCropLeftOffset;
+      uint64_t frameCropRightOffset;
+      uint64_t frameCropTopOffset;
+      uint64_t frameCropBottomOffset;
+      bool vuiParametersPresentFlag;
+
+      vui_parameters vuiParams;
+
+      //DERIVATIVE VALUES
+      uint8_t derived_subWidthC;
+      uint8_t derived_subHeightC;
+      uint8_t derived_mbWidthC;
+      uint8_t derived_mbHeightC;
+      uint64_t derived_bitDepth_Y; 
+      uint64_t derived_qpBdOffset_Y; 
+      uint64_t derived_bitDepth_C; 
+      uint64_t derived_qpBdOffset_C; 
+      uint64_t derived_rawMbBits;
+      uint64_t derived_maxFrameNum;
+      uint64_t derived_maxPicOrderCntLsb;
+      uint64_t derived_picWidthInMbs;
+      uint64_t derived_picWidthInSamples_L;
+      uint64_t derived_picWidthInSamples_C;
+      uint64_t derived_picHeightInMapUnits;
+      uint64_t derived_picSizeInMapUnits;
+      uint64_t derived_frameHeightInMbs;
+      size_t derived_scalingListSize;
+      size_t derived_scalingList4x4Amount;
+      size_t derived_scalingList8x8Amount;
+  };
+  class ppsUnit : public nalUnit {
+    public:
+      ppsUnit(const char * data, size_t len, uint8_t chromaFormatIdc = 0);
+      ~ppsUnit(){
+        if (picScalingMatrixPresentFlags != NULL){
+          free(picScalingMatrixPresentFlags);
+        }
+      }
+      void scalingList(uint64_t * scalingList, size_t sizeOfScalingList, bool & useDefaultScalingMatrixFlag, Utils::bitstream & bs);
+      void setPPSNumber(size_t newNumber);
+      void setSPSNumber(size_t newNumber);
+      void toPrettyString(std::ostream & out);
+      std::string generate();
+      
+      uint64_t picParameterSetId;
+      uint64_t seqParameterSetId;
+      bool entropyCodingModeFlag;
+      bool bottomFieldPicOrderInFramePresentFlag;
+      uint64_t numSliceGroupsMinus1;
+      uint64_t numrefIdx10DefaultActiveMinus1;
+      uint64_t numrefIdx11DefaultActiveMinus1;
+      bool weightedPredFlag;
+      uint8_t weightedBipredIdc;
+      int64_t picInitQpMinus26;
+      int64_t picInitQsMinus26;
+      int64_t chromaQpIndexOffset;
+      bool deblockingFilterControlPresentFlag;
+      bool constrainedIntraPredFlag;
+      bool redundantPicCntPresentFlag;
+      bool transform8x8ModeFlag;
+      bool picScalingMatrixPresentFlag;
+      //Here go scaling lists
+      uint8_t * picScalingMatrixPresentFlags;
+
+      uint64_t ** scalingList4x4;
+      bool * useDefaultScalingMatrix4x4Flag;
+      uint64_t ** scalingList8x8;
+      bool * useDefaultScalingMatrix8x8Flag;
+
+      int64_t secondChromaQpIndexOffset;
+
+      size_t derived_scalingListSize;
+      size_t derived_scalingList4x4Amount;
+      size_t derived_scalingList8x8Amount;
+
+      bool status_moreRBSP;
+  };
+  class codedSliceUnit : public nalUnit {
+    public:
+      codedSliceUnit(const char * data, size_t len);
+      void setPPSNumber(size_t newNumber);
+      void toPrettyString(std::ostream & out);
+
+      uint64_t firstMbInSlice;
+      uint64_t sliceType;
+      uint64_t picParameterSetId;
+  };
+
+  class seiUnit : public nalUnit {
+    public:
+      seiUnit(const char * data, size_t len);
+      void toPrettyString(std::ostream & out);
+
+      uint32_t payloadType;
+      uint32_t payloadSize;
+  };
+
+
+
+  nalUnit * nalFactory(FILE * in, bool annexb = true);
+  nalUnit * nalFactory(char * data, size_t len, size_t & offset, bool annexb = true);
 }

src/analysers/h264_analyser.cpp

@@ -0,0 +1,30 @@
+/// \file h264_analyser.cpp
+/// Reads an H264 file and prints all readable data about it
+
+#include <string>
+#include <iostream>
+#include <cstdio>
+
+#include <mist/config.h>
+#include <mist/defines.h>
+#include <mist/h264.h>
+#include <mist/bitfields.h>
+#include <mist/bitstream.h>
+
+int main(int argc, char ** argv){
+  Util::Config conf = Util::Config(argv[0]);
+  conf.addOption("filename", JSON::fromString("{\"arg_num\":1, \"arg\":\"string\", \"help\":\"Full path of the file to analyse.\"}"));
+  conf.parseArgs(argc, argv);
+  FILE * F = fopen(conf.getString("filename").c_str(), "r+b");
+  if (!F){
+    FAIL_MSG("No such file");
+  }
+
+  h264::nalUnit * nalPtr = h264::nalFactory(F);
+  while (nalPtr){
+    nalPtr->toPrettyString(std::cout);
+    nalPtr = h264::nalFactory(F);
+  }
+  return 0;
+}
+