AV1 including jitter buffer refactoring

.github/workflows/release.yaml

@@ -187,6 +187,7 @@ jobs:
             artifacts/srtc_subscribe-macos/srtc_subscribe-macos.tar.gz
             artifacts/srtc_publish-windows/srtc_publish-windows.zip
             artifacts/srtc_subscribe-windows/srtc_subscribe-windows.zip
+            artifacts/video-files/sintel-av1.webm
             artifacts/video-files/sintel-vp8.webm
             artifacts/video-files/sintel.h264
             artifacts/video-files/sintel_with_slices.h264

CMakeLists.txt

@@ -33,16 +33,20 @@ endif ()
 
 add_library(srtc
         # Headers
+        include/srtc/bit_reader.h
         include/srtc/byte_buffer.h
         include/srtc/depacketizer.h
+        include/srtc/depacketizer_av1.h
         include/srtc/depacketizer_h264.h
         include/srtc/depacketizer_h265.h
         include/srtc/depacketizer_opus.h
+        include/srtc/depacketizer_video.h
         include/srtc/depacketizer_vp8.h
         include/srtc/error.h
         include/srtc/event_loop.h
         include/srtc/extension_map.h
         include/srtc/extended_value.h
+        include/srtc/codec_av1.h
         include/srtc/codec_h264.h
         include/srtc/codec_h265.h
         include/srtc/ice_agent.h
@@ -99,17 +103,21 @@ add_library(srtc
         include/srtc/x509_certificate.h
         include/srtc/x509_hash.h
         # Sources
+        src/bit_reader.cpp
         src/byte_buffer.cpp
+        src/codec_av1.cpp
+        src/codec_h264.cpp
+        src/codec_h265.cpp
         src/depacketizer.cpp
+        src/depacketizer_av1.cpp
         src/depacketizer_h264.cpp
         src/depacketizer_h265.cpp
         src/depacketizer_opus.cpp
+        src/depacketizer_video.cpp
         src/depacketizer_vp8.cpp
         src/error.cpp
         src/extension_map.cpp
         src/extended_value.cpp
-        src/codec_h264.cpp
-        src/codec_h265.cpp
         src/ice_agent.cpp
         src/jitter_buffer.cpp
         src/logging.cpp
@@ -160,8 +168,6 @@ add_library(srtc
         src/util.cpp
         src/x509_certificate.cpp
         src/x509_hash.cpp
-        include/srtc/codec_av1.h
-        src/codec_av1.cpp
 )
 
 if (APPLE)
@@ -437,12 +443,16 @@ if (NOT CMAKE_CROSSCOMPILING AND SRTC_BUILD_TOOLS)
                 tools/http_whip_whep.h
                 tools/media_writer.h
                 tools/media_writer.cpp
+                tools/media_writer_av1.h
+                tools/media_writer_av1.cpp
                 tools/media_writer_h26x.h
                 tools/media_writer_h26x.cpp
                 tools/media_writer_ogg.h
                 tools/media_writer_ogg.cpp
                 tools/media_writer_vp8.h
                 tools/media_writer_vp8.cpp
+                tools/media_writer_webm.h
+                tools/media_writer_webm.cpp
         )
 
         add_subdirectory(tools/libogg)

README.md

@@ -7,7 +7,7 @@ This is srtc, a "simple" WebRTC library (publish side is done and working quite
 - Depends on OpenSSL (or BoringSSL) only, nothing else.
 - Portable code in "conservative" C++: language level is C++ 17, and no exceptions or RTTI.
 - Only one worker thread per PeerConnection.
-- Video codecs: VP8, H264 (any profile id), H265. AV1 is coming.
+- Video codecs: VP8, H264 (any profile id), H265, AV1.
 - Audo codec: Opus.
 - SDP offer generation and SDP response parsing.
 - ICE / STUN negotiation, DTLS negotiation, SRTP and SRTCP.
@@ -45,6 +45,9 @@ Has a command line tool to subscribe to audio and/or video, which can save media
 Media encoding / decoding and presentation are deliberately out of scope of this library. For publishing, the application needs to
 provide encoded media samples. For subscribing, the application receives encoded media samples which it needs to decode and present.
 
+The srtc library does handle packetization of media frames into RTP packets when publishing and the reconstruction of media
+frames from RTP packets when subscribing (a jitter buffer).
+
 The API is deliberately not compatible with Google's, but the concepts are similar. The Google WebRTC library is inteded
 for browsers, and therefore its API has to match the API defined for JavaScript and cannot be changed. I decided that it's
 not necessary to follow the JavaScript API.
@@ -167,6 +170,20 @@ To send video to Pion, run the publish sample like this:
 ./build/srtc_publish[.exe] -f sintel.h265
 ```
 
+#### Using an AV1 input file
+
+First please run the Pion WebRTC server like this to use AV1 (by default it uses H264):
+
+```bash
+./run.sh -codec av1
+```
+
+To send video to Pion, run the publish sample like this:
+
+```bash
+./build/srtc_publish[.exe] -f sintel-av1.webm
+```
+
 ### A command line tool for subscribing
 
 ```bash
@@ -229,6 +246,24 @@ And then subscribe like this:
 ./build/srtc_subscribe[.exe] --ov output.h265
 ```
 
+#### Running in AV1 mode
+
+Run the Pion server like this, just like for publishing, and use the web page to publish media.
+
+```bash
+./run.sh -codec av1
+```
+
+And then subscribe like this:
+
+```bash
+./build/srtc_subscribe[.exe] --ov output.webm
+```
+
+The resulting webm file will not contain any audio, just video - if you'd like to capture audio as well,
+please add `--oa output.ogg`.
+
+
 ### An Android demo / sample
 
 There is an Android demo:

convert_with_slices_h265.sh

@@ -2,7 +2,7 @@
 
 ffmpeg -i sintel_trailer-720p.mp4 \
 	-c:v libx265 \
-	-x265-params "slices=4" \
+	-x265-params "slices=2" \
 	-crf 23 \
 	-c:a copy \
 	sintel_with_slices.h265

include/srtc/bit_reader.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <cinttypes>
+#include <cstddef>
+
+namespace srtc
+{
+
+class BitReader
+{
+public:
+    BitReader(const uint8_t* buffer, size_t size)
+        : data(buffer)
+        , dataSize(size)
+        , bitPos(0)
+    {
+    }
+
+    uint32_t readBit();
+    uint32_t readBits(size_t n);
+    uint32_t readUnsignedExpGolomb();
+
+    const uint8_t* const data;
+    const size_t dataSize;
+    size_t bitPos;
+};
+
+
+}

include/srtc/byte_buffer.h

@@ -91,6 +91,7 @@ class ByteReader
     [[nodiscard]] uint8_t readU8();
     [[nodiscard]] uint16_t readU16();
     [[nodiscard]] uint32_t readU32();
+    [[nodiscard]] uint32_t readLEB128();
 
 	[[nodiscard]] ByteBuffer readByteBuffer(size_t size);
 

include/srtc/codec_h264.h

@@ -55,27 +55,10 @@ class NaluParser
 //////////
 
 bool isParameterNalu(uint8_t naluType);
+bool isKeyFrameNalu(uint8_t naluType);
+bool isFrameStart(const uint8_t* nalu, size_t size);
 
-//////////
-
-class BitReader
-{
-private:
-    const uint8_t* const data;
-    const size_t dataSize;
-    size_t bitPos;
-
-public:
-    BitReader(const uint8_t* buffer, size_t size)
-        : data(buffer)
-        , dataSize(size)
-        , bitPos(0)
-    {
-    }
-
-    uint32_t readBit();
-    uint32_t readBits(size_t n);
-    uint32_t readUnsignedExpGolomb();
-};
+bool isSliceNalu(uint8_t naluType);
+bool isSliceFrameStart(const uint8_t* data, size_t size);
 
 } // namespace srtc::h264

include/srtc/codec_h265.h

@@ -55,7 +55,10 @@ class NaluParser
 //////////
 
 bool isParameterNalu(uint8_t naluType);
-bool isKeyFrameNalu(uint8_t nalu_type);
-bool isFrameStart(const uint8_t* frame, size_t size);
+bool isKeyFrameNalu(uint8_t naluType);
+bool isFrameStart(const uint8_t* nalu, size_t size);
+
+bool isSliceNalu(uint8_t naluType);
+bool isSliceFrameStart(const uint8_t* data, size_t size);
 
 } // namespace srtc::h265

include/srtc/depacketizer.h

@@ -24,7 +24,6 @@ class Depacketizer
 
     virtual void reset() = 0;
 
-    virtual void extract(std::vector<ByteBuffer>& out, const JitterBufferItem* packet) = 0;
     virtual void extract(std::vector<ByteBuffer>& out, const std::vector<const JitterBufferItem*>& packetList) = 0;
 
     static std::pair<std::shared_ptr<Depacketizer>, Error> make(const std::shared_ptr<Track>& track);

include/srtc/depacketizer_av1.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "srtc/depacketizer_video.h"
+
+namespace srtc
+{
+
+class DepacketizerAV1 final : public DepacketizerVideo
+{
+public:
+    explicit DepacketizerAV1(const std::shared_ptr<Track>& track);
+    ~DepacketizerAV1() override;
+
+    void reset() override;
+
+    void extract(std::vector<ByteBuffer>& out, const std::vector<const JitterBufferItem*>& packetList) override;
+
+protected:
+    [[nodiscard]] bool isFrameStart(const ByteBuffer& payload) const override;
+
+private:
+    bool mSeenNewSequence;
+
+    void extractImpl(std::vector<ByteBuffer>& out, const JitterBufferItem* packet, ByteBuffer&& frame);
+};
+
+} // namespace srtc

include/srtc/depacketizer_h264.h

@@ -1,30 +1,30 @@
 #pragma once
 
 #include "srtc/byte_buffer.h"
-#include "srtc/depacketizer.h"
+#include "srtc/depacketizer_video.h"
 
 namespace srtc
 {
 
-class DepacketizerH264 final : public Depacketizer
+class DepacketizerH264 final : public DepacketizerVideo
 {
 public:
     explicit DepacketizerH264(const std::shared_ptr<Track>& track);
     ~DepacketizerH264() override;
 
-    [[nodiscard]] PacketKind getPacketKind(const ByteBuffer& payload, bool marker) const override;
-
     void reset() override;
 
-    void extract(std::vector<ByteBuffer>& out, const JitterBufferItem* packet) override;
     void extract(std::vector<ByteBuffer>& out, const std::vector<const JitterBufferItem*>& packetList) override;
 
+protected:
+    [[nodiscard]] bool isFrameStart(const ByteBuffer& payload) const override;
+
 private:
     uint8_t mHaveBits;
     ByteBuffer mFrameBuffer;
     uint64_t mLastRtpTimestamp;
 
-    void extractImpl(std::vector<ByteBuffer>& out, const JitterBufferItem* packet, ByteBuffer&& frame);
+    void extractImpl(std::vector<ByteBuffer>& out, const JitterBufferItem* packet, ByteBuffer&& nalu);
 };
 
 } // namespace srtc

include/srtc/depacketizer_h265.h

@@ -1,30 +1,30 @@
 #pragma once
 
 #include "srtc/byte_buffer.h"
-#include "srtc/depacketizer.h"
+#include "srtc/depacketizer_video.h"
 
 namespace srtc
 {
 
-class DepacketizerH265 final : public Depacketizer
+class DepacketizerH265 final : public DepacketizerVideo
 {
 public:
     explicit DepacketizerH265(const std::shared_ptr<Track>& track);
     ~DepacketizerH265() override;
 
-    [[nodiscard]] PacketKind getPacketKind(const ByteBuffer& payload, bool marker) const override;
-
     void reset() override;
 
-    void extract(std::vector<ByteBuffer>& out, const JitterBufferItem* packet) override;
     void extract(std::vector<ByteBuffer>& out, const std::vector<const JitterBufferItem*>& packetList) override;
 
+protected:
+    [[nodiscard]] bool isFrameStart(const ByteBuffer& payload) const override;
+
 private:
     uint8_t mHaveBits;
     ByteBuffer mFrameBuffer;
     uint64_t mLastRtpTimestamp;
 
-    void extractImpl(std::vector<ByteBuffer>& out, const JitterBufferItem* packet, ByteBuffer&& frame);
+    void extractImpl(std::vector<ByteBuffer>& out, const JitterBufferItem* packet, ByteBuffer&& nalu);
 };
 
 } // namespace srtc

include/srtc/depacketizer_opus.h

@@ -15,7 +15,6 @@ class DepacketizerOpus final : public Depacketizer
 
     void reset() override;
 
-    void extract(std::vector<ByteBuffer>& out, const JitterBufferItem* packet) override;
     void extract(std::vector<ByteBuffer>& out, const std::vector<const JitterBufferItem*>& packetList) override;
 };
 

include/srtc/depacketizer_video.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "srtc/byte_buffer.h"
+#include "srtc/depacketizer.h"
+
+namespace srtc
+{
+
+class DepacketizerVideo : public Depacketizer
+{
+public:
+    explicit DepacketizerVideo(const std::shared_ptr<Track>& track);
+    ~DepacketizerVideo() override;
+
+    [[nodiscard]] PacketKind getPacketKind(const ByteBuffer& payload, bool marker) const final;
+
+protected:
+    [[nodiscard]] virtual bool isFrameStart(const ByteBuffer& payload) const = 0;
+};
+} // namespace srtc