Skip to content

Commit e339453

Browse files
sgfnsgfn
authored
AV1 payloader (#222)
1 parent 067ebd4 commit e339453

File tree

15 files changed

+793
-14
lines changed

15 files changed

+793
-14
lines changed

‎lib/ex_webrtc/rtp/av1/leb128.ex

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
defmodule ExWebRTC.RTP.AV1.LEB128 do
2+
@moduledoc false
3+
# Utilities for handling unsigned Little Endian Base 128 integers
4+
5+
import Bitwise
6+
7+
# see https://chromium.googlesource.com/external/webrtc/+/HEAD/modules/rtp_rtcp/source/rtp_packetizer_av1.cc#61
8+
@spec encode(non_neg_integer(), [bitstring()]) :: binary()
9+
def encode(value, acc \\ [])
10+
11+
def encode(value, acc) when value < 0x80 do
12+
for group <- Enum.reverse([value | acc]), into: <<>> do
13+
<<group>>
14+
end
15+
end
16+
17+
def encode(value, acc) do
18+
group = 0x80 ||| (value &&& 0x7F)
19+
encode(value >>> 7, [group | acc])
20+
end
21+
22+
# see https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/rtc_base/byte_buffer.cc;drc=8e78783dc1f7007bad46d657c9f332614e240fd8;l=107
23+
@spec read(binary(), non_neg_integer(), non_neg_integer(), non_neg_integer()) ::
24+
{:ok, pos_integer(), non_neg_integer()} | {:error, :invalid_leb128_data}
25+
def read(data, read_bits \\ 0, leb128_size \\ 0, value \\ 0)
26+
27+
def read(<<0::1, group::7, _rest::binary>>, read_bits, leb128_size, value) do
28+
{:ok, leb128_size + 1, value ||| group <<< read_bits}
29+
end
30+
31+
def read(<<1::1, group::7, rest::binary>>, read_bits, leb128_size, value) do
32+
read(rest, read_bits + 7, leb128_size + 1, value ||| group <<< read_bits)
33+
end
34+
35+
def read(_, _, _, _), do: {:error, :invalid_leb128_data}
36+
end

‎lib/ex_webrtc/rtp/av1/obu.ex

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
defmodule ExWebRTC.RTP.AV1.OBU do
2+
@moduledoc false
3+
# Defines the Open Bitstream Unit, the base packetization unit of all structures present in the AV1 bitstream.
4+
#
5+
# Based on [the AV1 spec](https://aomediacodec.github.io/av1-spec/av1-spec.pdf).
6+
#
7+
# OBU syntax:
8+
# 0 1 2 3 4 5 6 7
9+
# +-+-+-+-+-+-+-+-+
10+
# |0| type |X|S|-| (REQUIRED)
11+
# +-+-+-+-+-+-+-+-+
12+
# X: | TID |SID|-|-|-| (OPTIONAL)
13+
# +-+-+-+-+-+-+-+-+
14+
# |1| |
15+
# +-+ OBU payload |
16+
# S: |1| | (OPTIONAL, variable length leb128 encoded)
17+
# +-+ size |
18+
# |0| |
19+
# +-+-+-+-+-+-+-+-+
20+
# | OBU payload |
21+
# | ... |
22+
23+
alias ExWebRTC.RTP.AV1.LEB128
24+
25+
@obu_sequence_header 1
26+
@obu_temporal_delimiter 2
27+
@obu_padding 15
28+
29+
@type t :: %__MODULE__{
30+
type: 0..15,
31+
x: 0 | 1,
32+
s: 0 | 1,
33+
tid: 0..7 | nil,
34+
sid: 0..3 | nil,
35+
payload: binary()
36+
}
37+
38+
@enforce_keys [:type, :x, :s, :payload]
39+
defstruct @enforce_keys ++ [:tid, :sid]
40+
41+
@doc """
42+
Parses the low overhead bitstream format defined in AV1 spec section 5.2.
43+
On success, returns the parsed OBU as well as the remainder of the AV1 bitstream.
44+
"""
45+
@spec parse(binary()) :: {:ok, t(), binary()} | {:error, :invalid_av1_bitstream}
46+
def parse(av1_bitstream_binary)
47+
48+
def parse(<<0::1, type::4, x::1, s::1, 0::1, rest::binary>>) do
49+
with {:ok, tid, sid, rest} <- parse_extension_header(x, rest),
50+
{:ok, payload, rest} <- parse_payload(s, rest),
51+
:ok <- validate_payload(type, payload) do
52+
{:ok,
53+
%__MODULE__{
54+
type: type,
55+
x: x,
56+
s: s,
57+
tid: tid,
58+
sid: sid,
59+
payload: payload
60+
}, rest}
61+
else
62+
{:error, _} = err -> err
63+
end
64+
end
65+
66+
def parse(_), do: {:error, :invalid_av1_bitstream}
67+
68+
defp parse_extension_header(0, rest), do: {:ok, nil, nil, rest}
69+
70+
defp parse_extension_header(1, <<tid::3, sid::2, 0::3, rest::binary>>),
71+
do: {:ok, tid, sid, rest}
72+
73+
defp parse_extension_header(_, _), do: {:error, :invalid_av1_bitstream}
74+
75+
defp parse_payload(0, rest), do: {:ok, rest, <<>>}
76+
77+
defp parse_payload(1, rest) do
78+
with {:ok, leb128_size, payload_size} <- LEB128.read(rest),
79+
<<_::binary-size(leb128_size), payload::binary-size(payload_size), rest::binary>> <- rest do
80+
{:ok, payload, rest}
81+
else
82+
_ -> {:error, :invalid_av1_bitstream}
83+
end
84+
end
85+
86+
defp validate_payload(@obu_padding, _), do: :ok
87+
defp validate_payload(@obu_temporal_delimiter, <<>>), do: :ok
88+
defp validate_payload(type, data) when type != @obu_temporal_delimiter and data != <<>>, do: :ok
89+
defp validate_payload(_, _), do: {:error, :invalid_av1_bitstream}
90+
91+
@spec serialize(t()) :: binary()
92+
def serialize(%__MODULE__{type: type, x: x, s: s, payload: payload} = obu) do
93+
obu_binary =
94+
<<0::1, type::4, x::1, s::1, 0::1>>
95+
|> add_extension_header(obu)
96+
|> add_payload_size(obu)
97+
98+
<<obu_binary::binary, payload::binary>>
99+
end
100+
101+
defp add_extension_header(obu_binary, %__MODULE__{x: 0, tid: nil, sid: nil}), do: obu_binary
102+
103+
defp add_extension_header(obu_binary, %__MODULE__{x: 1, tid: tid, sid: sid})
104+
when tid != nil and sid != nil do
105+
<<obu_binary::binary, tid::3, sid::2, 0::3>>
106+
end
107+
108+
defp add_extension_header(_obu_binary, _invalid_obu),
109+
do: raise("AV1 TID and SID must be set if, and only if X bit is set")
110+
111+
defp add_payload_size(obu_binary, %__MODULE__{s: 0}), do: obu_binary
112+
113+
defp add_payload_size(obu_binary, %__MODULE__{s: 1, payload: payload}) do
114+
payload_size = payload |> byte_size() |> LEB128.encode()
115+
<<obu_binary::binary, payload_size::binary>>
116+
end
117+
118+
@doc """
119+
Rewrites a specific case of the sequence header OBU to disable OBU dropping in the AV1 decoder
120+
in accordance with av1-rtp-spec sec. 5. Leaves other OBUs unchanged.
121+
"""
122+
@spec disable_dropping_in_decoder_if_applicable(t()) :: t()
123+
def disable_dropping_in_decoder_if_applicable(obu)
124+
125+
# We're handling the following case:
126+
# - still_picture = 0
127+
# - reduced_still_picture_header = 0
128+
# - timing_info_present_flag = 0
129+
# - operating_points_cnt_minus_1 = 0
130+
# - seq_level_idx[0] = 0
131+
# and setting operating_point_idc[0] = 0xFFF
132+
#
133+
# For the sequence header OBU syntax, refer to the AV1 spec sec. 5.5.
134+
def disable_dropping_in_decoder_if_applicable(
135+
%__MODULE__{
136+
type: @obu_sequence_header,
137+
payload: <<seq_profile::3, 0::3, iddpf::1, 0::5, _op_idc_0::12, 0::5, rest::bitstring>>
138+
} = obu
139+
) do
140+
%{obu | payload: <<seq_profile::3, 0::3, iddpf::1, 0::5, 0xFFF::12, 0::5, rest::bitstring>>}
141+
end
142+
143+
def disable_dropping_in_decoder_if_applicable(obu), do: obu
144+
end

‎lib/ex_webrtc/rtp/av1/payload.ex

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
defmodule ExWebRTC.RTP.AV1.Payload do
2+
@moduledoc false
3+
# Defines AV1 payload structure stored in RTP packet payload.
4+
#
5+
# Based on [RTP Payload Format for AV1](https://aomediacodec.github.io/av1-rtp-spec/v1.0.0.html).
6+
#
7+
# RTP payload syntax:
8+
# 0 1 2 3 4 5 6 7
9+
# +-+-+-+-+-+-+-+-+
10+
# |Z|Y| W |N|-|-|-| (REQUIRED)
11+
# +=+=+=+=+=+=+=+=+ (REPEATED W-1 times, or any times if W = 0)
12+
# |1| |
13+
# +-+ OBU fragment|
14+
# |1| | (REQUIRED, leb128 encoded)
15+
# +-+ size |
16+
# |0| |
17+
# +-+-+-+-+-+-+-+-+
18+
# | OBU fragment |
19+
# | ... |
20+
# +=+=+=+=+=+=+=+=+
21+
# | ... |
22+
# +=+=+=+=+=+=+=+=+ if W > 0, last fragment MUST NOT have size field
23+
# | OBU fragment |
24+
# | ... |
25+
# +=+=+=+=+=+=+=+=+
26+
27+
@type t :: %__MODULE__{
28+
z: 0 | 1,
29+
y: 0 | 1,
30+
w: 0 | 1 | 2 | 3,
31+
n: 0 | 1,
32+
payload: binary()
33+
}
34+
35+
@enforce_keys [:z, :y, :w, :n, :payload]
36+
defstruct @enforce_keys ++ []
37+
38+
@doc """
39+
Parses RTP payload as AV1 payload.
40+
"""
41+
@spec parse(binary()) :: {:ok, t()} | {:error, :invalid_packet}
42+
def parse(rtp_payload)
43+
44+
def parse(<<z::1, y::1, w::2, n::1, 0::3, payload::binary>>) do
45+
if payload == <<>> do
46+
{:error, :invalid_packet}
47+
else
48+
{:ok,
49+
%__MODULE__{
50+
z: z,
51+
y: y,
52+
w: w,
53+
n: n,
54+
payload: payload
55+
}}
56+
end
57+
end
58+
59+
def parse(_), do: {:error, :invalid_packet}
60+
61+
@spec serialize(t()) :: binary()
62+
def serialize(%__MODULE__{
63+
z: z,
64+
y: y,
65+
w: w,
66+
n: n,
67+
payload: payload
68+
}) do
69+
<<z::1, y::1, w::2, n::1, 0::3, payload::binary>>
70+
end
71+
72+
@doc """
73+
Payloads chunked fragments of single OBU and sets Z, Y bits.
74+
"""
75+
@spec payload_obu_fragments([binary()], 0 | 1) :: [t()]
76+
def payload_obu_fragments(obu_fragments, n_bit \\ 0)
77+
78+
def payload_obu_fragments([entire_obu], n_bit) do
79+
[%__MODULE__{z: 0, y: 0, w: 1, n: n_bit, payload: entire_obu}]
80+
end
81+
82+
def payload_obu_fragments([first_obu_fragment | next_obu_fragments], n_bit) do
83+
# First fragment of OBU: set Y bit
84+
first_obu_payload = %__MODULE__{z: 0, y: 1, w: 1, n: n_bit, payload: first_obu_fragment}
85+
86+
next_obu_payloads =
87+
next_obu_fragments
88+
# Middle fragments of OBU: set Z, Y bits
89+
# av1-rtp-spec sec. 4.4: N is set only for the first packet of the coded video sequence.
90+
|> Enum.map(&%__MODULE__{z: 1, y: 1, w: 1, n: 0, payload: &1})
91+
# Last fragment of OBU: set Z bit only (unset Y)
92+
|> List.update_at(-1, &%{&1 | y: 0})
93+
94+
[first_obu_payload | next_obu_payloads]
95+
end
96+
end

‎lib/ex_webrtc/rtp/av1/payloader.ex

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
defmodule ExWebRTC.RTP.Payloader.AV1 do
2+
@moduledoc false
3+
# Encapsulates AV1 video temporal units into RTP packets.
4+
#
5+
# Resources:
6+
# * [RTP Payload Format for AV1 (av1-rtp-spec)](https://aomediacodec.github.io/av1-rtp-spec/v1.0.0.html)
7+
# * [AV1 spec](https://aomediacodec.github.io/av1-spec/av1-spec.pdf).
8+
# * https://norkin.org/research/av1_decoder_model/index.html
9+
# * https://chromium.googlesource.com/external/webrtc/+/HEAD/modules/rtp_rtcp/source/video_rtp_depacketizer_av1.cc
10+
11+
@behaviour ExWebRTC.RTP.Payloader.Behaviour
12+
13+
alias ExWebRTC.RTP.AV1.{OBU, Payload}
14+
alias ExWebRTC.Utils
15+
16+
@obu_sequence_header 1
17+
@obu_temporal_delimiter 2
18+
19+
@aggregation_header_size_bytes 1
20+
21+
@type t :: %__MODULE__{
22+
max_payload_size: non_neg_integer()
23+
}
24+
25+
@enforce_keys [:max_payload_size]
26+
defstruct @enforce_keys
27+
28+
@impl true
29+
def new(max_payload_size) when max_payload_size > 100 do
30+
%__MODULE__{max_payload_size: max_payload_size}
31+
end
32+
33+
@impl true
34+
def payload(payloader, temporal_unit) when temporal_unit != <<>> do
35+
# In AV1, a temporal unit consists of all OBUs associated with a specific time instant.
36+
# Temporal units always start with a temporal delimiter OBU. They may contain multiple AV1 frames.
37+
# av1-rtp-spec sec. 5: The temporal delimiter OBU should be removed when transmitting.
38+
obus =
39+
case parse_obus(temporal_unit) do
40+
[%OBU{type: @obu_temporal_delimiter} | next_obus] ->
41+
next_obus
42+
43+
_ ->
44+
raise "Invalid AV1 temporal unit: does not start with temporal delimiter OBU"
45+
end
46+
47+
# With the current implementation, each RTP packet will contain one OBU element.
48+
# This element can be an entire OBU, or a fragment of an OBU bigger than max_payload_size.
49+
rtp_packets =
50+
Stream.flat_map(obus, fn obu ->
51+
n_bit = Utils.to_int(obu.type == @obu_sequence_header)
52+
53+
obu
54+
|> OBU.disable_dropping_in_decoder_if_applicable()
55+
|> OBU.serialize()
56+
|> Utils.chunk(payloader.max_payload_size - @aggregation_header_size_bytes)
57+
|> Payload.payload_obu_fragments(n_bit)
58+
end)
59+
|> Stream.map(&Payload.serialize/1)
60+
|> Enum.map(&ExRTP.Packet.new/1)
61+
|> List.update_at(-1, &%{&1 | marker: true})
62+
63+
{rtp_packets, payloader}
64+
end
65+
66+
defp parse_obus(data, obus \\ [])
67+
defp parse_obus(<<>>, obus), do: Enum.reverse(obus)
68+
69+
defp parse_obus(data, obus) do
70+
case OBU.parse(data) do
71+
{:ok, obu, rest} ->
72+
parse_obus(rest, [obu | obus])
73+
74+
{:error, :invalid_av1_bitstream} ->
75+
raise "Invalid AV1 bitstream: unable to parse OBU"
76+
end
77+
end
78+
end

‎lib/ex_webrtc/rtp/payloader.ex

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ defmodule ExWebRTC.RTP.Payloader do
3939
defp to_payloader_module(mime_type) do
4040
case String.downcase(mime_type) do
4141
"video/vp8" -> {:ok, ExWebRTC.RTP.Payloader.VP8}
42+
"video/av1" -> {:ok, ExWebRTC.RTP.Payloader.AV1}
4243
"audio/opus" -> {:ok, ExWebRTC.RTP.Payloader.Opus}
4344
"audio/pcma" -> {:ok, ExWebRTC.RTP.Payloader.G711}
4445
"audio/pcmu" -> {:ok, ExWebRTC.RTP.Payloader.G711}

0 commit comments

Comments
 (0)