CPU optimizations for Opus encoder to reduce consumption, especially for idle publishers like agents

- Default channel layout: "stereo" -> "mono"
- Default bitrate: 96kbps -> 64kbps
- Default Opus compression level: "10" -> "7"

Author: dangusev

getstream/video/rtc/encoders_patches.py

@@ -7,8 +7,12 @@
 from aiortc import RTCRtpCodecParameters
 from aiortc.codecs.h264 import MAX_FRAME_RATE as H264_MAX_FRAME_RATE
 from aiortc.codecs.h264 import H264Encoder
+from aiortc.codecs.opus import SAMPLE_RATE as OPUS_SAMPLE_RATE
+from aiortc.codecs.opus import SAMPLES_PER_FRAME as OPUS_SAMPLES_PER_FRAME
+from aiortc.codecs.opus import OpusEncoder
 from aiortc.codecs.vpx import Vp8Encoder
 from aiortc.rtcrtpsender import RTCEncodedFrame, RTCRtpSender
+from av import AudioResampler
 
 logger = logging.getLogger(__name__)
 
@@ -18,6 +22,13 @@
 STREAM_VIDEO_MIN_BITRATE = 1_500_000  # 1.5 Mbps
 STREAM_VIDEO_MAX_BITRATE = 3_000_000  # 3 Mbps
 
+# Override hardcoded Opus defaults to reduce CPU usage. Lower than aiortc's 96 kbps stereo default; tuned for
+# voice. Both are configurable per StreamOpusEncoder() instance.
+STREAM_OPUS_DEFAULT_BITRATE = 64_000  # 96kbps -> 64kbps
+STREAM_OPUS_DEFAULT_LAYOUT = "mono"  # stereo -> mono
+# Compression level <= 7 reduces the CPU work for encoder with minimal quality impact at 64kbps
+STREAM_OPUS_DEFAULT_COMPRESSION_LEVEL = 7  # 10 -> 7
+
 
 # Check if the Stream bitrate patching is disabled via environment variable
 BITRATE_PATCH_DISABLED = os.getenv(
@@ -36,6 +47,7 @@
 
 
 try:
+    # TODO: Implement a way to configure encoders per track.
     # Verify the name-mangled attributes we depend on still exist.
     assert hasattr(Vp8Encoder(), "_Vp8Encoder__target_bitrate")
     assert hasattr(H264Encoder(), "_H264Encoder__target_bitrate")
@@ -124,16 +136,38 @@ def _encode_frame(self, frame, force_keyframe):
     StreamH264Encoder = None  # type: ignore[assignment, misc]
 
 
+class StreamOpusEncoder(OpusEncoder):
+    """OpusEncoder subclass with configurable bitrate and channel layout."""
+
+    def __init__(
+        self,
+        bitrate: int = STREAM_OPUS_DEFAULT_BITRATE,
+        layout: str = STREAM_OPUS_DEFAULT_LAYOUT,
+        compression_level: int = STREAM_OPUS_DEFAULT_COMPRESSION_LEVEL,
+    ) -> None:
+        super().__init__()
+        self.codec.bit_rate = bitrate
+        self.codec.layout = layout
+        self.codec.options = {
+            "application": "voip",
+            "compression_level": str(compression_level),
+        }
+        # Resampler layout must match codec layout; parent hard-codes stereo.
+        self.resampler = AudioResampler(
+            format="s16",
+            layout=layout,
+            rate=OPUS_SAMPLE_RATE,
+            frame_size=OPUS_SAMPLES_PER_FRAME,
+        )
+
+
 def patch_sender_encoder(sender: RTCRtpSender) -> None:
     """Patch a sender to use Stream's tuned encoders for the negotiated codec.
 
-    Works for video (VP8/H264) senders. If anything
+    Works for video (VP8/H264) and audio (Opus) senders. If anything
     goes wrong (e.g. aiortc internals changed), the sender is left untouched
     and will use the stock encoder via get_encoder().
     """
-    if StreamVp8Encoder is None or StreamH264Encoder is None:
-        return
-
     try:
         _orig_next = sender._next_encoded_frame
 
@@ -142,10 +176,12 @@ async def _next_with_stream_encoder(
         ) -> Optional[RTCEncodedFrame]:
             if sender._RTCRtpSender__encoder is None:  # type: ignore[attr-defined]
                 mime = codec.mimeType.lower()
-                if mime == "video/vp8":
+                if mime == "video/vp8" and StreamVp8Encoder is not None:
                     sender._RTCRtpSender__encoder = StreamVp8Encoder()  # type: ignore[attr-defined]
-                elif mime == "video/h264":
+                elif mime == "video/h264" and StreamH264Encoder is not None:
                     sender._RTCRtpSender__encoder = StreamH264Encoder()  # type: ignore[attr-defined]
+                elif mime == "audio/opus":
+                    sender._RTCRtpSender__encoder = StreamOpusEncoder()  # type: ignore[attr-defined]
             return await _orig_next(codec)
 
         sender._next_encoded_frame = _next_with_stream_encoder  # type: ignore[method-assign]

getstream/video/rtc/pc.py

@@ -80,8 +80,8 @@ def addTrack(self, track: MediaStreamTrack) -> RTCRtpSender:
                 if transceiver.sender is sender:
                     transceiver.setCodecPreferences(publish_codec_preferences())
                     break
-            if not BITRATE_PATCH_DISABLED:
-                patch_sender_encoder(sender)
+        if not BITRATE_PATCH_DISABLED:
+            patch_sender_encoder(sender)
         return sender
 
     async def handle_answer(self, response):

tests/rtc/test_encoders_patches.py

@@ -3,13 +3,18 @@
 
 import pytest
 from aiortc.codecs.h264 import H264Encoder
+from aiortc.codecs.opus import OpusEncoder
 from aiortc.codecs.vpx import Vp8Encoder
 
 from getstream.video.rtc.encoders_patches import (
+    STREAM_OPUS_DEFAULT_BITRATE,
+    STREAM_OPUS_DEFAULT_COMPRESSION_LEVEL,
+    STREAM_OPUS_DEFAULT_LAYOUT,
     STREAM_VIDEO_DEFAULT_BITRATE,
     STREAM_VIDEO_MAX_BITRATE,
     STREAM_VIDEO_MIN_BITRATE,
     StreamH264Encoder,
+    StreamOpusEncoder,
     StreamVp8Encoder,
     patch_sender_encoder,
 )
@@ -68,6 +73,31 @@ def test_accepts_in_range(self):
         assert enc.target_bitrate == 2_000_000
 
 
+class TestStreamOpusEncoder:
+    def test_defaults(self):
+        enc = StreamOpusEncoder()
+        assert enc.codec.bit_rate == STREAM_OPUS_DEFAULT_BITRATE
+        assert enc.codec.layout.name == STREAM_OPUS_DEFAULT_LAYOUT
+        assert enc.codec.options["compression_level"] == str(
+            STREAM_OPUS_DEFAULT_COMPRESSION_LEVEL
+        )
+        assert enc.codec.options["application"] == "voip"
+
+    def test_custom_bitrate(self):
+        enc = StreamOpusEncoder(bitrate=32_000)
+        assert enc.codec.bit_rate == 32_000
+
+    def test_custom_layout_updates_resampler(self):
+        """Resampler layout must track codec layout — encode() asserts on it."""
+        enc = StreamOpusEncoder(layout="stereo")
+        assert enc.codec.layout.name == "stereo"
+        assert enc.resampler.layout.name == "stereo"
+
+    def test_custom_compression_level(self):
+        enc = StreamOpusEncoder(compression_level=10)
+        assert enc.codec.options["compression_level"] == "10"
+
+
 class TestBitratePatchDisabled:
     @pytest.mark.parametrize("env_val", [None, "", "1", "true", "yes", "on"])
     def test_enabled_by_default(self, monkeypatch, env_val):
@@ -165,6 +195,25 @@ async def _orig_coro(codec):
 
         assert isinstance(sender._RTCRtpSender__encoder, H264Cls)
 
+    @pytest.mark.asyncio
+    async def test_installs_opus_encoder(self):
+        sender = MagicMock()
+        sender._RTCRtpSender__encoder = None
+
+        async def _orig_coro(codec):
+            return None
+
+        sender._next_encoded_frame = _orig_coro
+        patch_sender_encoder(sender)
+
+        codec = MagicMock()
+        codec.mimeType = "audio/opus"
+        await sender._next_encoded_frame(codec)
+
+        from getstream.video.rtc.encoders_patches import StreamOpusEncoder as OpusCls
+
+        assert isinstance(sender._RTCRtpSender__encoder, OpusCls)
+
     @pytest.mark.asyncio
     async def test_does_not_replace_existing_encoder(self):
         """Already-set encoder is not replaced."""
@@ -183,15 +232,25 @@ async def _orig_coro(codec):
         await sender._next_encoded_frame(codec)
         assert sender._RTCRtpSender__encoder is existing_encoder
 
-    def test_noop_when_encoders_none(self, monkeypatch):
-        """patch_sender_encoder is a no-op when encoder classes failed to load."""
+    @pytest.mark.asyncio
+    async def test_skips_mime_when_encoder_class_is_none(self, monkeypatch):
+        """If a Stream* class failed to load, that mime branch leaves the encoder unset."""
         import getstream.video.rtc.encoders_patches as mod
 
         monkeypatch.setattr(mod, "StreamVp8Encoder", None)
         sender = MagicMock()
-        orig = sender._next_encoded_frame
+        sender._RTCRtpSender__encoder = None
+
+        async def _orig_coro(codec):
+            return None
+
+        sender._next_encoded_frame = _orig_coro
         mod.patch_sender_encoder(sender)
-        assert sender._next_encoded_frame is orig
+
+        codec = MagicMock()
+        codec.mimeType = "video/VP8"
+        await sender._next_encoded_frame(codec)
+        assert sender._RTCRtpSender__encoder is None
 
 
 class TestUpstreamAssumptions:
@@ -216,6 +275,11 @@ def test_upstream_assumptions_hold(self):
             "H264Encoder name-mangled __target_bitrate changed"
         )
 
+        # OpusEncoder still exposes the public attrs StreamOpusEncoder overrides.
+        opus = OpusEncoder()
+        assert hasattr(opus, "codec"), "OpusEncoder lost codec"
+        assert hasattr(opus, "resampler"), "OpusEncoder lost resampler"
+
         # RTCRtpSender has _next_encoded_frame and uses __encoder
         import inspect