feat: iOS video renderer improvements

example/ios/Podfile

@@ -1,5 +1,5 @@
 # Uncomment this line to define a global platform for your project
-platform :ios, '13.0'
+platform :ios, '15.0'
 
 # CocoaPods analytics sends network stats synchronously affecting flutter build latency.
 ENV['COCOAPODS_DISABLE_STATS'] = 'true'

example/ios/Runner.xcodeproj/project.pbxproj

@@ -472,7 +472,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 15.0;
 				MTL_ENABLE_DEBUG_INFO = NO;
 				SDKROOT = iphoneos;
 				SUPPORTED_PLATFORMS = iphoneos;
@@ -589,7 +589,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 15.0;
 				MTL_ENABLE_DEBUG_INFO = YES;
 				ONLY_ACTIVE_ARCH = YES;
 				SDKROOT = iphoneos;
@@ -638,7 +638,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 15.0;
 				MTL_ENABLE_DEBUG_INFO = NO;
 				SDKROOT = iphoneos;
 				SUPPORTED_PLATFORMS = iphoneos;

example/macos/Podfile

@@ -1,4 +1,4 @@
-platform :osx, '11.0'
+platform :osx, '12.0'
 
 # CocoaPods analytics sends network stats synchronously affecting flutter build latency.
 ENV['COCOAPODS_DISABLE_STATS'] = 'true'

example/macos/Runner.xcodeproj/project.pbxproj

@@ -545,7 +545,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				MACOSX_DEPLOYMENT_TARGET = 10.15;
+				MACOSX_DEPLOYMENT_TARGET = 12.0;
 				MTL_ENABLE_DEBUG_INFO = NO;
 				SDKROOT = macosx;
 				SWIFT_COMPILATION_MODE = wholemodule;
@@ -625,7 +625,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				MACOSX_DEPLOYMENT_TARGET = 10.15;
+				MACOSX_DEPLOYMENT_TARGET = 12.0;
 				MTL_ENABLE_DEBUG_INFO = YES;
 				ONLY_ACTIVE_ARCH = YES;
 				SDKROOT = macosx;
@@ -672,7 +672,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				MACOSX_DEPLOYMENT_TARGET = 10.15;
+				MACOSX_DEPLOYMENT_TARGET = 12.0;
 				MTL_ENABLE_DEBUG_INFO = NO;
 				SDKROOT = macosx;
 				SWIFT_COMPILATION_MODE = wholemodule;

ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoPlatformView.m

@@ -1,134 +1,38 @@
 #import "include/stream_webrtc_flutter/FlutterRTCVideoPlatformView.h"
 
 @implementation FlutterRTCVideoPlatformView {
-  CGSize _videoSize;
-  AVSampleBufferDisplayLayer* _videoLayer;
-  CGSize _remoteVideoSize;
-  CATransform3D _bufferTransform;
-  RTCVideoRotation _lastVideoRotation;
+  // The shared-Metal renderer, frames are handed straight to it (RTCVideoRenderer).
+  RTCVideoRenderingView* _renderingView;
 }
 
 - (instancetype)initWithFrame:(CGRect)frame {
   if (self = [super initWithFrame:frame]) {
-    _videoLayer = [[AVSampleBufferDisplayLayer alloc] init];
-    _videoLayer.videoGravity = AVLayerVideoGravityResizeAspectFill;
-    _videoLayer.frame = CGRectZero;
-    _bufferTransform = CATransform3DIdentity;
-    _lastVideoRotation = RTCVideoRotation_0;
-    [self.layer addSublayer:_videoLayer];
+    // Mirror the Swift SDK's VideoRenderer configuration
+    _renderingView = [[RTCVideoRenderingView alloc] initWithFrame:self.bounds];
+    _renderingView.renderingBackend = RTCVideoRenderingBackendSharedMetal;
+    _renderingView.maxInFlightFrames = 2;
+    _renderingView.videoContentMode = UIViewContentModeScaleAspectFill;
+    _renderingView.enabled = YES;
+    _renderingView.autoresizingMask =
+        UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
+    [self addSubview:_renderingView];
     self.opaque = NO;
   }
   return self;
 }
 
 - (void)layoutSubviews {
-  _videoLayer.frame = self.bounds;
-  [_videoLayer removeAllAnimations];
+  [super layoutSubviews];
+  _renderingView.frame = self.bounds;
 }
 
 - (void)setSize:(CGSize)size {
-  _remoteVideoSize = size;
+  [_renderingView setSize:size];
 }
 
 - (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
-  CVPixelBufferRef pixelBuffer = nil;
-  if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
-    pixelBuffer = ((RTCCVPixelBuffer*)frame.buffer).pixelBuffer;
-    CFRetain(pixelBuffer);
-  } else if ([frame.buffer isKindOfClass:[RTCI420Buffer class]]) {
-    pixelBuffer = [self toCVPixelBuffer:frame];
-  }
-
-  if (_lastVideoRotation != frame.rotation) {
-    _bufferTransform = [self fromFrameRotation:frame.rotation];
-    _videoLayer.transform = _bufferTransform;
-    [_videoLayer layoutIfNeeded];
-    _lastVideoRotation = frame.rotation;
-  }
-
-  CMSampleBufferRef sampleBuffer = [self sampleBufferFromPixelBuffer:pixelBuffer];
-  if (sampleBuffer) {
-    if (@available(iOS 14.0, *)) {
-      if ([_videoLayer requiresFlushToResumeDecoding]) {
-        [_videoLayer flushAndRemoveImage];
-      }
-    } else {
-      // Fallback on earlier versions
-    }
-    [_videoLayer enqueueSampleBuffer:sampleBuffer];
-    CFRelease(sampleBuffer);
-  }
-
-  CFRelease(pixelBuffer);
-}
-
-- (CVPixelBufferRef)toCVPixelBuffer:(RTCVideoFrame*)frame {
-  CVPixelBufferRef outputPixelBuffer;
-  NSDictionary* pixelAttributes = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}};
-  CVPixelBufferCreate(kCFAllocatorDefault, frame.width, frame.height,
-                      kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
-                      (__bridge CFDictionaryRef)(pixelAttributes), &outputPixelBuffer);
-  id<RTCI420Buffer> i420Buffer = (RTCI420Buffer*)frame.buffer;
-
-  CVPixelBufferLockBaseAddress(outputPixelBuffer, 0);
-  // NV12
-  uint8_t* dstY = CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0);
-  const size_t dstYStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0);
-  uint8_t* dstUV = CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1);
-  const size_t dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1);
-
-  [RTCYUVHelper I420ToNV12:i420Buffer.dataY
-                srcStrideY:i420Buffer.strideY
-                      srcU:i420Buffer.dataU
-                srcStrideU:i420Buffer.strideU
-                      srcV:i420Buffer.dataV
-                srcStrideV:i420Buffer.strideV
-                      dstY:dstY
-                dstStrideY:(int)dstYStride
-                     dstUV:dstUV
-               dstStrideUV:(int)dstUVStride
-                     width:i420Buffer.width
-                    height:i420Buffer.height];
-
-  CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0);
-  return outputPixelBuffer;
-}
-
-- (CMSampleBufferRef)sampleBufferFromPixelBuffer:(CVPixelBufferRef)pixelBuffer {
-  CMSampleBufferRef sampleBuffer = NULL;
-  OSStatus err = noErr;
-  CMVideoFormatDescriptionRef formatDesc = NULL;
-  err = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &formatDesc);
-  if (err != noErr) {
-    return nil;
-  }
-  CMSampleTimingInfo sampleTimingInfo = kCMTimingInfoInvalid;
-  err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBuffer, formatDesc,
-                                                 &sampleTimingInfo, &sampleBuffer);
-  if (sampleBuffer) {
-    CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES);
-    CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
-    CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
-  }
-  if (err != noErr) {
-    return nil;
-  }
-  formatDesc = nil;
-  return sampleBuffer;
-}
-
-- (CATransform3D)fromFrameRotation:(RTCVideoRotation)rotation {
-  switch (rotation) {
-    case RTCVideoRotation_0:
-      return CATransform3DIdentity;
-    case RTCVideoRotation_90:
-      return CATransform3DMakeRotation(M_PI / 2.0, 0, 0, 1);
-    case RTCVideoRotation_180:
-      return CATransform3DMakeRotation(M_PI, 0, 0, 1);
-    case RTCVideoRotation_270:
-      return CATransform3DMakeRotation(-M_PI / 0, 0, 0, 1);
-  }
-  return CATransform3DIdentity;
+  // Hand the frame straight to the shared-Metal renderer
+  [_renderingView renderFrame:frame];
 }
 
 @end

ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoPlatformViewController.m

@@ -46,7 +46,8 @@ - (void)setVideoTrack:(RTCVideoTrack*)videoTrack {
   }
   _videoTrack = videoTrack;
   _isFirstFrameRendered = false;
-  if (!oldValue) {
+  if (oldValue) {
+    // Detach from the previous track.
     [oldValue removeRenderer:(id<RTCVideoRenderer>)self];
     _videoView.frame = CGRectZero;
   }