From fdfec37ca98ed5acdc3f4fa38a7e3ce9f01a8755 Mon Sep 17 00:00:00 2001 From: Ilias Pavlidakis Date: Wed, 17 Jul 2024 19:13:29 +0300 Subject: [PATCH] Fix stretchy tracks (#462) --- .../VideoRenderer/LocalVideoView.swift | 47 +++++ .../{ => VideoRenderer}/VideoRenderer.swift | 109 ++---------- .../VideoRenderer/VideoRendererView.swift | 160 ++++++++++++++++++ StreamVideo.xcodeproj/project.pbxproj | 18 +- 4 files changed, 239 insertions(+), 95 deletions(-) create mode 100644 Sources/StreamVideoSwiftUI/CallView/VideoRenderer/LocalVideoView.swift rename Sources/StreamVideoSwiftUI/CallView/{ => VideoRenderer}/VideoRenderer.swift (64%) create mode 100644 Sources/StreamVideoSwiftUI/CallView/VideoRenderer/VideoRendererView.swift diff --git a/Sources/StreamVideoSwiftUI/CallView/VideoRenderer/LocalVideoView.swift b/Sources/StreamVideoSwiftUI/CallView/VideoRenderer/LocalVideoView.swift new file mode 100644 index 000000000..82cfd51e0 --- /dev/null +++ b/Sources/StreamVideoSwiftUI/CallView/VideoRenderer/LocalVideoView.swift @@ -0,0 +1,47 @@ +// +// Copyright © 2024 Stream.io Inc. All rights reserved. +// + +import Foundation +import StreamVideo +import SwiftUI + +public struct LocalVideoView: View { + + @Injected(\.streamVideo) var streamVideo + + private let callSettings: CallSettings + private var viewFactory: Factory + private var participant: CallParticipant + private var idSuffix: String + private var call: Call? + private var availableFrame: CGRect + + public init( + viewFactory: Factory, + participant: CallParticipant, + idSuffix: String = "local", + callSettings: CallSettings, + call: Call?, + availableFrame: CGRect + ) { + self.viewFactory = viewFactory + self.participant = participant + self.idSuffix = idSuffix + self.callSettings = callSettings + self.call = call + self.availableFrame = availableFrame + } + + public var body: some View { + viewFactory.makeVideoParticipantView( + participant: participant, + id: "\(streamVideo.user.id)-\(idSuffix)", + availableFrame: availableFrame, + contentMode: .scaleAspectFill, + customData: ["videoOn": .bool(callSettings.videoOn)], + call: call + ) + .adjustVideoFrame(to: availableFrame.width, ratio: availableFrame.width / availableFrame.height) + } +} diff --git a/Sources/StreamVideoSwiftUI/CallView/VideoRenderer.swift b/Sources/StreamVideoSwiftUI/CallView/VideoRenderer/VideoRenderer.swift similarity index 64% rename from Sources/StreamVideoSwiftUI/CallView/VideoRenderer.swift rename to Sources/StreamVideoSwiftUI/CallView/VideoRenderer/VideoRenderer.swift index 0e47a2be2..267adaaf2 100644 --- a/Sources/StreamVideoSwiftUI/CallView/VideoRenderer.swift +++ b/Sources/StreamVideoSwiftUI/CallView/VideoRenderer/VideoRenderer.swift @@ -8,103 +8,18 @@ import StreamVideo import StreamWebRTC import SwiftUI -public struct LocalVideoView: View { - - @Injected(\.streamVideo) var streamVideo - - private let callSettings: CallSettings - private var viewFactory: Factory - private var participant: CallParticipant - private var idSuffix: String - private var call: Call? - private var availableFrame: CGRect - - public init( - viewFactory: Factory, - participant: CallParticipant, - idSuffix: String = "local", - callSettings: CallSettings, - call: Call?, - availableFrame: CGRect - ) { - self.viewFactory = viewFactory - self.participant = participant - self.idSuffix = idSuffix - self.callSettings = callSettings - self.call = call - self.availableFrame = availableFrame - } - - public var body: some View { - viewFactory.makeVideoParticipantView( - participant: participant, - id: "\(streamVideo.user.id)-\(idSuffix)", - availableFrame: availableFrame, - contentMode: .scaleAspectFill, - customData: ["videoOn": .bool(callSettings.videoOn)], - call: call - ) - .adjustVideoFrame(to: availableFrame.width, ratio: availableFrame.width / availableFrame.height) - } -} - -public struct VideoRendererView: UIViewRepresentable { - - public typealias UIViewType = VideoRenderer - - @Injected(\.utils) var utils - @Injected(\.colors) var colors - @Injected(\.videoRendererPool) private var videoRendererPool - - var id: String - - var size: CGSize - - var contentMode: UIView.ContentMode - - /// The parameter is used as an optimisation that works with the ViewRenderer Cache that's in place. - /// In cases where there is no video available, we will render a dummy VideoRenderer that won't try - /// to get a handle on the cached VideoRenderer, resolving the issue where video tracks may get dark. - var showVideo: Bool - - var handleRendering: (VideoRenderer) -> Void - - public init( - id: String, - size: CGSize, - contentMode: UIView.ContentMode = .scaleAspectFill, - showVideo: Bool = true, - handleRendering: @escaping (VideoRenderer) -> Void - ) { - self.id = id - self.size = size - self.handleRendering = handleRendering - self.showVideo = showVideo - self.contentMode = contentMode - } - - public func makeUIView(context: Context) -> VideoRenderer { - let view = videoRendererPool.acquireRenderer(size: size) - view.videoContentMode = contentMode - view.backgroundColor = colors.participantBackground - if showVideo { - handleRendering(view) - } - return view - } - - public func updateUIView(_ uiView: VideoRenderer, context: Context) { - if showVideo { - handleRendering(uiView) - } - } -} - /// A custom video renderer based on RTCMTLVideoView for rendering RTCVideoTrack objects. public class VideoRenderer: RTCMTLVideoView { @Injected(\.thermalStateObserver) private var thermalStateObserver - @Injected(\.videoRendererPool) private var videoRendererPool + + private let _windowSubject: PassthroughSubject = .init() + private let _superviewSubject: PassthroughSubject = .init() + private let _frameSubject: PassthroughSubject = .init() + + var windowPublisher: AnyPublisher { _windowSubject.eraseToAnyPublisher() } + var superviewPublisher: AnyPublisher { _superviewSubject.eraseToAnyPublisher() } + var framePublisher: AnyPublisher { _frameSubject.eraseToAnyPublisher() } /// DispatchQueue for synchronizing access to the video track. let queue = DispatchQueue(label: "video-track") @@ -186,13 +101,19 @@ public class VideoRenderer: RTCMTLVideoView { override public func layoutSubviews() { super.layoutSubviews() viewSize = bounds.size + _frameSubject.send(frame) + } + + override public func willMove(toWindow newWindow: UIWindow?) { + _windowSubject.send(newWindow) + super.willMove(toWindow: newWindow) } /// Overrides the willMove(toSuperview:) method to release the renderer when removed from its superview. override public func willMove(toSuperview newSuperview: UIView?) { + _superviewSubject.send(newSuperview) super.willMove(toSuperview: newSuperview) if newSuperview == nil { - videoRendererPool.releaseRenderer(self) // Clean up any rendered frames. setSize(.zero) } diff --git a/Sources/StreamVideoSwiftUI/CallView/VideoRenderer/VideoRendererView.swift b/Sources/StreamVideoSwiftUI/CallView/VideoRenderer/VideoRendererView.swift new file mode 100644 index 000000000..2b3b74e2d --- /dev/null +++ b/Sources/StreamVideoSwiftUI/CallView/VideoRenderer/VideoRendererView.swift @@ -0,0 +1,160 @@ +// +// Copyright © 2024 Stream.io Inc. All rights reserved. +// + +import Combine +import Foundation +import StreamVideo +import SwiftUI + +/// A view that wraps a `VideoRenderer` and integrates with SwiftUI. +public struct VideoRendererView: UIViewRepresentable { + + /// The type of the `UIView` being represented. + public typealias UIViewType = VideoRenderer + + /// Injected dependency for accessing color configurations. + @Injected(\.colors) var colors + + /// The identifier for the video renderer. + var id: String + + /// The size of the video renderer view. + var size: CGSize + + /// The content mode for the video renderer. + var contentMode: UIView.ContentMode + + /// A flag to determine whether video should be shown. Optimizes rendering by using a dummy renderer when false. + var showVideo: Bool + + /// A closure to handle the rendering of the video. + var handleRendering: (VideoRenderer) -> Void + + /// Initializes a new instance of `VideoRendererView`. + /// - Parameters: + /// - id: The identifier for the video renderer. + /// - size: The size of the video renderer view. + /// - contentMode: The content mode for the video renderer. Default is `.scaleAspectFill`. + /// - showVideo: A flag to determine whether video should be shown. Default is `true`. + /// - handleRendering: A closure to handle the rendering of the video. + public init( + id: String, + size: CGSize, + contentMode: UIView.ContentMode = .scaleAspectFill, + showVideo: Bool = true, + handleRendering: @escaping (VideoRenderer) -> Void + ) { + self.id = id + self.size = size + self.handleRendering = handleRendering + self.showVideo = showVideo + self.contentMode = contentMode + } + + /// Dismantles the `UIView` when it is no longer needed. + /// - Parameters: + /// - uiView: The `VideoRenderer` to dismantle. + /// - coordinator: The coordinator associated with the view. + public static func dismantleUIView( + _ uiView: VideoRenderer, + coordinator: Coordinator + ) { + coordinator.dismantle() + } + + /// Creates the `VideoRenderer` view. + /// - Parameter context: The context containing information about the current state of the system. + /// - Returns: A configured `VideoRenderer` instance. + public func makeUIView(context: Context) -> VideoRenderer { + context.coordinator.renderer.frame = .init( + origin: context.coordinator.renderer.frame.origin, + size: size + ) + context.coordinator.renderer.videoContentMode = contentMode + context.coordinator.renderer.backgroundColor = colors.participantBackground + + if showVideo { + handleRendering(context.coordinator.renderer) + } + return context.coordinator.renderer + } + + /// Updates the `VideoRenderer` view when the state changes. + /// - Parameters: + /// - uiView: The `VideoRenderer` to update. + /// - context: The context containing information about the current state of the system. + public func updateUIView(_ uiView: VideoRenderer, context: Context) { + if showVideo { + handleRendering(uiView) + } + } + + /// Creates the coordinator for managing the view. + /// - Returns: A new `Coordinator` instance. + public func makeCoordinator() -> Coordinator { + Coordinator(handleRendering: handleRendering) + } +} + +/// Extension for `VideoRendererView` to define the `Coordinator` class. +extension VideoRendererView { + /// A class to coordinate the `VideoRendererView` and manage its lifecycle. + public final class Coordinator { + /// Injected dependency for accessing the video renderer pool. + @Injected(\.videoRendererPool) private var videoRendererPool + + /// A closure to handle the rendering of the video. + private let handleRendering: ((VideoRenderer) -> Void)? + /// A disposable bag to manage cancellable subscriptions. + private let disposableBag = DisposableBag() + + /// The video renderer managed by this coordinator. + fileprivate private(set) lazy var renderer: VideoRenderer = videoRendererPool + .acquireRenderer(size: .zero) + + /// Initializes a new instance of the coordinator. + /// - Parameter handleRendering: A closure to handle the rendering of the video. + init(handleRendering: ((VideoRenderer) -> Void)?) { + self.handleRendering = handleRendering + _ = renderer + setupRendererObservation() + } + + deinit { + dismantle() + } + + /// Dismantles the video renderer and releases resources. + func dismantle() { + renderer.track?.remove(renderer) + disposableBag.removeAll() + videoRendererPool.releaseRenderer(renderer) + } + + // MARK: Private API + + /// Sets up observation for the renderer's window and superview. + private func setupRendererObservation() { + renderer + .windowPublisher + .map { $0 != nil } + .removeDuplicates() + .sink { [weak self] in + guard let self else { return } + if $0 { handleRendering?(renderer) } + } + .store(in: disposableBag) + + renderer + .superviewPublisher + .map { $0 != nil } + .removeDuplicates() + .sink { [weak self] in + guard let self else { return } + if $0 { handleRendering?(renderer) } + } + .store(in: disposableBag) + } + } +} diff --git a/StreamVideo.xcodeproj/project.pbxproj b/StreamVideo.xcodeproj/project.pbxproj index 1975e3909..514aa82b7 100644 --- a/StreamVideo.xcodeproj/project.pbxproj +++ b/StreamVideo.xcodeproj/project.pbxproj @@ -15,6 +15,8 @@ 40073B7A2C456E44006A2867 /* StreamPictureInPictureFixedWindowSizePolicy_Tests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40073B792C456E44006A2867 /* StreamPictureInPictureFixedWindowSizePolicy_Tests.swift */; }; 40073B7D2C456F08006A2867 /* MockStreamAVPictureInPictureViewControlling.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40073B7C2C456F08006A2867 /* MockStreamAVPictureInPictureViewControlling.swift */; }; 40073B7F2C456F30006A2867 /* StreamPictureInPictureAdaptiveWindowSizePolicy_Tests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40073B7E2C456F30006A2867 /* StreamPictureInPictureAdaptiveWindowSizePolicy_Tests.swift */; }; + 40073B882C458250006A2867 /* VideoRendererView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40073B872C458250006A2867 /* VideoRendererView.swift */; }; + 40073B8A2C458259006A2867 /* LocalVideoView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40073B892C458259006A2867 /* LocalVideoView.swift */; }; 400D63F72AC3273F0000BB30 /* ThermalStateObserverTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 400D63F62AC3273F0000BB30 /* ThermalStateObserverTests.swift */; }; 400E50532BD2A900008C939E /* StreamAudioFilterCapturePostProcessingModule.swift in Sources */ = {isa = PBXBuildFile; fileRef = 400E50522BD2A900008C939E /* StreamAudioFilterCapturePostProcessingModule.swift */; }; 400E50552BD2AAD0008C939E /* StreamAudioProcessingModule.swift in Sources */ = {isa = PBXBuildFile; fileRef = 400E50542BD2AAD0008C939E /* StreamAudioProcessingModule.swift */; }; @@ -1206,6 +1208,8 @@ 40073B792C456E44006A2867 /* StreamPictureInPictureFixedWindowSizePolicy_Tests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamPictureInPictureFixedWindowSizePolicy_Tests.swift; sourceTree = ""; }; 40073B7C2C456F08006A2867 /* MockStreamAVPictureInPictureViewControlling.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MockStreamAVPictureInPictureViewControlling.swift; sourceTree = ""; }; 40073B7E2C456F30006A2867 /* StreamPictureInPictureAdaptiveWindowSizePolicy_Tests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamPictureInPictureAdaptiveWindowSizePolicy_Tests.swift; sourceTree = ""; }; + 40073B872C458250006A2867 /* VideoRendererView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VideoRendererView.swift; sourceTree = ""; }; + 40073B892C458259006A2867 /* LocalVideoView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LocalVideoView.swift; sourceTree = ""; }; 400D63F62AC3273F0000BB30 /* ThermalStateObserverTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ThermalStateObserverTests.swift; sourceTree = ""; }; 400E50522BD2A900008C939E /* StreamAudioFilterCapturePostProcessingModule.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamAudioFilterCapturePostProcessingModule.swift; sourceTree = ""; }; 400E50542BD2AAD0008C939E /* StreamAudioProcessingModule.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamAudioProcessingModule.swift; sourceTree = ""; }; @@ -2179,6 +2183,16 @@ path = Mocks; sourceTree = ""; }; + 40073B862C458240006A2867 /* VideoRenderer */ = { + isa = PBXGroup; + children = ( + 84231E4628B2506B007985EF /* VideoRenderer.swift */, + 40073B872C458250006A2867 /* VideoRendererView.swift */, + 40073B892C458259006A2867 /* LocalVideoView.swift */, + ); + path = VideoRenderer; + sourceTree = ""; + }; 401480332A5423C70029166A /* Utils */ = { isa = PBXGroup; children = ( @@ -3575,12 +3589,12 @@ 8434C523289AA2B60001490A /* CallView */ = { isa = PBXGroup; children = ( + 40073B862C458240006A2867 /* VideoRenderer */, 40245F2E2BE269B400FCF075 /* CallControls */, 40C7B82A2B612D5100FB9DB2 /* ViewModifiers */, 40AA2EDF2ADFF61B000DCA5C /* LayoutComponents */, 84429930293FA4680037232A /* ScreenSharing */, 846FBE8728AAD81E00147F6E /* Participants */, - 84231E4628B2506B007985EF /* VideoRenderer.swift */, 401480352A5447C50029166A /* LocalParticipantViewModifier.swift */, 8457CF9028BB835F00E8CF50 /* CallView.swift */, 846E4AFE29D236EA003733AB /* CallTopView.swift */, @@ -5773,6 +5787,7 @@ 8442993C294232360037232A /* IncomingCallView_iOS13.swift in Sources */, 84E86D4F2905E731004BA44C /* Utils.swift in Sources */, 84D419B828E7155100F574F9 /* CallContainer.swift in Sources */, + 40073B8A2C458259006A2867 /* LocalVideoView.swift in Sources */, 8434C531289AA8770001490A /* StreamVideoUI.swift in Sources */, 8457BF7C2A5BF9E0000AE567 /* ToastView.swift in Sources */, 844299412942394C0037232A /* VideoView_iOS13.swift in Sources */, @@ -5853,6 +5868,7 @@ 40A941782B4DC576006D6965 /* StreamPictureInPictureTrackStateAdapter.swift in Sources */, 408D29A42B6D251600885473 /* UIView+Snapshot.swift in Sources */, 84A6CD6128D49A7700318EC3 /* CallBackgrounds.swift in Sources */, + 40073B882C458250006A2867 /* VideoRendererView.swift in Sources */, 40A941722B4D9750006D6965 /* StreamAVPictureInPictureVideoCallViewController.swift in Sources */, 40A941702B4D96E6006D6965 /* StreamPictureInPictureController.swift in Sources */, );