diff --git a/Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCClient.swift b/Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCClient.swift index f3ec68cbf..da6dba6de 100644 --- a/Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCClient.swift +++ b/Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCClient.swift @@ -270,10 +270,14 @@ final class PlaybackOnlyRTCAudioDevice: NSObject, RTCAudioDevice { final class WebRTCClient: NSObject { private static let playbackOnlyAudioDevice = PlaybackOnlyRTCAudioDevice() + private static let sslInitialized: Void = { + RTCInitializeSSL() + }() + // The `RTCPeerConnectionFactory` is in charge of creating new RTCPeerConnection instances. // A new RTCPeerConnection should be created every new call, but the factory is shared. - private static let factory: RTCPeerConnectionFactory = { - RTCInitializeSSL() + private static let playbackFactory: RTCPeerConnectionFactory = { + _ = sslInitialized let videoEncoderFactory = RTCDefaultVideoEncoderFactory() let videoDecoderFactory = RTCDefaultVideoDecoderFactory() return RTCPeerConnectionFactory( @@ -283,7 +287,44 @@ final class WebRTCClient: NSObject { ) }() + private static let recordingFactory: RTCPeerConnectionFactory = { + _ = sslInitialized + let videoEncoderFactory = RTCDefaultVideoEncoderFactory() + let videoDecoderFactory = RTCDefaultVideoDecoderFactory() + return RTCPeerConnectionFactory( + encoderFactory: videoEncoderFactory, + decoderFactory: videoDecoderFactory + ) + }() + + private static func configureRecordingAudioSession() { + let configuration = RTCAudioSessionConfiguration.webRTC() + configuration.category = AVAudioSession.Category.playAndRecord.rawValue + configuration.mode = AVAudioSession.Mode.videoChat.rawValue + configuration.categoryOptions = [.defaultToSpeaker, .allowBluetooth, .allowBluetoothA2DP] + RTCAudioSessionConfiguration.setWebRTC(configuration) + } + + private static func restorePlaybackAudioSession() { + let configuration = RTCAudioSessionConfiguration.webRTC() + configuration.category = AVAudioSession.Category.playback.rawValue + configuration.mode = AVAudioSession.Mode.moviePlayback.rawValue + configuration.categoryOptions = [.mixWithOthers] + RTCAudioSessionConfiguration.setWebRTC(configuration) + + let session = RTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + do { + try session.setActive(false) + } catch { + Current.Log.error("Failed to release talkback audio session on close: \(error.localizedDescription)") + } + } + weak var delegate: WebRTCClientDelegate? + private let factory: RTCPeerConnectionFactory + private let supportsTalkback: Bool private let peerConnection: RTCPeerConnection private let mediaConstrains = [ kRTCMediaConstraintsOfferToReceiveAudio: kRTCMediaConstraintsValueTrue, @@ -291,6 +332,7 @@ final class WebRTCClient: NSObject { ] private var remoteVideoTrack: RTCVideoTrack? private var remoteAudioTrack: RTCAudioTrack? + private var localAudioTrack: RTCAudioTrack? private var remoteDataChannel: RTCDataChannel? @available(*, unavailable) @@ -298,7 +340,14 @@ final class WebRTCClient: NSObject { fatalError("WebRTCClient:init is unavailable") } - required init(iceServers: [String]) { + required init(iceServers: [String], supportsTalkback: Bool = false) { + self.supportsTalkback = supportsTalkback + let factory = supportsTalkback ? WebRTCClient.recordingFactory : WebRTCClient.playbackFactory + self.factory = factory + if supportsTalkback { + WebRTCClient.configureRecordingAudioSession() + } + let config = RTCConfiguration() config.iceServers = [RTCIceServer(urlStrings: iceServers)] @@ -316,7 +365,7 @@ final class WebRTCClient: NSObject { optionalConstraints: ["DtlsSrtpKeyAgreement": kRTCMediaConstraintsValueTrue] ) - guard let peerConnection = WebRTCClient.factory.peerConnection( + guard let peerConnection = factory.peerConnection( with: config, constraints: constraints, delegate: nil @@ -333,6 +382,8 @@ final class WebRTCClient: NSObject { func closeConnection() { peerConnection.close() + guard supportsTalkback else { return } + WebRTCClient.restorePlaybackAudioSession() } // MARK: Signaling @@ -401,20 +452,29 @@ final class WebRTCClient: NSObject { return !remoteAudioTrack.isEnabled } + func setMicrophoneEnabled(_ enabled: Bool) { + localAudioTrack?.isEnabled = enabled + } + private func createMediaTracks() { let streamId = "stream" let videoTrack = createVideoTrack() peerConnection.add(videoTrack, streamIds: [streamId]) remoteVideoTrack = peerConnection.transceivers.first { $0.mediaType == .video }?.receiver .track as? RTCVideoTrack + guard supportsTalkback else { return } + let audioTrack = factory.audioTrack(with: factory.audioSource(with: nil), trackId: "audio0") + audioTrack.isEnabled = false + localAudioTrack = audioTrack + peerConnection.add(audioTrack, streamIds: [streamId]) } private func createVideoTrack() -> RTCVideoTrack { // The local track only exists to establish the transceiver we read the remote track from; // we never send video, so there's no capturer. RTCCameraVideoCapturer is also unavailable in // app extensions, which is why a capturer here broke the device build of the notification ext. - let videoSource = WebRTCClient.factory.videoSource() - let videoTrack = WebRTCClient.factory.videoTrack(with: videoSource, trackId: "video0") + let videoSource = factory.videoSource() + let videoTrack = factory.videoTrack(with: videoSource, trackId: "video0") return videoTrack } @@ -427,8 +487,9 @@ final class WebRTCClient: NSObject { Current.Log.warning("Remote track is not an RTCAudioTrack") return } + let wasMuted = remoteAudioTrack.map { !$0.isEnabled } ?? true remoteAudioTrack = audioTrack - remoteAudioTrack?.isEnabled = false + remoteAudioTrack?.isEnabled = !wasMuted Current.Log.info("Remote audio track set successfully") } } diff --git a/Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCVideoPlayerViewController.swift b/Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCVideoPlayerViewController.swift index 893afcae5..8c27dadf9 100644 --- a/Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCVideoPlayerViewController.swift +++ b/Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCVideoPlayerViewController.swift @@ -23,10 +23,14 @@ class WebRTCVideoPlayerViewController: UIViewController { override func viewDidLoad() { super.viewDidLoad() setupVideoView() - viewModel.start() - if let client = viewModel.webRTCClient { - client.renderRemoteVideo(to: remoteVideoView) + viewModel.onClientReady = { [weak self] in + self?.attachRenderer() } + viewModel.start() + } + + private func attachRenderer() { + viewModel.webRTCClient?.renderRemoteVideo(to: remoteVideoView) } override func viewWillDisappear(_ animated: Bool) { diff --git a/Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCViewPlayerViewModel.swift b/Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCViewPlayerViewModel.swift index 1550c51cd..8a532c451 100644 --- a/Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCViewPlayerViewModel.swift +++ b/Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCViewPlayerViewModel.swift @@ -1,3 +1,4 @@ +import AVFoundation import Foundation import HAKit import HAKit_PromiseKit @@ -5,6 +6,10 @@ import Shared import SwiftUI import WebRTC +private enum CameraEntityFeature { + static let twoWayAudio = 4 +} + enum WebRTCSignalType: String { case session case answer @@ -28,13 +33,16 @@ final class WebRTCViewPlayerViewModel: ObservableObject { private let server: Server private let cameraEntityId: String private let supportsTalkback: Bool + private var statesToken: HACancellable? + + var onClientReady: (() -> Void)? @Published var failureReason: String? @Published var showLoader: Bool = true @Published var isMuted: Bool = true @Published var isWebRTCUnsupported: Bool = false - @Published var isTalkbackSupported: Bool = false @Published var isTalking: Bool = false + @Published var isTalkbackSupported: Bool = false /// Invoked on offer rejection or ICE failure. Used by the notification extension to fall back; /// the SwiftUI player leaves it `nil` and observes the published properties instead. @@ -46,7 +54,9 @@ final class WebRTCViewPlayerViewModel: ObservableObject { self.supportsTalkback = supportsTalkback } - func toggleTalkback() {} + deinit { + statesToken?.cancel() + } func toggleMute() { guard let webRTCClient else { return } @@ -62,13 +72,29 @@ final class WebRTCViewPlayerViewModel: ObservableObject { // MARK: - WebRTC func start() { - webRTCClient = nil - webRTCClient = WebRTCClient(iceServers: AppConstants.WebRTC.iceServers) + guard supportsTalkback else { + connect(withTalkback: false) + return + } + determineTalkbackSupport { [weak self] supported in + self?.connect(withTalkback: supported) + } + } + + private func connect(withTalkback: Bool) { + webRTCClient?.closeConnection() + sessionId = nil + pendingCandidates.removeAll() + webRTCClient = WebRTCClient( + iceServers: AppConstants.WebRTC.iceServers, + supportsTalkback: withTalkback + ) guard let webRTCClient else { assertionFailure("WebRTCClient initialization failed") return } webRTCClient.delegate = self + onClientReady?() webRTCClient.offer { [weak self] sdp in guard let self else { assertionFailure("Self is nil in WebRTCViewPlayerViewModel.start") @@ -98,26 +124,29 @@ final class WebRTCViewPlayerViewModel: ObservableObject { self?.onFailure?() } } handler: { [weak self] _, data in - guard let self else { return } - guard let typeString: String = try? data.decode("type") else { - assertionFailure("Failed to decode type from data") - return - } - let type = WebRTCSignalType(typeString) - switch type { - case .session: - handleSession(data) - case .answer: - handleAnswer(data) - case .candidate: - handleCandidate(data) - case .unknown: - debugPrint("Unknown type: \(typeString)") - } + self?.handleSignal(data) } } } + private func handleSignal(_ data: HAData) { + guard let typeString: String = try? data.decode("type") else { + assertionFailure("Failed to decode type from data") + return + } + let type = WebRTCSignalType(typeString) + switch type { + case .session: + handleSession(data) + case .answer: + handleAnswer(data) + case .candidate: + handleCandidate(data) + case .unknown: + debugPrint("Unknown type: \(typeString)") + } + } + private func handleSession(_ data: HAData) { guard let sessionId: String = try? data.decode("session_id") else { assertionFailure("Failed to decode session_id from data") @@ -194,6 +223,79 @@ final class WebRTCViewPlayerViewModel: ObservableObject { } } } + + // MARK: - Talkback + + func toggleTalkback() { + if isTalking { + stopTalkback() + } else { + startTalkback() + } + } + + private func startTalkback() { + Task { @MainActor [weak self] in + guard let self else { return } + let granted = await self.requestMicrophonePermission() + guard granted else { + self.failureReason = L10n.CameraPlayer.Talkback.microphoneDenied + return + } + self.webRTCClient?.setMicrophoneEnabled(true) + self.isTalking = true + } + } + + private func stopTalkback() { + webRTCClient?.setMicrophoneEnabled(false) + isTalking = false + } + + private func requestMicrophonePermission() async -> Bool { + await withCheckedContinuation { continuation in + if #available(iOS 17.0, *) { + AVAudioApplication.requestRecordPermission { granted in + continuation.resume(returning: granted) + } + } else { + AVAudioSession.sharedInstance().requestRecordPermission { granted in + continuation.resume(returning: granted) + } + } + } + } + + private func determineTalkbackSupport(completion: @escaping (Bool) -> Void) { + guard let api = Current.api(for: server) else { + completion(false) + return + } + var finished = false + let finish: (Bool) -> Void = { [weak self] supported in + Task { @MainActor [weak self] in + guard !finished else { return } + finished = true + self?.statesToken?.cancel() + self?.statesToken = nil + self?.isTalkbackSupported = supported + completion(supported) + } + } + statesToken = api.connection.caches.states().subscribe { [weak self] token, states in + guard let self else { + token.cancel() + return + } + guard let entity = states[cameraEntityId] else { return } + let features = (entity.attributes["supported_features"] as? Int) ?? 0 + finish((features & CameraEntityFeature.twoWayAudio) != 0) + } + Task { @MainActor in + try? await Task.sleep(nanoseconds: 3_000_000_000) + finish(false) + } + } } extension WebRTCViewPlayerViewModel: WebRTCClientDelegate {