Add two-way audio to the WebRTC camera player

Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCClient.swift

@@ -270,10 +270,14 @@ final class PlaybackOnlyRTCAudioDevice: NSObject, RTCAudioDevice {
 final class WebRTCClient: NSObject {
     private static let playbackOnlyAudioDevice = PlaybackOnlyRTCAudioDevice()
 
+    private static let sslInitialized: Void = {
+        RTCInitializeSSL()
+    }()
+
     // The `RTCPeerConnectionFactory` is in charge of creating new RTCPeerConnection instances.
     // A new RTCPeerConnection should be created every new call, but the factory is shared.
-    private static let factory: RTCPeerConnectionFactory = {
-        RTCInitializeSSL()
+    private static let playbackFactory: RTCPeerConnectionFactory = {
+        _ = sslInitialized
         let videoEncoderFactory = RTCDefaultVideoEncoderFactory()
         let videoDecoderFactory = RTCDefaultVideoDecoderFactory()
         return RTCPeerConnectionFactory(
@@ -283,22 +287,67 @@ final class WebRTCClient: NSObject {
         )
     }()
 
+    private static let recordingFactory: RTCPeerConnectionFactory = {
+        _ = sslInitialized
+        let videoEncoderFactory = RTCDefaultVideoEncoderFactory()
+        let videoDecoderFactory = RTCDefaultVideoDecoderFactory()
+        return RTCPeerConnectionFactory(
+            encoderFactory: videoEncoderFactory,
+            decoderFactory: videoDecoderFactory
+        )
+    }()
+
+    private static func configureRecordingAudioSession() {
+        let configuration = RTCAudioSessionConfiguration.webRTC()
+        configuration.category = AVAudioSession.Category.playAndRecord.rawValue
+        configuration.mode = AVAudioSession.Mode.videoChat.rawValue
+        configuration.categoryOptions = [.defaultToSpeaker, .allowBluetooth, .allowBluetoothA2DP]
+        RTCAudioSessionConfiguration.setWebRTC(configuration)
+    }
+
+    private static func restorePlaybackAudioSession() {
+        let configuration = RTCAudioSessionConfiguration.webRTC()
+        configuration.category = AVAudioSession.Category.playback.rawValue
+        configuration.mode = AVAudioSession.Mode.moviePlayback.rawValue
+        configuration.categoryOptions = [.mixWithOthers]
+        RTCAudioSessionConfiguration.setWebRTC(configuration)
+
+        let session = RTCAudioSession.sharedInstance()
+        session.lockForConfiguration()
+        defer { session.unlockForConfiguration() }
+        do {
+            try session.setActive(false)
+        } catch {
+            Current.Log.error("Failed to release talkback audio session on close: \(error.localizedDescription)")
+        }
+    }
+
     weak var delegate: WebRTCClientDelegate?
+    private let factory: RTCPeerConnectionFactory
+    private let supportsTalkback: Bool
     private let peerConnection: RTCPeerConnection
     private let mediaConstrains = [
         kRTCMediaConstraintsOfferToReceiveAudio: kRTCMediaConstraintsValueTrue,
         kRTCMediaConstraintsOfferToReceiveVideo: kRTCMediaConstraintsValueTrue,
     ]
     private var remoteVideoTrack: RTCVideoTrack?
     private var remoteAudioTrack: RTCAudioTrack?
+    private var localAudioTrack: RTCAudioTrack?
     private var remoteDataChannel: RTCDataChannel?
 
     @available(*, unavailable)
     override init() {
         fatalError("WebRTCClient:init is unavailable")
     }
 
-    required init(iceServers: [String]) {
+    required init(iceServers: [String], supportsTalkback: Bool = false) {
+        self.supportsTalkback = supportsTalkback
+        let factory = supportsTalkback ? WebRTCClient.recordingFactory : WebRTCClient.playbackFactory
+        self.factory = factory
+        if supportsTalkback {
+            WebRTCClient.configureRecordingAudioSession()
+        }
+
         let config = RTCConfiguration()
         config.iceServers = [RTCIceServer(urlStrings: iceServers)]
 
@@ -316,7 +365,7 @@ final class WebRTCClient: NSObject {
             optionalConstraints: ["DtlsSrtpKeyAgreement": kRTCMediaConstraintsValueTrue]
         )
 
-        guard let peerConnection = WebRTCClient.factory.peerConnection(
+        guard let peerConnection = factory.peerConnection(
             with: config,
             constraints: constraints,
             delegate: nil
@@ -333,6 +382,8 @@ final class WebRTCClient: NSObject {
 
     func closeConnection() {
         peerConnection.close()
+        guard supportsTalkback else { return }
+        WebRTCClient.restorePlaybackAudioSession()
     }
 
     // MARK: Signaling
@@ -401,20 +452,29 @@ final class WebRTCClient: NSObject {
         return !remoteAudioTrack.isEnabled
     }
 
+    func setMicrophoneEnabled(_ enabled: Bool) {
+        localAudioTrack?.isEnabled = enabled
+    }
+
     private func createMediaTracks() {
         let streamId = "stream"
         let videoTrack = createVideoTrack()
         peerConnection.add(videoTrack, streamIds: [streamId])
         remoteVideoTrack = peerConnection.transceivers.first { $0.mediaType == .video }?.receiver
             .track as? RTCVideoTrack
+        guard supportsTalkback else { return }
+        let audioTrack = factory.audioTrack(with: factory.audioSource(with: nil), trackId: "audio0")
+        audioTrack.isEnabled = false
+        localAudioTrack = audioTrack
+        peerConnection.add(audioTrack, streamIds: [streamId])
     }
 
     private func createVideoTrack() -> RTCVideoTrack {
         // The local track only exists to establish the transceiver we read the remote track from;
         // we never send video, so there's no capturer. RTCCameraVideoCapturer is also unavailable in
         // app extensions, which is why a capturer here broke the device build of the notification ext.
-        let videoSource = WebRTCClient.factory.videoSource()
-        let videoTrack = WebRTCClient.factory.videoTrack(with: videoSource, trackId: "video0")
+        let videoSource = factory.videoSource()
+        let videoTrack = factory.videoTrack(with: videoSource, trackId: "video0")
         return videoTrack
     }
 
@@ -427,8 +487,9 @@ final class WebRTCClient: NSObject {
             Current.Log.warning("Remote track is not an RTCAudioTrack")
             return
         }
+        let wasMuted = remoteAudioTrack.map { !$0.isEnabled } ?? true
         remoteAudioTrack = audioTrack
-        remoteAudioTrack?.isEnabled = false
+        remoteAudioTrack?.isEnabled = !wasMuted
         Current.Log.info("Remote audio track set successfully")
     }
 }

Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCVideoPlayerViewController.swift

@@ -23,10 +23,14 @@ class WebRTCVideoPlayerViewController: UIViewController {
     override func viewDidLoad() {
         super.viewDidLoad()
         setupVideoView()
-        viewModel.start()
-        if let client = viewModel.webRTCClient {
-            client.renderRemoteVideo(to: remoteVideoView)
+        viewModel.onClientReady = { [weak self] in
+            self?.attachRenderer()
         }
+        viewModel.start()
+    }
+
+    private func attachRenderer() {
+        viewModel.webRTCClient?.renderRemoteVideo(to: remoteVideoView)
     }
 
     override func viewWillDisappear(_ animated: Bool) {

Sources/App/Cameras/CameraPlayer/WebRTC/WebRTCViewPlayerViewModel.swift

@@ -1,10 +1,15 @@
+import AVFoundation
 import Foundation
 import HAKit
 import HAKit_PromiseKit
 import Shared
 import SwiftUI
 import WebRTC
 
+private enum CameraEntityFeature {
+    static let twoWayAudio = 4
+}
+
 enum WebRTCSignalType: String {
     case session
     case answer
@@ -28,13 +33,16 @@ final class WebRTCViewPlayerViewModel: ObservableObject {
     private let server: Server
     private let cameraEntityId: String
     private let supportsTalkback: Bool
+    private var statesToken: HACancellable?
+
+    var onClientReady: (() -> Void)?
 
     @Published var failureReason: String?
     @Published var showLoader: Bool = true
     @Published var isMuted: Bool = true
     @Published var isWebRTCUnsupported: Bool = false
-    @Published var isTalkbackSupported: Bool = false
     @Published var isTalking: Bool = false
+    @Published var isTalkbackSupported: Bool = false
 
     /// Invoked on offer rejection or ICE failure. Used by the notification extension to fall back;
     /// the SwiftUI player leaves it `nil` and observes the published properties instead.
@@ -46,7 +54,9 @@ final class WebRTCViewPlayerViewModel: ObservableObject {
         self.supportsTalkback = supportsTalkback
     }
 
-    func toggleTalkback() {}
+    deinit {
+        statesToken?.cancel()
+    }
 
     func toggleMute() {
         guard let webRTCClient else { return }
@@ -62,13 +72,29 @@ final class WebRTCViewPlayerViewModel: ObservableObject {
     // MARK: - WebRTC
 
     func start() {
-        webRTCClient = nil
-        webRTCClient = WebRTCClient(iceServers: AppConstants.WebRTC.iceServers)
+        guard supportsTalkback else {
+            connect(withTalkback: false)
+            return
+        }
+        determineTalkbackSupport { [weak self] supported in
+            self?.connect(withTalkback: supported)
+        }
+    }
+
+    private func connect(withTalkback: Bool) {
+        webRTCClient?.closeConnection()
+        sessionId = nil
+        pendingCandidates.removeAll()
+        webRTCClient = WebRTCClient(
+            iceServers: AppConstants.WebRTC.iceServers,
+            supportsTalkback: withTalkback
+        )
         guard let webRTCClient else {
             assertionFailure("WebRTCClient initialization failed")
             return
         }
         webRTCClient.delegate = self
+        onClientReady?()
         webRTCClient.offer { [weak self] sdp in
             guard let self else {
                 assertionFailure("Self is nil in WebRTCViewPlayerViewModel.start")
@@ -98,26 +124,29 @@ final class WebRTCViewPlayerViewModel: ObservableObject {
                     self?.onFailure?()
                 }
             } handler: { [weak self] _, data in
-                guard let self else { return }
-                guard let typeString: String = try? data.decode("type") else {
-                    assertionFailure("Failed to decode type from data")
-                    return
-                }
-                let type = WebRTCSignalType(typeString)
-                switch type {
-                case .session:
-                    handleSession(data)
-                case .answer:
-                    handleAnswer(data)
-                case .candidate:
-                    handleCandidate(data)
-                case .unknown:
-                    debugPrint("Unknown type: \(typeString)")
-                }
+                self?.handleSignal(data)
             }
         }
     }
 
+    private func handleSignal(_ data: HAData) {
+        guard let typeString: String = try? data.decode("type") else {
+            assertionFailure("Failed to decode type from data")
+            return
+        }
+        let type = WebRTCSignalType(typeString)
+        switch type {
+        case .session:
+            handleSession(data)
+        case .answer:
+            handleAnswer(data)
+        case .candidate:
+            handleCandidate(data)
+        case .unknown:
+            debugPrint("Unknown type: \(typeString)")
+        }
+    }
+
     private func handleSession(_ data: HAData) {
         guard let sessionId: String = try? data.decode("session_id") else {
             assertionFailure("Failed to decode session_id from data")
@@ -194,6 +223,79 @@ final class WebRTCViewPlayerViewModel: ObservableObject {
             }
         }
     }
+
+    // MARK: - Talkback
+
+    func toggleTalkback() {
+        if isTalking {
+            stopTalkback()
+        } else {
+            startTalkback()
+        }
+    }
+
+    private func startTalkback() {
+        Task { @MainActor [weak self] in
+            guard let self else { return }
+            let granted = await self.requestMicrophonePermission()
+            guard granted else {
+                self.failureReason = L10n.CameraPlayer.Talkback.microphoneDenied
+                return
+            }
+            self.webRTCClient?.setMicrophoneEnabled(true)
+            self.isTalking = true
+        }
+    }
+
+    private func stopTalkback() {
+        webRTCClient?.setMicrophoneEnabled(false)
+        isTalking = false
+    }
+
+    private func requestMicrophonePermission() async -> Bool {
+        await withCheckedContinuation { continuation in
+            if #available(iOS 17.0, *) {
+                AVAudioApplication.requestRecordPermission { granted in
+                    continuation.resume(returning: granted)
+                }
+            } else {
+                AVAudioSession.sharedInstance().requestRecordPermission { granted in
+                    continuation.resume(returning: granted)
+                }
+            }
+        }
+    }
+
+    private func determineTalkbackSupport(completion: @escaping (Bool) -> Void) {
+        guard let api = Current.api(for: server) else {
+            completion(false)
+            return
+        }
+        var finished = false
+        let finish: (Bool) -> Void = { [weak self] supported in
+            Task { @MainActor [weak self] in
+                guard !finished else { return }
+                finished = true
+                self?.statesToken?.cancel()
+                self?.statesToken = nil
+                self?.isTalkbackSupported = supported
+                completion(supported)
+            }
+        }
+        statesToken = api.connection.caches.states().subscribe { [weak self] token, states in
+            guard let self else {
+                token.cancel()
+                return
+            }
+            guard let entity = states[cameraEntityId] else { return }
+            let features = (entity.attributes["supported_features"] as? Int) ?? 0
+            finish((features & CameraEntityFeature.twoWayAudio) != 0)
+        }
+        Task { @MainActor in
+            try? await Task.sleep(nanoseconds: 3_000_000_000)
+            finish(false)
+        }
+    }
 }
 
 extension WebRTCViewPlayerViewModel: WebRTCClientDelegate {