Skip to content

feat(sender): add per-receiver codec selection #175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Betafer wants to merge 1 commit into
base: maint-3.5
Choose a base branch
from
Draft

feat(sender): add per-receiver codec selection #175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions TargetBridge-Sender/TBDisplaySender/TBDisplaySenderContentView.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -615,6 +615,32 @@ private struct TBDisplaySenderSessionSettingsSheet: View {
.disabled(session.isConnected || session.isStreaming)
}

settingRow(
TBDisplaySenderL10n.videoCompression(service.language),
details: TBDisplaySenderL10n.codecPreferenceHint(
session.codecPreference,
language: service.language
)
) {
Picker(
TBDisplaySenderL10n.videoCompression(service.language),
selection: Binding(
get: { session.codecPreference },
set: { service.setCodecPreference($0, for: session) }
)
) {
ForEach(TBDisplayCodecPreference.allCases) { preference in
Text(TBDisplaySenderL10n.codecPreferenceTitle(
preference,
language: service.language
))
.tag(preference)
}
}
.pickerStyle(.menu)
.disabled(session.isConnected || session.isStreaming)
}

if session.captureSource == .extendedDesktop {
settingRow(renderMatchingTitle, details: renderMatchingDetails) {
Toggle("", isOn: $session.matchRenderToStream)
Expand Down
22 changes: 22 additions & 0 deletions TargetBridge-Sender/TBDisplaySender/TBDisplaySenderLocalization.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,28 @@ enum TBDisplaySenderL10n {
])
}

static func videoCompression(_ language: TBDisplaySenderLanguage) -> String {
text("sender.codec.label", language)
}

static func codecPreferenceTitle(
_ preference: TBDisplayCodecPreference,
language: TBDisplaySenderLanguage
) -> String {
text("sender.codec.\(preference.rawValue)", language)
}

static func codecPreferenceHint(
_ preference: TBDisplayCodecPreference,
language: TBDisplaySenderLanguage
) -> String {
text("sender.codec.hint.\(preference.rawValue)", language)
}

static func codecUnavailable(_ language: TBDisplaySenderLanguage) -> String {
text("sender.codec.unavailable", language)
}

static func missingScreenRecordingPermission(language: TBDisplaySenderLanguage) -> String {
text("sender.error.screen_recording_permission", language)
}
Expand Down
55 changes: 53 additions & 2 deletions TargetBridge-Sender/TBDisplaySender/TBDisplaySenderManager.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,9 @@ final class TBDisplaySenderService: ObservableObject {
receiverAddress: session.receiverIP.trimmingCharacters(in: .whitespacesAndNewlines),
receiverProfileAvailable: session.receiverSupportsHEVCDecodeHint != nil,
receiverSupportsHEVC: session.receiverSupportsHEVCDecodeHint,
requiresHEVC: session.capturePreset.codecName == "HEVC",
requiresHEVC: session.codecPreference == .hevc || (
session.codecPreference == .automatic && session.capturePreset.codecName == "HEVC"
),
cableRate: session.cableTestResult,
requiresSenderInputMonitoring: role == .senderMaster,
senderInputMonitoringGranted: localInputMonitoringTrusted,
Expand Down Expand Up @@ -317,6 +319,7 @@ final class TBDisplaySenderService: ObservableObject {

private static let persistedSessionsKey = "fd.tbdisplaysender.sessions.v1"
private static let receiverDisplayProfilesKey = "fd.tbdisplaysender.receiverDisplayProfiles.v1"
private static let receiverCodecPreferencesKey = "fd.tbdisplaysender.receiverCodecPreferences.v1"
/// Earlier builds persisted `false` when the audio addon had not finished
/// loading. Repair that ambiguous state exactly once without making future
/// deliberate choices reversible.
Expand All @@ -342,6 +345,7 @@ final class TBDisplaySenderService: ObservableObject {
var inputControlRole: String?
var inputBindings: [TBInputBinding]?
var matchRenderToStream: Bool?
var codecPreference: String?
}

private var lastPersistedData: Data?
Expand Down Expand Up @@ -376,7 +380,8 @@ final class TBDisplaySenderService: ObservableObject {
volume: session.volume,
inputControlRole: session.inputControlRole.rawValue,
inputBindings: session.inputBindings,
matchRenderToStream: session.matchRenderToStream
matchRenderToStream: session.matchRenderToStream,
codecPreference: session.codecPreference.rawValue
)
}
guard let data = try? JSONEncoder().encode(configs) else { return }
Expand Down Expand Up @@ -464,6 +469,9 @@ final class TBDisplaySenderService: ObservableObject {
session.brightness = config.brightness
session.volume = config.volume ?? 0.5
session.matchRenderToStream = config.matchRenderToStream ?? false
session.codecPreference = config.codecPreference
.flatMap(TBDisplayCodecPreference.init(rawValue:))
?? .automatic
}

func refreshLocalInterfaces() {
Expand All @@ -476,6 +484,7 @@ final class TBDisplaySenderService: ObservableObject {
func applyDiscoveredReceiver(_ receiver: TBDiscoveredReceiver, to session: TBDisplaySenderSession) {
session.receiverIP = receiver.ip(for: session.transportKind)
session.receiverSupportsHEVCDecodeHint = receiver.supportsHEVCDecode
restoreCodecPreference(for: receiver, to: session, defaultToAutomatic: true)
if session.localInterfaceIP.isEmpty {
session.localInterfaceIP = suggestedInterfaceForNewSession(transportKind: session.transportKind)?.ip
?? availableInterfaces(for: session.transportKind).first?.ip
Expand Down Expand Up @@ -505,6 +514,48 @@ final class TBDisplaySenderService: ObservableObject {
session.selectedReceiverID = receiver.id
session.receiverIP = receiver.ip(for: session.transportKind)
session.receiverSupportsHEVCDecodeHint = receiver.supportsHEVCDecode
restoreCodecPreference(for: receiver, to: session, defaultToAutomatic: false)
}
}

func setCodecPreference(_ preference: TBDisplayCodecPreference, for session: TBDisplaySenderSession) {
guard !session.isConnected, !session.isStreaming else { return }
session.codecPreference = preference
guard let key = receiverCodecPreferenceKey(for: session) else { return }
var preferences = persistedCodecPreferences
preferences[key] = preference.rawValue
UserDefaults.standard.set(preferences, forKey: Self.receiverCodecPreferencesKey)
}

private var persistedCodecPreferences: [String: String] {
UserDefaults.standard.dictionary(forKey: Self.receiverCodecPreferencesKey) as? [String: String] ?? [:]
}

private func receiverCodecPreferenceKey(for receiver: TBDiscoveredReceiver) -> String {
"service:\(receiver.serviceName)"
}

private func receiverCodecPreferenceKey(for session: TBDisplaySenderSession) -> String? {
let selectedID = session.selectedReceiverID.trimmingCharacters(in: .whitespacesAndNewlines)
if let serviceName = selectedID.split(separator: "|", maxSplits: 1).first,
!serviceName.isEmpty {
return "service:\(serviceName)"
}

let receiverIP = session.receiverIP.trimmingCharacters(in: .whitespacesAndNewlines)
return receiverIP.isEmpty ? nil : "ip:\(receiverIP)"
}

private func restoreCodecPreference(
for receiver: TBDiscoveredReceiver,
to session: TBDisplaySenderSession,
defaultToAutomatic: Bool
) {
let rawValue = persistedCodecPreferences[receiverCodecPreferenceKey(for: receiver)]
if let rawValue, let preference = TBDisplayCodecPreference(rawValue: rawValue) {
session.codecPreference = preference
} else if defaultToAutomatic {
session.codecPreference = .automatic
}
}

Expand Down
138 changes: 119 additions & 19 deletions TargetBridge-Sender/TBDisplaySender/TBDisplaySenderService.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,19 @@ struct TBFrameRatePacer {
}
}

enum TBDisplayCodecPreference: String, CaseIterable, Identifiable, Codable {
case automatic
case h264
case hevc

var id: String { rawValue }
}

struct TBDisplayCodecDecision: Equatable {
let codecType: CMVideoCodecType
let usedFallback: Bool
}

enum TBDisplayCaptureSource: String, CaseIterable, Identifiable {
case desktopMirror
case extendedDesktop
Expand Down Expand Up @@ -1178,6 +1191,17 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u
}
}
}
@Published var codecPreference: TBDisplayCodecPreference = .automatic {
didSet {
if !isStreaming {
streamResolutionText = TBDisplaySenderL10n.streamSummary(
preset: capturePreset,
source: captureSource,
language: language
)
}
}
}
@Published var captureSource: TBDisplayCaptureSource = .desktopMirror {
didSet {
if !isStreaming {
Expand Down Expand Up @@ -1268,7 +1292,7 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u
/// While a binding trigger key is held (matched), swallow its key-up so the
/// raw trigger key never reaches the slave.
private var suppressedTriggerKeyCode: UInt16?
private static var cachedSupportsHEVCHardwareEncode: Bool?
private static var cachedHardwareEncoderSupport: [String: Bool] = [:]
private var receivedInputEventCount: UInt64 = 0
var onRemoteSwitchRequest: ((Int) -> Void)?
var onRemoteDeactivateInputRequest: (() -> Void)?
Expand Down Expand Up @@ -1334,9 +1358,14 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u
statusText = state.text(language)
}

private static func probeHEVCHardwareEncoderSupport() -> Bool {
if let cachedSupportsHEVCHardwareEncode {
return cachedSupportsHEVCHardwareEncode
private static func probeHardwareEncoderSupport(
codecType: CMVideoCodecType,
width: Int,
height: Int
) -> Bool {
let cacheKey = "\(codecType)-\(width)x\(height)"
if let cached = cachedHardwareEncoderSupport[cacheKey] {
return cached
}

let encoderSpecification: CFDictionary = [
Expand All @@ -1347,9 +1376,9 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u
var session: VTCompressionSession?
let status = VTCompressionSessionCreate(
allocator: kCFAllocatorDefault,
width: 1920,
height: 1080,
codecType: kCMVideoCodecType_HEVC,
width: Int32(width),
height: Int32(height),
codecType: codecType,
encoderSpecification: encoderSpecification,
imageBufferAttributes: nil,
compressedDataAllocator: nil,
Expand All @@ -1362,21 +1391,74 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u
}

let supported = status == noErr
cachedSupportsHEVCHardwareEncode = supported
cachedHardwareEncoderSupport[cacheKey] = supported
return supported
}

private func resolvedCodecType(for preset: TBDisplayCapturePreset, profile: TBMonitorDisplayProfile?) -> CMVideoCodecType {
switch preset {
case .standard1440p, .smooth1440p60, .smooth1800p60:
let receiverSupportsHEVC = profile?.supportsHEVCDecode ?? receiverSupportsHEVCDecodeHint ?? false
if receiverSupportsHEVC, Self.probeHEVCHardwareEncoderSupport() {
return kCMVideoCodecType_HEVC
}
return kCMVideoCodecType_H264
case .crisp2160p60, .retina4k60, .native5k, .native5k60Experimental:
return preset.codecType
static func chooseCodec(
preference: TBDisplayCodecPreference,
preset: TBDisplayCapturePreset,
receiverSupportsHEVC: Bool,
senderSupportsH264: Bool,
senderSupportsHEVC: Bool
) -> TBDisplayCodecDecision? {
let h264Available = senderSupportsH264
let hevcAvailable = receiverSupportsHEVC && senderSupportsHEVC
let preferredCodec: CMVideoCodecType

switch preference {
case .automatic:
preferredCodec = preset.codecType == kCMVideoCodecType_HEVC || hevcAvailable
? kCMVideoCodecType_HEVC
: kCMVideoCodecType_H264
case .h264:
preferredCodec = kCMVideoCodecType_H264
case .hevc:
preferredCodec = kCMVideoCodecType_HEVC
}

if preferredCodec == kCMVideoCodecType_HEVC, hevcAvailable {
return TBDisplayCodecDecision(codecType: kCMVideoCodecType_HEVC, usedFallback: false)
}
if preferredCodec == kCMVideoCodecType_H264, h264Available {
return TBDisplayCodecDecision(codecType: kCMVideoCodecType_H264, usedFallback: false)
}
if preferredCodec == kCMVideoCodecType_HEVC, h264Available {
return TBDisplayCodecDecision(codecType: kCMVideoCodecType_H264, usedFallback: true)
}
if preferredCodec == kCMVideoCodecType_H264, hevcAvailable {
return TBDisplayCodecDecision(codecType: kCMVideoCodecType_HEVC, usedFallback: true)
}

return nil
}

private func resolvedCodecDecision(
for preset: TBDisplayCapturePreset,
profile: TBMonitorDisplayProfile?
) -> TBDisplayCodecDecision? {
let receiverSupportsHEVC = profile?.supportsHEVCDecode
?? receiverSupportsHEVCDecodeHint
?? false
return Self.chooseCodec(
preference: codecPreference,
preset: preset,
receiverSupportsHEVC: receiverSupportsHEVC,
senderSupportsH264: Self.probeHardwareEncoderSupport(
codecType: kCMVideoCodecType_H264,
width: preset.width,
height: preset.height
),
senderSupportsHEVC: Self.probeHardwareEncoderSupport(
codecType: kCMVideoCodecType_HEVC,
width: preset.width,
height: preset.height
)
)
}

private func resolvedCodecType(for preset: TBDisplayCapturePreset, profile: TBMonitorDisplayProfile?) -> CMVideoCodecType {
resolvedCodecDecision(for: preset, profile: profile)?.codecType ?? preset.codecType
}

private func codecName(for codecType: CMVideoCodecType) -> String {
Expand Down Expand Up @@ -2543,10 +2625,28 @@ final class TBDisplaySenderSession: NSObject, ObservableObject, Identifiable, @u

let preset = capturePreset
let usesRawNV12 = rawNV12Enabled(for: profile)
let codecType = resolvedCodecType(for: preset, profile: profile)
let codecDecision: TBDisplayCodecDecision
if usesRawNV12 {
codecDecision = TBDisplayCodecDecision(codecType: preset.codecType, usedFallback: false)
} else {
guard let resolvedDecision = resolvedCodecDecision(for: preset, profile: profile) else {
setStatus(.captureError(TBDisplaySenderL10n.codecUnavailable(language)))
TBLog.connection.error(
"capture: no compatible hardware codec preset=\(preset.rawValue, privacy: .public) preference=\(self.codecPreference.rawValue, privacy: .public)"
)
return false
}
codecDecision = resolvedDecision
}
let codecType = codecDecision.codecType
let codecName = usesRawNV12 ? "NV12 RAW" : codecName(for: codecType)
activeCodecType = usesRawNV12 ? nil : codecType
activeCodecName = codecName
if codecDecision.usedFallback {
TBLog.connection.notice(
"capture: codec fallback preference=\(self.codecPreference.rawValue, privacy: .public) selected=\(codecName, privacy: .public)"
)
}
guard let connection else { return false }

// The encode/send pipeline runs entirely on its own serial queue,
Expand Down
Loading
Loading