diff --git a/.changeset/vast-trains-care.md b/.changeset/vast-trains-care.md new file mode 100644 index 0000000000..36067550a9 --- /dev/null +++ b/.changeset/vast-trains-care.md @@ -0,0 +1,5 @@ +--- +'livekit-client': patch +--- + +Support simulcast for svc codecs (vp9/av1) diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index e8bd78f39e..0d86e9e822 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -173,6 +173,14 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit return !!this.reconnectTimeout; } + get serverVersion(): string | undefined { + return ( + this.latestJoinResponse?.serverInfo?.version || + this.latestJoinResponse?.serverVersion || + undefined + ); + } + /** * Owns the data channels: the three flow-controlled publisher wrappers (engine-lifetime; the * RTCDataChannel handles underneath are attached/detached as peer connections come and go, with diff --git a/src/room/Room.ts b/src/room/Room.ts index 75488f4726..6a3462c624 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -339,7 +339,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.log, this.outgoingDataStreamManager, this.getRemoteParticipantClientProtocol, - () => this.engine.latestJoinResponse?.serverInfo?.version, + () => this.engine?.serverVersion, ); this.rpcClientManager.on('sendDataPacket', ({ packet }) => { this.engine?.sendDataPacket(packet, DataChannelKind.RELIABLE); diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index 1993a1d154..645482c224 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -19,6 +19,7 @@ import { TrackInfo, TrackUnpublishedResponse, UserPacket, + VideoLayer_Mode, protoInt64, } from '@livekit/protocol'; import { SignalConnectionState } from '../../api/SignalClient'; @@ -101,6 +102,8 @@ import { isLocalTrack, isLocalVideoTrack, isSVCCodec, + isSVCSimulcast, + isSVCSimulcastSupportedByServer, isSafari17Based, isVideoCodec, isVideoTrack, @@ -108,12 +111,14 @@ import { sleep, supportsAV1, supportsVP9, + usesLegacySVCEncodings, } from '../utils'; import Participant from './Participant'; import type { ParticipantTrackPermission } from './ParticipantTrackPermission'; import { trackPermissionToProto } from './ParticipantTrackPermission'; import type RemoteParticipant from './RemoteParticipant'; import { + computeStartTargetBitrate, computeTrackBackupEncodings, computeVideoEncodings, getDefaultDegradationPreference, @@ -1135,7 +1140,19 @@ export default class LocalParticipant extends Participant { req.height = dims.height; // for svc codecs, disable simulcast and use vp8 for backup codec if (isLocalVideoTrack(track)) { - if (isSVCCodec(videoCodec)) { + if ( + isSVCSimulcast(videoCodec, opts) && + (usesLegacySVCEncodings() || !isSVCSimulcastSupportedByServer(this.engine?.serverVersion)) + ) { + opts.simulcast = false; + this.log.info( + 'SVC simulcast is not supported, disabling simulcast.', + getLogContextFromTrack(track), + ); + } + + const svcSimulcast = isSVCSimulcast(videoCodec, opts); + if (isSVCCodec(videoCodec) && !svcSimulcast) { if (track.source === Track.Source.ScreenShare) { // vp9 svc with screenshare cannot encode multiple spatial layers // doing so reduces publish resolution to minimal resolution @@ -1157,12 +1174,14 @@ export default class LocalParticipant extends Participant { opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY'; } - req.simulcastCodecs = [ - new SimulcastCodec({ - codec: videoCodec, - cid: track.mediaStreamTrack.id, - }), - ]; + const primaryCodec = new SimulcastCodec({ + codec: videoCodec, + cid: track.mediaStreamTrack.id, + }); + if (svcSimulcast) { + primaryCodec.videoLayerMode = VideoLayer_Mode.ONE_SPATIAL_LAYER_PER_STREAM; + } + req.simulcastCodecs = [primaryCodec]; // set up backup if (opts.backupCodec === true) { @@ -1197,7 +1216,7 @@ export default class LocalParticipant extends Participant { req.width, req.height, encodings, - isSVCCodec(opts.videoCodec), + isSVCCodec(opts.videoCodec) && !isSVCSimulcast(opts.videoCodec, opts), ); } else if (track.kind === Track.Kind.Audio) { encodings = [ @@ -1253,12 +1272,9 @@ export default class LocalParticipant extends Participant { }); } } else if (track.codec && isVideoCodec(track.codec)) { - // Apply start bitrate for all video codecs to prevent initial blurriness. - // - SVC codecs: use first encoding's bitrate (single stream with built-in layers) - // - Simulcast: sum all encoding bitrates (independent streams, BWE needs total) - const targetBitrate = isSVCCodec(track.codec) - ? (encodings[0]?.maxBitrate ?? 0) - : encodings.reduce((sum, enc) => sum + (enc.maxBitrate ?? 0), 0); + // Apply start bitrate for all video codecs to prevent initial blurriness, + // see computeStartTargetBitrate + const targetBitrate = computeStartTargetBitrate(track.codec, opts, encodings); if (targetBitrate > 0) { this.engine.pcManager.publisher.setTrackCodecBitrate({ cid: req.cid, diff --git a/src/room/participant/publishUtils.test.ts b/src/room/participant/publishUtils.test.ts index e04c3f9f89..9a75626927 100644 --- a/src/room/participant/publishUtils.test.ts +++ b/src/room/participant/publishUtils.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43 } from '../track/options'; import { computeDefaultScreenShareSimulcastPresets, + computeStartTargetBitrate, computeVideoEncodings, determineAppropriateEncoding, presets43, @@ -116,6 +117,78 @@ describe('computeVideoEncodings', () => { expect(encodings![0].scaleResolutionDownBy).toBe(1); }); + // svc carries the scalabilityMode on the first encoding only (whether it emits a + // single encoding or the legacy multi-encoding shape), simulcast carries it on all + const countScalabilityModes = (encodings?: RTCRtpEncodingParameters[]) => + /* @ts-ignore */ + encodings!.filter((encoding) => encoding.scalabilityMode !== undefined).length; + + it('keeps svc for an svc codec without simulcast', () => { + const encodings = computeVideoEncodings(false, 960, 540, { + simulcast: false, + videoCodec: 'vp9', + scalabilityMode: 'L3T3_KEY', + }); + /* @ts-ignore */ + expect(encodings![0].scalabilityMode).toBe('L3T3_KEY'); + expect(countScalabilityModes(encodings)).toBe(1); + }); + + it('keeps svc for an svc codec with a multi spatial layer mode even if simulcast is set', () => { + const encodings = computeVideoEncodings(false, 960, 540, { + simulcast: true, + videoCodec: 'vp9', + scalabilityMode: 'L3T3_KEY', + }); + /* @ts-ignore */ + expect(encodings![0].scalabilityMode).toBe('L3T3_KEY'); + expect(countScalabilityModes(encodings)).toBe(1); + }); + + it('returns a simulcast ladder for an svc codec with simulcast and an L1Tx mode', () => { + for (const videoCodec of ['vp9', 'av1'] as const) { + const encodings = computeVideoEncodings(false, 960, 540, { + simulcast: true, + videoCodec, + scalabilityMode: 'L1T2', + }); + expect(encodings).toHaveLength(3); + expect(encodings!.map((e) => e.rid)).toEqual(['q', 'h', 'f']); + // every encoding needs both scalabilityMode and scaleResolutionDownBy for chrome + // M113+ to treat them as real simulcast rather than legacy svc + encodings!.forEach((encoding) => { + /* @ts-ignore */ + expect(encoding.scalabilityMode).toBe('L1T2'); + expect(encoding.scaleResolutionDownBy).toBeGreaterThanOrEqual(1); + }); + } + }); + + it('sets the scalability mode on a single encoding svc simulcast ladder', () => { + const encodings = computeVideoEncodings(false, 100, 120, { + simulcast: true, + videoCodec: 'vp9', + scalabilityMode: 'L1T3', + }); + expect(encodings).toHaveLength(1); + expect(encodings![0].rid).toBe('q'); + /* @ts-ignore */ + expect(encodings![0].scalabilityMode).toBe('L1T3'); + }); + + it('does not set a scalability mode for non-svc simulcast', () => { + const encodings = computeVideoEncodings(false, 960, 540, { + simulcast: true, + videoCodec: 'vp8', + scalabilityMode: 'L1T2', + }); + expect(encodings).toHaveLength(3); + encodings!.forEach((encoding) => { + /* @ts-ignore */ + expect(encoding.scalabilityMode).toBeUndefined(); + }); + }); + // it('respects default backup codec encoding', () => { // const vp8Encodings = computeTrackBackupEncodings(false, 100, 120, { simulcast: true }); // const h264Encodings = computeVideoEncodings(false, 100, 120, { @@ -193,3 +266,63 @@ describe('screenShareSimulcastDefaults', () => { expect(defaultSimulcastLayers[0].encoding.maxBitrate).toBe(375000); }); }); + +describe('computeStartTargetBitrate', () => { + // ordered q..f, as encodingsFromPresets builds them + const simulcastLadder: RTCRtpEncodingParameters[] = [ + { rid: 'q', maxBitrate: 160_000 }, + { rid: 'h', maxBitrate: 450_000 }, + { rid: 'f', maxBitrate: 680_000 }, + ]; + + it('sums the ladder for plain simulcast', () => { + expect(computeStartTargetBitrate('vp8', { simulcast: true }, simulcastLadder)).toBe(1_290_000); + }); + + it('sums the ladder for vp9/av1 published as rid simulcast', () => { + // encodings[0] is the *smallest* layer here, so taking it would under-hint BWE + for (const codec of ['vp9', 'av1'] as const) { + expect( + computeStartTargetBitrate( + codec, + { simulcast: true, videoCodec: codec, scalabilityMode: 'L1T2' }, + simulcastLadder, + ), + ).toBe(1_290_000); + } + }); + + it('uses the single encoding for vp9/av1 svc', () => { + for (const codec of ['vp9', 'av1'] as const) { + expect( + computeStartTargetBitrate( + codec, + { simulcast: false, videoCodec: codec, scalabilityMode: 'L3T3_KEY' }, + [{ maxBitrate: 680_000 }], + ), + ).toBe(680_000); + } + }); + + it('uses the first encoding for the legacy svc shape, which is ordered f..q', () => { + // legacy SVC pushes videoRids[2 - i], so encodings[0] carries the full bitrate + const legacySvc: RTCRtpEncodingParameters[] = [ + { rid: 'f', maxBitrate: 680_000 }, + { rid: 'h', maxBitrate: 226_667 }, + { rid: 'q', maxBitrate: 75_556 }, + ]; + expect( + computeStartTargetBitrate( + 'vp9', + { simulcast: false, videoCodec: 'vp9', scalabilityMode: 'L3T3_KEY' }, + legacySvc, + ), + ).toBe(680_000); + }); + + it('handles missing bitrates and empty encodings', () => { + expect(computeStartTargetBitrate('vp8', { simulcast: true }, [])).toBe(0); + expect(computeStartTargetBitrate('vp9', undefined, [])).toBe(0); + expect(computeStartTargetBitrate('vp8', { simulcast: true }, [{ rid: 'q' }])).toBe(0); + }); +}); diff --git a/src/room/participant/publishUtils.ts b/src/room/participant/publishUtils.ts index 3bb868b81c..c2a097efad 100644 --- a/src/room/participant/publishUtils.ts +++ b/src/room/participant/publishUtils.ts @@ -13,12 +13,12 @@ import type { import { ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43 } from '../track/options'; import type { LoggerOptions } from '../types'; import { - compareVersions, getReactNativeOs, isReactNative, isSVCCodec, - isSafariBased, + isSVCSimulcast, isSafariSvcApi, + usesLegacySVCEncodings, } from '../utils'; /** @internal */ @@ -111,6 +111,8 @@ export function computeVideoEncodings( const useSimulcast = options?.simulcast; const scalabilityMode = options?.scalabilityMode; const videoCodec = options?.videoCodec; + // VP9/AV1 published as rid based simulcast rather than SVC, see isSVCSimulcast + const useSVCSimulcast = isSVCSimulcast(videoCodec, options); if ((!videoEncoding && !useSimulcast && !scalabilityMode) || !width || !height) { // when we aren't simulcasting or svc, will need to return a single encoding without @@ -134,7 +136,7 @@ export function computeVideoEncodings( videoEncoding.priority, ); - if (scalabilityMode && isSVCCodec(videoCodec)) { + if (scalabilityMode && isSVCCodec(videoCodec) && !useSVCSimulcast) { const sm = new ScalabilityMode(scalabilityMode); const encodings: RTCRtpEncodingParameters[] = []; @@ -142,20 +144,10 @@ export function computeVideoEncodings( if (sm.spatial > 3) { throw new Error(`unsupported scalabilityMode: ${scalabilityMode}`); } - // Before M113 in Chrome, defining multiple encodings with an SVC codec indicated - // that SVC mode should be used. Safari still works this way. - // This is a bit confusing but is due to how libwebrtc interpreted the encodings field - // before M113. - // Announced here: https://groups.google.com/g/discuss-webrtc/c/-QQ3pxrl-fw?pli=1 + // Browsers that read multiple encodings on an SVC codec as SVC rather than as + // simulcast, see usesLegacySVCEncodings const browser = getBrowser(); - if ( - isSafariBased() || - // Even tho RN runs M114, it does not produce SVC layers when a single encoding - // is provided. So we'll use the legacy SVC specification for now. - // TODO: when we upstream libwebrtc, this will need additional verification - isReactNative() || - (browser?.name === 'Chrome' && compareVersions(browser?.version, '113') < 0) - ) { + if (usesLegacySVCEncodings()) { const bitratesRatio = sm.suffix == 'h' ? 2 : 3; // safari 18.4 uses a different svc API that requires scaleResolutionDownBy to be set. const requireScale = isSafariSvcApi(browser); @@ -193,6 +185,19 @@ export function computeVideoEncodings( return [videoEncoding]; } + // Chrome M113+ only treats multiple encodings on an SVC capable codec as real + // simulcast when every encoding carries an explicit scalabilityMode next to its + // scaleResolutionDownBy. Without it the encodings are interpreted as legacy SVC. + const applySVCSimulcastMode = (encodings: RTCRtpEncodingParameters[]) => { + if (useSVCSimulcast) { + encodings.forEach((encoding) => { + /* @ts-ignore */ + encoding.scalabilityMode = scalabilityMode; + }); + } + return encodings; + }; + let presets: Array; if (isScreenShare) { presets = @@ -220,13 +225,43 @@ export function computeVideoEncodings( // based on other conditions. const size = Math.max(width, height); if (size >= 960 && midPreset) { - return encodingsFromPresets(width, height, [lowPreset, midPreset, original], sourceFramerate); + return applySVCSimulcastMode( + encodingsFromPresets(width, height, [lowPreset, midPreset, original], sourceFramerate), + ); } if (size >= 480) { - return encodingsFromPresets(width, height, [lowPreset, original], sourceFramerate); + return applySVCSimulcastMode( + encodingsFromPresets(width, height, [lowPreset, original], sourceFramerate), + ); } } - return encodingsFromPresets(width, height, [original]); + return applySVCSimulcastMode(encodingsFromPresets(width, height, [original])); +} + +/** + * Bitrate to hint to the bandwidth estimator through `x-google-start-bitrate`, so that + * a publish does not spend its first seconds ramping up from a very low rate. + * + * It has to be the total the encoder will put on the wire, which means picking the + * encoding that carries the inclusive bitrate: + * - SVC publishes a single stream with the layers built in. `encodings[0]` holds the + * full bitrate — the legacy SVC shape orders its encodings `f`..`q`, so that holds + * for both SVC shapes. + * - Simulcast publishes independent streams ordered `q`..`f`, so the total is the sum. + * This includes VP9/AV1 published as rid based simulcast, where `encodings[0]` is + * the *smallest* layer even though the codec is SVC capable. + * + * @internal + */ +export function computeStartTargetBitrate( + codec: string, + options: TrackPublishOptions | undefined, + encodings: RTCRtpEncodingParameters[], +): number { + if (isSVCCodec(codec) && !isSVCSimulcast(codec, options)) { + return encodings[0]?.maxBitrate ?? 0; + } + return encodings.reduce((sum, enc) => sum + (enc.maxBitrate ?? 0), 0); } export function computeTrackBackupEncodings( diff --git a/src/room/track/LocalVideoTrack.ts b/src/room/track/LocalVideoTrack.ts index 5c99fc9b0c..31683db86a 100644 --- a/src/room/track/LocalVideoTrack.ts +++ b/src/room/track/LocalVideoTrack.ts @@ -16,7 +16,7 @@ import { import type { VideoSenderStats } from '../stats'; import { computeBitrate, monitorFrequency } from '../stats'; import type { LoggerOptions } from '../types'; -import { isFireFox, isMobile, isSVCCodec, isWeb } from '../utils'; +import { isFireFox, isMobile, isSVCCodec, isSVCSimulcast, isWeb } from '../utils'; import LocalTrack from './LocalTrack'; import { Track, VideoQuality } from './Track'; import type { TrackPublishOptions, VideoCaptureOptions, VideoCodec } from './options'; @@ -243,6 +243,16 @@ export default class LocalVideoTrack extends LocalTrack { return items; } + /** + * Whether `codec` is being published as SVC (a single stream carrying all spatial + * layers) as opposed to rid based simulcast. VP9/AV1 are SVC unless the publisher + * opted into simulcast, in which case each rid is an independent stream and the + * layers can be enabled/disabled individually. + */ + private isSvcPublish(codec?: string): boolean { + return isSVCCodec(codec) && !isSVCSimulcast(codec, this.publishOptions); + } + setPublishingQuality(maxQuality: VideoQuality) { const qualities: SubscribedQuality[] = []; for (let q = VideoQuality.LOW; q <= VideoQuality.HIGH; q += 1) { @@ -254,7 +264,7 @@ export default class LocalVideoTrack extends LocalTrack { ); } this.log.debug(`setting publishing quality. max quality ${maxQuality}`, this.logContext); - this.setPublishingLayers(isSVCCodec(this.codec), qualities); + this.setPublishingLayers(this.isSvcPublish(this.codec), qualities); } async restartTrack(options?: VideoCaptureOptions) { @@ -491,7 +501,7 @@ export default class LocalVideoTrack extends LocalTrack { }); // only enable simulcast codec for preference codec setted if (!this.codec && codecs.length > 0) { - await this.setPublishingLayers(isSVCCodec(codecs[0].codec), codecs[0].qualities); + await this.setPublishingLayers(this.isSvcPublish(codecs[0].codec), codecs[0].qualities); return []; } @@ -501,7 +511,7 @@ export default class LocalVideoTrack extends LocalTrack { const newCodecs: VideoCodec[] = []; for await (const codec of codecs) { if (!this.codec || this.codec === codec.codec) { - await this.setPublishingLayers(isSVCCodec(codec.codec), codec.qualities); + await this.setPublishingLayers(this.isSvcPublish(codec.codec), codec.qualities); } else { const simulcastCodecInfo = this.simulcastCodecs.get(codec.codec as VideoCodec); this.log.debug(`try setPublishingCodec for ${codec.codec}`, { @@ -522,7 +532,7 @@ export default class LocalVideoTrack extends LocalTrack { simulcastCodecInfo.encodings!, codec.qualities, this.senderLock, - isSVCCodec(codec.codec), + this.isSvcPublish(codec.codec), this.log, this.logContext, ); diff --git a/src/room/track/options.ts b/src/room/track/options.ts index d10bc6955f..7d449a6f98 100644 --- a/src/room/track/options.ts +++ b/src/room/track/options.ts @@ -79,7 +79,7 @@ export interface TrackPublishDefaults { /** * scalability mode for svc codecs, defaults to 'L3T3_KEY'. - * for svc codecs, simulcast is disabled. + * for svc codecs, simulcast is disabled if more than one spatial layer is used ('L2Tx' or 'L3Tx'). */ scalabilityMode?: ScalabilityMode; diff --git a/src/room/utils.test.ts b/src/room/utils.test.ts index 3614abefad..a6b3541df0 100644 --- a/src/room/utils.test.ts +++ b/src/room/utils.test.ts @@ -4,10 +4,13 @@ import { ddExtensionURI, extractMaxAgeFromRequestHeaders, getClientInfo, + isSVCSimulcast, + isSVCSimulcastSupportedByServer, negotiateDependencyDescriptor, splitUtf8, supportsAdaptiveStream, toWebsocketUrl, + usesLegacySVCEncodings, } from './utils'; describe('toWebsocketUrl', () => { @@ -280,3 +283,87 @@ describe('supportsAdaptiveStream', () => { expect(supportsAdaptiveStream()).toBe(false); }); }); + +describe('isSVCSimulcast', () => { + it('requires an svc capable codec, simulcast and a single spatial layer mode', () => { + expect(isSVCSimulcast('vp9', { simulcast: true, scalabilityMode: 'L1T2' })).toBe(true); + expect(isSVCSimulcast('av1', { simulcast: true, scalabilityMode: 'L1T3' })).toBe(true); + }); + + it('stays on svc without the opt in', () => { + expect(isSVCSimulcast('vp9', { simulcast: false, scalabilityMode: 'L1T2' })).toBe(false); + expect(isSVCSimulcast('vp9', { simulcast: true })).toBe(false); + // a multi spatial layer mode is svc by definition + expect(isSVCSimulcast('vp9', { simulcast: true, scalabilityMode: 'L3T3_KEY' })).toBe(false); + expect(isSVCSimulcast('vp9', undefined)).toBe(false); + }); + + it('does not apply to non svc codecs', () => { + expect(isSVCSimulcast('vp8', { simulcast: true, scalabilityMode: 'L1T2' })).toBe(false); + expect(isSVCSimulcast('h264', { simulcast: true, scalabilityMode: 'L1T2' })).toBe(false); + expect(isSVCSimulcast(undefined, { simulcast: true, scalabilityMode: 'L1T2' })).toBe(false); + }); +}); + +describe('isSVCSimulcastSupportedByServer', () => { + it('requires a server newer than 1.13.6', () => { + expect(isSVCSimulcastSupportedByServer('1.13.7')).toBe(true); + expect(isSVCSimulcastSupportedByServer('1.14.0')).toBe(true); + expect(isSVCSimulcastSupportedByServer('2.0.0')).toBe(true); + }); + + it('rejects 1.13.6 and older', () => { + expect(isSVCSimulcastSupportedByServer('1.13.6')).toBe(false); + expect(isSVCSimulcastSupportedByServer('1.13.5')).toBe(false); + expect(isSVCSimulcastSupportedByServer('1.9.0')).toBe(false); + expect(isSVCSimulcastSupportedByServer('0.15.1')).toBe(false); + }); + + it('treats an unknown version as unsupported', () => { + expect(isSVCSimulcastSupportedByServer(undefined)).toBe(false); + expect(isSVCSimulcastSupportedByServer('')).toBe(false); + }); +}); + +describe('usesLegacySVCEncodings', () => { + const stubUserAgent = (userAgent: string) => + vi.stubGlobal('navigator', { userAgent, product: 'Gecko' }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + const chrome = (v: string) => + `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${v} Safari/537.36`; + + it('is legacy on chrome before M113', () => { + stubUserAgent(chrome('112.0.0.0')); + expect(usesLegacySVCEncodings()).toBe(true); + }); + + it('is not legacy from chrome M113 onwards', () => { + stubUserAgent(chrome('113.0.0.0')); + expect(usesLegacySVCEncodings()).toBe(false); + stubUserAgent(chrome('120.0.0.0')); + expect(usesLegacySVCEncodings()).toBe(false); + }); + + it('is legacy on safari, which has no rid based svc simulcast', () => { + stubUserAgent( + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15', + ); + expect(usesLegacySVCEncodings()).toBe(true); + }); + + it('is legacy on iOS, where every browser is webkit', () => { + stubUserAgent( + 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/120.0.0.0 Mobile/15E148 Safari/604.1', + ); + expect(usesLegacySVCEncodings()).toBe(true); + }); + + it('is legacy on react native regardless of chrome version', () => { + vi.stubGlobal('navigator', { userAgent: chrome('120.0.0.0'), product: 'ReactNative' }); + expect(usesLegacySVCEncodings()).toBe(true); + }); +}); diff --git a/src/room/utils.ts b/src/room/utils.ts index 3ab39e4410..f3f21a0987 100644 --- a/src/room/utils.ts +++ b/src/room/utils.ts @@ -185,6 +185,65 @@ export function negotiateDependencyDescriptor(transceiver: RTCRtpTransceiver): b } } +/** + * VP9 and AV1 are published as SVC (a single RTP stream carrying every spatial layer) + * by default. They can instead be published as real, rid based simulcast — one + * independent stream per rid, each carrying a single spatial layer — when the caller + * opts in with `simulcast: true` and a single spatial layer scalability mode (`L1Tx`). + * + * The SFU has to be told about this: without an explicit + * `SimulcastCodec.videoLayerMode` it assumes `MULTIPLE_SPATIAL_LAYERS_PER_STREAM` for + * any SVC capable codec. + */ +export function isSVCSimulcast( + codec?: string, + options?: { simulcast?: boolean; scalabilityMode?: string }, +): boolean { + return isSVCCodec(codec) && !!options?.simulcast && !!options.scalabilityMode?.startsWith('L1T'); +} + +/** + * Whether the browser reads multiple encodings on an SVC capable codec as *legacy SVC* + * rather than as real simulcast. + * + * Before Chrome M113, supplying more than one encoding for VP9/AV1 selected SVC mode; + * only from M113 does libwebrtc treat such encodings as simulcast, and only when each + * one carries its own scalabilityMode. Safari (and anything WebKit based, i. e. every + * browser on iOS) still uses the old interpretation, as does React Native's libwebrtc. + * Announced at https://groups.google.com/g/discuss-webrtc/c/-QQ3pxrl-fw + * + * Where this is true the rids would not exist on the wire, so VP9/AV1 must be published + * as SVC no matter what the caller asked for. + */ +export function usesLegacySVCEncodings(): boolean { + const browser = getBrowser(); + return ( + isSafariBased() || + // Even tho RN runs M114, it does not produce SVC layers when a single encoding + // is provided. So we'll use the legacy SVC specification for now. + // TODO: when we upstream libwebrtc, this will need additional verification + isReactNative() || + (browser?.name === 'Chrome' && compareVersions(browser.version, '113') < 0) + ); +} + +/** + * Last server version that doesn't support vp9/av1 simulcast. + */ +const svcSimulcastMinServerVersion = '1.13.6'; + +/** + * Whether the connected server honours `SimulcastCodec.videoLayerMode`, i. e. whether + * VP9/AV1 can be published as rid based simulcast. An unknown version is treated as + * unsupported so the publish falls back to SVC. + */ +export function isSVCSimulcastSupportedByServer(serverVersion?: string): boolean { + if (!serverVersion) { + return false; + } + return compareVersions(serverVersion, svcSimulcastMinServerVersion) > 0; +} + export function supportsSetSinkId(elm?: HTMLMediaElement): boolean { if (!document || isSafariBased()) { return false;