Support simulcast for svc codecs (vp9/av1)

.changeset/vast-trains-care.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Support simulcast for svc codecs (vp9/av1)

src/room/participant/LocalParticipant.ts

@@ -19,6 +19,7 @@ import {
   TrackInfo,
   TrackUnpublishedResponse,
   UserPacket,
+  VideoLayer_Mode,
   protoInt64,
 } from '@livekit/protocol';
 import { SignalConnectionState } from '../../api/SignalClient';
@@ -101,19 +102,23 @@ import {
   isLocalTrack,
   isLocalVideoTrack,
   isSVCCodec,
+  isSVCSimulcast,
+  isSVCSimulcastSupportedByServer,
   isSafari17Based,
   isVideoCodec,
   isVideoTrack,
   isWeb,
   sleep,
   supportsAV1,
   supportsVP9,
+  usesLegacySVCEncodings,
 } from '../utils';
 import Participant from './Participant';
 import type { ParticipantTrackPermission } from './ParticipantTrackPermission';
 import { trackPermissionToProto } from './ParticipantTrackPermission';
 import type RemoteParticipant from './RemoteParticipant';
 import {
+  computeStartTargetBitrate,
   computeTrackBackupEncodings,
   computeVideoEncodings,
   getDefaultDegradationPreference,
@@ -240,6 +245,11 @@ export default class LocalParticipant extends Participant {
     }
   }
 
+  private getServerVersion(): string | undefined {
+    const joinResponse = this.engine?.latestJoinResponse;
+    return joinResponse?.serverInfo?.version || joinResponse?.serverVersion || undefined;
+  }
+
   /**
    * @internal
    */
@@ -1135,7 +1145,15 @@ export default class LocalParticipant extends Participant {
       req.height = dims.height;
       // for svc codecs, disable simulcast and use vp8 for backup codec
       if (isLocalVideoTrack(track)) {
-        if (isSVCCodec(videoCodec)) {
+        if (
+          isSVCSimulcast(videoCodec, opts) &&
+          (usesLegacySVCEncodings() || !isSVCSimulcastSupportedByServer(this.getServerVersion()))
+        ) {
+          opts.simulcast = false;
+        }
+
+        const svcSimulcast = isSVCSimulcast(videoCodec, opts);
+        if (isSVCCodec(videoCodec) && !svcSimulcast) {
           if (track.source === Track.Source.ScreenShare) {
             // vp9 svc with screenshare cannot encode multiple spatial layers
             // doing so reduces publish resolution to minimal resolution
@@ -1157,12 +1175,14 @@ export default class LocalParticipant extends Participant {
           opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY';
         }
 
-        req.simulcastCodecs = [
-          new SimulcastCodec({
-            codec: videoCodec,
-            cid: track.mediaStreamTrack.id,
-          }),
-        ];
+        const primaryCodec = new SimulcastCodec({
+          codec: videoCodec,
+          cid: track.mediaStreamTrack.id,
+        });
+        if (svcSimulcast) {
+          primaryCodec.videoLayerMode = VideoLayer_Mode.ONE_SPATIAL_LAYER_PER_STREAM;
+        }
+        req.simulcastCodecs = [primaryCodec];
 
         // set up backup
         if (opts.backupCodec === true) {
@@ -1197,7 +1217,7 @@ export default class LocalParticipant extends Participant {
         req.width,
         req.height,
         encodings,
-        isSVCCodec(opts.videoCodec),
+        isSVCCodec(opts.videoCodec) && !isSVCSimulcast(opts.videoCodec, opts),
       );
     } else if (track.kind === Track.Kind.Audio) {
       encodings = [
@@ -1253,12 +1273,9 @@ export default class LocalParticipant extends Participant {
             });
           }
         } else if (track.codec && isVideoCodec(track.codec)) {
-          // Apply start bitrate for all video codecs to prevent initial blurriness.
-          // - SVC codecs: use first encoding's bitrate (single stream with built-in layers)
-          // - Simulcast: sum all encoding bitrates (independent streams, BWE needs total)
-          const targetBitrate = isSVCCodec(track.codec)
-            ? (encodings[0]?.maxBitrate ?? 0)
-            : encodings.reduce((sum, enc) => sum + (enc.maxBitrate ?? 0), 0);
+          // Apply start bitrate for all video codecs to prevent initial blurriness,
+          // see computeStartTargetBitrate
+          const targetBitrate = computeStartTargetBitrate(track.codec, opts, encodings);
           if (targetBitrate > 0) {
             this.engine.pcManager.publisher.setTrackCodecBitrate({
               cid: req.cid,

src/room/participant/publishUtils.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest';
 import { ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43 } from '../track/options';
 import {
   computeDefaultScreenShareSimulcastPresets,
+  computeStartTargetBitrate,
   computeVideoEncodings,
   determineAppropriateEncoding,
   presets43,
@@ -116,6 +117,78 @@ describe('computeVideoEncodings', () => {
     expect(encodings![0].scaleResolutionDownBy).toBe(1);
   });
 
+  // svc carries the scalabilityMode on the first encoding only (whether it emits a
+  // single encoding or the legacy multi-encoding shape), simulcast carries it on all
+  const countScalabilityModes = (encodings?: RTCRtpEncodingParameters[]) =>
+    /* @ts-ignore */
+    encodings!.filter((encoding) => encoding.scalabilityMode !== undefined).length;
+
+  it('keeps svc for an svc codec without simulcast', () => {
+    const encodings = computeVideoEncodings(false, 960, 540, {
+      simulcast: false,
+      videoCodec: 'vp9',
+      scalabilityMode: 'L3T3_KEY',
+    });
+    /* @ts-ignore */
+    expect(encodings![0].scalabilityMode).toBe('L3T3_KEY');
+    expect(countScalabilityModes(encodings)).toBe(1);
+  });
+
+  it('keeps svc for an svc codec with a multi spatial layer mode even if simulcast is set', () => {
+    const encodings = computeVideoEncodings(false, 960, 540, {
+      simulcast: true,
+      videoCodec: 'vp9',
+      scalabilityMode: 'L3T3_KEY',
+    });
+    /* @ts-ignore */
+    expect(encodings![0].scalabilityMode).toBe('L3T3_KEY');
+    expect(countScalabilityModes(encodings)).toBe(1);
+  });
+
+  it('returns a simulcast ladder for an svc codec with simulcast and an L1Tx mode', () => {
+    for (const videoCodec of ['vp9', 'av1'] as const) {
+      const encodings = computeVideoEncodings(false, 960, 540, {
+        simulcast: true,
+        videoCodec,
+        scalabilityMode: 'L1T2',
+      });
+      expect(encodings).toHaveLength(3);
+      expect(encodings!.map((e) => e.rid)).toEqual(['q', 'h', 'f']);
+      // every encoding needs both scalabilityMode and scaleResolutionDownBy for chrome
+      // M113+ to treat them as real simulcast rather than legacy svc
+      encodings!.forEach((encoding) => {
+        /* @ts-ignore */
+        expect(encoding.scalabilityMode).toBe('L1T2');
+        expect(encoding.scaleResolutionDownBy).toBeGreaterThanOrEqual(1);
+      });
+    }
+  });
+
+  it('sets the scalability mode on a single encoding svc simulcast ladder', () => {
+    const encodings = computeVideoEncodings(false, 100, 120, {
+      simulcast: true,
+      videoCodec: 'vp9',
+      scalabilityMode: 'L1T3',
+    });
+    expect(encodings).toHaveLength(1);
+    expect(encodings![0].rid).toBe('q');
+    /* @ts-ignore */
+    expect(encodings![0].scalabilityMode).toBe('L1T3');
+  });
+
+  it('does not set a scalability mode for non-svc simulcast', () => {
+    const encodings = computeVideoEncodings(false, 960, 540, {
+      simulcast: true,
+      videoCodec: 'vp8',
+      scalabilityMode: 'L1T2',
+    });
+    expect(encodings).toHaveLength(3);
+    encodings!.forEach((encoding) => {
+      /* @ts-ignore */
+      expect(encoding.scalabilityMode).toBeUndefined();
+    });
+  });
+
   //   it('respects default backup codec encoding', () => {
   //     const vp8Encodings = computeTrackBackupEncodings(false, 100, 120, { simulcast: true });
   //     const h264Encodings = computeVideoEncodings(false, 100, 120, {
@@ -193,3 +266,63 @@ describe('screenShareSimulcastDefaults', () => {
     expect(defaultSimulcastLayers[0].encoding.maxBitrate).toBe(375000);
   });
 });
+
+describe('computeStartTargetBitrate', () => {
+  // ordered q..f, as encodingsFromPresets builds them
+  const simulcastLadder: RTCRtpEncodingParameters[] = [
+    { rid: 'q', maxBitrate: 160_000 },
+    { rid: 'h', maxBitrate: 450_000 },
+    { rid: 'f', maxBitrate: 680_000 },
+  ];
+
+  it('sums the ladder for plain simulcast', () => {
+    expect(computeStartTargetBitrate('vp8', { simulcast: true }, simulcastLadder)).toBe(1_290_000);
+  });
+
+  it('sums the ladder for vp9/av1 published as rid simulcast', () => {
+    // encodings[0] is the *smallest* layer here, so taking it would under-hint BWE
+    for (const codec of ['vp9', 'av1'] as const) {
+      expect(
+        computeStartTargetBitrate(
+          codec,
+          { simulcast: true, videoCodec: codec, scalabilityMode: 'L1T2' },
+          simulcastLadder,
+        ),
+      ).toBe(1_290_000);
+    }
+  });
+
+  it('uses the single encoding for vp9/av1 svc', () => {
+    for (const codec of ['vp9', 'av1'] as const) {
+      expect(
+        computeStartTargetBitrate(
+          codec,
+          { simulcast: false, videoCodec: codec, scalabilityMode: 'L3T3_KEY' },
+          [{ maxBitrate: 680_000 }],
+        ),
+      ).toBe(680_000);
+    }
+  });
+
+  it('uses the first encoding for the legacy svc shape, which is ordered f..q', () => {
+    // legacy SVC pushes videoRids[2 - i], so encodings[0] carries the full bitrate
+    const legacySvc: RTCRtpEncodingParameters[] = [
+      { rid: 'f', maxBitrate: 680_000 },
+      { rid: 'h', maxBitrate: 226_667 },
+      { rid: 'q', maxBitrate: 75_556 },
+    ];
+    expect(
+      computeStartTargetBitrate(
+        'vp9',
+        { simulcast: false, videoCodec: 'vp9', scalabilityMode: 'L3T3_KEY' },
+        legacySvc,
+      ),
+    ).toBe(680_000);
+  });
+
+  it('handles missing bitrates and empty encodings', () => {
+    expect(computeStartTargetBitrate('vp8', { simulcast: true }, [])).toBe(0);
+    expect(computeStartTargetBitrate('vp9', undefined, [])).toBe(0);
+    expect(computeStartTargetBitrate('vp8', { simulcast: true }, [{ rid: 'q' }])).toBe(0);
+  });
+});

src/room/participant/publishUtils.ts

@@ -13,12 +13,12 @@ import type {
 import { ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43 } from '../track/options';
 import type { LoggerOptions } from '../types';
 import {
-  compareVersions,
   getReactNativeOs,
   isReactNative,
   isSVCCodec,
-  isSafariBased,
+  isSVCSimulcast,
   isSafariSvcApi,
+  usesLegacySVCEncodings,
 } from '../utils';
 
 /** @internal */
@@ -111,6 +111,8 @@ export function computeVideoEncodings(
   const useSimulcast = options?.simulcast;
   const scalabilityMode = options?.scalabilityMode;
   const videoCodec = options?.videoCodec;
+  // VP9/AV1 published as rid based simulcast rather than SVC, see isSVCSimulcast
+  const useSVCSimulcast = isSVCSimulcast(videoCodec, options);
 
   if ((!videoEncoding && !useSimulcast && !scalabilityMode) || !width || !height) {
     // when we aren't simulcasting or svc, will need to return a single encoding without
@@ -134,28 +136,18 @@ export function computeVideoEncodings(
     videoEncoding.priority,
   );
 
-  if (scalabilityMode && isSVCCodec(videoCodec)) {
+  if (scalabilityMode && isSVCCodec(videoCodec) && !useSVCSimulcast) {
     const sm = new ScalabilityMode(scalabilityMode);
 
     const encodings: RTCRtpEncodingParameters[] = [];
 
     if (sm.spatial > 3) {
       throw new Error(`unsupported scalabilityMode: ${scalabilityMode}`);
     }
-    // Before M113 in Chrome, defining multiple encodings with an SVC codec indicated
-    // that SVC mode should be used. Safari still works this way.
-    // This is a bit confusing but is due to how libwebrtc interpreted the encodings field
-    // before M113.
-    // Announced here: https://groups.google.com/g/discuss-webrtc/c/-QQ3pxrl-fw?pli=1
+    // Browsers that read multiple encodings on an SVC codec as SVC rather than as
+    // simulcast, see usesLegacySVCEncodings
     const browser = getBrowser();
-    if (
-      isSafariBased() ||
-      // Even tho RN runs M114, it does not produce SVC layers when a single encoding
-      // is provided. So we'll use the legacy SVC specification for now.
-      // TODO: when we upstream libwebrtc, this will need additional verification
-      isReactNative() ||
-      (browser?.name === 'Chrome' && compareVersions(browser?.version, '113') < 0)
-    ) {
+    if (usesLegacySVCEncodings()) {
       const bitratesRatio = sm.suffix == 'h' ? 2 : 3;
       // safari 18.4 uses a different svc API that requires scaleResolutionDownBy to be set.
       const requireScale = isSafariSvcApi(browser);
@@ -193,6 +185,19 @@ export function computeVideoEncodings(
     return [videoEncoding];
   }
 
+  // Chrome M113+ only treats multiple encodings on an SVC capable codec as real
+  // simulcast when every encoding carries an explicit scalabilityMode next to its
+  // scaleResolutionDownBy. Without it the encodings are interpreted as legacy SVC.
+  const applySVCSimulcastMode = (encodings: RTCRtpEncodingParameters[]) => {
+    if (useSVCSimulcast) {
+      encodings.forEach((encoding) => {
+        /* @ts-ignore */
+        encoding.scalabilityMode = scalabilityMode;
+      });
+    }
+    return encodings;
+  };
+
   let presets: Array<VideoPreset>;
   if (isScreenShare) {
     presets =
@@ -220,13 +225,43 @@ export function computeVideoEncodings(
     //      based on other conditions.
     const size = Math.max(width, height);
     if (size >= 960 && midPreset) {
-      return encodingsFromPresets(width, height, [lowPreset, midPreset, original], sourceFramerate);
+      return applySVCSimulcastMode(
+        encodingsFromPresets(width, height, [lowPreset, midPreset, original], sourceFramerate),
+      );
     }
     if (size >= 480) {
-      return encodingsFromPresets(width, height, [lowPreset, original], sourceFramerate);
+      return applySVCSimulcastMode(
+        encodingsFromPresets(width, height, [lowPreset, original], sourceFramerate),
+      );
     }
   }
-  return encodingsFromPresets(width, height, [original]);
+  return applySVCSimulcastMode(encodingsFromPresets(width, height, [original]));
+}
+
+/**
+ * Bitrate to hint to the bandwidth estimator through `x-google-start-bitrate`, so that
+ * a publish does not spend its first seconds ramping up from a very low rate.
+ *
+ * It has to be the total the encoder will put on the wire, which means picking the
+ * encoding that carries the inclusive bitrate:
+ *  - SVC publishes a single stream with the layers built in. `encodings[0]` holds the
+ *    full bitrate — the legacy SVC shape orders its encodings `f`..`q`, so that holds
+ *    for both SVC shapes.
+ *  - Simulcast publishes independent streams ordered `q`..`f`, so the total is the sum.
+ *    This includes VP9/AV1 published as rid based simulcast, where `encodings[0]` is
+ *    the *smallest* layer even though the codec is SVC capable.
+ *
+ * @internal
+ */
+export function computeStartTargetBitrate(
+  codec: string,
+  options: TrackPublishOptions | undefined,
+  encodings: RTCRtpEncodingParameters[],
+): number {
+  if (isSVCCodec(codec) && !isSVCSimulcast(codec, options)) {
+    return encodings[0]?.maxBitrate ?? 0;
+  }
+  return encodings.reduce((sum, enc) => sum + (enc.maxBitrate ?? 0), 0);
 }
 
 export function computeTrackBackupEncodings(