Support simulcast for svc codecs (vp9/av1)

.changeset/vast-trains-care.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Support simulcast for svc codecs (vp9/av1)

src/room/RTCEngine.ts

@@ -173,6 +173,14 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     return !!this.reconnectTimeout;
   }
 
+  get serverVersion(): string | undefined {
+    return (
+      this.latestJoinResponse?.serverInfo?.version ||
+      this.latestJoinResponse?.serverVersion ||
+      undefined
+    );
+  }
+
   /**
    * Owns the data channels: the three flow-controlled publisher wrappers (engine-lifetime; the
    * RTCDataChannel handles underneath are attached/detached as peer connections come and go, with

src/room/Room.ts

@@ -339,7 +339,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       this.log,
       this.outgoingDataStreamManager,
       this.getRemoteParticipantClientProtocol,
-      () => this.engine.latestJoinResponse?.serverInfo?.version,
+      () => this.engine?.serverVersion,
     );
     this.rpcClientManager.on('sendDataPacket', ({ packet }) => {
       this.engine?.sendDataPacket(packet, DataChannelKind.RELIABLE);

src/room/participant/LocalParticipant.ts

@@ -19,6 +19,7 @@ import {
   TrackInfo,
   TrackUnpublishedResponse,
   UserPacket,
+  VideoLayer_Mode,
   protoInt64,
 } from '@livekit/protocol';
 import { SignalConnectionState } from '../../api/SignalClient';
@@ -101,19 +102,23 @@ import {
   isLocalTrack,
   isLocalVideoTrack,
   isSVCCodec,
+  isSVCSimulcast,
+  isSVCSimulcastSupportedByServer,
   isSafari17Based,
   isVideoCodec,
   isVideoTrack,
   isWeb,
   sleep,
   supportsAV1,
   supportsVP9,
+  usesLegacySVCEncodings,
 } from '../utils';
 import Participant from './Participant';
 import type { ParticipantTrackPermission } from './ParticipantTrackPermission';
 import { trackPermissionToProto } from './ParticipantTrackPermission';
 import type RemoteParticipant from './RemoteParticipant';
 import {
+  computeStartTargetBitrate,
   computeTrackBackupEncodings,
   computeVideoEncodings,
   getDefaultDegradationPreference,
@@ -1135,7 +1140,19 @@ export default class LocalParticipant extends Participant {
       req.height = dims.height;
       // for svc codecs, disable simulcast and use vp8 for backup codec
       if (isLocalVideoTrack(track)) {
-        if (isSVCCodec(videoCodec)) {
+        if (
+          isSVCSimulcast(videoCodec, opts) &&
+          (usesLegacySVCEncodings() || !isSVCSimulcastSupportedByServer(this.engine?.serverVersion))
+        ) {
+          opts.simulcast = false;
+          this.log.info(
+            'SVC simulcast is not supported, disabling simulcast.',
+            getLogContextFromTrack(track),
+          );
+        }
+
+        const svcSimulcast = isSVCSimulcast(videoCodec, opts);
+        if (isSVCCodec(videoCodec) && !svcSimulcast) {
           if (track.source === Track.Source.ScreenShare) {
             // vp9 svc with screenshare cannot encode multiple spatial layers
             // doing so reduces publish resolution to minimal resolution
@@ -1157,12 +1174,14 @@ export default class LocalParticipant extends Participant {
           opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY';
         }
 
-        req.simulcastCodecs = [
-          new SimulcastCodec({
-            codec: videoCodec,
-            cid: track.mediaStreamTrack.id,
-          }),
-        ];
+        const primaryCodec = new SimulcastCodec({
+          codec: videoCodec,
+          cid: track.mediaStreamTrack.id,
+        });
+        if (svcSimulcast) {
+          primaryCodec.videoLayerMode = VideoLayer_Mode.ONE_SPATIAL_LAYER_PER_STREAM;
+        }
+        req.simulcastCodecs = [primaryCodec];
 
         // set up backup
         if (opts.backupCodec === true) {
@@ -1197,7 +1216,7 @@ export default class LocalParticipant extends Participant {
         req.width,
         req.height,
         encodings,
-        isSVCCodec(opts.videoCodec),
+        isSVCCodec(opts.videoCodec) && !isSVCSimulcast(opts.videoCodec, opts),
       );
     } else if (track.kind === Track.Kind.Audio) {
       encodings = [
@@ -1253,12 +1272,9 @@ export default class LocalParticipant extends Participant {
             });
           }
         } else if (track.codec && isVideoCodec(track.codec)) {
-          // Apply start bitrate for all video codecs to prevent initial blurriness.
-          // - SVC codecs: use first encoding's bitrate (single stream with built-in layers)
-          // - Simulcast: sum all encoding bitrates (independent streams, BWE needs total)
-          const targetBitrate = isSVCCodec(track.codec)
-            ? (encodings[0]?.maxBitrate ?? 0)
-            : encodings.reduce((sum, enc) => sum + (enc.maxBitrate ?? 0), 0);
+          // Apply start bitrate for all video codecs to prevent initial blurriness,
+          // see computeStartTargetBitrate
+          const targetBitrate = computeStartTargetBitrate(track.codec, opts, encodings);
           if (targetBitrate > 0) {
             this.engine.pcManager.publisher.setTrackCodecBitrate({
               cid: req.cid,

src/room/participant/publishUtils.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest';
 import { ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43 } from '../track/options';
 import {
   computeDefaultScreenShareSimulcastPresets,
+  computeStartTargetBitrate,
   computeVideoEncodings,
   determineAppropriateEncoding,
   presets43,
@@ -116,6 +117,78 @@ describe('computeVideoEncodings', () => {
     expect(encodings![0].scaleResolutionDownBy).toBe(1);
   });
 
+  // svc carries the scalabilityMode on the first encoding only (whether it emits a
+  // single encoding or the legacy multi-encoding shape), simulcast carries it on all
+  const countScalabilityModes = (encodings?: RTCRtpEncodingParameters[]) =>
+    /* @ts-ignore */
+    encodings!.filter((encoding) => encoding.scalabilityMode !== undefined).length;
+
+  it('keeps svc for an svc codec without simulcast', () => {
+    const encodings = computeVideoEncodings(false, 960, 540, {
+      simulcast: false,
+      videoCodec: 'vp9',
+      scalabilityMode: 'L3T3_KEY',
+    });
+    /* @ts-ignore */
+    expect(encodings![0].scalabilityMode).toBe('L3T3_KEY');
+    expect(countScalabilityModes(encodings)).toBe(1);
+  });
+
+  it('keeps svc for an svc codec with a multi spatial layer mode even if simulcast is set', () => {
+    const encodings = computeVideoEncodings(false, 960, 540, {
+      simulcast: true,
+      videoCodec: 'vp9',
+      scalabilityMode: 'L3T3_KEY',
+    });
+    /* @ts-ignore */
+    expect(encodings![0].scalabilityMode).toBe('L3T3_KEY');
+    expect(countScalabilityModes(encodings)).toBe(1);
+  });
+
+  it('returns a simulcast ladder for an svc codec with simulcast and an L1Tx mode', () => {
+    for (const videoCodec of ['vp9', 'av1'] as const) {
+      const encodings = computeVideoEncodings(false, 960, 540, {
+        simulcast: true,
+        videoCodec,
+        scalabilityMode: 'L1T2',
+      });
+      expect(encodings).toHaveLength(3);
+      expect(encodings!.map((e) => e.rid)).toEqual(['q', 'h', 'f']);
+      // every encoding needs both scalabilityMode and scaleResolutionDownBy for chrome
+      // M113+ to treat them as real simulcast rather than legacy svc
+      encodings!.forEach((encoding) => {
+        /* @ts-ignore */
+        expect(encoding.scalabilityMode).toBe('L1T2');
+        expect(encoding.scaleResolutionDownBy).toBeGreaterThanOrEqual(1);
+      });
+    }
+  });
+
+  it('sets the scalability mode on a single encoding svc simulcast ladder', () => {
+    const encodings = computeVideoEncodings(false, 100, 120, {
+      simulcast: true,
+      videoCodec: 'vp9',
+      scalabilityMode: 'L1T3',
+    });
+    expect(encodings).toHaveLength(1);
+    expect(encodings![0].rid).toBe('q');
+    /* @ts-ignore */
+    expect(encodings![0].scalabilityMode).toBe('L1T3');
+  });
+
+  it('does not set a scalability mode for non-svc simulcast', () => {
+    const encodings = computeVideoEncodings(false, 960, 540, {
+      simulcast: true,
+      videoCodec: 'vp8',
+      scalabilityMode: 'L1T2',
+    });
+    expect(encodings).toHaveLength(3);
+    encodings!.forEach((encoding) => {
+      /* @ts-ignore */
+      expect(encoding.scalabilityMode).toBeUndefined();
+    });
+  });
+
   //   it('respects default backup codec encoding', () => {
   //     const vp8Encodings = computeTrackBackupEncodings(false, 100, 120, { simulcast: true });
   //     const h264Encodings = computeVideoEncodings(false, 100, 120, {
@@ -193,3 +266,63 @@ describe('screenShareSimulcastDefaults', () => {
     expect(defaultSimulcastLayers[0].encoding.maxBitrate).toBe(375000);
   });
 });
+
+describe('computeStartTargetBitrate', () => {
+  // ordered q..f, as encodingsFromPresets builds them
+  const simulcastLadder: RTCRtpEncodingParameters[] = [
+    { rid: 'q', maxBitrate: 160_000 },
+    { rid: 'h', maxBitrate: 450_000 },
+    { rid: 'f', maxBitrate: 680_000 },
+  ];
+
+  it('sums the ladder for plain simulcast', () => {
+    expect(computeStartTargetBitrate('vp8', { simulcast: true }, simulcastLadder)).toBe(1_290_000);
+  });
+
+  it('sums the ladder for vp9/av1 published as rid simulcast', () => {
+    // encodings[0] is the *smallest* layer here, so taking it would under-hint BWE
+    for (const codec of ['vp9', 'av1'] as const) {
+      expect(
+        computeStartTargetBitrate(
+          codec,
+          { simulcast: true, videoCodec: codec, scalabilityMode: 'L1T2' },
+          simulcastLadder,
+        ),
+      ).toBe(1_290_000);
+    }
+  });
+
+  it('uses the single encoding for vp9/av1 svc', () => {
+    for (const codec of ['vp9', 'av1'] as const) {
+      expect(
+        computeStartTargetBitrate(
+          codec,
+          { simulcast: false, videoCodec: codec, scalabilityMode: 'L3T3_KEY' },
+          [{ maxBitrate: 680_000 }],
+        ),
+      ).toBe(680_000);
+    }
+  });
+
+  it('uses the first encoding for the legacy svc shape, which is ordered f..q', () => {
+    // legacy SVC pushes videoRids[2 - i], so encodings[0] carries the full bitrate
+    const legacySvc: RTCRtpEncodingParameters[] = [
+      { rid: 'f', maxBitrate: 680_000 },
+      { rid: 'h', maxBitrate: 226_667 },
+      { rid: 'q', maxBitrate: 75_556 },
+    ];
+    expect(
+      computeStartTargetBitrate(
+        'vp9',
+        { simulcast: false, videoCodec: 'vp9', scalabilityMode: 'L3T3_KEY' },
+        legacySvc,
+      ),
+    ).toBe(680_000);
+  });
+
+  it('handles missing bitrates and empty encodings', () => {
+    expect(computeStartTargetBitrate('vp8', { simulcast: true }, [])).toBe(0);
+    expect(computeStartTargetBitrate('vp9', undefined, [])).toBe(0);
+    expect(computeStartTargetBitrate('vp8', { simulcast: true }, [{ rid: 'q' }])).toBe(0);
+  });
+});