Skip to content

Commit 55bfa20

Browse files
authored
Add web support for pre-connect audio buffer (#1000)
- Extract `AudioFrameCapture` abstraction with conditional imports (native/web) - Implement web audio capture using AudioWorklet API
1 parent 4f33259 commit 55bfa20

7 files changed

Lines changed: 869 additions & 53 deletions

File tree

‎.changes/pre-connect-audio-web‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
patch type="added" "Add web support for pre-connect audio buffer"
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright 2025 LiveKit, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import 'dart:typed_data' show Uint8List;
16+
17+
import 'package:flutter_webrtc/flutter_webrtc.dart' show MediaStreamTrack;
18+
19+
import 'audio_frame_capture_native.dart' if (dart.library.js_interop) 'audio_frame_capture_web.dart';
20+
21+
/// A single frame of raw PCM audio data.
22+
class AudioFrame {
23+
final int sampleRate;
24+
final int channels;
25+
final Uint8List data;
26+
final String commonFormat;
27+
28+
const AudioFrame({
29+
required this.sampleRate,
30+
required this.channels,
31+
required this.data,
32+
required this.commonFormat,
33+
});
34+
}
35+
36+
/// Platform-agnostic interface for capturing raw PCM audio frames from a track.
37+
///
38+
/// On native (iOS/Android), this uses MethodChannel + EventChannel.
39+
/// On web, this uses Web Audio API with AudioWorklet.
40+
abstract class AudioFrameCapture {
41+
/// Stream of raw PCM audio frames.
42+
Stream<AudioFrame> get frameStream;
43+
44+
/// Start capturing audio from the given [track].
45+
///
46+
/// Returns `true` if the renderer started successfully.
47+
Future<bool> start({
48+
required MediaStreamTrack track,
49+
required String rendererId,
50+
required int sampleRate,
51+
required int channels,
52+
required String commonFormat,
53+
});
54+
55+
/// Stop capturing and release resources.
56+
Future<void> stop();
57+
}
58+
59+
/// Factory that returns the platform-appropriate implementation.
60+
AudioFrameCapture createAudioFrameCapture() => createAudioFrameCaptureImpl();
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Copyright 2025 LiveKit, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import 'dart:async';
16+
import 'dart:typed_data' show Uint8List;
17+
18+
import 'package:flutter/services.dart' show EventChannel;
19+
20+
import 'package:flutter_webrtc/flutter_webrtc.dart' show MediaStreamTrack;
21+
22+
import '../logger.dart';
23+
import '../support/native.dart';
24+
import 'audio_frame_capture.dart';
25+
26+
/// Native (iOS/Android/macOS) implementation using MethodChannel + EventChannel.
27+
class AudioFrameCaptureNative implements AudioFrameCapture {
28+
EventChannel? _eventChannel;
29+
StreamSubscription? _streamSubscription;
30+
final _controller = StreamController<AudioFrame>.broadcast();
31+
String? _rendererId;
32+
33+
@override
34+
Stream<AudioFrame> get frameStream => _controller.stream;
35+
36+
@override
37+
Future<bool> start({
38+
required MediaStreamTrack track,
39+
required String rendererId,
40+
required int sampleRate,
41+
required int channels,
42+
required String commonFormat,
43+
}) async {
44+
final result = await Native.startAudioRenderer(
45+
trackId: track.id!,
46+
rendererId: rendererId,
47+
format: {
48+
'commonFormat': commonFormat,
49+
'sampleRate': sampleRate,
50+
'channels': channels,
51+
},
52+
);
53+
54+
if (result != true) return false;
55+
56+
_rendererId = rendererId;
57+
58+
_eventChannel = EventChannel('io.livekit.audio.renderer/channel-$rendererId');
59+
_streamSubscription = _eventChannel?.receiveBroadcastStream().listen((event) {
60+
try {
61+
_controller.add(AudioFrame(
62+
sampleRate: event['sampleRate'] as int,
63+
channels: event['channels'] as int,
64+
data: event['data'] as Uint8List,
65+
commonFormat: (event['commonFormat'] as String?) ?? commonFormat,
66+
));
67+
} catch (e) {
68+
logger.warning('[AudioFrameCapture] Error parsing native event: $e');
69+
}
70+
});
71+
72+
return true;
73+
}
74+
75+
@override
76+
Future<void> stop() async {
77+
await _streamSubscription?.cancel();
78+
_streamSubscription = null;
79+
_eventChannel = null;
80+
81+
final rendererId = _rendererId;
82+
if (rendererId != null) {
83+
await Native.stopAudioRenderer(rendererId: rendererId);
84+
_rendererId = null;
85+
}
86+
87+
if (!_controller.isClosed) {
88+
await _controller.close();
89+
}
90+
}
91+
}
92+
93+
AudioFrameCapture createAudioFrameCaptureImpl() => AudioFrameCaptureNative();
Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
// Copyright 2025 LiveKit, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import 'dart:async';
16+
import 'dart:js_interop';
17+
import 'dart:js_interop_unsafe';
18+
import 'dart:typed_data' show Uint8List;
19+
20+
import 'package:dart_webrtc/dart_webrtc.dart' show MediaStreamTrackWeb;
21+
import 'package:flutter_webrtc/flutter_webrtc.dart' show MediaStreamTrack;
22+
import 'package:web/web.dart' as web;
23+
24+
import '../logger.dart';
25+
import '../support/audio_pcm_utils.dart';
26+
import 'audio_frame_capture.dart';
27+
28+
/// JavaScript source for the AudioWorkletProcessor.
29+
///
30+
/// Runs on the audio rendering thread. Forwards raw float32 input samples
31+
/// to the main thread via MessagePort.
32+
const _workletProcessorJs = '''
33+
class AudioRendererProcessor extends AudioWorkletProcessor {
34+
process(inputs, outputs, parameters) {
35+
const input = inputs[0];
36+
if (input && input.length > 0 && input[0].length > 0) {
37+
const channels = input.length;
38+
const frames = input[0].length;
39+
40+
// Interleave channels into a single Float32Array.
41+
const interleaved = new Float32Array(frames * channels);
42+
for (let frame = 0; frame < frames; frame++) {
43+
for (let ch = 0; ch < channels; ch++) {
44+
interleaved[frame * channels + ch] = input[ch][frame];
45+
}
46+
}
47+
48+
this.port.postMessage({
49+
samples: interleaved.buffer,
50+
channels: channels,
51+
frames: frames,
52+
}, [interleaved.buffer]);
53+
}
54+
return true;
55+
}
56+
}
57+
registerProcessor('audio-renderer-processor', AudioRendererProcessor);
58+
''';
59+
60+
/// Web implementation using AudioWorklet to capture raw PCM frames.
61+
class AudioFrameCaptureWeb implements AudioFrameCapture {
62+
web.AudioContext? _audioContext;
63+
web.AudioWorkletNode? _workletNode;
64+
web.AudioNode? _sourceNode;
65+
StreamController<AudioFrame>? _controller;
66+
String _targetFormat = 'int16';
67+
int _targetChannels = 1;
68+
69+
@override
70+
Stream<AudioFrame> get frameStream => (_controller ??= StreamController<AudioFrame>.broadcast()).stream;
71+
72+
@override
73+
Future<bool> start({
74+
required MediaStreamTrack track,
75+
required String rendererId,
76+
required int sampleRate,
77+
required int channels,
78+
required String commonFormat,
79+
}) async {
80+
_targetFormat = commonFormat;
81+
_targetChannels = channels;
82+
_controller ??= StreamController<AudioFrame>.broadcast();
83+
84+
try {
85+
// 1. Get the underlying JS MediaStreamTrack.
86+
final jsTrack = (track as MediaStreamTrackWeb).jsTrack;
87+
final mediaStream = web.MediaStream([jsTrack].toJS);
88+
89+
// 2. Create AudioContext.
90+
_audioContext = web.AudioContext();
91+
final ctx = _audioContext!;
92+
93+
// 3. Register worklet processor via Blob URL.
94+
final blob = web.Blob(
95+
[_workletProcessorJs.toJS].toJS,
96+
web.BlobPropertyBag(type: 'application/javascript'),
97+
);
98+
final blobUrl = web.URL.createObjectURL(blob);
99+
try {
100+
await ctx.audioWorklet.addModule(blobUrl).toDart;
101+
} finally {
102+
web.URL.revokeObjectURL(blobUrl);
103+
}
104+
105+
// 4. Create audio pipeline: source → worklet → destination.
106+
_sourceNode = ctx.createMediaStreamSource(mediaStream);
107+
_workletNode = web.AudioWorkletNode(ctx, 'audio-renderer-processor');
108+
109+
_sourceNode!.connect(_workletNode!);
110+
// No destination connection needed — process() is called as long as
111+
// there is input flowing and it returns true. Connecting to
112+
// ctx.destination would route mic audio to speakers (echo/feedback).
113+
114+
// 5. Listen for PCM frames from the worklet thread.
115+
_workletNode!.port.onmessage = _onWorkletMessage.toJS;
116+
117+
return true;
118+
} catch (e) {
119+
logger.warning('[AudioFrameCapture] Failed to start web capture: $e');
120+
await stop();
121+
return false;
122+
}
123+
}
124+
125+
void _onWorkletMessage(web.MessageEvent event) {
126+
final controller = _controller;
127+
if (controller == null || controller.isClosed) return;
128+
129+
try {
130+
final data = event.data as JSObject;
131+
final samplesBuffer = (data.getProperty('samples'.toJS) as JSArrayBuffer).toDart;
132+
final channels = (data.getProperty('channels'.toJS) as JSNumber).toDartInt;
133+
final frames = (data.getProperty('frames'.toJS) as JSNumber).toDartInt;
134+
135+
final outChannels = _targetChannels.clamp(1, channels);
136+
final actualSampleRate = _audioContext?.sampleRate.toInt() ?? 48000;
137+
final srcFloat32 = samplesBuffer.asFloat32List();
138+
139+
final Uint8List bytes;
140+
if (_targetFormat == 'float32') {
141+
bytes = float32ToFloat32Bytes(srcFloat32, channels, outChannels, frames);
142+
} else {
143+
bytes = float32ToInt16Bytes(srcFloat32, channels, outChannels, frames);
144+
}
145+
146+
controller.add(AudioFrame(
147+
sampleRate: actualSampleRate,
148+
channels: outChannels,
149+
data: bytes,
150+
commonFormat: _targetFormat,
151+
));
152+
} catch (e) {
153+
logger.warning('[AudioFrameCapture] Error processing worklet frame: $e');
154+
}
155+
}
156+
157+
@override
158+
Future<void> stop() async {
159+
_workletNode?.port.onmessage = null;
160+
161+
try {
162+
_workletNode?.disconnect();
163+
} catch (_) {}
164+
_workletNode = null;
165+
166+
try {
167+
_sourceNode?.disconnect();
168+
} catch (_) {}
169+
_sourceNode = null;
170+
171+
try {
172+
if (_audioContext?.state != 'closed') {
173+
await _audioContext?.close().toDart;
174+
}
175+
} catch (_) {}
176+
_audioContext = null;
177+
178+
final controller = _controller;
179+
_controller = null;
180+
if (controller != null && !controller.isClosed) {
181+
await controller.close();
182+
}
183+
}
184+
}
185+
186+
AudioFrameCapture createAudioFrameCaptureImpl() => AudioFrameCaptureWeb();

0 commit comments

Comments
 (0)