|
| 1 | +// Copyright 2025 LiveKit, Inc. |
| 2 | +// |
| 3 | +// Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +// you may not use this file except in compliance with the License. |
| 5 | +// You may obtain a copy of the License at |
| 6 | +// |
| 7 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +// |
| 9 | +// Unless required by applicable law or agreed to in writing, software |
| 10 | +// distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +// See the License for the specific language governing permissions and |
| 13 | +// limitations under the License. |
| 14 | + |
| 15 | +import 'dart:async'; |
| 16 | +import 'dart:js_interop'; |
| 17 | +import 'dart:js_interop_unsafe'; |
| 18 | +import 'dart:typed_data' show Uint8List; |
| 19 | + |
| 20 | +import 'package:dart_webrtc/dart_webrtc.dart' show MediaStreamTrackWeb; |
| 21 | +import 'package:flutter_webrtc/flutter_webrtc.dart' show MediaStreamTrack; |
| 22 | +import 'package:web/web.dart' as web; |
| 23 | + |
| 24 | +import '../logger.dart'; |
| 25 | +import '../support/audio_pcm_utils.dart'; |
| 26 | +import 'audio_frame_capture.dart'; |
| 27 | + |
| 28 | +/// JavaScript source for the AudioWorkletProcessor. |
| 29 | +/// |
| 30 | +/// Runs on the audio rendering thread. Forwards raw float32 input samples |
| 31 | +/// to the main thread via MessagePort. |
| 32 | +const _workletProcessorJs = ''' |
| 33 | +class AudioRendererProcessor extends AudioWorkletProcessor { |
| 34 | + process(inputs, outputs, parameters) { |
| 35 | + const input = inputs[0]; |
| 36 | + if (input && input.length > 0 && input[0].length > 0) { |
| 37 | + const channels = input.length; |
| 38 | + const frames = input[0].length; |
| 39 | +
|
| 40 | + // Interleave channels into a single Float32Array. |
| 41 | + const interleaved = new Float32Array(frames * channels); |
| 42 | + for (let frame = 0; frame < frames; frame++) { |
| 43 | + for (let ch = 0; ch < channels; ch++) { |
| 44 | + interleaved[frame * channels + ch] = input[ch][frame]; |
| 45 | + } |
| 46 | + } |
| 47 | +
|
| 48 | + this.port.postMessage({ |
| 49 | + samples: interleaved.buffer, |
| 50 | + channels: channels, |
| 51 | + frames: frames, |
| 52 | + }, [interleaved.buffer]); |
| 53 | + } |
| 54 | + return true; |
| 55 | + } |
| 56 | +} |
| 57 | +registerProcessor('audio-renderer-processor', AudioRendererProcessor); |
| 58 | +'''; |
| 59 | + |
| 60 | +/// Web implementation using AudioWorklet to capture raw PCM frames. |
| 61 | +class AudioFrameCaptureWeb implements AudioFrameCapture { |
| 62 | + web.AudioContext? _audioContext; |
| 63 | + web.AudioWorkletNode? _workletNode; |
| 64 | + web.AudioNode? _sourceNode; |
| 65 | + StreamController<AudioFrame>? _controller; |
| 66 | + String _targetFormat = 'int16'; |
| 67 | + int _targetChannels = 1; |
| 68 | + |
| 69 | + @override |
| 70 | + Stream<AudioFrame> get frameStream => (_controller ??= StreamController<AudioFrame>.broadcast()).stream; |
| 71 | + |
| 72 | + @override |
| 73 | + Future<bool> start({ |
| 74 | + required MediaStreamTrack track, |
| 75 | + required String rendererId, |
| 76 | + required int sampleRate, |
| 77 | + required int channels, |
| 78 | + required String commonFormat, |
| 79 | + }) async { |
| 80 | + _targetFormat = commonFormat; |
| 81 | + _targetChannels = channels; |
| 82 | + _controller ??= StreamController<AudioFrame>.broadcast(); |
| 83 | + |
| 84 | + try { |
| 85 | + // 1. Get the underlying JS MediaStreamTrack. |
| 86 | + final jsTrack = (track as MediaStreamTrackWeb).jsTrack; |
| 87 | + final mediaStream = web.MediaStream([jsTrack].toJS); |
| 88 | + |
| 89 | + // 2. Create AudioContext. |
| 90 | + _audioContext = web.AudioContext(); |
| 91 | + final ctx = _audioContext!; |
| 92 | + |
| 93 | + // 3. Register worklet processor via Blob URL. |
| 94 | + final blob = web.Blob( |
| 95 | + [_workletProcessorJs.toJS].toJS, |
| 96 | + web.BlobPropertyBag(type: 'application/javascript'), |
| 97 | + ); |
| 98 | + final blobUrl = web.URL.createObjectURL(blob); |
| 99 | + try { |
| 100 | + await ctx.audioWorklet.addModule(blobUrl).toDart; |
| 101 | + } finally { |
| 102 | + web.URL.revokeObjectURL(blobUrl); |
| 103 | + } |
| 104 | + |
| 105 | + // 4. Create audio pipeline: source → worklet → destination. |
| 106 | + _sourceNode = ctx.createMediaStreamSource(mediaStream); |
| 107 | + _workletNode = web.AudioWorkletNode(ctx, 'audio-renderer-processor'); |
| 108 | + |
| 109 | + _sourceNode!.connect(_workletNode!); |
| 110 | + // No destination connection needed — process() is called as long as |
| 111 | + // there is input flowing and it returns true. Connecting to |
| 112 | + // ctx.destination would route mic audio to speakers (echo/feedback). |
| 113 | + |
| 114 | + // 5. Listen for PCM frames from the worklet thread. |
| 115 | + _workletNode!.port.onmessage = _onWorkletMessage.toJS; |
| 116 | + |
| 117 | + return true; |
| 118 | + } catch (e) { |
| 119 | + logger.warning('[AudioFrameCapture] Failed to start web capture: $e'); |
| 120 | + await stop(); |
| 121 | + return false; |
| 122 | + } |
| 123 | + } |
| 124 | + |
| 125 | + void _onWorkletMessage(web.MessageEvent event) { |
| 126 | + final controller = _controller; |
| 127 | + if (controller == null || controller.isClosed) return; |
| 128 | + |
| 129 | + try { |
| 130 | + final data = event.data as JSObject; |
| 131 | + final samplesBuffer = (data.getProperty('samples'.toJS) as JSArrayBuffer).toDart; |
| 132 | + final channels = (data.getProperty('channels'.toJS) as JSNumber).toDartInt; |
| 133 | + final frames = (data.getProperty('frames'.toJS) as JSNumber).toDartInt; |
| 134 | + |
| 135 | + final outChannels = _targetChannels.clamp(1, channels); |
| 136 | + final actualSampleRate = _audioContext?.sampleRate.toInt() ?? 48000; |
| 137 | + final srcFloat32 = samplesBuffer.asFloat32List(); |
| 138 | + |
| 139 | + final Uint8List bytes; |
| 140 | + if (_targetFormat == 'float32') { |
| 141 | + bytes = float32ToFloat32Bytes(srcFloat32, channels, outChannels, frames); |
| 142 | + } else { |
| 143 | + bytes = float32ToInt16Bytes(srcFloat32, channels, outChannels, frames); |
| 144 | + } |
| 145 | + |
| 146 | + controller.add(AudioFrame( |
| 147 | + sampleRate: actualSampleRate, |
| 148 | + channels: outChannels, |
| 149 | + data: bytes, |
| 150 | + commonFormat: _targetFormat, |
| 151 | + )); |
| 152 | + } catch (e) { |
| 153 | + logger.warning('[AudioFrameCapture] Error processing worklet frame: $e'); |
| 154 | + } |
| 155 | + } |
| 156 | + |
| 157 | + @override |
| 158 | + Future<void> stop() async { |
| 159 | + _workletNode?.port.onmessage = null; |
| 160 | + |
| 161 | + try { |
| 162 | + _workletNode?.disconnect(); |
| 163 | + } catch (_) {} |
| 164 | + _workletNode = null; |
| 165 | + |
| 166 | + try { |
| 167 | + _sourceNode?.disconnect(); |
| 168 | + } catch (_) {} |
| 169 | + _sourceNode = null; |
| 170 | + |
| 171 | + try { |
| 172 | + if (_audioContext?.state != 'closed') { |
| 173 | + await _audioContext?.close().toDart; |
| 174 | + } |
| 175 | + } catch (_) {} |
| 176 | + _audioContext = null; |
| 177 | + |
| 178 | + final controller = _controller; |
| 179 | + _controller = null; |
| 180 | + if (controller != null && !controller.isClosed) { |
| 181 | + await controller.close(); |
| 182 | + } |
| 183 | + } |
| 184 | +} |
| 185 | + |
| 186 | +AudioFrameCapture createAudioFrameCaptureImpl() => AudioFrameCaptureWeb(); |
0 commit comments