Skip to content

Commit d1fe342

Browse files
authored
Apply audio processing options when local capture starts (#1115)
## Summary - move create-time audio processing setup into the local capture start path so publish/preconnect prepares platform audio processing before WebRTC opens the microphone, matching the Swift SDK flow - keep `LocalAudioTrack.setAudioProcessingOptions` as a command-style runtime API that returns on success and throws `AudioProcessingException` on failure - expose structured failure reasons for invalid combinations, unavailable platform support, native apply failures, and unknown failures - clean up local audio tracks when capture/publish startup fails and keep cleanup failures from masking the original error - bump `flutter_webrtc` to `1.5.2` for Android audio device module access ## Behavior - `LocalAudioTrack.create(...)` stores requested audio processing options; it no longer attempts to apply them immediately. - `track.start()` starts local capture and applies stored processing options before the microphone opens on supported platforms. - Publish and preconnect paths fail during capture start if the exposed native platform API reports audio processing setup failure. - Runtime `setAudioProcessingOptions(...)` still applies immediately for active local audio tracks and throws on failed native apply/store. - Android uses `JavaAudioDeviceModule.prewarmRecording(options)`. That WebRTC API returns `void`, so this PR can surface thrown failures, but clean internal `false` returns from `initRecordingIfNeeded()` / `prewarmRecordingIfNeeded()` are not observable until WebRTC/flutter_webrtc exposes a resultful API. ## Testing - `flutter pub get` - `flutter pub get` in `example` - `flutter analyze --no-pub` - `flutter test --no-pub` - `flutter build apk --debug --no-pub` in `example` - Android emulator smoke test: example joined a room, published local microphone audio, muted/unpublished, and showed no ADM/platform-unavailable errors
1 parent 55af814 commit d1fe342

19 files changed

Lines changed: 670 additions & 203 deletions
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
patch type="fixed" "Clean up local audio tracks when recording start fails"
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
patch type="fixed" "Throw platformUnavailable when runtime audio processing is unsupported"
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
patch type="fixed" "Apply create-time audio processing when local recording is prepared"

‎.changes/runtime-audio-options‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
minor type="added" "Runtime audio processing options for local audio tracks and engine-wide audio processing state read-back on AudioManager"
1+
minor type="added" "Runtime audio processing controls for local audio tracks"

‎android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt‎

Lines changed: 99 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
package io.livekit.plugin
1818

1919
import android.annotation.SuppressLint
20+
import android.os.Handler
21+
import android.os.Looper
2022
import androidx.annotation.NonNull
2123

2224
import io.flutter.embedding.engine.plugins.FlutterPlugin
@@ -37,13 +39,18 @@ import org.webrtc.audio.AudioProcessingMode
3739
import org.webrtc.audio.AudioProcessingOptions
3840
import org.webrtc.audio.AudioProcessingOptionsResult
3941
import org.webrtc.audio.AudioProcessingState
42+
import java.util.concurrent.ExecutorService
43+
import java.util.concurrent.Executors
44+
import java.util.concurrent.RejectedExecutionException
4045

4146
/** LiveKitPlugin */
4247
class LiveKitPlugin : FlutterPlugin, MethodCallHandler {
4348
private var audioProcessors = mutableMapOf<String, AudioProcessors>()
4449
private var flutterWebRTCPlugin = FlutterWebRTCPlugin.sharedSingleton
4550
private var binaryMessenger: BinaryMessenger? = null
4651
private var audioSwitchManager: LKAudioSwitchManager? = null
52+
private var audioDeviceModuleExecutor: ExecutorService? = null
53+
private val mainHandler = Handler(Looper.getMainLooper())
4754

4855
/// The MethodChannel that will the communication between Flutter and native Android
4956
///
@@ -59,6 +66,8 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler {
5966
channel.setMethodCallHandler(this)
6067
binaryMessenger = flutterPluginBinding.binaryMessenger
6168
audioSwitchManager = LKAudioSwitchManager(flutterPluginBinding.applicationContext)
69+
audioDeviceModuleExecutor?.shutdown()
70+
audioDeviceModuleExecutor = Executors.newSingleThreadExecutor()
6271
}
6372

6473
@SuppressLint("SuspiciousIndentation")
@@ -236,7 +245,84 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler {
236245
return
237246
}
238247

239-
val options = AudioProcessingOptions(
248+
val options = audioProcessingOptions(call)
249+
val processingResult = mediaTrack.setAudioProcessingOptions(options)
250+
result.success(
251+
mapOf(
252+
"result" to processingResult.isSuccess,
253+
"code" to audioProcessingResultCodeString(processingResult.code),
254+
"message" to processingResult.message,
255+
),
256+
)
257+
}
258+
259+
private fun handleStartLocalRecording(call: MethodCall, result: Result) {
260+
val audioDeviceModule = flutterWebRTCPlugin.audioDeviceModule
261+
if (audioDeviceModule == null) {
262+
result.error("rejectedPlatformUnavailable", "audio device module is unavailable", null)
263+
return
264+
}
265+
266+
val executor = audioDeviceModuleExecutorOrError(result, "rejectedPlatformUnavailable") ?: return
267+
val options = audioProcessingOptions(call)
268+
try {
269+
executor.execute {
270+
try {
271+
// prewarmRecording applies Android platform AP and prepares recording
272+
// without setting the client-start flag. WebRTC exposes this as void,
273+
// so only thrown failures can be surfaced here.
274+
audioDeviceModule.prewarmRecording(options)
275+
mainHandler.post {
276+
result.success(null)
277+
}
278+
} catch (error: Throwable) {
279+
mainHandler.post {
280+
result.error("applyFailed", error.message, null)
281+
}
282+
}
283+
}
284+
} catch (error: RejectedExecutionException) {
285+
result.error("rejectedPlatformUnavailable", "audio device module executor is unavailable", null)
286+
}
287+
}
288+
289+
private fun handleStopLocalRecording(result: Result) {
290+
val audioDeviceModule = flutterWebRTCPlugin.audioDeviceModule
291+
if (audioDeviceModule == null) {
292+
result.error("stopLocalRecording", "audio device module is unavailable", null)
293+
return
294+
}
295+
296+
val executor = audioDeviceModuleExecutorOrError(result, "stopLocalRecording") ?: return
297+
try {
298+
executor.execute {
299+
try {
300+
audioDeviceModule.requestStopRecording()
301+
mainHandler.post {
302+
result.success(null)
303+
}
304+
} catch (error: Throwable) {
305+
mainHandler.post {
306+
result.error("stopLocalRecording", error.message, null)
307+
}
308+
}
309+
}
310+
} catch (error: RejectedExecutionException) {
311+
result.error("stopLocalRecording", "audio device module executor is unavailable", null)
312+
}
313+
}
314+
315+
private fun audioDeviceModuleExecutorOrError(result: Result, code: String): ExecutorService? {
316+
val executor = audioDeviceModuleExecutor
317+
if (executor == null || executor.isShutdown) {
318+
result.error(code, "audio device module executor is unavailable", null)
319+
return null
320+
}
321+
return executor
322+
}
323+
324+
private fun audioProcessingOptions(call: MethodCall): AudioProcessingOptions =
325+
AudioProcessingOptions(
240326
AudioProcessingComponentOptions(
241327
call.argument<Boolean>("echoCancellation") ?: true,
242328
audioProcessingMode(call.argument<String>("echoCancellationMode")),
@@ -255,16 +341,6 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler {
255341
),
256342
)
257343

258-
val processingResult = mediaTrack.setAudioProcessingOptions(options)
259-
result.success(
260-
mapOf(
261-
"result" to processingResult.isSuccess,
262-
"code" to audioProcessingResultCodeString(processingResult.code),
263-
"message" to processingResult.message,
264-
),
265-
)
266-
}
267-
268344
private fun audioProcessingMode(value: String?): AudioProcessingMode = when (value) {
269345
"platform" -> AudioProcessingMode.PLATFORM
270346
"software" -> AudioProcessingMode.SOFTWARE
@@ -274,7 +350,7 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler {
274350
private fun audioProcessingResultCodeString(code: AudioProcessingOptionsResult.Code): String = when (code) {
275351
AudioProcessingOptionsResult.Code.APPLIED -> "applied"
276352
AudioProcessingOptionsResult.Code.STORED -> "stored"
277-
AudioProcessingOptionsResult.Code.REJECTED_REMOTE_TRACK -> "rejectedRemoteTrack"
353+
AudioProcessingOptionsResult.Code.REJECTED_REMOTE_TRACK -> "unknown"
278354
AudioProcessingOptionsResult.Code.REJECTED_INVALID_COMBINATION -> "rejectedInvalidCombination"
279355
AudioProcessingOptionsResult.Code.REJECTED_PLATFORM_UNAVAILABLE -> "rejectedPlatformUnavailable"
280356
AudioProcessingOptionsResult.Code.APPLY_FAILED -> "applyFailed"
@@ -352,6 +428,14 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler {
352428
handleSetAudioProcessingOptions(call, result)
353429
}
354430

431+
"startLocalRecording" -> {
432+
handleStartLocalRecording(call, result)
433+
}
434+
435+
"stopLocalRecording" -> {
436+
handleStopLocalRecording(result)
437+
}
438+
355439
"getAudioProcessingState" -> {
356440
handleGetAudioProcessingState(result)
357441
}
@@ -388,6 +472,9 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler {
388472
audioSwitchManager?.dispose()
389473
audioSwitchManager = null
390474

475+
audioDeviceModuleExecutor?.shutdown()
476+
audioDeviceModuleExecutor = null
477+
391478
// Cleanup all processors
392479
audioProcessors.values.forEach { it.cleanup() }
393480
audioProcessors.clear()

‎docs/audio.md‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -108,24 +108,34 @@ final now = AudioManager.instance.audioEngineState;
108108

109109
Session management is one half of the audio stack. The other half is audio processing, the signal processing applied to captured audio such as echo cancellation, noise suppression, auto gain control, and the high pass filter. `AudioManager` is the home for both, and they compose cleanly.
110110

111-
The session intent decides how the platform treats audio. Processing options decide what happens to captured local microphone audio:
111+
The session intent decides how the platform treats audio. Capture options decide what happens to local microphone audio:
112112

113113
- A call usually uses the automatic `communication` session. The default `AudioCaptureOptions` enable echo cancellation, noise suppression, and auto gain control, while leaving the high pass filter off.
114114
- Use `AudioProcessingOptions.communication()` when you want all four voice filters on for an existing local audio track.
115115
- Use `AudioProcessingOptions.noProcessing()` for local capture where you want minimal processing, such as high quality recording or app-managed audio effects.
116116

117-
Processing is applied per local audio track. A malformed request (an incompatible mode, or a remote track) throws, while a request the platform could not honor comes back as an unsuccessful result:
117+
Create-time processing is configured through `AudioCaptureOptions`. `LocalAudioTrack.create(...)` stores these options, and LiveKit prepares them when local recording starts, such as during publish or preconnect. If the exposed native platform API reports that capture-time setup failed, the start path throws `AudioProcessingException` before publish creates a server-side publication.
118+
119+
```dart
120+
final track = await LocalAudioTrack.create(
121+
const AudioCaptureOptions(
122+
echoCancellation: true,
123+
noiseSuppression: true,
124+
autoGainControl: true,
125+
highPassFilter: false,
126+
),
127+
);
128+
```
129+
130+
For an existing local audio track, call `setAudioProcessingOptions`. It returns when the native layer applies or stores the options. If the request is invalid, unsupported, or cannot be applied, it throws `AudioProcessingException` and the track keeps its previous processing options.
118131

119132
```dart
120133
try {
121-
final result = await localAudioTrack.setAudioProcessingOptions(
134+
await localAudioTrack.setAudioProcessingOptions(
122135
const AudioProcessingOptions.noProcessing(),
123136
);
124-
if (!result.isSuccess) {
125-
print('audio processing not applied: ${result.code.value} ${result.message}');
126-
}
127137
} on AudioProcessingException catch (error) {
128-
print('invalid audio processing request: ${error.code.value} ${error.message}');
138+
print('audio processing not applied: ${error.reason.name} ${error.message}');
129139
}
130140
```
131141

‎lib/src/audio/audio_manager.dart‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -291,8 +291,8 @@ class AudioManager {
291291
///
292292
/// The audio processing module is owned by the native peer connection factory
293293
/// and shared engine-wide, so this reflects what is actually applied across
294-
/// the engine rather than any single track — use it to verify what a
295-
/// `LocalAudioTrack.setAudioProcessingOptions` request resolved to. Returns
294+
/// the engine rather than any single track. Use it to verify native state
295+
/// after a `LocalAudioTrack.setAudioProcessingOptions` request. Returns
296296
/// `null` when the native side cannot provide it.
297297
Future<AudioProcessingState?> getAudioProcessingState() async {
298298
final response = await Native.getAudioProcessingState();

‎lib/src/participant/local.dart‎

Lines changed: 75 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -179,80 +179,94 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
179179
final audioEncoding = publishOptions.encoding ?? AudioEncoding.presetMusic;
180180
final List<rtc.RTCRtpEncoding> encodings = [audioEncoding.toRTCRtpEncoding()];
181181

182-
final req = lk_rtc.AddTrackRequest(
183-
cid: track.getCid(),
184-
name: publishOptions.name ?? AudioPublishOptions.defaultMicrophoneName,
185-
type: track.kind.toPBType(),
186-
source: track.source.toPBType(),
187-
muted: track.muted,
188-
stream: buildStreamId(publishOptions, track.source),
189-
disableDtx: !publishOptions.dtx,
190-
disableRed: room.e2eeManager != null ? true : publishOptions.red ?? true,
191-
encryption: room.roomOptions.lkEncryptionType,
192-
);
182+
final shouldStopOnFailure = !track.isActive;
183+
try {
184+
// Start capture before signaling so create-time audio processing failures
185+
// abort publish without creating a server-side publication.
186+
await track.start();
193187

194-
// Populate audio features (e.g., TF_NO_DTX, TF_PRECONNECT_BUFFER)
195-
req.audioFeatures.addAll([
196-
if (!publishOptions.dtx) lk_models.AudioTrackFeature.TF_NO_DTX,
197-
if (publishOptions.preConnect) lk_models.AudioTrackFeature.TF_PRECONNECT_BUFFER,
198-
]);
188+
final req = lk_rtc.AddTrackRequest(
189+
cid: track.getCid(),
190+
name: publishOptions.name ?? AudioPublishOptions.defaultMicrophoneName,
191+
type: track.kind.toPBType(),
192+
source: track.source.toPBType(),
193+
muted: track.muted,
194+
stream: buildStreamId(publishOptions, track.source),
195+
disableDtx: !publishOptions.dtx,
196+
disableRed: room.e2eeManager != null ? true : publishOptions.red ?? true,
197+
encryption: room.roomOptions.lkEncryptionType,
198+
);
199199

200-
Future<lk_models.TrackInfo> negotiate() async {
201-
track.transceiver = await room.engine.createTransceiverRTCRtpSender(track, publishOptions!, encodings);
202-
await room.engine.negotiate();
203-
return lk_models.TrackInfo();
204-
}
200+
// Populate audio features (e.g., TF_NO_DTX, TF_PRECONNECT_BUFFER)
201+
req.audioFeatures.addAll([
202+
if (!publishOptions.dtx) lk_models.AudioTrackFeature.TF_NO_DTX,
203+
if (publishOptions.preConnect) lk_models.AudioTrackFeature.TF_PRECONNECT_BUFFER,
204+
]);
205205

206-
late lk_models.TrackInfo trackInfo;
207-
if (room.engine.enabledPublishCodecs?.isNotEmpty ?? false) {
208-
final rets = await Future.wait<lk_models.TrackInfo>([room.engine.addTrack(req), negotiate()]);
209-
trackInfo = rets[0];
210-
} else {
211-
trackInfo = await room.engine.addTrack(req);
206+
Future<lk_models.TrackInfo> negotiate() async {
207+
track.transceiver = await room.engine.createTransceiverRTCRtpSender(track, publishOptions!, encodings);
208+
await room.engine.negotiate();
209+
return lk_models.TrackInfo();
210+
}
212211

213-
final transceiverInit = rtc.RTCRtpTransceiverInit(
214-
direction: rtc.TransceiverDirection.SendOnly,
215-
sendEncodings: encodings,
216-
);
217-
// addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web)
218-
track.transceiver = await room.engine.publisher?.pc.addTransceiver(
219-
track: track.mediaStreamTrack,
220-
kind: rtc.RTCRtpMediaType.RTCRtpMediaTypeAudio,
221-
init: transceiverInit,
222-
);
212+
late lk_models.TrackInfo trackInfo;
213+
if (room.engine.enabledPublishCodecs?.isNotEmpty ?? false) {
214+
final rets = await Future.wait<lk_models.TrackInfo>([room.engine.addTrack(req), negotiate()]);
215+
trackInfo = rets[0];
216+
} else {
217+
trackInfo = await room.engine.addTrack(req);
223218

224-
await room.engine.negotiate();
225-
}
219+
final transceiverInit = rtc.RTCRtpTransceiverInit(
220+
direction: rtc.TransceiverDirection.SendOnly,
221+
sendEncodings: encodings,
222+
);
223+
// addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web)
224+
track.transceiver = await room.engine.publisher?.pc.addTransceiver(
225+
track: track.mediaStreamTrack,
226+
kind: rtc.RTCRtpMediaType.RTCRtpMediaTypeAudio,
227+
init: transceiverInit,
228+
);
226229

227-
logger.fine('publishAudioTrack engine.addTrack response: ${trackInfo}');
230+
await room.engine.negotiate();
231+
}
228232

229-
track.lastPublishOptions = publishOptions;
233+
logger.fine('publishAudioTrack engine.addTrack response: ${trackInfo}');
230234

231-
final pub = LocalTrackPublication<LocalAudioTrack>(
232-
participant: this,
233-
info: trackInfo,
234-
track: track,
235-
);
236-
addTrackPublication(pub);
235+
track.lastPublishOptions = publishOptions;
237236

238-
// did publish
239-
await track.onPublish();
240-
await track.processor?.onPublish(room);
237+
final pub = LocalTrackPublication<LocalAudioTrack>(
238+
participant: this,
239+
info: trackInfo,
240+
track: track,
241+
);
242+
addTrackPublication(pub);
241243

242-
final listener = track.createListener();
243-
listener.on((TrackEndedEvent event) async {
244-
logger.fine('TrackEndedEvent: ${event.track}');
245-
await removePublishedTrack(pub.sid);
246-
});
244+
// did publish
245+
await track.onPublish();
246+
await track.processor?.onPublish(room);
247247

248-
[events, room.events].emit(LocalTrackPublishedEvent(
249-
participant: this,
250-
publication: pub,
251-
));
248+
final listener = track.createListener();
249+
listener.on((TrackEndedEvent event) async {
250+
logger.fine('TrackEndedEvent: ${event.track}');
251+
await removePublishedTrack(pub.sid);
252+
});
252253

253-
await track.start();
254+
[events, room.events].emit(LocalTrackPublishedEvent(
255+
participant: this,
256+
publication: pub,
257+
));
254258

255-
return pub;
259+
return pub;
260+
} catch (error) {
261+
if (shouldStopOnFailure) {
262+
try {
263+
await track.stop();
264+
} catch (stopError) {
265+
logger.warning('failed to stop audio track after publish failure: $stopError');
266+
}
267+
}
268+
rethrow;
269+
}
256270
}
257271

258272
/// Publish a [LocalVideoTrack] to the [Room].

0 commit comments

Comments
 (0)