Skip to content

Commit 8e148f1

Browse files
authored
Improve session api related docs (#946)
1 parent f1c3de9 commit 8e148f1

7 files changed

Lines changed: 140 additions & 10 deletions

File tree

‎.changes/agent-session-docs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
patch type="docs" "Improve Session API docs

‎lib/src/agent/chat/transcription_stream_receiver.dart‎

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,48 @@ import 'message_receiver.dart';
2525

2626
/// Converts LiveKit transcription text streams into [ReceivedMessage]s.
2727
///
28-
/// Each stream corresponds to a single message (agent or user). The stream
29-
/// yields textual updates which are aggregated until the message is finalized.
30-
/// When a new message for the same participant arrives, previous partial
31-
/// content is purged so that memory usage remains bounded.
28+
/// This receiver is intended for agent-powered transcription streams produced by
29+
/// the LiveKit Agents framework (text streams require `livekit-agents >= 1.0.0`).
30+
///
31+
/// The receiver listens on a text stream [topic] (default: `'lk.transcription'`)
32+
/// and aggregates chunked updates into a single [ReceivedMessage] per transcript
33+
/// segment.
34+
///
35+
/// ## Text stream semantics
36+
///
37+
/// - **Agent transcripts**: the agent emits a new text stream for each message.
38+
/// The stream yields chunks that should be appended until the transcript is
39+
/// finalized.
40+
/// - **User transcripts**: the agent may resend the full transcription text for
41+
/// a segment on each update (until finalized).
42+
///
43+
/// ## Message identity / diffing
44+
///
45+
/// The segment id (`lk.segment_id`) is stable across the lifetime of a transcript
46+
/// segment. This receiver uses it as [ReceivedMessage.id], which makes it safe
47+
/// to use for UI diffing (e.g. `ListView` keys). If `lk.segment_id` is missing,
48+
/// the text stream id is used as a fallback.
49+
///
50+
/// When a new segment for the same participant arrives, older partial segments
51+
/// are removed to keep memory usage bounded.
52+
///
53+
/// ## Example (agent transcript chunks)
54+
///
55+
/// Incoming chunks (same segment id):
56+
/// ```text
57+
/// { segment_id: "1", content: "Hello" }
58+
/// { segment_id: "1", content: " world" }
59+
/// { segment_id: "1", content: "!" }
60+
/// ```
61+
///
62+
/// Output messages:
63+
/// ```text
64+
/// ReceivedMessage(id: "1", content: AgentTranscript("Hello"))
65+
/// ReceivedMessage(id: "1", content: AgentTranscript("Hello world"))
66+
/// ReceivedMessage(id: "1", content: AgentTranscript("Hello world!"))
67+
/// ```
68+
///
69+
/// - SeeAlso: https://docs.livekit.io/agents/
3270
class TranscriptionStreamReceiver implements MessageReceiver {
3371
TranscriptionStreamReceiver({
3472
required Room room,

‎lib/src/agent/room_agent.dart‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ import 'constants.dart';
2121

2222
extension AgentRoom on Room {
2323
/// All agent participants currently in the room.
24+
///
25+
/// - Note: This excludes participants that are publishing on behalf of
26+
/// another participant (for example, "avatar worker" participants). Those
27+
/// workers can be discovered by filtering [remoteParticipants] for
28+
/// participants whose `lk.publish_on_behalf` attribute matches the agent's
29+
/// identity.
2430
Iterable<RemoteParticipant> get agentParticipants => remoteParticipants.values.where(
2531
(participant) {
2632
if (participant.kind != ParticipantKind.AGENT) {

‎lib/src/agent/session.dart‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,14 @@ import 'session_options.dart';
4949
/// be sent with [sendText], and the message history can be inspected or restored
5050
/// via [messages], [getMessageHistory], and [restoreMessageHistory].
5151
///
52+
/// Message transport is pluggable: you can provide custom [MessageSender] and
53+
/// [MessageReceiver] implementations to integrate with different channels. By
54+
/// default, [Session] uses:
55+
/// - [TextMessageSender] (topic `'lk.chat'`) to send user text and emit loopback
56+
/// messages for immediate UI updates.
57+
/// - [TranscriptionStreamReceiver] (topic `'lk.transcription'`) to receive agent
58+
/// and user transcripts as a message stream.
59+
///
5260
/// The session is designed to be observed from Flutter widgets (it extends
5361
/// [ChangeNotifier] through [DisposableChangeNotifier]) in the same way that the
5462
/// Swift implementation conforms to `ObservableObject`.

‎lib/src/core/room_preconnect.dart‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,31 @@ import '../preconnect/pre_connect_audio_buffer.dart';
1919
import 'room.dart';
2020

2121
extension RoomPreConnect on Room {
22-
/// Wrap an async operation while a pre-connect audio buffer records.
23-
/// Stops and flushes on error.
22+
/// Runs an async [operation] while a pre-connect audio buffer records.
23+
///
24+
/// This is primarily used for voice agent experiences to reduce perceived
25+
/// latency: the microphone starts recording before [Room.connect] completes,
26+
/// and the buffered audio is sent to the agent once it becomes active.
27+
///
28+
/// If [operation] throws, recording is stopped and the buffer is reset
29+
/// (discarding any buffered audio).
30+
///
31+
/// Example:
32+
/// ```dart
33+
/// await room.withPreConnectAudio(
34+
/// () async {
35+
/// final creds = await tokenService.fetch();
36+
/// await room.connect(creds.serverUrl, creds.participantToken);
37+
/// return true;
38+
/// },
39+
/// timeout: const Duration(seconds: 20),
40+
/// onError: (error) => logger.warning('Preconnect failed: $error'),
41+
/// );
42+
/// ```
43+
///
44+
/// - Note: Ensure microphone permissions are granted early in your app
45+
/// lifecycle so pre-connect can start without additional prompts.
46+
/// - SeeAlso: [PreConnectAudioBuffer]
2447
Future<T> withPreConnectAudio<T>(
2548
Future<T> Function() operation, {
2649
Duration timeout = const Duration(seconds: 10),

‎lib/src/json/agent_attributes.dart‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,18 +76,32 @@ enum AgentState {
7676

7777
@JsonSerializable()
7878
class TranscriptionAttributes {
79+
/// Schema for transcription-related text stream attributes.
80+
///
81+
/// These attributes are attached to LiveKit text streams used for agent and
82+
/// user transcriptions (typically on the `'lk.transcription'` topic).
83+
///
84+
/// - Note: Some agent implementations may encode `lk.transcription_final` as a
85+
/// boolean or a string (`"true"`/`"false"`/`"1"`/`"0"`). This model accepts
86+
/// both forms.
7987
const TranscriptionAttributes({
8088
this.lkSegmentId,
8189
this.lkTranscribedTrackId,
8290
this.lkTranscriptionFinal,
8391
});
8492

93+
/// Stable identifier for the transcript segment (`lk.segment_id`).
94+
///
95+
/// A segment id remains stable for the lifetime of a transcript segment and
96+
/// can be used to reconcile incremental updates in UIs.
8597
@JsonKey(name: 'lk.segment_id')
8698
final String? lkSegmentId;
8799

100+
/// Track id associated with the transcription (`lk.transcribed_track_id`).
88101
@JsonKey(name: 'lk.transcribed_track_id')
89102
final String? lkTranscribedTrackId;
90103

104+
/// Whether this segment is finalized (`lk.transcription_final`).
91105
@JsonKey(name: 'lk.transcription_final', fromJson: _boolFromJson, toJson: _boolToJson)
92106
final bool? lkTranscriptionFinal;
93107

‎lib/src/preconnect/pre_connect_audio_buffer.dart‎

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,17 @@ import '../types/participant_state.dart';
3434

3535
typedef PreConnectOnError = void Function(Object error);
3636

37+
/// Captures and buffers microphone audio before a room connection completes.
38+
///
39+
/// This is used by `Room.withPreConnectAudio` to reduce perceived latency for
40+
/// voice agent experiences: the microphone can begin recording while the app is
41+
/// still connecting and dispatching an agent, then the buffered audio is sent
42+
/// once the agent becomes active.
43+
///
44+
/// Audio is buffered in memory (bounded by [defaultMaxSize]); if it overflows,
45+
/// the oldest audio is dropped until the agent is ready.
3746
class PreConnectAudioBuffer {
47+
/// Topic used to send the buffered audio stream to agents.
3848
static const String dataTopic = 'lk.agent.pre-connect-audio-buffer';
3949

4050
static const int defaultMaxSize = 10 * 1024 * 1024; // 10MB
@@ -72,16 +82,33 @@ class PreConnectAudioBuffer {
7282
}) : _onError = onError,
7383
_requestSampleRate = sampleRate;
7484

75-
// Getters
85+
/// Whether pre-connect recording is currently active.
7686
bool get isRecording => _isRecording;
87+
88+
/// Number of buffered bytes currently stored.
7789
int get bufferedSize => _buffer.length;
90+
91+
/// The local audio track used for recording while pre-connecting.
7892
LocalAudioTrack? get localTrack => _localTrack;
7993

8094
Future<LocalTrackPublishedEvent>? _localTrackPublishedEvent;
8195

82-
/// Future that completes when an agent is ready.
96+
/// Completes when an agent becomes active and the buffer has been sent.
97+
///
98+
/// If the configured timeout elapses before an agent becomes active, this
99+
/// future completes with an error.
83100
Future<void> get agentReadyFuture => _agentReadyManager.future;
84101

102+
/// Starts capturing audio into an in-memory buffer.
103+
///
104+
/// [timeout] controls how long to wait for an agent to become active. When the
105+
/// agent becomes active, buffered audio is sent automatically and
106+
/// [agentReadyFuture] completes. If the timeout is reached first,
107+
/// [agentReadyFuture] completes with an error and callers should [reset] the
108+
/// buffer.
109+
///
110+
/// - Note: Ensure microphone permissions are granted before calling this, or
111+
/// audio capture may fail depending on platform.
85112
Future<void> startRecording({
86113
Duration timeout = const Duration(seconds: 20),
87114
}) async {
@@ -180,6 +207,11 @@ class PreConnectAudioBuffer {
180207
));
181208
}
182209

210+
/// Stops recording and releases native audio capture resources.
211+
///
212+
/// If [withError] is provided, [agentReadyFuture] completes with that error.
213+
/// Otherwise, [agentReadyFuture] completes successfully (if not already
214+
/// completed).
183215
Future<void> stopRecording({Object? withError}) async {
184216
if (!_isRecording) return;
185217
_isRecording = false;
@@ -219,7 +251,8 @@ class PreConnectAudioBuffer {
219251
logger.info('[Preconnect audio] stopped recording');
220252
}
221253

222-
// Clean-up & reset for re-use
254+
/// Stops recording and clears buffered audio and listeners so the instance
255+
/// can be reused.
223256
Future<void> reset() async {
224257
await stopRecording();
225258
_timeoutTimer?.cancel();
@@ -240,12 +273,19 @@ class PreConnectAudioBuffer {
240273
logger.info('[Preconnect audio] reset');
241274
}
242275

243-
// Dispose the audio buffer and clean up all resources.
276+
/// Disposes this buffer (alias for [reset]).
244277
Future<void> dispose() async {
245278
await reset();
246279
logger.info('[Preconnect audio] disposed');
247280
}
248281

282+
/// Sends the currently buffered audio to one or more agent identities.
283+
///
284+
/// This is a one-shot operation; repeated calls are ignored after the buffer
285+
/// has been sent.
286+
///
287+
/// The stream is written to [topic] (default: [dataTopic]) and includes
288+
/// attributes that help the agent interpret the raw audio payload.
249289
Future<void> sendAudioData({
250290
required List<String> agents,
251291
String topic = dataTopic,

0 commit comments

Comments
 (0)