Answer undeliverable client tool results with an error result

.changeset/client-tool-result-delivery.md

@@ -0,0 +1,5 @@
+---
+"@fishaudio/agent-client": patch
+---
+
+Client tool results that cannot be delivered no longer leave the agent waiting on its server-side tool timeout. A result larger than about 60 KB serialized (`MAX_CLIENT_TOOL_RESULT_BYTES`), a non-JSON-serializable value, or a transport error on send is replaced by an error result that tells the agent what happened, alongside the existing `tool_failed` error event.

docs/client-tools.md

@@ -26,6 +26,8 @@ session.registerClientTool("get_cart", async () => ({ items: await loadCart() })
 
 - A handler that throws (or times out after `clientToolTimeoutMs`, default 15 s) reports an error result to the agent and emits an `error` event (`tool_failed` / `tool_timeout`).
 - A call to a tool with no registered handler is answered with an error result — the conversation continues.
+- The serialized result must fit in one realtime message: about 60 KB (`MAX_CLIENT_TOOL_RESULT_BYTES`). A larger or non-JSON-serializable return value is replaced by an error result telling the agent why, and a `tool_failed` error is emitted. Return a summary or a reference (an id, a URL) instead of a large payload.
+- If a result cannot be sent at all (transport error), the SDK sends a short error result in its place so the agent does not wait out its server-side tool timeout.
 
 ## Observing tool activity
 

packages/client/src/__tests__/agentSession.test.ts

@@ -6,6 +6,7 @@ import type {
 } from "@fishaudio/agent-protocol";
 import { FishAgentError } from "../errors.js";
 import { startAgentSession, type AgentSessionOptions } from "../session/agentSession.js";
+import { MAX_CLIENT_TOOL_RESULT_BYTES } from "../session/toolDispatcher.js";
 import type {
   TranscriptionSegmentUpdate,
   Transport,
@@ -78,7 +79,14 @@ class MockTransport implements Transport {
     this.micEnabled = enabled;
   }
 
+  /** Rejects the next `count` sends with this error, then delivers normally. */
+  sendError?: { error: unknown; count: number };
+
   async sendClientEvent(message: ClientSessionMessage): Promise<void> {
+    if (this.sendError && this.sendError.count > 0) {
+      this.sendError.count -= 1;
+      throw this.sendError.error;
+    }
     this.sent.push(message);
   }
 
@@ -815,6 +823,71 @@ describe("client tools", () => {
     await tick();
     expect(transport.sent).toEqual([{ type: "client_tool.result", callId: "call-1" }]);
   });
+
+  it("replaces an oversized result with an error result", async () => {
+    const { session, transport } = await start({
+      clientTools: { lookup: () => ({ blob: "x".repeat(MAX_CLIENT_TOOL_RESULT_BYTES) }) },
+    });
+    const errors: FishAgentError[] = [];
+    session.on("error", (error) => errors.push(error));
+    transport.agent(call());
+    await tick();
+    expect(transport.sent).toHaveLength(1);
+    expect(transport.sent[0]).toMatchObject({
+      type: "client_tool.result",
+      callId: "call-1",
+      isError: true,
+      result: expect.stringContaining("too large"),
+    });
+    expect(errors[0]).toMatchObject({ code: "tool_failed" });
+  });
+
+  it("replaces a non-serializable result with an error result", async () => {
+    const circular: Record<string, unknown> = {};
+    circular.self = circular;
+    const { transport } = await start({ clientTools: { lookup: () => circular } });
+    transport.agent(call());
+    await tick();
+    expect(transport.sent[0]).toMatchObject({
+      isError: true,
+      result: expect.stringContaining("not JSON-serializable"),
+    });
+  });
+
+  it("falls back to an error result when the result cannot be sent", async () => {
+    const { session, transport } = await start({
+      clientTools: { lookup: () => ({ ok: true }) },
+    });
+    const errors: FishAgentError[] = [];
+    session.on("error", (error) => errors.push(error));
+    transport.sendError = { error: new Error("packet too large"), count: 1 };
+    transport.agent(call());
+    await tick();
+    expect(transport.sent).toEqual([
+      {
+        type: "client_tool.result",
+        callId: "call-1",
+        isError: true,
+        result: expect.stringContaining("could not be delivered"),
+      },
+    ]);
+    expect(errors).toHaveLength(1);
+    expect(errors[0]).toMatchObject({ code: "tool_failed" });
+  });
+
+  it("reports, without looping, when even the error result cannot be sent", async () => {
+    const { session, transport } = await start({
+      clientTools: { lookup: () => ({ ok: true }) },
+    });
+    const errors: FishAgentError[] = [];
+    session.on("error", (error) => errors.push(error));
+    transport.sendError = { error: new Error("disconnected"), count: 5 };
+    transport.agent(call());
+    await tick();
+    expect(transport.sent).toEqual([]);
+    expect(transport.sendError.count).toBe(3);
+    expect(errors.map((error) => error.code)).toEqual(["tool_failed", "tool_failed"]);
+  });
 });
 
 describe("lifecycle", () => {

packages/client/src/index.ts

@@ -19,6 +19,7 @@ export type {
   UserTranscriptEvent,
 } from "./events.js";
 export type { ClientToolHandler } from "./session/toolDispatcher.js";
+export { MAX_CLIENT_TOOL_RESULT_BYTES } from "./session/toolDispatcher.js";
 export type {
   AgentSessionCreateRequest,
   SessionOverrides,

packages/client/src/session/toolDispatcher.ts

@@ -8,6 +8,16 @@ export type ClientToolHandler = (
 
 export const DEFAULT_CLIENT_TOOL_TIMEOUT_MS = 15_000;
 
+/**
+ * Upper bound on the serialized `client_tool.result` message. The realtime
+ * data channel rejects packets above 64 000 bytes; this leaves headroom for
+ * the packet envelope. Larger results are replaced by an error result so the
+ * agent hears back instead of waiting out its server-side timeout.
+ */
+export const MAX_CLIENT_TOOL_RESULT_BYTES = 60_000;
+
+const textEncoder = new TextEncoder();
+
 type ClientToolCall = Extract<AgentSessionMessage, { type: "client_tool.call" }>;
 
 export class ClientToolDispatcher {
@@ -66,14 +76,64 @@ export class ClientToolDispatcher {
     callId: string,
     body: { result?: unknown; isError?: boolean },
   ): Promise<void> {
+    const message: ClientSessionMessage = { type: "client_tool.result", callId, ...body };
+    const rejected = this.undeliverable(message);
+    if (rejected !== undefined) {
+      this.onError(new FishAgentError("tool_failed", rejected));
+      await this.deliver({ type: "client_tool.result", callId, isError: true, result: rejected });
+      return;
+    }
     try {
-      await this.send({ type: "client_tool.result", callId, ...body });
+      await this.send(message);
     } catch (error) {
       this.onError(
         new FishAgentError("tool_failed", "Failed to deliver a client tool result", {
           cause: error,
         }),
       );
+      if (body.isError) {
+        return;
+      }
+      // The agent is still waiting on this call. Trade the lost result for a
+      // short error result so it can recover in conversation now, rather than
+      // after its server-side tool timeout.
+      await this.deliver({
+        type: "client_tool.result",
+        callId,
+        isError: true,
+        result: `Client tool result could not be delivered: ${String(error)}`,
+      });
+    }
+  }
+
+  /** Why the message cannot go over the wire as-is, or undefined if it can. */
+  private undeliverable(message: ClientSessionMessage): string | undefined {
+    let serialized: string;
+    try {
+      serialized = JSON.stringify(message);
+    } catch (error) {
+      return `Client tool result is not JSON-serializable: ${String(error)}`;
+    }
+    const bytes = textEncoder.encode(serialized).byteLength;
+    if (bytes > MAX_CLIENT_TOOL_RESULT_BYTES) {
+      return (
+        `Client tool result is too large to send (${bytes} bytes serialized; ` +
+        `limit ${MAX_CLIENT_TOOL_RESULT_BYTES}). Return a summary or a reference instead.`
+      );
+    }
+    return undefined;
+  }
+
+  /** Last-resort send of an error result; a failure here is reported, not retried. */
+  private async deliver(message: ClientSessionMessage): Promise<void> {
+    try {
+      await this.send(message);
+    } catch (error) {
+      this.onError(
+        new FishAgentError("tool_failed", "Failed to deliver a client tool error result", {
+          cause: error,
+        }),
+      );
     }
   }