diff --git a/.changeset/dynamic-tool-schema-replay.md b/.changeset/dynamic-tool-schema-replay.md new file mode 100644 index 0000000000..ef98282600 --- /dev/null +++ b/.changeset/dynamic-tool-schema-replay.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Reject dynamic tool input schemas containing Zod transformations or custom validation that JSON Schema replay would silently discard. Errors now identify the tool and explain how to move runtime validation into its durable executor; opaque Standard Schema validators must provide JSON Schema explicitly. diff --git a/docs/guides/dynamic-capabilities.md b/docs/guides/dynamic-capabilities.md index fe21376b74..708a9c6905 100644 --- a/docs/guides/dynamic-capabilities.md +++ b/docs/guides/dynamic-capabilities.md @@ -247,6 +247,49 @@ export default defineDynamic({ }); ``` +### Input schemas and replay + +Dynamic tools persist `inputSchema` as JSON Schema and rebuild validation from +that snapshot. Use Zod 3 or Zod 4 constraints that JSON Schema can express, such +as `.min()`, `.max()`, `.regex()`, and `.optional()`, or provide plain JSON Schema. + +eve rejects Zod transformations, including `.trim()`, case conversion, +`.transform()`, preprocessing, coercion, custom refinements, `.pipe()`, and +`.catch()`, because JSON Schema cannot preserve their runtime behavior. Other +Standard Schema validators are opaque to this check; provide their JSON Schema +explicitly and perform custom validation in `execute()`. A rejected input +schema logs an error with the tool name and omits the resolver's complete +result. + +Keep normalization and custom validation inside a replayable executor. For +example, this tool rejects whitespace-only input and trims valid input every +time it executes, including after replay: + +```ts title="agent/tools/normalize.ts" +import { defineDynamic, defineTool } from "eve/tools"; +import { z } from "zod"; + +const runtimeInput = z.object({ value: z.string().trim().min(1) }); + +export default defineDynamic({ + events: { + "session.started": () => + defineTool({ + description: "Normalize a nonempty value.", + inputSchema: z.object({ value: z.string().min(1).regex(/\S/) }), + execute(input) { + return runtimeInput.parse(input); + }, + }), + }, +}); +``` + +The module-level validator remains available to the durable callback without +being captured in session state. This restriction applies only to dynamic tool +inputs. Sessions that already persisted a lossy schema must be restarted or re-resolved by a new +deployment; the missing transformation cannot be recovered from JSON Schema. + ### Author replayable callbacks Write callback properties as inline function expressions, arrows, method shorthand, or module-level function references. eve transforms authored modules that import `defineTool`, including helper modules outside `agent/tools/`, and stores each callback's referenced closure values independently. diff --git a/docs/tools/overview.mdx b/docs/tools/overview.mdx index 6dc4ee578f..c3785b01b9 100644 --- a/docs/tools/overview.mdx +++ b/docs/tools/overview.mdx @@ -32,6 +32,10 @@ A tool definition needs: When a tool returns structured data, add an optional `outputSchema`. With Zod or Standard Schema it also types the `execute` return. +Tools returned by `defineDynamic` persist their input schemas as JSON Schema. +See [Input schemas and replay](/docs/guides/dynamic-capabilities#input-schemas-and-replay) +for supported validators and where to put transformations or custom validation. + ### Stream preliminary tool results An async generator lets a long-running tool stream complete output snapshots diff --git a/e2e/fixtures/agent-tools/agent/agent.ts b/e2e/fixtures/agent-tools/agent/agent.ts index 384bb0e2aa..b10b480a90 100644 --- a/e2e/fixtures/agent-tools/agent/agent.ts +++ b/e2e/fixtures/agent-tools/agent/agent.ts @@ -1,7 +1,9 @@ import { e2eAgentConfig } from "@eve-e2e/config"; import { defineAgent } from "eve"; +import { respond } from "./lib/schema-replay-responder"; + export default defineAgent({ - ...e2eAgentConfig(), + ...e2eAgentConfig({ mock: respond }), reasoning: "high", }); diff --git a/e2e/fixtures/agent-tools/agent/lib/schema-replay-responder.ts b/e2e/fixtures/agent-tools/agent/lib/schema-replay-responder.ts new file mode 100644 index 0000000000..a55947a9a8 --- /dev/null +++ b/e2e/fixtures/agent-tools/agent/lib/schema-replay-responder.ts @@ -0,0 +1,14 @@ +import type { MockModelRequest, MockModelResponse } from "eve/evals"; + +export function respond(request: MockModelRequest): MockModelResponse | string { + const message = request.lastUserMessage ?? ""; + if (!message.includes("schema replay regression")) return `Mock reply: ${message}`; + if (request.tools.some((tool) => tool.name === "invalid_dynamic_schema")) { + throw new Error("A dynamic schema with an unsupported transformation reached the model."); + } + + const value = message.includes("whitespace") ? " " : " accepted "; + const id = `schema-replay-${request.userMessageCount}`; + if (request.toolResults.some((result) => result.id === id)) return "Schema replay checked."; + return { toolCalls: [{ id, name: "normalize_dynamic", input: { value } }] }; +} diff --git a/e2e/fixtures/agent-tools/agent/tools/dynamic-schema-replay.ts b/e2e/fixtures/agent-tools/agent/tools/dynamic-schema-replay.ts new file mode 100644 index 0000000000..c5da7a13af --- /dev/null +++ b/e2e/fixtures/agent-tools/agent/tools/dynamic-schema-replay.ts @@ -0,0 +1,30 @@ +import { defineDynamic, defineTool } from "eve/tools"; +import { z } from "zod"; + +const runtimeInput = z.object({ value: z.string().trim().min(1) }); + +export default defineDynamic({ + events: { + "session.started": () => ({ + normalize_dynamic: defineTool({ + description: "Call only when asked to normalize_dynamic for the schema replay regression.", + inputSchema: z.object({ value: z.string().min(1) }), + execute(input) { + return runtimeInput.parse(input); + }, + }), + }), + "turn.started": (_event, ctx) => { + if (!JSON.stringify(ctx.messages).includes("schema replay regression")) return null; + return { + invalid_dynamic_schema: defineTool({ + description: "Must be rejected before the model sees it.", + inputSchema: z.object({ value: z.string().trim().min(1).optional() }), + execute(input) { + return input; + }, + }), + }; + }, + }, +}); diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/schema-replay.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/schema-replay.eval.ts new file mode 100644 index 0000000000..72d2366c47 --- /dev/null +++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/schema-replay.eval.ts @@ -0,0 +1,29 @@ +import { defineEval } from "eve/evals"; + +export default defineEval({ + description: + "Dynamic schemas reject lossy validation and durable executors retain normalization.", + async test(t) { + if (process.env.EVE_E2E_MODEL !== "mock") { + t.skip( + "Requires the deterministic mock model to inspect available tools and send exact whitespace.", + ); + return; + } + + const first = await t.send("Run the schema replay regression with accepted input."); + first.expectOk(); + first.calledTool("normalize_dynamic", { output: { value: "accepted" } }); + + const replayed = await t.send("Run the schema replay regression again with accepted input."); + replayed.expectOk(); + replayed.calledTool("normalize_dynamic", { output: { value: "accepted" } }); + + const rejected = await t.send("Run the schema replay regression with whitespace input."); + rejected.expectOk(); + rejected.calledTool("normalize_dynamic", { status: "failed" }); + + t.notCalledTool("invalid_dynamic_schema"); + t.succeeded(); + }, +}); diff --git a/packages/eve/src/context/dynamic-tool-lifecycle.test.ts b/packages/eve/src/context/dynamic-tool-lifecycle.test.ts index 51904e8ffd..b80f9eefc7 100644 --- a/packages/eve/src/context/dynamic-tool-lifecycle.test.ts +++ b/packages/eve/src/context/dynamic-tool-lifecycle.test.ts @@ -768,6 +768,36 @@ describe("dispatchDynamicToolEvent", () => { expect(ctx.get(SessionDynamicToolRuntimeRevisionKey)).toBe("deployment:dpl_new"); }); + it.each(["session.started", "turn.started", "step.started"] as const)( + "omits a resolver with a lossy input schema at %s, including after durable replay", + async (eventType) => { + const ctx = createCtx(); + const execute = vi.fn(async () => ({ ok: true })); + const resolver = createResolver("normalize", [eventType], () => ({ + unsafe: stampTestTool( + defineTool({ + description: "Must not accept whitespace without trimming it first", + inputSchema: z.object({ value: z.string().trim().min(1).optional() }), + execute, + }), + ), + sibling: createReplayableTool(), + })); + + await dispatchDynamicToolEvent({ + ctx, + resolvers: [resolver], + messages: [], + event: makeEvent(eventType), + }); + + expect(buildDynamicTools(ctx)).toEqual([]); + const replayed = await deserializeContext(JSON.parse(JSON.stringify(serializeContext(ctx)))); + expect(buildDynamicTools(replayed)).toEqual([]); + expect(execute).not.toHaveBeenCalled(); + }, + ); + it("leaves unsupported connection input schemas for the MCP server to validate", async () => { const ctx = createCtx(); const inputSchema: JsonObject = { diff --git a/packages/eve/src/context/dynamic-tool-lifecycle.ts b/packages/eve/src/context/dynamic-tool-lifecycle.ts index 0a80a9a0bd..46b20aaeda 100644 --- a/packages/eve/src/context/dynamic-tool-lifecycle.ts +++ b/packages/eve/src/context/dynamic-tool-lifecycle.ts @@ -36,7 +36,8 @@ import { } from "#tools/durable-callbacks.js"; import { toErrorMessage } from "#shared/errors.js"; import { parseJsonObject } from "#shared/json.js"; -import { serializeInputSchema, serializeOutputSchema } from "#tools/schema.js"; +import { serializeOutputSchema } from "#tools/schema.js"; +import { serializeDynamicToolInputSchema } from "#context/dynamic-tool-schema.js"; import type { ResolvedDynamicToolResolver } from "#runtime/types.js"; const log = createLogger("dynamic-tools"); @@ -227,12 +228,13 @@ function createMetadata(input: { readonly name: string; readonly resolver: ResolvedDynamicToolResolver; }): CurrentDynamicToolMetadata { + const inputSchema = serializeDynamicToolInputSchema(input.name, input.entry.inputSchema); return { callbacks: validateDurableDynamicToolCallbacks(input.name, input.entry), description: input.entry.description, execution: input.entry.execution === "background" ? "background" : undefined, entryKey: input.entryKey, - inputSchema: serializeInputSchema(input.entry.inputSchema), + inputSchema, name: input.name, outputSchema: serializeOutputSchema(input.entry.outputSchema), resolverSlug: input.resolver.slug, diff --git a/packages/eve/src/context/dynamic-tool-schema.test.ts b/packages/eve/src/context/dynamic-tool-schema.test.ts new file mode 100644 index 0000000000..af990b32a0 --- /dev/null +++ b/packages/eve/src/context/dynamic-tool-schema.test.ts @@ -0,0 +1,108 @@ +import { asSchema } from "ai"; +import { describe, expect, it } from "vitest"; +import { z } from "zod"; +import { z as z3 } from "zod/v3"; + +import { serializeDynamicToolInputSchema } from "#context/dynamic-tool-schema.js"; +import { toInputSchema } from "#tools/schema.js"; + +describe("dynamic tool input schemas", () => { + it.each([ + ["Zod 4 trim", z.object({ value: z.string().trim().min(1).optional() })], + ["Zod 3 trim", z3.object({ value: z3.string().trim().min(1).optional() })], + ])("rejects %s before replay can accept whitespace", async (_label, schema) => { + expect(await schema["~standard"].validate({ value: " " })).toHaveProperty("issues"); + expect(() => serializeDynamicToolInputSchema("normalize", schema)).toThrow( + /Dynamic tool "normalize" inputSchema contains Zod .*cannot be preserved.*move normalization or custom validation into execute\(\)/, + ); + }); + + it.each([ + ["transform", z.string().transform((value) => value.length)], + ["overwrite", z.string().overwrite((value) => value.trim())], + ["lowercase", z.string().toLowerCase()], + ["uppercase", z.string().toUpperCase()], + ["preprocess", z.preprocess(String, z.string())], + ["coercion", z.coerce.number()], + ["refinement", z.string().refine((value) => value !== "forbidden")], + ["super refinement", z.string().superRefine(() => {})], + ["custom", z.custom(() => true)], + ["pipe", z.string().pipe(z.string().min(3))], + ["catch", z.string().catch("fallback")], + ["Zod 3 transform", z3.string().transform((value) => value.length)], + ["Zod 3 refinement", z3.string().refine((value) => value !== "forbidden")], + ["Zod 3 preprocess", z3.preprocess(String, z3.string())], + ["Zod 3 lowercase", z3.string().toLowerCase()], + ["Zod 3 uppercase", z3.string().toUpperCase()], + ["Zod 3 coercion", z3.coerce.number()], + ["Zod 3 pipe", z3.string().pipe(z3.string().min(3))], + ["Zod 3 catch", z3.string().catch("fallback")], + ])("rejects %s", (_label, schema) => { + expect(() => serializeDynamicToolInputSchema("tool", schema)).toThrow( + "cannot be preserved in JSON Schema during replay", + ); + }); + + it.each([ + ["array", z.array(z.string().trim())], + ["tuple rest", z.tuple([]).rest(z.string().trim())], + ["union", z.union([z.number(), z.string().trim()])], + ["intersection", z.intersection(z.object({}), z.object({ value: z.string().trim() }))], + ["record key", z.record(z.string().trim(), z.string())], + ["record value", z.record(z.string(), z.string().trim())], + ["catchall", z.object({}).catchall(z.string().trim())], + ["nullable", z.string().trim().nullable()], + ["default", z.string().trim().default("value")], + ["lazy", z.lazy(() => z.string().trim())], + ["Zod 3 array", z3.array(z3.string().trim())], + ["Zod 3 lazy", z3.lazy(() => z3.string().trim())], + [ + "schema-like property names", + z.object({ _def: z.string(), _zod: z.string(), value: z.string().trim() }), + ], + ])("finds transformations inside %s", (_label, schema) => { + expect(() => serializeDynamicToolInputSchema("nested", schema)).toThrow( + "cannot be preserved in JSON Schema during replay", + ); + }); + + it.each([ + ["Zod 4", z.object({ value: z.string().min(1).regex(/\S/).optional() })], + ["Zod 3", z3.object({ value: z3.string().min(1).regex(/\S/).optional() })], + ])("replays supported %s validation after a JSON round trip", async (_label, source) => { + const serialized = serializeDynamicToolInputSchema("validate", source); + const validate = asSchema(toInputSchema(JSON.parse(JSON.stringify(serialized)))).validate; + await expect(validate?.({ value: " " })).resolves.toMatchObject({ success: false }); + await expect(validate?.({ value: "valid" })).resolves.toMatchObject({ success: true }); + await expect(validate?.({})).resolves.toMatchObject({ success: true }); + }); + + it("handles recursive schemas without revisiting them", () => { + const schema = z.object({ + value: z.string(), + get children() { + return z.array(schema); + }, + }); + expect(serializeDynamicToolInputSchema("tree", schema)).toMatchObject({ type: "object" }); + }); + + it("requires opaque Standard Schema validators to declare their replay contract explicitly", () => { + const schema = { + "~standard": { + version: 1 as const, + vendor: "custom", + validate: (value: unknown) => ({ value }), + jsonSchema: { input: () => ({ type: "string" }), output: () => ({ type: "string" }) }, + }, + }; + expect(() => serializeDynamicToolInputSchema("custom", schema)).toThrow( + "an opaque Standard Schema validator", + ); + }); + + it("preserves plain JSON Schema without interpreting its data as Zod internals", () => { + const schema = { type: "object", properties: { _def: { const: { type: "transform" } } } }; + expect(serializeDynamicToolInputSchema("remote", schema)).toEqual(schema); + }); +}); diff --git a/packages/eve/src/context/dynamic-tool-schema.ts b/packages/eve/src/context/dynamic-tool-schema.ts new file mode 100644 index 0000000000..42efd505f4 --- /dev/null +++ b/packages/eve/src/context/dynamic-tool-schema.ts @@ -0,0 +1,96 @@ +import type { JsonObject } from "#shared/json.js"; +import { serializeInputSchema, type ToolSchemaSource } from "#tools/schema.js"; + +/** JSON Schema cannot persist executable validation or normalization logic. */ +export function serializeDynamicToolInputSchema( + name: string, + source: ToolSchemaSource, +): JsonObject { + const seen = new Set(); + + function unsupported(feature: string): never { + throw new Error( + `Dynamic tool "${name}" inputSchema contains ${feature}, which cannot be preserved in JSON Schema during replay. ` + + "Use a JSON Schema-compatible inputSchema and move normalization or custom validation into execute().", + ); + } + + function visit(value: unknown): void { + if (typeof value !== "object" || value === null || seen.has(value)) return; + seen.add(value); + if (Array.isArray(value)) { + for (const item of value) visit(item); + return; + } + + const record = value as Record; + const standard = record["~standard"] as { validate?: unknown } | undefined; + if (typeof standard?.validate !== "function") { + // Object shapes are maps of schemas, including getter-backed recursive fields. + for (const item of Object.values(record)) visit(item); + return; + } + const zod = record._zod as { def?: Record } | undefined; + const def = zod?.def ?? (record._def as Record | undefined); + if (def === undefined) unsupported("an unrecognized Zod validator"); + + const type = def.typeName ?? def.type; + if ( + type === "transform" || + type === "pipe" || + type === "custom" || + type === "catch" || + type === "ZodEffects" || + type === "ZodPipeline" || + type === "ZodCatch" + ) { + unsupported(`Zod ${String(type)}`); + } + if (def.coerce === true) unsupported("Zod coercion"); + + for (const check of (def.checks ?? []) as Array>) { + const checkDef = (check._zod as { def?: Record } | undefined)?.def; + const kind = checkDef?.check ?? check.kind; + if ( + kind === "overwrite" || + kind === "custom" || + kind === "trim" || + kind === "toLowerCase" || + kind === "toUpperCase" + ) { + unsupported(`Zod ${String(kind)}`); + } + } + + if (type === "lazy" || type === "ZodLazy") { + visit((def.getter as () => unknown)()); + } + if (type === "ZodObject") visit((def.shape as () => unknown)()); + for (const key of [ + "shape", + "innerType", + "schema", + "type", + "element", + "items", + "options", + "left", + "right", + "keyType", + "valueType", + "rest", + "catchall", + ]) { + visit(def[key]); + } + } + + // Plain JSON Schema already declares the complete replay contract. Opaque + // Standard Schema validators cannot prove that their emitted schema does. + if ("~standard" in source) { + const standard = source["~standard"] as { vendor?: string }; + if (standard.vendor !== "zod") unsupported("an opaque Standard Schema validator"); + visit(source); + } + return serializeInputSchema(source); +}